[update]添加 定时任务

DB/DbClient.py

@@ -15,6 +15,7 @@ __author__ = 'JHao'
 import os
 import sys
 from Util.GetConfig import GetConfig
+from Util.utilClass import Singleton
 
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 
@@ -24,6 +25,8 @@ class DbClient(object):
     DbClient
     """
 
+    __metaclass__ = Singleton
+
     def __init__(self):
         """
         init
@@ -67,5 +70,8 @@ class DbClient(object):
 
 
 if __name__ == "__main__":
-    account = DbClient().delete('123')
+    account = DbClient()
+    print account.get()
+    account.changeTable('use')
+    account.put('ac')
     print(account)

DB/SsdbClient.py

@@ -40,7 +40,7 @@ class SsdbClient(object):
         :return:
         """
         values = self.__conn.hgetall(name=self.name)
-        return random.choice(values.keys())
+        return random.choice(values.keys()) if values else None
 
     def put(self, value):
         """
@@ -57,7 +57,8 @@ class SsdbClient(object):
         :return:
         """
         key = self.get()
-        self.__conn.hdel(self.name, key)
+        if key:
+            self.__conn.hdel(self.name, key)
         return key
 
     def delete(self, key):

Manager/ProxyManager.py

@@ -13,7 +13,8 @@
 __author__ = 'JHao'
 
 from DB.DbClient import DbClient
-
+from Util.GetConfig import GetConfig
+from ProxyGetter.GetFreeProxy import GetFreeProxy
 
 class ProxyManager(object):
     """
@@ -21,4 +22,53 @@ class ProxyManager(object):
     """
 
     def __init__(self):
-        self.db = DbClient
+        self.db = DbClient()
+        self.config = GetConfig()
+        self.raw_proxy_queue = 'raw_proxy'
+        self.useful_proxy_queue = 'useful_proxy_queue'
+
+    def refresh(self):
+        """
+        fetch proxy into Db by ProxyGetter
+        :return:
+        """
+        for proxyGetter in self.config.proxy_getter_functions:
+            proxy_set = set()
+            # fetch raw proxy
+            for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
+                proxy_set.add(proxy)
+
+            # store raw proxy
+            self.db.changeTable(self.raw_proxy_queue)
+            for proxy in proxy_set:
+                self.db.put(proxy)
+
+    def get(self):
+        """
+        return a useful proxy
+        :return:
+        """
+        self.db.changeTable(self.useful_proxy_queue)
+        return self.db.pop()
+
+    def delete(self, proxy):
+        """
+        delete proxy from pool
+        :param proxy:
+        :return:
+        """
+        self.db.changeTable(self.useful_proxy_queue)
+        self.db.delete(proxy)
+
+    def getAll(self):
+        """
+        get all proxy from pool
+        :return:
+        """
+        self.db.changeTable(self.useful_proxy_queue)
+        return self.db.getAll()
+
+
+if __name__ == '__main__':
+    pp = ProxyManager()
+    pp.refresh()

Schedule/ProxyRefreshSchedule.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     ProxyRefreshSchedule.py  
+   Description :  代理定时刷新
+   Author :       JHao
+   date：          2016/12/4
+-------------------------------------------------
+   Change Activity:
+                   2016/12/4: 代理定时刷新
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from apscheduler.schedulers.blocking import BlockingScheduler
+from multiprocessing import Process
+import requests
+import time
+
+from Manager.ProxyManager import ProxyManager
+
+
+class ProxyRefreshSchedule(ProxyManager):
+    """
+    代理定时刷新
+    """
+
+    def __init__(self):
+        ProxyManager.__init__(self)
+
+    def validProxy(self):
+        self.db.changeTable(self.raw_proxy_queue)
+        raw_proxy = self.db.pop()
+        while raw_proxy:
+            proxies = {"http": "http://{proxy}".format(proxy=raw_proxy),
+                       "https": "https://{proxy}".format(proxy=raw_proxy)}
+            try:
+                r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=50, verify=False)
+                if r.status_code == 200:
+                    self.db.changeTable(self.useful_proxy_queue)
+                    self.db.put(raw_proxy)
+            except Exception as e:
+                # print e
+                pass
+            self.db.changeTable(self.raw_proxy_queue)
+            raw_proxy = self.db.pop()
+
+
+def refreshPool():
+    pp = ProxyRefreshSchedule()
+    pp.validProxy()
+
+
+def main(process_num=100):
+    p = ProxyRefreshSchedule()
+    p.refresh()
+
+    for num in range(process_num):
+        P = Process(target=refreshPool, args=())
+        P.start()
+    print '{time}: refresh complete!'.format(time=time.ctime())
+
+
+if __name__ == '__main__':
+    # pp = ProxyRefreshSchedule()
+    # pp.main()
+    main()
+    sched = BlockingScheduler()
+    sched.add_job(main, 'interval', minute=20)
+    sched.start()

Util/utilClass.py

@@ -8,6 +8,7 @@
 -------------------------------------------------
    Change Activity:
                    2016/12/3: Class LazyProperty
+                   2016/12/4: rewrite ConfigParser
 -------------------------------------------------
 """
 __author__ = 'JHao'
@@ -44,3 +45,16 @@ class ConfigParse(ConfigParser):
 
     def optionxform(self, optionstr):
         return optionstr
+
+
+class Singleton(type):
+    """
+    Singleton Metaclass
+    """
+
+    _inst = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._inst:
+            cls._inst[cls] = super(Singleton, cls).__call__(*args)
+        return cls._inst[cls]