[update] 修改日志

Manager/ProxyManager.py

@@ -15,6 +15,7 @@ __author__ = 'JHao'
 
 from DB.DbClient import DbClient
 from Util.GetConfig import GetConfig
+from Util.LogHandler import LogHandler
 from ProxyGetter.getFreeProxy import GetFreeProxy
 
 
@@ -27,6 +28,7 @@ class ProxyManager(object):
         self.db = DbClient()
         self.config = GetConfig()
         self.raw_proxy_queue = 'raw_proxy'
+        self.log = LogHandler('proxy_manager')
         self.useful_proxy_queue = 'useful_proxy_queue'
 
     def refresh(self):
@@ -39,6 +41,7 @@ class ProxyManager(object):
             # fetch raw proxy
             for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
                 if proxy.strip():
+                    self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy))
                     proxy_set.add(proxy.strip())
 
             # store raw proxy
@@ -81,5 +84,5 @@ class ProxyManager(object):
 
 if __name__ == '__main__':
     pp = ProxyManager()
-    # pp.refresh()
-    print pp.get_status()
+    pp.refresh()
+    # print pp.get_status()

Schedule/ProxyRefreshSchedule.py

@@ -15,7 +15,7 @@
 
 import sys
 import time
-from multiprocessing import Process
+from threading import Thread
 from apscheduler.schedulers.blocking import BlockingScheduler
 
 sys.path.append('../')
@@ -48,7 +48,7 @@ class ProxyRefreshSchedule(ProxyManager):
             if validUsefulProxy(raw_proxy):
                 self.db.changeTable(self.useful_proxy_queue)
                 self.db.put(raw_proxy)
-                self.log.debug('proxy: %s validation passes' % raw_proxy)
+                self.log.info('proxy: %s validation pass' % raw_proxy)
             else:
                 self.log.debug('proxy: %s validation fail' % raw_proxy)
                 pass
@@ -62,7 +62,7 @@ def refresh_pool():
     pp.valid_proxy()
 
 
-def main(process_num=10):
+def main(process_num=30):
     p = ProxyRefreshSchedule()
 
     # 获取新代理
@@ -71,7 +71,7 @@ def main(process_num=10):
     # 检验新代理
     pl = []
     for num in range(process_num):
-        proc = Process(target=refresh_pool, args=())
+        proc = Thread(target=refresh_pool, args=())
         pl.append(proc)
 
     for num in range(process_num):

Util/utilFunction.py

@@ -64,11 +64,10 @@ def validUsefulProxy(proxy):
     :param proxy:
     :return:
     """
-    proxies = {"http": "http://{proxy}".format(proxy=proxy),
-               "https": "https://{proxy}".format(proxy=proxy)}
+    proxies = {"https": "https://{proxy}".format(proxy=proxy)}
     try:
         # 超过30秒的代理就不要了
-        r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=30, verify=False)
+        r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=20, verify=False)
         if r.status_code == 200:
             return True
     except Exception, e: