1.flask支持多进程处理任务

2.优化 proxy 采集、校验流程，加快 userfull proxy 校验速度

Api/ProxyApi.py

@@ -2,13 +2,13 @@
 # !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     ProxyApi.py  
-   Description :  
+   File Name：     ProxyApi.py
+   Description :
    Author :       JHao
    date：          2016/12/4
 -------------------------------------------------
    Change Activity:
-                   2016/12/4: 
+                   2016/12/4:
 -------------------------------------------------
 """
 __author__ = 'JHao'
@@ -26,7 +26,6 @@ app = Flask(__name__)
 
 
 class JsonResponse(Response):
-
     @classmethod
     def force_type(cls, response, environ=None):
         if isinstance(response, (dict, list)):
@@ -86,7 +85,7 @@ def getStatus():
 
 def run():
     config = GetConfig()
-    app.run(host=config.host_ip, port=config.host_port)
+    app.run(host=config.host_ip, port=config.host_port, threaded=False, processes=config.processes)
 
 
 if __name__ == '__main__':

Config.ini

@@ -9,11 +9,11 @@ name = proxy
 
 [ProxyGetter]
 ;register the proxy getter function
-freeProxyFirst  = 1
+freeProxyFirst = 1
 freeProxySecond = 1
 ;freeProxyThird  = 1
 freeProxyFourth = 1
-freeProxyFifth  = 1
+freeProxyFifth = 1
 freeProxySixth = 1
 freeProxySeventh = 1
 freeProxyEight = 1
@@ -26,7 +26,9 @@ freeProxyWallFirst = 1
 freeProxyWallSecond = 1
 freeProxyWallThird = 1
 
-[HOST]
+[API]
 ; API接口配置 http://127.0.0.1:5010
 ip = 0.0.0.0
 port = 5010
+; flask多进程处理请求
+processes = 10

Manager/ProxyManager.py

@@ -2,13 +2,13 @@
 # !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     ProxyManager.py  
-   Description :  
+   File Name：     ProxyManager.py
+   Description :
    Author :       JHao
    date：          2016/12/3
 -------------------------------------------------
    Change Activity:
-                   2016/12/3: 
+                   2016/12/3:
 -------------------------------------------------
 """
 __author__ = 'JHao'
@@ -40,30 +40,22 @@ class ProxyManager(object):
         fetch proxy into Db by ProxyGetter
         :return:
         """
+        self.db.changeTable(self.raw_proxy_queue)
         for proxyGetter in self.config.proxy_getter_functions:
             # fetch
-            proxy_set = set()
             try:
                 self.log.info("{func}: fetch proxy start".format(func=proxyGetter))
-                proxy_iter = [_ for _ in getattr(GetFreeProxy, proxyGetter.strip())()]
+                for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
+                    # 挨个存储 proxy，优化raw 队列的 push 速度，进而加快 check proxy 的速度
+                    proxy = proxy.strip()
+                    if proxy and verifyProxyFormat(proxy):
+                        self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy))
+                        self.db.put(proxy)
+                    else:
+                        self.log.error('{func}: fetch proxy {proxy} error'.format(func=proxyGetter, proxy=proxy))
             except Exception as e:
                 self.log.error("{func}: fetch proxy fail".format(func=proxyGetter))
                 continue
-            for proxy in proxy_iter:
-                proxy = proxy.strip()
-                if proxy and verifyProxyFormat(proxy):
-                    self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy))
-                    proxy_set.add(proxy)
-                else:
-                    self.log.error('{func}: fetch proxy {proxy} error'.format(func=proxyGetter, proxy=proxy))
-
-            # store
-            for proxy in proxy_set:
-                self.db.changeTable(self.useful_proxy_queue)
-                if self.db.exists(proxy):
-                    continue
-                self.db.changeTable(self.raw_proxy_queue)
-                self.db.put(proxy)
 
     def get(self):
         """

Schedule/ProxyRefreshSchedule.py

@@ -18,7 +18,8 @@ import sys
 import time
 import logging
 from threading import Thread
-from apscheduler.schedulers.blocking import BlockingScheduler
+# 使用后台调度，不使用阻塞式~
+from apscheduler.schedulers.background import BackgroundScheduler as Sch
 
 sys.path.append('../')
 
@@ -73,12 +74,7 @@ def refreshPool():
     pp.validProxy()
 
 
-def main(process_num=30):
-    p = ProxyRefreshSchedule()
-
-    # 获取新代理
-    p.refresh()
-
+def batch_refresh(process_num=30):
     # 检验新代理
     pl = []
     for num in range(process_num):
@@ -93,11 +89,18 @@ def main(process_num=30):
         pl[num].join()
 
 
+def fetch_all():
+    p = ProxyRefreshSchedule()
+    # 获取新代理
+    p.refresh()
+
+
 def run():
-    main()
-    sch = BlockingScheduler()
-    sch.add_job(main, 'interval', minutes=10)  # 每10分钟抓取一次
+    sch = Sch()
+    sch.add_job(fetch_all, 'interval', minutes=5)  # 每5分钟抓取一次
+    sch.add_job(batch_refresh, "interval", minutes=1)  # 每分钟检查一次
     sch.start()
+    fetch_all()
 
 
 if __name__ == '__main__':

Util/GetConfig.py

@@ -2,7 +2,7 @@
 # !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     GetConfig.py  
+   File Name：     GetConfig.py
    Description :  fetch config from config.ini
    Author :       JHao
    date：          2016/12/3
@@ -51,11 +51,15 @@ class GetConfig(object):
 
     @LazyProperty
     def host_ip(self):
-        return self.config_file.get('HOST','ip')
+        return self.config_file.get('API','ip')
 
     @LazyProperty
     def host_port(self):
-        return int(self.config_file.get('HOST', 'port'))
+        return int(self.config_file.get('API', 'port'))
+
+    @LazyProperty
+    def processes(self):
+        return int(self.config_file.get('API', 'processes'))
 
 if __name__ == '__main__':
     gg = GetConfig()
@@ -66,3 +70,4 @@ if __name__ == '__main__':
     print(gg.proxy_getter_functions)
     print(gg.host_ip)
     print(gg.host_port)
+    print(gg.processes)