[update] 代理抓取时添加异常捕获

Api/ProxyApi.py

@@ -20,13 +20,10 @@ sys.path.append('../')
 from flask import Flask, jsonify, request
 from Util.GetConfig import GetConfig
 
-
-
 from Manager.ProxyManager import ProxyManager
 
 app = Flask(__name__)
 
-
 api_list = {
     'get': u'get an usable proxy',
     # 'refresh': u'refresh proxy pool',
@@ -78,5 +75,6 @@ def run():
     config = GetConfig()
     app.run(host=config.host_ip, port=config.host_port)
 
+
 if __name__ == '__main__':
     run()

ProxyGetter/getFreeProxy.py

@@ -65,7 +65,7 @@ class GetFreeProxy(object):
         :return:
         """
         url = "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format(
-                proxy_number)
+            proxy_number)
         request = WebRequest()
         # html = request.get(url).content
         # content为未解码，text为解码后的字符串
@@ -162,6 +162,7 @@ class GetFreeProxy(object):
             for tr in proxy_list[1:]:
                 yield ':'.join(tr.xpath('./td/text()')[0:2])
 
+
 if __name__ == '__main__':
     gg = GetFreeProxy()
     # for e in gg.freeProxyFirst():
@@ -171,12 +172,12 @@ if __name__ == '__main__':
     #     print(e)
     #
     # for e in gg.freeProxyThird():
-        # print(e)
+    # print(e)
 
     # for e in gg.freeProxyFourth():
     #     print(e)
 
-    #for e in gg.freeProxyFifth():
+    # for e in gg.freeProxyFifth():
     #    print(e)
 
     # for e in gg.freeProxySixth():

Run/main.py

@@ -15,7 +15,6 @@ __author__ = 'JHao'
 import sys
 from multiprocessing import Process
 
-
 sys.path.append('../')
 
 from Api.ProxyApi import run as ProxyApiRun
@@ -38,5 +37,6 @@ def run():
     for p in p_list:
         p.join()
 
+
 if __name__ == '__main__':
-   run()
+    run()

Schedule/ProxyCheck.py

@@ -37,7 +37,7 @@ class ProxyCheck(ProxyManager, Thread):
             proxy_item = self.db.pop()
             while proxy_item:
                 proxy = proxy_item.get('proxy')
-                counter = proxy_item.get('value')
+                counter = proxy_item.get('value', 1)
                 if validUsefulProxy(proxy):
                     # 验证通过计数器加1
                     if counter and int(counter) < 1:
@@ -48,7 +48,7 @@ class ProxyCheck(ProxyManager, Thread):
                 else:
                     self.log.info('ProxyCheck: {} validation fail'.format(proxy))
                     # 验证失败，计数器减1
-                    if counter and int(counter) <= -FAIL_COUNT:
+                    if counter and int(counter) <= FAIL_COUNT:
                         self.log.info('ProxyCheck: {} fail too many, delete!'.format(proxy))
                         self.db.delete(proxy)
                     else: