[update] valid proxy

Schedule/ProxyCheck.py

@@ -13,7 +13,6 @@
 __author__ = 'J_hao'
 
 import sys
-import threading
 from threading import Thread
 
 sys.path.append('../')
@@ -22,14 +21,14 @@ from Util.utilFunction import validUsefulProxy
 from Manager.ProxyManager import ProxyManager
 from Util.LogHandler import LogHandler
 
-FAIL_COUNT = 2  # 校验失败次数， 超过次数删除代理
+FAIL_COUNT = 1  # 校验失败次数， 超过次数删除代理
 
 
 class ProxyCheck(ProxyManager, Thread):
     def __init__(self, queue, item_dict):
         ProxyManager.__init__(self)
         Thread.__init__(self)
-        self.log = LogHandler('proxy_check')
+        self.log = LogHandler('proxy_check', file=False)  # 多线程同时写一个日志文件会有问题
         self.queue = queue
         self.item_dict = item_dict
 
@@ -44,11 +43,11 @@ class ProxyCheck(ProxyManager, Thread):
                     self.db.put(proxy, num=int(count) - 1)
                 else:
                     pass
-                print('ProxyCheck: {} validation pass'.format(proxy))
+                self.log.info('ProxyCheck: {} validation pass'.format(proxy))
             else:
-                print('ProxyCheck: {} validation fail'.format(proxy))
-                if count and int(count) > FAIL_COUNT:
-                    print('ProxyCheck: {} fail too many, delete!'.format(proxy))
+                self.log.info('ProxyCheck: {} validation fail'.format(proxy))
+                if count and int(count) + 1 >= FAIL_COUNT:
+                    self.log.info('ProxyCheck: {} fail too many, delete!'.format(proxy))
                     self.db.delete(proxy)
                 else:
                     self.db.put(proxy, num=int(count) + 1)

Schedule/ProxyValidSchedule.py

@@ -13,19 +13,24 @@
 __author__ = 'JHao'
 
 import sys
+import time
+
+try:
+    from Queue import Queue  # py3
+except:
+    from queue import Queue  # py2
 
 sys.path.append('../')
 
 from Schedule.ProxyCheck import ProxyCheck
 from Manager.ProxyManager import ProxyManager
-from queue import Queue
-import time
 
 
 class ProxyValidSchedule(ProxyManager, object):
     def __init__(self):
         ProxyManager.__init__(self)
         self.queue = Queue()
+        self.proxy_item = dict()
 
     def __validProxy(self, threads=10):
         """
@@ -35,7 +40,7 @@ class ProxyValidSchedule(ProxyManager, object):
         """
         thread_list = list()
         for index in range(threads):
-            thread_list.append(ProxyCheck(self.queue, self.item_dict))
+            thread_list.append(ProxyCheck(self.queue, self.proxy_item))
 
         for thread in thread_list:
             thread.daemon = True
@@ -45,19 +50,20 @@ class ProxyValidSchedule(ProxyManager, object):
             thread.join()
 
     def main(self):
-        self.put_queue()
+        self.putQueue()
         while True:
-            if self.queue.qsize():
+            if not self.queue.empty():
+                self.log.info("Start valid useful proxy")
                 self.__validProxy()
             else:
-                print('Time sleep 5 minutes.')
-                time.sleep(60 * 1)
-                self.put_queue()
+                self.log.info('Valid Complete! sleep 5 minutes.')
+                time.sleep(60 * 5)
+                self.putQueue()
 
-    def put_queue(self):
+    def putQueue(self):
         self.db.changeTable(self.useful_proxy_queue)
-        self.item_dict = self.db.getAll()
-        for item in self.item_dict:
+        self.proxy_item = self.db.getAll()
+        for item in self.proxy_item:
             self.queue.put(item)
 
 
@@ -68,4 +74,4 @@ def run():
 
 if __name__ == '__main__':
     p = ProxyValidSchedule()
-    p.main()
+    p.main()