[update] 修改启动方式

Config.ini

@@ -1,14 +1,14 @@
 [DB]
-type = REDIS
-host = localhost
-port = 6379
+type = SSDB
+host = 42.123.99.64
+port = 8889
 name = proxy
 
 [ProxyGetter]
 ;register the proxy getter function
-freeProxyFirst  = 1
+;freeProxyFirst  = 1
 freeProxySecond = 1
 freeProxyThird  = 1
-;freeProxyFourth = 1
+freeProxyFourth = 1
 freeProxyFifth  = 1
 

Run/main.py

@@ -19,14 +19,21 @@ sys.path.append('../')
 
 from Api.ProxyApi import run as ProxyApiRun
 from Schedule.ProxyValidSchedule import run as ValidRun
+from Schedule.ProxyRefreshSchedule import run as RefreshRun
 
 
 def run():
-    p1 = Process(target=ProxyApiRun)
-    p1.start()
-    p2 = Process(target=ValidRun())
-    p2.start()
-
+    p_list = list()
+    p1 = Process(target=ProxyApiRun, name='ProxyApiRun')
+    p_list.append(p1)
+    p2 = Process(target=ValidRun, name='ValidRun')
+    p_list.append(p2)
+    p3 = Process(target=RefreshRun, name='RefreshRun')
+    p_list.append(p3)
+    for p in p_list:
+        p.start()
+    for p in p_list:
+        p.join()
 
 if __name__ == '__main__':
     run()

Schedule/ProxyRefreshSchedule.py

@@ -36,30 +36,29 @@ class ProxyRefreshSchedule(ProxyManager):
         ProxyManager.__init__(self)
         self.log = LogHandler('refresh_schedule')
 
-    def valid_proxy(self):
+    def validProxy(self):
         """
-        valid_proxy
+        验证raw_proxy_queue中的代理, 将可用的代理放入useful_proxy_queue
         :return:
         """
         self.db.changeTable(self.raw_proxy_queue)
         raw_proxy = self.db.pop()
-        self.log.info('%s start valid proxy' % time.ctime())
+        self.log.info('%s start validProxy_a' % time.ctime())
         while raw_proxy:
             if validUsefulProxy(raw_proxy):
                 self.db.changeTable(self.useful_proxy_queue)
                 self.db.put(raw_proxy)
-                self.log.info('proxy: %s validation pass' % raw_proxy)
+                self.log.info('validProxy_a: %s validation pass' % raw_proxy)
             else:
-                self.log.debug('proxy: %s validation fail' % raw_proxy)
-                pass
+                self.log.debug('validProxy_a: %s validation fail' % raw_proxy)
             self.db.changeTable(self.raw_proxy_queue)
             raw_proxy = self.db.pop()
-        self.log.info('%s valid proxy complete' % time.ctime())
+        self.log.info('%s validProxy_a complete' % time.ctime())
 
 
-def refresh_pool():
+def refreshPool():
     pp = ProxyRefreshSchedule()
-    pp.valid_proxy()
+    pp.validProxy()
 
 
 def main(process_num=30):
@@ -71,7 +70,7 @@ def main(process_num=30):
     # 检验新代理
     pl = []
     for num in range(process_num):
-        proc = Thread(target=refresh_pool, args=())
+        proc = Thread(target=refreshPool, args=())
         pl.append(proc)
 
     for num in range(process_num):
@@ -81,8 +80,12 @@ def main(process_num=30):
         pl[num].join()
 
 
-if __name__ == '__main__':
+def run():
     # main()
     sched = BlockingScheduler()
     sched.add_job(main, 'interval', minutes=10)
     sched.start()
+
+
+if __name__ == '__main__':
+    run()

Schedule/ProxyValidSchedule.py

@@ -2,7 +2,7 @@
 """
 -------------------------------------------------
    File Name：     ProxyValidSchedule.py  
-   Description :  代理验证
+   Description :  验证useful_proxy_queue中的代理,将不可用的移出
    Author :       JHao
    date：          2017/3/31
 -------------------------------------------------
@@ -26,7 +26,7 @@ class ProxyValidSchedule(ProxyManager):
         ProxyManager.__init__(self)
         self.log = LogHandler('valid_schedule')
 
-    def __validProxy__(self):
+    def __validProxy(self):
         """
         验证代理
         :return:
@@ -35,20 +35,21 @@ class ProxyValidSchedule(ProxyManager):
             self.db.changeTable(self.useful_proxy_queue)
             for each_proxy in self.db.getAll():
                 if validUsefulProxy(each_proxy):
-                    self.log.debug('proxy: {} validation pass'.format(each_proxy))
+                    self.log.debug('validProxy_b: {} validation pass'.format(each_proxy))
                 else:
                     self.db.delete(each_proxy)
-                    self.log.info('proxy: {} validation fail'.format(each_proxy))
-        self.log.info(u'代理验证程序运行正常')
+                    self.log.info('validProxy_b: {} validation fail'.format(each_proxy))
+        self.log.info('validProxy_a running normal')
 
     def main(self):
-        self.__validProxy__()
+        self.__validProxy()
 
 
 def run():
     p = ProxyValidSchedule()
     p.main()
 
+
 if __name__ == '__main__':
     p = ProxyValidSchedule()
     p.main()

Util/utilFunction.py

@@ -12,6 +12,9 @@
 -------------------------------------------------
 """
 import requests
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+
+requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
 
 # noinspection PyPep8Naming