add some test

Api/__init__.py

@@ -10,4 +10,5 @@
                    2016/12/3: 
 -------------------------------------------------
 """
-__author__ = 'JHao'
+__author__ = 'JHao'
+

Config.ini

@@ -1,7 +1,7 @@
 [DB]
 type = SSDB
 host = security
-port = 57888
+port = 8888
 name = proxy
 
 [ProxyGetter]

ProxyGetter/getFreeProxy.py

@@ -20,6 +20,8 @@ sys.setdefaultencoding('utf-8')
 
 from Util.utilFunction import robustCrawl, getHtmlTree
 
+# for debug to disable insecureWarning
+requests.packages.urllib3.disable_warnings()
 
 class GetFreeProxy(object):
     """
@@ -44,6 +46,7 @@ class GetFreeProxy(object):
             proxy_list = tree.xpath('.//div[@id="index_free_list"]//tbody/tr')
             for proxy in proxy_list:
                 yield ':'.join(proxy.xpath('./td/text()')[0:2])
+        print 'finish kuaidaili fetching proxy ip'
 
     @staticmethod
     @robustCrawl
@@ -58,6 +61,7 @@ class GetFreeProxy(object):
         html = requests.get(url).content
         for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
             yield proxy
+        print 'finish 66ip fetching proxy ip'
 
     @staticmethod
     @robustCrawl
@@ -75,6 +79,7 @@ class GetFreeProxy(object):
             proxy_list = re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html)
             for proxy in proxy_list:
                 yield proxy
+        print 'finish youdaili fetching proxy ip'
 
     @staticmethod
     @robustCrawl
@@ -87,6 +92,7 @@ class GetFreeProxy(object):
         html = requests.get(url).content
         for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
             yield proxy
+        print 'finish xici fetching proxy ip'
 
     @staticmethod
     @robustCrawl
@@ -102,6 +108,7 @@ class GetFreeProxy(object):
             d = tree.xpath('.//table[@class="table"]/tbody/tr[{}]/td'.format(i + 1))[0]
             o = d.xpath('.//span/text() | .//div/text()')
             yield ''.join(o[:-1]) + ':' + o[-1]
+        print 'finish guobanjia fetching proxy ip'
 
 
 if __name__ == '__main__':

Schedule/ProxyRefreshSchedule.py

@@ -13,11 +13,12 @@
 """
 __author__ = 'JHao'
 
-from apscheduler.schedulers.blocking import BlockingScheduler
+import sys
+import time
 from multiprocessing import Process
+
 import requests
-import time
-import sys
+from apscheduler.schedulers.blocking import BlockingScheduler
 
 sys.path.append('../')
 
@@ -48,6 +49,7 @@ class ProxyRefreshSchedule(ProxyManager):
                 pass
             self.db.changeTable(self.raw_proxy_queue)
             raw_proxy = self.db.pop()
+            print 'validate a  proxy'
 
 
 def refreshPool():
@@ -58,17 +60,41 @@ def refreshPool():
 def main(process_num=100):
     p = ProxyRefreshSchedule()
     p.refresh()
+    pl = []
+    for num in range(process_num):
+        P = Process(target=refreshPool, args=())
+        P.daemon = True
+        pl.append(P)
 
+    for num in range(process_num):
+        pl[num].start()
+
+    print 'all proc start'
+
+    for num in range(process_num):
+        pl[num].join()
+
+
+#
+#     print '{time}: refresh complete!'.format(time=time.ctime())
+
+
+def main(process_num=100):
+    p = ProxyRefreshSchedule()
+    p.refresh()
     for num in range(process_num):
         P = Process(target=refreshPool, args=())
+        P.daemon = True
         P.start()
-    print '{time}: refresh complete!'.format(time=time.ctime())
+        P.join()
+
 
+print '{time}: refresh complete!'.format(time=time.ctime())
 
 if __name__ == '__main__':
     # pp = ProxyRefreshSchedule()
     # pp.main()
     # main()
     sched = BlockingScheduler()
-    sched.add_job(main, 'interval', minutes=20)
+    sched.add_job(main, 'interval', seconds=2)
     sched.start()