fix fifth proxy getter

Api/ProxyApi.py

@@ -13,9 +13,10 @@
 """
 __author__ = 'JHao'
 
-from flask import Flask, jsonify, request
 import sys
 
+from flask import Flask, jsonify, request
+
 sys.path.append('../')
 
 from Manager.ProxyManager import ProxyManager
@@ -59,12 +60,12 @@ def delete():
     ProxyManager().delete(proxy)
     return 'success'
 
+
 @app.route('/get_status/')
 def get_status():
     status = ProxyManager().get_status()
     return jsonify(status)
 
 
-
 if __name__ == '__main__':
     app.run(host='0.0.0.0', port=65432)

Config.ini

@@ -10,5 +10,5 @@ freeProxyFirst  = 1
 freeProxySecond = 1
 freeProxyThird  = 1
 ;freeProxyFourth = 1
-;freeProxyFifth  = 1
+freeProxyFifth  = 1
 

ProxyGetter/getFreeProxy.py

@@ -145,25 +145,29 @@ class GetFreeProxy(object):
         tree = getHtmlTree(url, header=header)
         # 现在每天最多放15个（一页）
         for i in xrange(15):
-            d = tree.xpath('.//table[@class="table"]/tbody/tr[{}]/td'.format(i + 1))[0]
+            d = tree.xpath('//*[@id="list"]/table/tbody/tr[{}]/td'.format(i + 1))[0]
+            # print d
+
             o = d.xpath('.//span/text() | .//div/text()')
+            # print o
+
             yield ''.join(o[:-1]) + ':' + o[-1]
         print 'finish guobanjia fetching proxy ip'
 
 
 if __name__ == '__main__':
     gg = GetFreeProxy()
-    for e in gg.freeProxyFirst():
-        print e
-
-    for e in gg.freeProxySecond():
-        print e
-
-    for e in gg.freeProxyThird():
-        print e
+    # for e in gg.freeProxyFirst():
+    #     print e
+    #
+    # for e in gg.freeProxySecond():
+    #     print e
+    #
+    # for e in gg.freeProxyThird():
+    #     print e
 
     # for e in gg.freeProxyFourth():
     #     print e
     #
-    # for e in gg.freeProxyFifth():
-    #     print e
+    for e in gg.freeProxyFifth():
+        print e

Schedule/ProxyRefreshSchedule.py

@@ -12,18 +12,23 @@
 -------------------------------------------------
 """
 import logging
+import os
 import random
+import sys
+import time
 from multiprocessing import Process
 
 import requests
 from apscheduler.schedulers.blocking import BlockingScheduler
-import sys
+
 sys.path.append('../')
 from DB.DbClient import DbClient
 from Manager.ProxyManager import ProxyManager
 
 __author__ = 'JHao'
 
+log = logging.getLogger('apscheduler')
+
 
 class ProxyRefreshSchedule(ProxyManager):
     """
@@ -34,10 +39,18 @@ class ProxyRefreshSchedule(ProxyManager):
         ProxyManager.__init__(self)
 
     def valid_proxy(self):
+        logger_raw = logging.getLogger('apscheduler.raw_check-{}'.format(os.getpid()))
+
+        fmt = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
+        fh = logging.FileHandler(filename='../log/log.txt')
+        fh.setFormatter(fmt=fmt)
+        logger_raw.addHandler(fh)
+
         self.db.changeTable(self.raw_proxy_queue)
         raw_proxy = self.db.pop()
-        print '[*]check raw proxy {} ...'.format(raw_proxy)
         while raw_proxy:
+            logger_raw.debug('[*] check raw proxy {} ...'.format(raw_proxy))
+            # print '[*] check raw proxy {} ...'.format(raw_proxy)
             proxies = {"http": "http://{proxy}".format(proxy=raw_proxy),
                        "https": "https://{proxy}".format(proxy=raw_proxy)}
             try:
@@ -46,26 +59,43 @@ class ProxyRefreshSchedule(ProxyManager):
                 if r.status_code == 200:
                     self.db.changeTable(self.useful_proxy_queue)
                     self.db.put(raw_proxy)
+                    logger_raw.debug('[+] raw proxy {} succeed validating...'.format(raw_proxy))
+                    # print '[+] raw proxy {} succeed validating...'.format(raw_proxy)
             except Exception, e:
                 print e
                 pass
             self.db.changeTable(self.raw_proxy_queue)
             raw_proxy = self.db.pop()
-            print 'validate a  proxy'
+            logger_raw.debug('[-] raw proxy {} invalid'.format(raw_proxy))
+            # print '[-] raw proxy {} invalid'.format(raw_proxy)
 
     def validate_useful_proxy(self, proxy_list):
+        logger_avail = logging.getLogger('apscheduler.avail_check-{}'.format(os.getpid()))
+
+        fmt = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
+        fh = logging.FileHandler(filename='../log/log.txt')
+        fh.setFormatter(fmt=fmt)
+        logger_avail.addHandler(fh)
+
+        len_proxy = len(proxy_list)
+
         self.db.changeTable(self.useful_proxy_queue)
-        for proxy in proxy_list:
-            print '[*]validating proxy : {} ...({} remained)'.format(proxy, len(proxy_list))
+        while proxy_list:
+            proxy = proxy_list.pop()
+            logger_avail.debug('[*] check available proxy : {} ...({} remained)'.format(proxy, len(proxy_list)))
+            # print '[*] check available proxy : {} ...({} remained)'.format(proxy, len(proxy_list))
             proxies = {"http": "http://{proxy}".format(proxy=proxy),
                        "https": "https://{proxy}".format(proxy=proxy)}
             try:
                 r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=30, verify=False)
                 if r.status_code == 200:
+                    logger_avail.debug('[+] proxy {} is still available'.format(proxy))
                     continue
             except Exception, e:
                 self.db.delete(proxy)
-                print '[-]delete proxy {}'.format(proxy)
+                logger_avail.debug('[-] A checked proxy {} has been removed from useful_proxy_queue'.format(proxy))
+                # print '[-] A checked proxy {} has been removed from useful_proxy_queue'.format(proxy)
+        logger_avail.debug('Process {} finished checking {} proxies.'.format(os.getpid(), len_proxy))
 
 
 def refresh_pool():
@@ -84,17 +114,17 @@ def main(process_num=10):
     pl = []
     for num in range(process_num):
         proc = Process(target=refresh_pool, args=())
-        proc.daemon = True
+        # proc.daemon = True
         pl.append(proc)
 
     for num in range(process_num):
         pl[num].start()
 
-    print 'All raw_proxy_crawler sub-processes start.'
-
     for num in range(process_num):
         pl[num].join()
 
+    log.debug('Process main completed.')
+
 
 def main_check(process_num=10):
     db = DbClient()
@@ -109,39 +139,47 @@ def main_check(process_num=10):
     pl = []
     for num in range(process_num):
         proc = Process(target=validate_user_proxy, args=(uncheck_list[num],))
-        proc.daemon = True
+        # proc.daemon = True
         pl.append(proc)
 
     for num in range(process_num):
         pl[num].start()
 
-    print 'All proxy validator sub-processes start.'
-
     for num in range(process_num):
         pl[num].join()
 
+    log.debug('Process main_check completed.')
+
+
+to_time = time.time()
+
+
+def test():
+    print 'start test', time.time() - to_time
+    time.sleep(20)
+    print 'end test', time.time() - to_time
+
+
+def test2():
+    print 'start test2', time.time() - to_time
+    time.sleep(40)
+    print 'end test2', time.time() - to_time
 
-# def main(process_num=100):
-#     p = ProxyRefreshSchedule()
-#     p.refresh()
-#     for num in range(process_num):
-#         P = Process(target=refreshPool, args=())
-#         P.daemon = True
-#         P.start()
-#         P.join()
-# print '{time}: refresh complete!'.format(time=time.ctime())
 
 if __name__ == '__main__':
-    log = logging.getLogger('apscheduler')
-    log.setLevel(logging.INFO)  # DEBUG
+    log.setLevel(logging.DEBUG)  # DEBUG
 
     fmt = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
     h = logging.StreamHandler()
     h.setFormatter(fmt)
     log.addHandler(h)
 
+    fh = logging.FileHandler(filename='../log/log.txt')
+    fh.setFormatter(fmt=fmt)
+    log.addHandler(fh)
+
     # main()
     sched = BlockingScheduler()
-    # sched.add_job(main, 'interval', seconds=10)
-    sched.add_job(main_check, 'interval', seconds=15)
+    sched.add_job(main, 'interval', minutes=10)
+    sched.add_job(main_check, 'interval', minutes=15)
     sched.start()