[update] add log to ProxyRefreshSchedule

Schedule/ProxyRefreshSchedule.py

@@ -9,25 +9,23 @@
 -------------------------------------------------
    Change Activity:
                    2016/12/4: 代理定时刷新
+                   2017/03/06: 使用LogHandler添加日志
 -------------------------------------------------
 """
-import logging
-import os
-import random
+
 import sys
 import time
-from multiprocessing import Process
-
 import requests
+from multiprocessing import Process
 from apscheduler.schedulers.blocking import BlockingScheduler
 
 sys.path.append('../')
-from DB.DbClient import DbClient
+
 from Manager.ProxyManager import ProxyManager
+from Util.LogHandler import LogHandler
 
-__author__ = 'JHao'
 
-log = logging.getLogger('apscheduler')
+__author__ = 'JHao'
 
 
 class ProxyRefreshSchedule(ProxyManager):
@@ -37,20 +35,17 @@ class ProxyRefreshSchedule(ProxyManager):
 
     def __init__(self):
         ProxyManager.__init__(self)
+        self.log = LogHandler('refresh_schedule')
 
     def valid_proxy(self):
-        logger_raw = logging.getLogger('apscheduler.raw_check-{}'.format(os.getpid()))
-
-        fmt = logging.Formatter('%(asctime)s - %(levelname)s : %(name)s : %(message)s')
-        fh = logging.FileHandler(filename='../log/raw_proxy_log.txt')
-        fh.setFormatter(fmt=fmt)
-        logger_raw.addHandler(fh)
-
+        """
+        valid_proxy
+        :return:
+        """
         self.db.changeTable(self.raw_proxy_queue)
         raw_proxy = self.db.pop()
         while raw_proxy:
-            logger_raw.debug('[*] check raw proxy {} ...'.format(raw_proxy))
-            # print '[*] check raw proxy {} ...'.format(raw_proxy)
+            self.log.info('%s start valid proxy' % time.ctime())
             proxies = {"http": "http://{proxy}".format(proxy=raw_proxy),
                        "https": "https://{proxy}".format(proxy=raw_proxy)}
             try:
@@ -59,43 +54,13 @@ class ProxyRefreshSchedule(ProxyManager):
                 if r.status_code == 200:
                     self.db.changeTable(self.useful_proxy_queue)
                     self.db.put(raw_proxy)
-                    logger_raw.debug('[+] raw proxy {} succeed validating...'.format(raw_proxy))
-                    # print '[+] raw proxy {} succeed validating...'.format(raw_proxy)
+                    self.log.info('proxy: %s validation passes')
             except Exception, e:
                 print e
                 pass
             self.db.changeTable(self.raw_proxy_queue)
             raw_proxy = self.db.pop()
-            logger_raw.debug('[-] raw proxy {} invalid'.format(raw_proxy))
-            # print '[-] raw proxy {} invalid'.format(raw_proxy)
-
-    def validate_useful_proxy(self, proxy_list):
-        logger_avail = logging.getLogger('apscheduler.avail_check-{}'.format(os.getpid()))
-
-        fmt = logging.Formatter('%(asctime)s - %(levelname)s : %(name)s : %(message)s')
-        fh = logging.FileHandler(filename='../log/available_proxy_log.txt')
-        fh.setFormatter(fmt=fmt)
-        logger_avail.addHandler(fh)
-
-        len_proxy = len(proxy_list)
-
-        self.db.changeTable(self.useful_proxy_queue)
-        while proxy_list:
-            proxy = proxy_list.pop()
-            logger_avail.debug('[*] check available proxy : {} ...({} remained)'.format(proxy, len(proxy_list)))
-            # print '[*] check available proxy : {} ...({} remained)'.format(proxy, len(proxy_list))
-            proxies = {"http": "http://{proxy}".format(proxy=proxy),
-                       "https": "https://{proxy}".format(proxy=proxy)}
-            try:
-                r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=30, verify=False)
-                if r.status_code == 200:
-                    logger_avail.debug('[+] proxy {} is still available'.format(proxy))
-                    continue
-            except Exception, e:
-                self.db.delete(proxy)
-                logger_avail.debug('[-] A checked proxy {} has been removed from useful_proxy_queue'.format(proxy))
-                # print '[-] A checked proxy {} has been removed from useful_proxy_queue'.format(proxy)
-        logger_avail.debug('Process {} finished checking {} proxies.'.format(os.getpid(), len_proxy))
+        self.log.info('%s valid proxy complete' % time.ctime())
 
 
 def refresh_pool():
@@ -103,11 +68,6 @@ def refresh_pool():
     pp.valid_proxy()
 
 
-def validate_user_proxy(proxy_list):
-    pp = ProxyRefreshSchedule()
-    pp.validate_useful_proxy(proxy_list)
-
-
 def main(process_num=10):
     p = ProxyRefreshSchedule()
     p.refresh()
@@ -123,63 +83,9 @@ def main(process_num=10):
     for num in range(process_num):
         pl[num].join()
 
-    log.debug('Process main completed.')
-
-
-def main_check(process_num=10):
-    db = DbClient()
-    useful_proxy_queue = 'useful_proxy_queue'
-    db.changeTable(useful_proxy_queue)
-    proxy_list = db.getAll()
-
-    uncheck_list = [list() for i in xrange(process_num)]
-    for proxy in proxy_list:
-        uncheck_list[random.randint(0, process_num - 1)].append(proxy)
-
-    pl = []
-    for num in range(process_num):
-        proc = Process(target=validate_user_proxy, args=(uncheck_list[num],))
-        # proc.daemon = True
-        pl.append(proc)
-
-    for num in range(process_num):
-        pl[num].start()
-
-    for num in range(process_num):
-        pl[num].join()
-
-    log.debug('Process main_check completed.')
-
-
-to_time = time.time()
-
-
-def test():
-    print 'start test', time.time() - to_time
-    time.sleep(20)
-    print 'end test', time.time() - to_time
-
-
-def test2():
-    print 'start test2', time.time() - to_time
-    time.sleep(40)
-    print 'end test2', time.time() - to_time
-
 
 if __name__ == '__main__':
-    log.setLevel(logging.DEBUG)  # DEBUG
-
-    fmt = logging.Formatter('%(asctime)s - %(levelname)s : %(name)s : %(message)s')
-    h = logging.StreamHandler()
-    h.setFormatter(fmt)
-    log.addHandler(h)
-
-    fh = logging.FileHandler(filename='../log/log.txt')
-    fh.setFormatter(fmt=fmt)
-    log.addHandler(fh)
-
-    # main()
+    main()
     sched = BlockingScheduler()
     sched.add_job(main, 'interval', minutes=10)
-    sched.add_job(main_check, 'interval', minutes=15)
     sched.start()

Util/LogHandler.py

@@ -75,6 +75,6 @@ class LogHandler(logging.Logger):
 
 
 if __name__ == '__main__':
-    # log = get_logger("aa")
-    # log.error("aa")
+    log = LogHandler('test')
+    log.info('this is a test msg')
     pass

log/test

@@ -0,0 +1 @@
+2017-03-06 15:37:55,073 LogHandler.py[line:79] INFO this is a test msg