Merge pull request #70 from jhao104/improve#63

Improve#63

Config.ini

@@ -15,5 +15,6 @@ freeProxyFourth = 1
 freeProxyFifth  = 1
 
 [HOST]
+; API接口配置 http://127.0.0.1:5051
 ip = 0.0.0.0
 port = 5010

Manager/ProxyManager.py

@@ -40,7 +40,7 @@ class ProxyManager(object):
             proxy_set = set()
             # fetch raw proxy
             for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
-                if proxy.strip():
+                if proxy:
                     self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy))
                     proxy_set.add(proxy.strip())
 
@@ -76,6 +76,7 @@ class ProxyManager(object):
         return self.db.getAll()
 
     def get_status(self):
+        # TODO rename get_count..
         self.db.changeTable(self.raw_proxy_queue)
         total_raw_proxy = self.db.get_status()
         self.db.changeTable(self.useful_proxy_queue)

Run/main.py

@@ -21,7 +21,7 @@ sys.path.append('../')
 from Api.ProxyApi import run as ProxyApiRun
 from Schedule.ProxyValidSchedule import run as ValidRun
 from Schedule.ProxyRefreshSchedule import run as RefreshRun
-from Util.GetConfig import GetConfig
+
 
 def run():
     p_list = list()

Schedule/ProxyRefreshSchedule.py

@@ -47,20 +47,24 @@ class ProxyRefreshSchedule(ProxyManager):
         """
         self.db.changeTable(self.raw_proxy_queue)
         raw_proxy = self.db.pop()
-        self.log.info('%s start validProxy_a' % time.ctime())
+        self.log.info('ProxyRefreshSchedule: %s start validProxy' % time.ctime())
         # 计算剩余代理，用来减少重复计算
         remaining_proxies = self.db.getAll()
         while raw_proxy:
+            if isinstance(raw_proxy, bytes):
+                # 兼容Py3
+                raw_proxy = raw_proxy.decode('utf8')
+
             if (raw_proxy not in remaining_proxies) and validUsefulProxy(raw_proxy):
                 self.db.changeTable(self.useful_proxy_queue)
                 self.db.put(raw_proxy)
-                self.log.info('validProxy_a: %s validation pass' % raw_proxy)
+                self.log.info('ProxyRefreshSchedule: %s validation pass' % raw_proxy)
             else:
-                self.log.debug('validProxy_a: %s validation fail' % raw_proxy)
+                self.log.info('ProxyRefreshSchedule: %s validation fail' % raw_proxy)
             self.db.changeTable(self.raw_proxy_queue)
             raw_proxy = self.db.pop()
             remaining_proxies = self.db.getAll()
-        self.log.info('%s validProxy_a complete' % time.ctime())
+        self.log.info('ProxyRefreshSchedule: %s validProxy complete' % time.ctime())
 
 
 def refreshPool():
@@ -88,9 +92,9 @@ def main(process_num=30):
 
 
 def run():
-    # main()
+    main()
     sched = BlockingScheduler()
-    sched.add_job(main, 'interval', minutes=5)
+    sched.add_job(main, 'interval', minutes=10)  # 每10分钟抓取一次
     sched.start()
 
 

Schedule/ProxyValidSchedule.py

@@ -36,22 +36,25 @@ class ProxyValidSchedule(ProxyManager):
             self.db.changeTable(self.useful_proxy_queue)
             for each_proxy in self.db.getAll():
                 if isinstance(each_proxy, bytes):
+                    # 兼容PY3
                     each_proxy = each_proxy.decode('utf-8')
 
+                value = self.db.getvalue(each_proxy)
                 if validUsefulProxy(each_proxy):
                     # 成功计数器加1
-                    self.db.inckey(each_proxy, 1)
-                    self.log.debug('validProxy_b: {} validation pass'.format(each_proxy))
+                    if value and int(value) < 1:
+                        self.db.inckey(each_proxy, 1)
+                    self.log.info('ProxyValidSchedule: {} validation pass'.format(each_proxy))
                 else:
                     # 失败计数器减一
-                    self.db.inckey(each_proxy, -1)
-                    # self.db.delete(each_proxy)
-                    self.log.info('validProxy_b: {} validation fail'.format(each_proxy))
-                value = self.db.getvalue(each_proxy)
-                if value and int(value) < -5:
-                    # 计数器小于-5删除该代理
-                    self.db.delete(each_proxy)
-            self.log.info('validProxy_a running normal')
+                    if value and int(value) < -5:
+                        # 计数器小于-5删除该代理
+                        self.db.delete(each_proxy)
+                    else:
+                        self.db.inckey(each_proxy, -1)
+                    self.log.info('ProxyValidSchedule: {} validation fail'.format(each_proxy))
+
+            self.log.info('ProxyValidSchedule running normal')
             sleep(60 * 1)
 
     def main(self):

Util/LogHandler.py

@@ -2,12 +2,13 @@
 """
 -------------------------------------------------
    File Name：     LogHandler.py
-   Description :
+   Description :  日志操作模块
    Author :       JHao
    date：          2017/3/6
 -------------------------------------------------
    Change Activity:
                    2017/3/6: log handler
+                   2017/9/21: 屏幕输出/文件输出 可选(默认屏幕和文件均输出)
 -------------------------------------------------
 """
 __author__ = 'JHao'
@@ -38,12 +39,14 @@ class LogHandler(logging.Logger):
     LogHandler
     """
 
-    def __init__(self, name, level=DEBUG):
+    def __init__(self, name, level=DEBUG, stream=True, file=True):
         self.name = name
         self.level = level
         logging.Logger.__init__(self, self.name, level=level)
-        self.__setFileHandler__()
-        self.__setStreamHandler__()
+        if stream:
+            self.__setStreamHandler__()
+        if file:
+            self.__setFileHandler__()
 
     def __setFileHandler__(self, level=None):
         """

Util/utilFunction.py

@@ -17,7 +17,7 @@ from lxml import etree
 from Util.LogHandler import LogHandler
 from Util.WebRequest import WebRequest
 
-logger = LogHandler(__name__)
+logger = LogHandler(__name__, stream=False)
 
 
 # noinspection PyPep8Naming
@@ -81,8 +81,8 @@ def validUsefulProxy(proxy):
         # 超过40秒的代理就不要了
         r = requests.get('https://www.baidu.com', proxies=proxies, timeout=40, verify=False)
         if r.status_code == 200:
-            logger.debug('%s is ok' % proxy)
+            logger.info('%s is ok' % proxy)
             return True
     except Exception as e:
-        logger.info(e)
+        logger.debug(e)
         return False