[fix] fix log

Manager/ProxyManager.py

@@ -38,7 +38,8 @@ class ProxyManager(object):
             proxy_set = set()
             # fetch raw proxy
             for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
-                proxy_set.add(proxy)
+                if proxy.strip():
+                    proxy_set.add(proxy.strip())
 
             # store raw proxy
             self.db.changeTable(self.raw_proxy_queue)

ProxyGetter/getFreeProxy.py

@@ -54,7 +54,6 @@ class GetFreeProxy(object):
         # 页数不用太多， 后面的全是历史IP， 可用性不高
 
         for url in url_list:
-            print url
             tree = getHtmlTree(url)
             proxy_list = tree.xpath('.//div[@id="index_free_list"]//tbody/tr')
             for proxy in proxy_list:

Schedule/ProxyRefreshSchedule.py

@@ -44,8 +44,8 @@ class ProxyRefreshSchedule(ProxyManager):
         """
         self.db.changeTable(self.raw_proxy_queue)
         raw_proxy = self.db.pop()
+        self.log.info('%s start valid proxy' % time.ctime())
         while raw_proxy:
-            self.log.info('%s start valid proxy' % time.ctime())
             proxies = {"http": "http://{proxy}".format(proxy=raw_proxy),
                        "https": "https://{proxy}".format(proxy=raw_proxy)}
             try:
@@ -54,9 +54,9 @@ class ProxyRefreshSchedule(ProxyManager):
                 if r.status_code == 200:
                     self.db.changeTable(self.useful_proxy_queue)
                     self.db.put(raw_proxy)
-                    self.log.info('proxy: %s validation passes')
+                    self.log.debug('proxy: %s validation passes' % raw_proxy)
             except Exception, e:
-                print e
+                self.log.debug('proxy: %s validation fail' % raw_proxy)
                 pass
             self.db.changeTable(self.raw_proxy_queue)
             raw_proxy = self.db.pop()
@@ -74,7 +74,6 @@ def main(process_num=10):
     pl = []
     for num in range(process_num):
         proc = Process(target=refresh_pool, args=())
-        # proc.daemon = True
         pl.append(proc)
 
     for num in range(process_num):