Revise some errors

1.redis get data type is bytes, transfer to str
2.requests get.content is bytes, add getHtmlText funtion

DB/DbClient.py

@@ -39,6 +39,7 @@ class DbClient(object):
 
         所有方法需要相应类去具体实现：
             SSDB：SsdbClient.py
+            REDIS:RedisClient.py    只是对redis set的操作，不一定好复用
 
     """
 

DB/RedisClient.py

@@ -32,7 +32,7 @@ class RedisClient(object):
         get random result
         :return:
         """
-        return self.__conn.srandmember(name=self.name)
+        return self.__conn.srandmember(name=self.name).decode('utf-8')         #redis return bytes
 
     def put(self, value):
         """
@@ -81,7 +81,7 @@ if __name__ == '__main__':
     # print(redis_con.getAll())
     redis_con.changeTable('raw_proxy')
 
-    redis_con.put('132.112.43.221:8888')
+    # redis_con.put('132.112.43.221:8888')
     # redis_con.changeTable('proxy')
     print(redis_con.get_status())
     print(redis_con.getAll())

Manager/ProxyManager.py

@@ -27,7 +27,7 @@ class ProxyManager(object):
         self.db = DbClient()
         self.config = GetConfig()
         self.raw_proxy_queue = 'raw_proxy'
-        self.useful_proxy_queue = 'useful_proxy_queue'
+        self.useful_proxy_queue = 'useful_proxy'
 
     def refresh(self):
         """
@@ -37,7 +37,7 @@ class ProxyManager(object):
         for proxyGetter in self.config.proxy_getter_functions:
             proxy_set = set()
             # fetch raw proxy
-            for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
+            for proxy in getattr(GetFreeProxy, proxyGetter.strip())():   #get GetFreeProxy.freeProxyFirst and it is a iteration
                 if proxy.strip():
                     proxy_set.add(proxy.strip())
 
@@ -77,9 +77,9 @@ class ProxyManager(object):
         quan_raw_proxy = self.db.get_status()
         self.db.changeTable(self.useful_proxy_queue)
         quan_useful_queue = self.db.get_status()
-        return {'raw_proxy': quan_raw_proxy, 'useful_proxy_queue': quan_useful_queue}
+        return {'raw_proxy': quan_raw_proxy, 'useful_proxy': quan_useful_queue}
 
 if __name__ == '__main__':
     pp = ProxyManager()
-    # pp.refresh()
+    pp.refresh()
     print(pp.get_status())

ProxyGetter/getFreeProxy.py

@@ -9,6 +9,7 @@
 -------------------------------------------------
    Change Activity:
                    2016/11/25: 
+                   这一部分考虑用scrapy框架代替
 -------------------------------------------------
 """
 import re
@@ -24,7 +25,7 @@ except:
 
 
 
-from Util.utilFunction import robustCrawl, getHtmlTree
+from Util.utilFunction import robustCrawl, getHtmlTree, getHTMLText
 
 # for debug to disable insecureWarning
 requests.packages.urllib3.disable_warnings()
@@ -48,7 +49,7 @@ class GetFreeProxy(object):
         pass
 
     @staticmethod
-    @robustCrawl
+    @robustCrawl    #decoration print error if exception happen
     def freeProxyFirst(page=10):
         """
         抓取快代理IP http://www.kuaidaili.com/
@@ -74,7 +75,8 @@ class GetFreeProxy(object):
         """
         url = "http://m.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format(
             proxy_number)
-        html = requests.get(url, headers=HEADER).content
+
+        html = getHTMLText(url, headers=HEADER)
         for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
             yield proxy
 

Schedule/ProxyValidSchedule.py

@@ -31,9 +31,13 @@ class ProxyValidSchedule(ProxyManager):
         验证代理
         :return:
         """
-        while 1:
+        while True:
             self.db.changeTable(self.useful_proxy_queue)
             for each_proxy in self.db.getAll():
+                if isinstance(each_proxy,bytes):
+                    each_proxy = each_proxy.decode('utf-8')
+                    self.log.info(u'数据库返回数据类型为bytes')
+
                 if validUsefulProxy(each_proxy):
                     self.log.debug('proxy: {} validation pass'.format(each_proxy))
                 else:

Util/LogHandler.py

@@ -75,6 +75,6 @@ class LogHandler(logging.Logger):
 
 
 if __name__ == '__main__':
-    log = LogHandler('test')
+    log = LogHandler(__name__)
     log.info('this is a test msg')
     pass

Util/utilFunction.py

@@ -13,6 +13,19 @@
 """
 import requests
 
+from Util.LogHandler import LogHandler
+
+logger = LogHandler(__name__)
+
+
+def getHTMLText(url, headers = {'user':'Mozilla/5.0'}):
+    try:
+        response = requests.get(url, headers = headers, timeout=10)
+        response.raise_for_status()
+        response.encoding = response.apparent_encoding
+        return response.text
+    except:
+        return response.status_code
 
 # noinspection PyPep8Naming
 def robustCrawl(func):
@@ -20,8 +33,8 @@ def robustCrawl(func):
         try:
             return func(*args, **kwargs)
         except Exception as e:
-            print(u"sorry, 抓取出错。错误原因:")
-            print(e)
+            logger.info(u"sorry, 抓取出错。错误原因:")
+            logger.info(e)
 
     return decorate
 
@@ -54,6 +67,7 @@ def getHtmlTree(url, **kwargs):
               'Accept-Encoding': 'gzip, deflate, sdch',
               'Accept-Language': 'zh-CN,zh;q=0.8',
               }
+    #取代理服务器用代理服务器访问
     html = requests.get(url=url, headers=header, timeout=30).content
     return etree.HTML(html)
 
@@ -70,7 +84,8 @@ def validUsefulProxy(proxy):
         # 超过30秒的代理就不要了
         r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=30, verify=False)
         if r.status_code == 200:
+            logger.debug('%s is ok' % proxy)
             return True
     except Exception as e:
-        print(e)
+        logger.info(e)
         return False