merge Edward_PC

DB/DbClient.py

@@ -39,6 +39,7 @@ class DbClient(object):
 
         所有方法需要相应类去具体实现：
             SSDB：SsdbClient.py
+            REDIS:RedisClient.py    只是对redis set的操作，不一定好复用
 
     """
 
@@ -93,7 +94,7 @@ class DbClient(object):
 
 if __name__ == "__main__":
     account = DbClient()
-    print account.get()
+    print(account.get())
     account.changeTable('use')
     account.put('ac')
-    print(account)
+    print(account.get())

DB/RedisClient.py

@@ -32,7 +32,7 @@ class RedisClient(object):
         get random result
         :return:
         """
-        return self.__conn.srandmember(name=self.name)
+        return self.__conn.srandmember(name=self.name).decode('utf-8')         #redis return bytes
 
     def put(self, value):
         """
@@ -40,7 +40,7 @@ class RedisClient(object):
         :param value:
         :return:
         """
-        value = json.dump(value, ensure_ascii=False).encode('utf-8') if isinstance(value, (dict, list)) else value
+        value = json.dumps(value) if isinstance(value, (dict, list)) else value
         return self.__conn.sadd(self.name, value)
 
     def pop(self):
@@ -48,7 +48,7 @@ class RedisClient(object):
         pop an item
         :return:
         """
-        return self.__conn.spop(self.name)
+        return self.__conn.spop(self.name).decode('utf-8')                    #redis return bytes
 
     def delete(self, value):
         """
@@ -73,14 +73,16 @@ if __name__ == '__main__':
     # redis_con.put('abc')
     # redis_con.put('123')
     # redis_con.put('123.115.235.221:8800')
-    # print redis_con.getAll()
+    # redis_con.put(['123', '115', '235.221:8800'])
+    # print(redis_con.getAll())
     # redis_con.delete('abc')
-    # print redis_con.getAll()
-    # redis_con.pop()
-    # print redis_con.getAll()
+    # print(redis_con.getAll())
+
+    # print(redis_con.getAll())
     redis_con.changeTable('raw_proxy')
+    redis_con.pop()
 
-    redis_con.put('132.112.43.221:8888')
+    # redis_con.put('132.112.43.221:8888')
     # redis_con.changeTable('proxy')
-    print redis_con.get_status()
-    print redis_con.getAll()
+    print(redis_con.get_status())
+    print(redis_con.getAll())

Manager/ProxyManager.py

@@ -29,7 +29,7 @@ class ProxyManager(object):
         self.config = GetConfig()
         self.raw_proxy_queue = 'raw_proxy'
         self.log = LogHandler('proxy_manager')
-        self.useful_proxy_queue = 'useful_proxy_queue'
+        self.useful_proxy_queue = 'useful_proxy'
 
     def refresh(self):
         """
@@ -39,7 +39,7 @@ class ProxyManager(object):
         for proxyGetter in self.config.proxy_getter_functions:
             proxy_set = set()
             # fetch raw proxy
-            for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
+            for proxy in getattr(GetFreeProxy, proxyGetter.strip())():   #get GetFreeProxy.freeProxyFirst and it is a iteration
                 if proxy.strip():
                     self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy))
                     proxy_set.add(proxy.strip())
@@ -80,9 +80,9 @@ class ProxyManager(object):
         quan_raw_proxy = self.db.get_status()
         self.db.changeTable(self.useful_proxy_queue)
         quan_useful_queue = self.db.get_status()
-        return {'raw_proxy': quan_raw_proxy, 'useful_proxy_queue': quan_useful_queue}
+        return {'raw_proxy': quan_raw_proxy, 'useful_proxy': quan_useful_queue}
 
 if __name__ == '__main__':
     pp = ProxyManager()
     pp.refresh()
-    # print pp.get_status()
+    print(pp.get_status())

ProxyGetter/getFreeProxy.py

@@ -9,17 +9,23 @@
 -------------------------------------------------
    Change Activity:
                    2016/11/25: 
+                   这一部分考虑用scrapy框架代替
 -------------------------------------------------
 """
 import re
-import sys
-
 import requests
 
-reload(sys)
-sys.setdefaultencoding('utf-8')
+try:
+    from importlib import reload   #py3 实际不会实用，只是为了不显示语法错误
+except:
+    import sys     # py2
+    reload(sys)
+    sys.setdefaultencoding('utf-8')
+
+
 
-from Util.utilFunction import robustCrawl, getHtmlTree
+
+from Util.utilFunction import robustCrawl, getHtmlTree, getHTMLText
 
 # for debug to disable insecureWarning
 requests.packages.urllib3.disable_warnings()
@@ -43,7 +49,7 @@ class GetFreeProxy(object):
         pass
 
     @staticmethod
-    @robustCrawl
+    @robustCrawl    #decoration print error if exception happen
     def freeProxyFirst(page=10):
         """
         抓取快代理IP http://www.kuaidaili.com/
@@ -69,7 +75,8 @@ class GetFreeProxy(object):
         """
         url = "http://m.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format(
             proxy_number)
-        html = requests.get(url, headers=HEADER).content
+
+        html = getHTMLText(url, headers=HEADER)
         for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
             yield proxy
 
@@ -137,4 +144,4 @@ if __name__ == '__main__':
     #     print e
 
     for e in gg.freeProxyFifth():
-        print e
+        print(e)

Schedule/ProxyValidSchedule.py

@@ -31,9 +31,13 @@ class ProxyValidSchedule(ProxyManager):
         验证代理
         :return:
         """
-        while 1:
+        while True:
             self.db.changeTable(self.useful_proxy_queue)
             for each_proxy in self.db.getAll():
+                if isinstance(each_proxy,bytes):
+                    each_proxy = each_proxy.decode('utf-8')
+                    self.log.info(u'数据库返回数据类型为bytes')
+
                 if validUsefulProxy(each_proxy):
                     self.log.debug('validProxy_b: {} validation pass'.format(each_proxy))
                 else:

Util/GetConfig.py

@@ -52,8 +52,8 @@ class GetConfig(object):
 
 if __name__ == '__main__':
     gg = GetConfig()
-    print gg.db_type
-    print gg.db_name
-    print gg.db_host
-    print gg.db_port
-    print gg.proxy_getter_functions
+    print(gg.db_type)
+    print(gg.db_name)
+    print(gg.db_host)
+    print(gg.db_port)
+    print(gg.proxy_getter_functions)

Util/LogHandler.py

@@ -81,6 +81,6 @@ class LogHandler(logging.Logger):
 
 
 if __name__ == '__main__':
-    log = LogHandler('test', level=WARNING)
-    log.error('this is a test msg')
-
+    log = LogHandler(__name__)
+    log.info('this is a test msg')
+    pass

Util/utilClass.py

@@ -33,7 +33,10 @@ class LazyProperty(object):
             return value
 
 
-from ConfigParser import ConfigParser
+try:
+    from configparser import ConfigParser # py3
+except:
+    from configparser import ConfigParser # py2
 
 
 class ConfigParse(ConfigParser):
@@ -44,6 +47,7 @@ class ConfigParse(ConfigParser):
     def __init__(self):
         ConfigParser.__init__(self)
 
+
     def optionxform(self, optionstr):
         return optionstr
 

Util/utilFunction.py

@@ -16,6 +16,19 @@ from requests.packages.urllib3.exceptions import InsecureRequestWarning
 
 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
+from Util.LogHandler import LogHandler
+
+logger = LogHandler(__name__)
+
+
+def getHTMLText(url, headers = {'user':'Mozilla/5.0'}):
+    try:
+        response = requests.get(url, headers = headers, timeout=10)
+        response.raise_for_status()
+        response.encoding = response.apparent_encoding
+        return response.text
+    except:
+        return response.status_code
 
 # noinspection PyPep8Naming
 def robustCrawl(func):
@@ -23,8 +36,8 @@ def robustCrawl(func):
         try:
             return func(*args, **kwargs)
         except Exception as e:
-            print u"sorry, 抓取出错。错误原因:"
-            print e
+            logger.info(u"sorry, 抓取出错。错误原因:")
+            logger.info(e)
 
     return decorate
 
@@ -57,6 +70,7 @@ def getHtmlTree(url, **kwargs):
               'Accept-Encoding': 'gzip, deflate, sdch',
               'Accept-Language': 'zh-CN,zh;q=0.8',
               }
+    #取代理服务器用代理服务器访问
     html = requests.get(url=url, headers=header, timeout=30).content
     return etree.HTML(html)
 
@@ -72,6 +86,8 @@ def validUsefulProxy(proxy):
         # 超过30秒的代理就不要了
         r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=20, verify=False)
         if r.status_code == 200:
+            logger.debug('%s is ok' % proxy)
             return True
-    except Exception, e:
+    except Exception as e:
+        logger.info(e)
         return False