Merge pull request #125 from jhao104/dev

Dev

Config.ini

@@ -2,7 +2,7 @@
 ;Configure the database information
 ;type: SSDB/REDIS/MONGODB if use redis, only modify the host port，the type should be SSDB
 type = SSDB
-host = localhost
+host = 127.0.0.1
 ;port = 6379
 port = 8888
 name = proxy
@@ -11,12 +11,16 @@ name = proxy
 ;register the proxy getter function
 freeProxyFirst  = 1
 freeProxySecond = 1
-freeProxyThird  = 1
+;freeProxyThird  = 1
 freeProxyFourth = 1
-freeProxyFifth  = 1
+;freeProxyFifth  = 1
 freeProxySixth = 1
 freeProxySeventh = 1
 
+;foreign website, outside the wall
+;freeProxyWallFirst = 1
+;freeProxyWallSecond = 1
+
 [HOST]
 ; API接口配置 http://127.0.0.1:5051
 ip = 0.0.0.0

DB/SsdbClient.py

@@ -63,7 +63,7 @@ class SsdbClient(object):
         :param num:
         :return:
         """
-        data = self.__conn.hincrby(self.name, proxy, num)
+        data = self.__conn.hset(self.name, proxy, num)
         return data
 
     def delete(self, key):

Manager/ProxyManager.py

@@ -19,6 +19,7 @@ from Util import EnvUtil
 from DB.DbClient import DbClient
 from Util.GetConfig import GetConfig
 from Util.LogHandler import LogHandler
+from Util.utilFunction import verifyProxyFormat
 from ProxyGetter.getFreeProxy import GetFreeProxy
 
 
@@ -40,14 +41,23 @@ class ProxyManager(object):
         :return:
         """
         for proxyGetter in self.config.proxy_getter_functions:
+            # fetch
             proxy_set = set()
-            # fetch raw proxy
-            for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
-                if proxy:
+            try:
+                self.log.info("{func}: fetch proxy start".format(func=proxyGetter))
+                proxy_iter = [_ for _ in getattr(GetFreeProxy, proxyGetter.strip())()]
+            except Exception as e:
+                self.log.error("{func}: fetch proxy fail".format(func=proxyGetter))
+                continue
+            for proxy in proxy_iter:
+                proxy = proxy.strip()
+                if proxy and verifyProxyFormat(proxy):
                     self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy))
-                    proxy_set.add(proxy.strip())
+                    proxy_set.add(proxy)
+                else:
+                    self.log.error('{func}: fetch proxy {proxy} error'.format(func=proxyGetter, proxy=proxy))
 
-            # store raw proxy
+            # store
             for proxy in proxy_set:
                 self.db.changeTable(self.useful_proxy_queue)
                 if self.db.exists(proxy):
@@ -97,6 +107,7 @@ class ProxyManager(object):
         total_useful_queue = self.db.getNumber()
         return {'raw_proxy': total_raw_proxy, 'useful_proxy': total_useful_queue}
 
+
 if __name__ == '__main__':
     pp = ProxyManager()
     pp.refresh()

ProxyGetter/getFreeProxy.py

@@ -12,16 +12,17 @@
 -------------------------------------------------
 """
 import re
+import sys
 import requests
 
 try:
     from importlib import reload  # py3 实际不会实用，只是为了不显示语法错误
 except:
-    import sys  # py2
-
     reload(sys)
     sys.setdefaultencoding('utf-8')
 
+sys.path.append('../')
+
 from Util.utilFunction import robustCrawl, getHtmlTree
 from Util.WebRequest import WebRequest
 
@@ -82,8 +83,6 @@ class GetFreeProxy(object):
         url = "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format(
             proxy_number)
         request = WebRequest()
-        # html = request.get(url).content
-        # content为未解码，text为解码后的字符串
         html = request.get(url).text
         for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
             yield proxy
@@ -179,61 +178,31 @@ class GetFreeProxy(object):
                 yield ':'.join(tr.xpath('./td/text()')[0:2])
 
     @staticmethod
-    def freeProxyEight():
+    def freeProxyWallFirst():
+        """
+        墙外网站 cn-proxy
+        :return:
+        """
         urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218']
         request = WebRequest()
         for url in urls:
-            r = requests.get(url)
+            r = request.get(url)
             proxies = re.findall(
-                '<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.content)
+                r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)
             for proxy in proxies:
                 yield ':'.join(proxy)
 
     @staticmethod
-    def freeProxyNight():
-        urls = ['http://www.mimiip.com/gngao/%s' % n for n in range(1, 10)]
-        request = WebRequest()
-        for url in urls:
-            r = requests.get(url)
-            proxies = re.findall(
-                '<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W].*<td>(\d+)</td>', r.content)
-            for proxy in proxies:
-                yield ':'.join(proxy)
-
-    @staticmethod
-    def freeProxyTenth():
+    def freeProxyWallSecond():
         urls = ['https://proxy-list.org/english/index.php?p=%s' %
                 n for n in range(1, 10)]
         request = WebRequest()
         import base64
         for url in urls:
-            r = requests.get(url)
-            proxies = re.findall("Proxy\('(.*?)'\)", r.content)
+            r = request.get(url)
+            proxies = re.findall(r"Proxy\('(.*?)'\)", r.text)
             for proxy in proxies:
-                yield base64.b64decode(proxy)
-
-    @staticmethod
-    def freeProxyEleventh():
-        urls = ['http://www.cz88.net/proxy/%s' % m for m in
-                ['index.shtml'] + ['http_%s.shtml' % n for n in range(2, 11)]]
-        request = WebRequest()
-        for url in urls:
-            r = requests.get(url)
-            proxies = re.findall(
-                '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</div><div class="port">(\d+)</div>', r.content)
-            for proxy in proxies:
-                yield ':'.join(proxy)
-
-    @staticmethod
-    def freeProxy12th():
-        urls = ['http://www.ip181.com/daili/%s.html' % n for n in range(1, 11)]
-        request = WebRequest()
-        for url in urls:
-            r = requests.get(url)
-            proxies = re.findall(
-                '<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]*?<td>(\d+)</td>', r.content)
-            for proxy in proxies:
-                yield ':'.join(proxy)
+                yield base64.b64decode(proxy).decode()
 
 
 if __name__ == '__main__':
@@ -245,15 +214,24 @@ if __name__ == '__main__':
     #     print(e)
     #
     # for e in gg.freeProxyThird():
-    # print(e)
-
+    #     print(e)
+    #
     # for e in gg.freeProxyFourth():
     #     print(e)
-
+    #
     # for e in gg.freeProxyFifth():
-    #    print(e)
-
+    #     print(e)
+    #
     # for e in gg.freeProxySixth():
     #     print(e)
-    for e in gg.freeProxySeventh():
+    #
+    # for e in gg.freeProxySeventh():
+    #     print(e)
+
+    #
+    #
+    # for e in gg.freeProxyWallFirst():
+    #     print(e)
+    #
+    for e in gg.freeProxyWallSecond():
         print(e)

README.md

@@ -178,8 +178,10 @@ freeProxyCustom  = 1  # 确保名字和你添加方法名字一致
 
 　　这里感谢以下contributor的无私奉献：
 
-　　[@kangnwh](https://github.com/kangnwh)| [@bobobo80](https://github.com/bobobo80)| [@halleywj](https://github.com/halleywj)| [@newlyedward](https://github.com/newlyedward)| [@wang-ye](https://github.com/wang-ye)| [@gladmo](https://github.com/gladmo)| [@bernieyangmh](https://github.com/bernieyangmh)| [@PythonYXY](https://github.com/PythonYXY)| [@zuijiawoniu](https://github.com/zuijiawoniu)| [@netAir](https://github.com/netAir)| [@gladmo](https://github.com/gladmo)| [@scil](https://github.com/scil)
+　　[@kangnwh](https://github.com/kangnwh)| [@bobobo80](https://github.com/bobobo80)| [@halleywj](https://github.com/halleywj)| [@newlyedward](https://github.com/newlyedward)| [@wang-ye](https://github.com/wang-ye)| [@gladmo](https://github.com/gladmo)| [@bernieyangmh](https://github.com/bernieyangmh)| [@PythonYXY](https://github.com/PythonYXY)| [@zuijiawoniu](https://github.com/zuijiawoniu)| [@netAir](https://github.com/netAir)| [@scil](https://github.com/scil)| [@tangrela](https://github.com/tangrela)
 
 
+### Release Notes
 
+   [release notes](https://github.com/jhao104/proxy_pool/blob/master/doc/release_notes.md)
 

Schedule/ProxyCheck.py

@@ -22,7 +22,7 @@ from Util.utilFunction import validUsefulProxy
 from Manager.ProxyManager import ProxyManager
 from Util.LogHandler import LogHandler
 
-FAIL_COUNT = 1  # 校验失败次数， 超过次数删除代理
+FAIL_COUNT = 2  # 校验失败次数， 超过次数删除代理
 
 
 class ProxyCheck(ProxyManager, Thread):
@@ -34,27 +34,21 @@ class ProxyCheck(ProxyManager, Thread):
     def run(self):
         self.db.changeTable(self.useful_proxy_queue)
         while True:
-            proxy_item = self.db.pop()
-            while proxy_item:
-                proxy = proxy_item.get('proxy')
-                counter = proxy_item.get('value', 1)
+            for proxy, count in self.db.getAll().items():
                 if validUsefulProxy(proxy):
-                    # 验证通过计数器加1
-                    if counter and int(counter) < 1:
-                        self.db.put(proxy, num=int(counter) + 1)
+                    # 验证通过计数器减1
+                    if count and int(count) > 0:
+                        self.db.put(proxy, num=int(count) - 1)
                     else:
-                        self.db.put(proxy)
+                        pass
                     self.log.info('ProxyCheck: {} validation pass'.format(proxy))
                 else:
                     self.log.info('ProxyCheck: {} validation fail'.format(proxy))
-                    # 验证失败，计数器减1
-                    if counter and int(counter) <= FAIL_COUNT:
+                    if count and int(count) > FAIL_COUNT:
                         self.log.info('ProxyCheck: {} fail too many, delete!'.format(proxy))
                         self.db.delete(proxy)
                     else:
-                        self.db.put(proxy, num=int(counter) - 1)
-
-                proxy_item = self.db.pop()
+                        self.db.put(proxy, num=int(count) + 1)
             sleep(60 * 5)
 
 

Schedule/ProxyRefreshSchedule.py

@@ -95,9 +95,9 @@ def main(process_num=30):
 
 def run():
     main()
-    sched = BlockingScheduler()
-    sched.add_job(main, 'interval', minutes=10)  # 每10分钟抓取一次
-    sched.start()
+    sch = BlockingScheduler()
+    sch.add_job(main, 'interval', minutes=10)  # 每10分钟抓取一次
+    sch.start()
 
 
 if __name__ == '__main__':

Util/utilFunction.py

@@ -18,7 +18,7 @@ from lxml import etree
 from Util.LogHandler import LogHandler
 from Util.WebRequest import WebRequest
 
-logger = LogHandler(__name__, stream=False)
+# logger = LogHandler(__name__, stream=False)
 
 
 # noinspection PyPep8Naming
@@ -27,8 +27,9 @@ def robustCrawl(func):
         try:
             return func(*args, **kwargs)
         except Exception as e:
-            logger.info(u"sorry, 抓取出错。错误原因:")
-            logger.info(e)
+            pass
+            # logger.info(u"sorry, 抓取出错。错误原因:")
+            # logger.info(e)
 
     return decorate
 
@@ -42,7 +43,8 @@ def verifyProxyFormat(proxy):
     """
     import re
     verify_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}"
-    return True if re.findall(verify_regex, proxy) else False
+    _proxy = re.findall(verify_regex, proxy)
+    return True if len(_proxy) == 1 and _proxy[0] == proxy else False
 
 
 # noinspection PyPep8Naming
@@ -97,10 +99,10 @@ def validUsefulProxy(proxy):
     proxies = {"http": "http://{proxy}".format(proxy=proxy)}
     try:
         # 超过20秒的代理就不要了
-        r = requests.get('http://httpbin.org/ip', proxies=proxies, timeout=20, verify=False)
+        r = requests.get('http://httpbin.org/ip', proxies=proxies, timeout=10, verify=False)
         if r.status_code == 200:
-            logger.info('%s is ok' % proxy)
+            # logger.info('%s is ok' % proxy)
             return True
     except Exception as e:
-        logger.debug(e)
+        # logger.error(str(e))
         return False

doc/release_notes.md

@@ -1,9 +1,21 @@
 ## Release Notes
 
-* newest
-　　1.使用多线程验证useful_pool
+* dev
+
+  1.优化代理格式检查;
+
+  2.增加代理源;
+
+  3. fix bug [#122](https://github.com/jhao104/proxy_pool/issues/122)
+
+* 1.11(2017.8)
+
+　　1.使用多线程验证useful_pool;
+
+* 1.10(2016.11)
 
-* 1.10
 　　1. 第一版；
+
 　　2. 支持PY2/PY3;
+
 　　3. 代理池基本功能；