[update] 优化抓去函数，每次少抓一些 减少耗时

ProxyGetter/getFreeProxy.py

@@ -15,17 +15,10 @@ import re
 import sys
 import requests
 
-try:
-    from importlib import reload  # py3 实际不会实用，只是为了不显示语法错误
-except:
-    reload(sys)
-    sys.setdefaultencoding('utf-8')
-
 sys.path.append('..')
 
 from Util.WebRequest import WebRequest
 from Util.utilFunction import getHtmlTree
-from Util.utilFunction import verifyProxyFormat
 
 # for debug to disable insecureWarning
 requests.packages.urllib3.disable_warnings()
@@ -48,9 +41,6 @@ class GetFreeProxy(object):
     proxy getter
     """
 
-    def __init__(self):
-        pass
-
     @staticmethod
     def freeProxyFirst(page=10):
         """
@@ -164,7 +154,7 @@ class GetFreeProxy(object):
         url = 'http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10'
         request = WebRequest()
         try:
-            res = request.get(url).json()
+            res = request.get(url, timeout=10).json()
             for row in res['RESULT']['rows']:
                 yield '{}:{}'.format(row['ip'], row['port'])
         except Exception as e:
@@ -180,7 +170,7 @@ class GetFreeProxy(object):
             'https://www.kuaidaili.com/free/intr/{page}/'
         ]
         for url in url_list:
-            for page in range(1, 5):
+            for page in range(1, 2):
                 page_url = url.format(page=page)
                 tree = getHtmlTree(page_url)
                 proxy_list = tree.xpath('.//table//tr')
@@ -192,14 +182,14 @@ class GetFreeProxy(object):
         """
         秘密代理 http://www.mimiip.com
         """
-        url_gngao = ['http://www.mimiip.com/gngao/%s' % n for n in range(1, 10)]  # 国内高匿
-        url_gnpu = ['http://www.mimiip.com/gnpu/%s' % n for n in range(1, 10)]  # 国内普匿
-        url_gntou = ['http://www.mimiip.com/gntou/%s' % n for n in range(1, 10)]  # 国内透明
+        url_gngao = ['http://www.mimiip.com/gngao/%s' % n for n in range(1, 2)]  # 国内高匿
+        url_gnpu = ['http://www.mimiip.com/gnpu/%s' % n for n in range(1, 2)]  # 国内普匿
+        url_gntou = ['http://www.mimiip.com/gntou/%s' % n for n in range(1, 2)]  # 国内透明
         url_list = url_gngao + url_gnpu + url_gntou
 
         request = WebRequest()
         for url in url_list:
-            r = request.get(url)
+            r = request.get(url, timeout=10)
             proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W].*<td>(\d+)</td>', r.text)
             for proxy in proxies:
                 yield ':'.join(proxy)
@@ -213,7 +203,7 @@ class GetFreeProxy(object):
         urls = ['https://proxy.coderbusy.com/classical/country/cn.aspx?page=1']
         request = WebRequest()
         for url in urls:
-            r = request.get(url)
+            r = request.get(url, timeout=10)
             proxies = re.findall('data-ip="(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})".+?>(\d+)</td>', r.text)
             for proxy in proxies:
                 yield ':'.join(proxy)
@@ -227,7 +217,7 @@ class GetFreeProxy(object):
         urls = ['http://www.ip3366.net/free/']
         request = WebRequest()
         for url in urls:
-            r = request.get(url)
+            r = request.get(url, timeout=10)
             proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
             for proxy in proxies:
                 yield ":".join(proxy)
@@ -246,14 +236,14 @@ class GetFreeProxy(object):
         ]
         request = WebRequest()
         for url in urls:
-            r = request.get(url)
+            r = request.get(url, timeout=10)
             proxies = re.findall(r'<td>\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?</td>[\s\S]*?<td>\s*?(\d+)\s*?</td>',
                                  r.text)
             for proxy in proxies:
                 yield ":".join(proxy)
 
     @staticmethod
-    def freeProxyTwelve(page_count=8):
+    def freeProxyTwelve(page_count=2):
         """
         guobanjia http://ip.jiangxianli.com/?page=
         免费代理库
@@ -278,7 +268,7 @@ class GetFreeProxy(object):
         urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218']
         request = WebRequest()
         for url in urls:
-            r = request.get(url)
+            r = request.get(url, timeout=10)
             proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)
             for proxy in proxies:
                 yield ':'.join(proxy)
@@ -293,7 +283,7 @@ class GetFreeProxy(object):
         request = WebRequest()
         import base64
         for url in urls:
-            r = request.get(url)
+            r = request.get(url, timeout=10)
             proxies = re.findall(r"Proxy\('(.*?)'\)", r.text)
             for proxy in proxies:
                 yield base64.b64decode(proxy).decode()
@@ -303,7 +293,7 @@ class GetFreeProxy(object):
         urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1']
         request = WebRequest()
         for url in urls:
-            r = request.get(url)
+            r = request.get(url, timeout=10)
             proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
             for proxy in proxies:
                 yield ':'.join(proxy)
@@ -312,7 +302,17 @@ class GetFreeProxy(object):
 if __name__ == '__main__':
     from CheckProxy import CheckProxy
 
-    CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFifth)
-    CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySecond)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFirst)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySecond)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyThird)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFourth)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyFifth)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySixth)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxySeventh)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyEight)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyNinth)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyTen)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyEleven)
+    CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxyTwelve)
 
-    CheckProxy.checkAllGetProxyFunc()
+    # CheckProxy.checkAllGetProxyFunc()