11111

fetcher/proxyFetcher.py

@@ -12,6 +12,8 @@
 """
 import base64
 
+import requests
+
 __author__ = 'JHao'
 
 import re
@@ -300,3 +302,21 @@ class ProxyFetcher(object):
                 r.text)
             for proxy in proxies:
                 yield ':'.join(proxy)
+
+    @staticmethod
+    def freeProxy16():
+
+        urls = [
+            'https://hidemy.name/en/proxy-list/?type=h&anon=4#list',
+            'https://hidemy.name/en/proxy-list/?type=h&anon=4&start=64#list',
+            'https://hidemy.name/en/proxy-list/?type=h&anon=4&start=128#list',
+            'https://hidemy.name/en/proxy-list/?type=h&anon=4&start=192#list',
+        ]
+        _proxies = {'http': 'http://myhome.97admin.com:34185', 'https': 'http://myhome.97admin.com:34185'}
+        for url in urls:
+            r = WebRequest().get(url, timeout=10, proxies=_proxies)
+            proxies = re.findall(
+                r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td><td>(\d+)</td>',
+                r.text)
+            for proxy in proxies:
+                yield ':'.join(proxy)

setting.py

@@ -51,16 +51,17 @@ PROXY_FETCHER = [
     "freeProxy04",
     "freeProxy05",
     # "freeProxy06",
-    "freeProxy07",
+    # "freeProxy07",
     # "freeProxy08",
     "freeProxy09",
     "freeProxy13",
     "freeProxy14",
     "freeProxy15",
+    "freeProxy16",
 ]
 
 # ############# proxy validator #################
-VERIFY_URL = "http://www.baidu.com"
+VERIFY_URL = "https://www.cloudflare.com/"
 
 VERIFY_TIMEOUT = 3
 

util/validators.py

@@ -40,14 +40,9 @@ def timeOutValidator(proxy):
                'Connection': 'keep-alive',
                'Accept-Language': 'zh-CN,zh;q=0.8'}
     try:
-        r=requests.get(url='http://myip.ipip.net',proxies=proxies,timeout=conf.verifyTimeout, verify=False)
-        proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})',
-                             r.text)
-        if len(proxies)>0:
+        r = requests.head(conf.verifyUrl, headers=headers, proxies=proxies, timeout=conf.verifyTimeout, verify=False)
+        if r.status_code == 200:
             return True
-        # r = requests.head(conf.verifyUrl, headers=headers, proxies=proxies, timeout=conf.verifyTimeout, verify=False)
-        # if r.status_code == 200:
-        #     return True
     except Exception as e:
         pass
     return False
@@ -60,5 +55,11 @@ def customValidator(proxy):
     :param proxy:
     :return:
     """
-
-    return True
+    proxies = {"http": "http://{proxy}".format(proxy=proxy), "https": "https://{proxy}".format(proxy=proxy)}
+    r = requests.get(url='http://myip.ipip.net', proxies=proxies, timeout=conf.verifyTimeout, verify=False)
+    proxies = re.findall(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})',
+                         r.text)
+    if len(proxies) > 0:
+        return True
+    else:
+        return False