Merge branch 'master' of https://github.com/kagxin/proxy_pool into kagxin-master

 Conflicts:

	Util/utilFunction.py

ProxyGetter/CheckProxy.py

@@ -12,7 +12,7 @@
 """
 __author__ = 'JHao'
 
-from getFreeProxy import GetFreeProxy
+from .getFreeProxy import GetFreeProxy
 from Util.utilFunction import verifyProxyFormat
 
 

Util/utilFunction.py

@@ -11,10 +11,10 @@
                    2016/11/25: 添加robustCrawl、verifyProxy、getHtmlTree
 -------------------------------------------------
 """
-import requests
 from lxml import etree
 
 from Util.WebRequest import WebRequest
+from .validators import validators
 from Config.ConfigGetter import config
 
 
@@ -98,3 +98,7 @@ def validUsefulProxy(proxy):
         pass
     return False
 
+    for v_func in validators:
+        if not v_func(proxy):
+            return False
+    return True

Util/validators.py

@@ -0,0 +1,38 @@
+import requests
+
+validators = []
+
+
+def validator(func):
+    validators.append(func)
+    return func
+
+
+@validator
+def timeOutValidator(proxy):
+    """
+    检测超时
+    :param proxy:
+    :return:
+    """
+    if isinstance(proxy, bytes):
+        proxy = proxy.decode("utf8")
+    proxies = {"http": "http://{proxy}".format(proxy=proxy)}
+    try:
+        r = requests.get('http://www.baidu.com', proxies=proxies, timeout=10, verify=False)
+        if r.status_code == 200:
+            return True
+    except Exception as e:
+        pass
+    return False
+
+
+@validator
+def customValidator(proxy):
+    """
+    自定义validator函数，校验代理是否可用
+    :param proxy:
+    :return:
+    """
+
+    return True