Merge pull request #350 from jhao104/develop

Develop

Config/setting.py

@@ -38,6 +38,7 @@ DB_HOST = getenv('db_host', '127.0.0.1')
 DB_PORT = getenv('db_port', '8080')
 DB_PASSWORD = getenv('db_password', '')
 
+
 """ 数据库配置 """
 DATABASES = {
     "default": {

ProxyGetter/getFreeProxy.py

@@ -285,11 +285,49 @@ class GetFreeProxy(object):
     #         for proxy in proxies:
     #             yield ':'.join(proxy)
 
+    @staticmethod
+    def freeProxy13(max_page=2):
+        """
+        http://www.qydaili.com/free/?action=china&page=1
+        齐云代理
+        :param max_page:
+        :return:
+        """
+        base_url = 'http://www.qydaili.com/free/?action=china&page='
+        request = WebRequest()
+        for page in range(1, max_page + 1):
+            url = base_url + str(page)
+            r = request.get(url, timeout=10)
+            proxies = re.findall(
+                r'<td.*?>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td.*?>(\d+)</td>',
+                r.text)
+            for proxy in proxies:
+                yield ':'.join(proxy)
+
+    @staticmethod
+    def freeProxy14(max_page=2):
+        """
+        http://www.89ip.cn/index.html
+        89免费代理
+        :param max_page:
+        :return:
+        """
+        base_url = 'http://www.89ip.cn/index_{}.html'
+        request = WebRequest()
+        for page in range(1, max_page + 1):
+            url = base_url.format(page)
+            r = request.get(url, timeout=10)
+            proxies = re.findall(
+                r'<td.*?>[\s\S]*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?</td>[\s\S]*?<td.*?>[\s\S]*?(\d+)[\s\S]*?</td>',
+                r.text)
+            for proxy in proxies:
+                yield ':'.join(proxy)
+
 
 if __name__ == '__main__':
     from CheckProxy import CheckProxy
 
-    CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01)
     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy02)
     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy03)
     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy04)
@@ -298,5 +336,7 @@ if __name__ == '__main__':
     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy07)
     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy08)
     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy09)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy13)
+    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy14)
 
     CheckProxy.checkAllGetProxyFunc()