11111

docker-compose.yml

@@ -5,10 +5,7 @@ services:
     container_name: proxy_pool
     ports:
       - "5010:5010"
-    links:
-      - proxy_redis
     environment:
-      DB_CONN: "redis://@proxy_redis:6379/0"
-  proxy_redis:
-    image: "redis"
-    container_name: proxy_redis
+      DB_CONN: "redis://@myhome.97admin.com:63790/1"
+    environment:
+      - TZ=Asia/Shanghai

fetcher/proxyFetcher.py

@@ -10,6 +10,9 @@
                    2016/11/25: proxyFetcher
 -------------------------------------------------
 """
+import base64
+
+
 __author__ = 'JHao'
 
 import re
@@ -279,3 +282,20 @@ class ProxyFetcher(object):
             ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", r.text)
             for ip in ips:
                 yield ip.strip()
+
+    @staticmethod
+    def freeProxy15():
+        urls=[
+            'https://www.xroxy.com/proxylist.php?port=&type=All_http&ssl=&country=&latency=&reliability=#table',
+            'https://www.xroxy.com/proxylist.php?port=&type=All_http&ssl=&country=&latency=&reliability=&sort=reliability&desc=true&pnum=1#table',
+            'https://www.xroxy.com/proxylist.php?port=&type=All_http&ssl=&country=&latency=&reliability=&sort=reliability&desc=true&pnum=2#table',
+            'https://www.xroxy.com/proxylist.php?port=&type=All_http&ssl=&country=&latency=&reliability=&sort=reliability&desc=true&pnum=3#table',
+            'https://www.xroxy.com/proxylist.php?port=&type=All_http&ssl=&country=&latency=&reliability=&sort=reliability&desc=true&pnum=4#table',
+        ]
+        for url in urls:
+            r = WebRequest().get(url, timeout=10)
+            proxies = re.findall(
+                r'<td.*?>[\s\S]*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?</td>[\s\S]*?<td.*?>[\s\S]*?(\d+)[\s\S]*?</td>',
+                r.text)
+            for proxy in proxies:
+                yield ':'.join(proxy)

proxyPool.py

@@ -10,6 +10,8 @@
                    2020/6/19:
 -------------------------------------------------
 """
+import os
+
 __author__ = 'JHao'
 
 import click
@@ -44,4 +46,5 @@ def server():
 
 
 if __name__ == '__main__':
+    os.environ['TZ'] = 'Asia/Shanghai'
     cli()

setting.py

@@ -37,7 +37,7 @@ PORT = 5010
 # example:
 #      Redis: redis://:password@ip:port/db
 #      Ssdb:  ssdb://:password@ip:port
-DB_CONN = 'redis://:pwd@127.0.0.1:6379/0'
+DB_CONN = 'redis://@myhome.97admin.com:63790/1'
 
 # proxy table name
 TABLE_NAME = 'use_proxy'
@@ -55,13 +55,14 @@ PROXY_FETCHER = [
     # "freeProxy08",
     "freeProxy09",
     "freeProxy13",
-    "freeProxy14"
+    "freeProxy14",
+    "freeProxy15",
 ]
 
 # ############# proxy validator #################
 VERIFY_URL = "http://www.baidu.com"
 
-VERIFY_TIMEOUT = 10
+VERIFY_TIMEOUT = 3
 
 MAX_FAIL_COUNT = 0