Merge branch 'master' of https://github.com/DustinPT/proxy_pool into DustinPT-master

Util/utilFunction.py

@@ -100,7 +100,7 @@ def validUsefulProxy(proxy):
     try:
         # 超过20秒的代理就不要了
         r = requests.get('http://httpbin.org/ip', proxies=proxies, timeout=10, verify=False)
-        if r.status_code == 200:
+        if r.status_code == 200 and r.headers['content-type'].lower().find('application/json') != -1 and r.json()['origin']:
             # logger.info('%s is ok' % proxy)
             return True
     except Exception as e:

my/Config.ini

@@ -0,0 +1,37 @@
+[DB]
+;Configure the database information
+;type: SSDB/MONGODB if use redis, only modify the host port，the type should be SSDB
+type = PROXY_POOL_DB_TYPE
+host = PROXY_POOL_DB_HOST
+port = PROXY_POOL_DB_PORT
+name = proxy
+;username = your_username (Only Mongodb)
+;password = your_password
+
+[ProxyGetter]
+;register the proxy getter function
+freeProxyFirst = 1
+freeProxySecond = 1
+;freeProxyThird  = 1
+freeProxyFourth = 1
+freeProxyFifth = 1
+;freeProxySixth = 1
+freeProxySeventh = 1
+;freeProxyEight = 1
+;freeProxyNinth = 1
+freeProxyTen = 1
+freeProxyEleven = 1
+freeProxyTwelve = 1
+;foreign website, outside the wall
+;freeProxyWallFirst = 1
+;freeProxyWallSecond = 1
+;freeProxyWallThird = 1
+
+[API]
+# API config http://127.0.0.1:5010
+# The ip specified when starting the web API
+ip = 0.0.0.0
+# he port on which to run the web API
+port = 8080
+# Flask processes option
+processes = 10

my/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.6
+
+WORKDIR /usr/src/app
+
+ENV TZ=Asia/Shanghai \
+	PROXY_POOL_DB_TYPE=SSDB \
+	PROXY_POOL_DB_HOST=redis \
+	PROXY_POOL_DB_PORT=6379
+
+COPY . .
+
+RUN pip install --no-cache-dir -r requirements.txt && cp my/Config.ini ./
+
+CMD [ "my/run.sh" ]
+
+EXPOSE 8080

my/build.sh

@@ -0,0 +1 @@
+docker build -t registry.cn-beijing.aliyuncs.com/ryttech/proxy_pool:1.12.20181114 -f my/Dockerfile .

my/run.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+for var in \
+	PROXY_POOL_DB_TYPE \
+	PROXY_POOL_DB_HOST \
+	PROXY_POOL_DB_PORT \
+; do
+	val="${!var}"
+	if [ "$val" ]; then
+		sed -ri "s/$var/$val/" Config.ini
+	fi
+done
+
+python Run/main.py