[update] proxy check :camera:

README.md

@@ -1,5 +1,5 @@
 
-爬虫IP代理池
+ProxyPool 代理IP池
 =======
 [![Build Status](https://travis-ci.org/jhao104/proxy_pool.svg?branch=master)](https://travis-ci.org/jhao104/proxy_pool)
 [![](https://img.shields.io/badge/Powered%20by-@j_hao104-green.svg)](http://www.spiderpy.cn/blog/)
@@ -17,78 +17,75 @@
                            __ / /
                           /___ /
 
-##### [介绍文档](https://github.com/jhao104/proxy_pool/blob/master/doc/introduce.md)
+### ProxyPool
+
+* 文档: [document](https://github.com/jhao104/proxy_pool/blob/master/doc/introduce.md)
 
 * 支持版本: ![](https://img.shields.io/badge/Python-2.x-green.svg) ![](https://img.shields.io/badge/Python-3.x-blue.svg)
 
 * 测试地址: http://118.24.52.95 (单机勿压, 感谢。 恶意访问关[小黑屋](https://github.com/jhao104/proxy_pool/blob/bff423dffe6e2881ee45d5b66d8a6ad682c8e4ab/doc/block_ips.md)哦)
 
-### 下载安装
+* 有闲心的朋友帮忙点点末尾的[广告](https://tracking.gitads.io/?repo=proxy_pool)哦 :stuck_out_tongue_winking_eye:
 
-* 下载源码:
+### 安装
 
-```shell
+#### 下载代码:
+
+* git clone
+
+```bash
 git clone git@github.com:jhao104/proxy_pool.git
+```
 
-或者直接到https://github.com/jhao104/proxy_pool/releases 下载zip文件
+* releases
+
+```bash
+https://github.com/jhao104/proxy_pool/releases 下载zip文件
 ```
 
 * 安装依赖:
 
-```shell
+```bash
 pip install -r requirements.txt
 ```
 
-* 配置Config/setting.py:
+* 修改配置:
 
-```shell
-# Config/setting.py 为项目配置文件
 
-# 配置DB     
-DATABASES = {
-    "default": {
-        "TYPE": "SSDB",        # 目前支持SSDB或REDIS数据库
-        "HOST": "127.0.0.1",   # db host
-        "PORT": 8888,          # db port，例如SSDB通常使用8888，REDIS通常默认使用6379
-        "NAME": "proxy",       # 默认配置
-        "PASSWORD": ""         # db password
+```python
+# setting.py 为项目配置文件
 
-    }
-}
+# 配置API服务
 
+HOST = "0.0.0.0"               # IP
+PORT = 5000                    # 监听端口
 
-# 配置 ProxyGetter
 
-PROXY_GETTER = [
-    "freeProxy01",      # 这里是启用的代理抓取函数名，可在ProxyGetter/getFreeProxy.py 扩展
-    "freeProxy02",
-    ....
-]
+# 配置数据库
 
+DB_CONN = 'ssdb://@127.0.0.1:8888'
 
-# 配置 API服务
 
-SERVER_API = {
-    "HOST": "0.0.0.0",  # 监听ip, 0.0.0.0 监听所有IP
-    "PORT": 5010        # 监听端口
-}
-       
-# 上面配置启动后，代理池访问地址为 http://127.0.0.1:5010
+# 配置 ProxyFetcher
 
+PROXY_FETCHER = [
+    "freeProxy01",      # 这里是启用的代理抓取方法名，所有fetch方法位于fetcher/proxyFetcher.py
+    "freeProxy02",
+    # ....
+]
 ```
 
 * 启动:
 
-```shell
-# 如果你的依赖已经安装完成并且具备运行条件,可以在cli目录下通过ProxyPool.py启。动
-# 程序分为: schedule 调度程序 和 webserver Api服务
-
-# 首先启动调度程序
->>>python proxyPool.py schedule
+```bash
+# 如果已经具备运行条件, 可用通过proxyPool.py启动。
+# 程序分为: schedule 调度程序 和 server Api服务
 
-# 然后启动webApi服务
->>>python proxyPool.py webserver
+# 启动调度程序
+python proxyPool.py schedule
 
+# 启动webApi服务
+python proxyPool.py server
 
 ```
 

api/proxyApi.py

@@ -22,6 +22,7 @@ from flask import Flask, jsonify, request
 from util.six import iteritems
 from handler.proxyHandler import ProxyHandler
 from handler.configHandler import ConfigHandler
+from helper.proxy import Proxy
 
 app = Flask(__name__)
 conf = ConfigHandler()
@@ -57,13 +58,13 @@ def index():
 @app.route('/get/')
 def get():
     proxy = proxy_handler.get()
-    return proxy.info_dict if proxy else {"code": 0, "src": "no proxy"}
+    return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"}
 
 
 @app.route('/pop/')
 def pop():
     proxy = proxy_handler.pop()
-    return proxy.info_dict if proxy else {"code": 0, "src": "no proxy"}
+    return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"}
 
 
 @app.route('/refresh/')
@@ -75,13 +76,13 @@ def refresh():
 @app.route('/get_all/')
 def getAll():
     proxies = proxy_handler.getAll()
-    return jsonify([_.info_dict for _ in proxies])
+    return jsonify([_.to_dict for _ in proxies])
 
 
 @app.route('/delete/', methods=['GET'])
 def delete():
     proxy = request.args.get('proxy')
-    status = proxy_handler.delete(proxy)
+    status = proxy_handler.delete(Proxy(proxy))
     return {"code": 0, "src": status}
 
 

doc/release_notes.md

@@ -1,12 +1,14 @@
 ## Release Notes
 
-* 2.1.0
+* 2.1.0 (202007)
 
     1. 新增免费代理源 `西拉代理`  （2020-03-30）
     2. Fix Bug [#401](https://github.com/jhao104/proxy_pool/issues/401) [#356](https://github.com/jhao104/proxy_pool/issues/356)
     3. 优化Docker镜像体积 (2020-06-19)
+    4. 优化配置
+    5. 优化代码结构
 
-* 2.0.1 
+* 2.0.1 (2020)
 
     1. 新增免费代理源 `89免费代理`;
     2. 新增免费代理源 `齐云代理` 

handler/configHandler.py

@@ -57,3 +57,7 @@ class ConfigHandler(object):
     @LazyProperty
     def verifyTimeout(self):
         return os.getenv("VERIFY_TIMEOUT", setting.VERIFY_TIMEOUT)
+
+    @LazyProperty
+    def maxFailCount(self):
+        return os.getenv("MAX_FAIL_COUNT", setting.MAX_FAIL_COUNT)

handler/proxyHandler.py

@@ -46,22 +46,22 @@ class ProxyHandler(object):
             return Proxy.createFromJson(proxy)
         return None
 
-    def put(self, proxy_obj):
+    def put(self, proxy):
         """
         put proxy into use proxy
         :return:
         """
         self.db.changeTable(self.conf.useProxy)
-        self.db.put(proxy_obj)
+        self.db.put(proxy)
 
-    def delete(self, proxy_str):
+    def delete(self, proxy):
         """
         delete useful proxy
-        :param proxy_str:
+        :param proxy:
         :return:
         """
         self.db.changeTable(self.conf.useProxy)
-        return self.db.delete(proxy_str)
+        return self.db.delete(proxy.proxy)
 
     def getAll(self):
         """
@@ -72,14 +72,14 @@ class ProxyHandler(object):
         proxies_dict = self.db.getAll()
         return [Proxy.createFromJson(value) for _, value in proxies_dict.items()]
 
-    def exists(self, proxy_str):
+    def exists(self, proxy):
         """
         check proxy exists
-        :param proxy_str:
+        :param proxy:
         :return:
         """
         self.db.changeTable(self.conf.useProxy)
-        return self.db.exists(proxy_str)
+        return self.db.exists(proxy.proxy)
 
     def getCount(self):
         """

helper/check.py

@@ -20,6 +20,7 @@ from helper.proxy import Proxy
 from util.validators import validators
 from handler.logHandler import LogHandler
 from handler.proxyHandler import ProxyHandler
+from handler.configHandler import ConfigHandler
 
 
 def proxyCheck(proxy_obj):
@@ -62,6 +63,7 @@ class Checker(Thread):
         self.log = LogHandler("checker")
         self.proxy_handler = ProxyHandler()
         self.queue = queue
+        self.conf = ConfigHandler()
 
     def run(self):
         self.log.info("ProxyCheck - {}  : start".format(self.name))
@@ -76,7 +78,7 @@ class Checker(Thread):
             proxy = proxyCheck(proxy)
             if self.type == "raw":
                 if proxy.last_status:
-                    if self.proxy_handler.exists(proxy.proxy):
+                    if self.proxy_handler.exists(proxy):
                         self.log.info('ProxyCheck - {}  : {} exists'.format(self.name, proxy.proxy.ljust(23)))
                     else:
                         self.log.info('ProxyCheck - {}  : {} success'.format(self.name, proxy.proxy.ljust(23)))
@@ -84,7 +86,20 @@ class Checker(Thread):
                 else:
                     self.log.info('ProxyCheck - {}  : {} fail'.format(self.name, proxy.proxy.ljust(23)))
             else:
-                pass
+                if proxy.last_status:
+                    self.log.info('ProxyCheck - {}  : {} pass'.format(self.name, proxy.proxy.ljust(23)))
+                    self.proxy_handler.put(proxy)
+                else:
+                    if proxy.fail_count > self.conf.maxFailCount:
+                        self.log.info('ProxyCheck - {}  : {} fail, count {} delete'.format(self.name,
+                                                                                           proxy.proxy.ljust(23),
+                                                                                           self.conf.maxFailCount))
+                        self.proxy_handler.delete(proxy)
+                    else:
+                        self.log.info('ProxyCheck - {}  : {} fail, count {} keep'.format(self.name,
+                                                                                         proxy.proxy.ljust(23),
+                                                                                         self.conf.maxFailCount))
+                    self.proxy_handler.put(proxy)
             self.queue.task_done()
 
 

setting.py

@@ -29,6 +29,7 @@ SHOW_CONFIG = True            # print key config
 
 # ############### server config ###############
 HOST = "0.0.0.0"
+
 PORT = 5000
 
 # ############### db config ###################
@@ -63,6 +64,9 @@ PROXY_FETCHER = [
 
 # ############# proxy validator #################
 VERIFY_RUL = "http://www.baidu.com"
+
 VERIFY_TIMEOUT = 10
 
+MAX_FAIL_COUNT = 0
+