Merge pull request #470 from jhao104/release-2.1.0

Release 2.1.0

.travis.yml

@@ -1,7 +1,10 @@
 language: python
 python:
-  - 2.7
-  # - nightly
+  - "2.7"
+  - "3.4"
+  - "3.5"
+  - "3.6"
+  - "3.7"
 os:
   - linux
 install:

Api/ProxyApi.py

@@ -1,128 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：     ProxyApi.py
-   Description :   WebApi
-   Author :       JHao
-   date：          2016/12/4
--------------------------------------------------
-   Change Activity:
-                   2016/12/04: WebApi
-                   2019/08/14: 集成Gunicorn启动方式
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-import sys
-import platform
-from werkzeug.wrappers import Response
-from flask import Flask, jsonify, request
-
-sys.path.append('../')
-
-from Config.ConfigGetter import config
-from Manager.ProxyManager import ProxyManager
-
-app = Flask(__name__)
-
-
-class JsonResponse(Response):
-    @classmethod
-    def force_type(cls, response, environ=None):
-        if isinstance(response, (dict, list)):
-            response = jsonify(response)
-
-        return super(JsonResponse, cls).force_type(response, environ)
-
-
-app.response_class = JsonResponse
-
-api_list = {
-    'get': u'get an useful proxy',
-    # 'refresh': u'refresh proxy pool',
-    'get_all': u'get all proxy from proxy pool',
-    'delete?proxy=127.0.0.1:8080': u'delete an unable proxy',
-    'get_status': u'proxy number'
-}
-
-
-@app.route('/')
-def index():
-    return api_list
-
-
-@app.route('/get/')
-def get():
-    proxy = ProxyManager().get()
-    return proxy.info_json if proxy else {"code": 0, "src": "no proxy"}
-
-
-@app.route('/refresh/')
-def refresh():
-    # TODO refresh会有守护程序定时执行，由api直接调用性能较差，暂不使用
-    # ProxyManager().refresh()
-    pass
-    return 'success'
-
-
-@app.route('/get_all/')
-def getAll():
-    proxies = ProxyManager().getAll()
-    return jsonify([_.info_dict for _ in proxies])
-
-
-@app.route('/delete/', methods=['GET'])
-def delete():
-    proxy = request.args.get('proxy')
-    ProxyManager().delete(proxy)
-    return {"code": 0, "src": "success"}
-
-
-@app.route('/get_status/')
-def getStatus():
-    status = ProxyManager().getNumber()
-    return status
-
-
-if platform.system() != "Windows":
-    import gunicorn.app.base
-    from six import iteritems
-
-
-    class StandaloneApplication(gunicorn.app.base.BaseApplication):
-
-        def __init__(self, app, options=None):
-            self.options = options or {}
-            self.application = app
-            super(StandaloneApplication, self).__init__()
-
-        def load_config(self):
-            _config = dict([(key, value) for key, value in iteritems(self.options)
-                            if key in self.cfg.settings and value is not None])
-            for key, value in iteritems(_config):
-                self.cfg.set(key.lower(), value)
-
-        def load(self):
-            return self.application
-
-
-def runFlask():
-    app.run(host=config.host_ip, port=config.host_port)
-
-
-def runFlaskWithGunicorn():
-    _options = {
-        'bind': '%s:%s' % (config.host_ip, config.host_port),
-        'workers': 4,
-        'accesslog': '-',  # log to stdout
-        'access_log_format': '%(h)s %(l)s %(t)s "%(r)s" %(s)s "%(a)s"'
-    }
-    StandaloneApplication(app, _options).run()
-
-
-if __name__ == '__main__':
-    if platform.system() == "Windows":
-        runFlask()
-    else:
-        runFlaskWithGunicorn()

Config/ConfigGetter.py

@@ -1,75 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     ConfigGetter
-   Description :   读取配置
-   Author :        JHao
-   date：          2019/2/15
--------------------------------------------------
-   Change Activity:
-                   2019/2/15:
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-
-from Util.utilClass import LazyProperty
-from Config.setting import *
-
-
-class ConfigGetter(object):
-    """
-    get config
-    """
-
-    def __init__(self):
-        pass
-
-    @LazyProperty
-    def db_type(self):
-        return DATABASES.get("default", {}).get("TYPE", "SSDB")
-
-    @LazyProperty
-    def db_name(self):
-        return DATABASES.get("default", {}).get("NAME", "proxy")
-
-    @LazyProperty
-    def db_host(self):
-        return DATABASES.get("default", {}).get("HOST", "127.0.0.1")
-
-    @LazyProperty
-    def db_port(self):
-        return DATABASES.get("default", {}).get("PORT", 8888)
-
-    @LazyProperty
-    def db_password(self):
-        return DATABASES.get("default", {}).get("PASSWORD", "")
-
-    @LazyProperty
-    def proxy_getter_functions(self):
-        return PROXY_GETTER
-
-    @LazyProperty
-    def host_ip(self):
-        return SERVER_API.get("HOST", "127.0.0.1")
-
-    @LazyProperty
-    def host_port(self):
-        return SERVER_API.get("PORT", 5010)
-
-    @LazyProperty
-    def verify_host(self):
-        return VERIFY_HOST
-
-
-config = ConfigGetter()
-
-if __name__ == '__main__':
-    print(config.db_type)
-    print(config.db_name)
-    print(config.db_host)
-    print(config.db_port)
-    print(config.proxy_getter_functions)
-    print(config.host_ip)
-    print(config.host_port)
-    print(config.db_password)

Config/__init__.py

@@ -1,12 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     __init__
-   Description :
-   Author :        JHao
-   date：          2019/2/15
--------------------------------------------------
-   Change Activity:
-                   2019/2/15:
--------------------------------------------------
-"""

Config/setting.py

@@ -1,95 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     setting.py
-   Description :   配置文件
-   Author :        JHao
-   date：          2019/2/15
--------------------------------------------------
-   Change Activity:
-                   2019/2/15:
--------------------------------------------------
-"""
-import sys
-from os import getenv
-from logging import getLogger
-
-log = getLogger(__name__)
-
-HEADER = """
-****************************************************************
-*** ______  ********************* ______ *********** _  ********
-*** | ___ \_ ******************** | ___ \ ********* | | ********
-*** | |_/ / \__ __   __  _ __   _ | |_/ /___ * ___  | | ********
-*** |  __/|  _// _ \ \ \/ /| | | ||  __// _ \ / _ \ | | ********
-*** | |   | | | (_) | >  < \ |_| || |  | (_) | (_) || |___  ****
-*** \_|   |_|  \___/ /_/\_\ \__  |\_|   \___/ \___/ \_____/ ****
-****                       __ / /                          *****
-************************* /___ / *******************************
-*************************       ********************************
-****************************************************************
-"""
-
-PY3 = sys.version_info >= (3,)
-
-DB_TYPE = getenv('db_type', 'SSDB').upper()
-DB_HOST = getenv('db_host', '127.0.0.1')
-DB_PORT = getenv('db_port', 8888)
-DB_PASSWORD = getenv('db_password', '')
-
-
-""" 数据库配置 """
-DATABASES = {
-    "default": {
-        "TYPE": DB_TYPE,
-        "HOST": DB_HOST,
-        "PORT": DB_PORT,
-        "NAME": "proxy",
-        "PASSWORD": DB_PASSWORD
-    }
-}
-
-# register the proxy getter function
-
-PROXY_GETTER = [
-    "freeProxy01",
-    # "freeProxy02",
-    "freeProxy03",
-    "freeProxy04",
-    "freeProxy05",
-    # "freeProxy06",
-    "freeProxy07",
-    # "freeProxy08",
-    "freeProxy09",
-    "freeProxy13",
-    "freeProxy14",
-    "freeProxy14",
-]
-
-""" API config http://127.0.0.1:5010 """
-SERVER_API = {
-    "HOST": "0.0.0.0",  # The ip specified which starting the web API
-    "PORT": 5010  # port number to which the server listens to
-}
-
-VERIFY_HOST = getenv('proxy_verify_host', 'http://www.baidu.com')
-
-
-class ConfigError(BaseException):
-    pass
-
-
-def checkConfig():
-    if DB_TYPE not in ["SSDB", "REDIS"]:
-        raise ConfigError('db_type Do not support: %s, must SSDB/REDIS .' % DB_TYPE)
-
-    if type(DB_PORT) == str and not DB_PORT.isdigit():
-        raise ConfigError('if db_port is string, it must be digit, not %s' % DB_PORT)
-
-    from ProxyGetter import getFreeProxy
-    illegal_getter = list(filter(lambda key: not hasattr(getFreeProxy.GetFreeProxy, key), PROXY_GETTER))
-    if len(illegal_getter) > 0:
-        raise ConfigError("ProxyGetter: %s does not exists" % "/".join(illegal_getter))
-
-
-checkConfig()

DB/DbClient.py

@@ -1,111 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：    DbClient.py
-   Description :  DB工厂类
-   Author :       JHao
-   date：          2016/12/2
--------------------------------------------------
-   Change Activity:
-                   2016/12/2:
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-import os
-import sys
-
-from Config.ConfigGetter import config
-from Util import Singleton
-
-sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-
-
-class DbClient(object):
-    """
-    DbClient DB工厂类 提供get/put/update/pop/delete/exists/getAll/clean/getNumber/changeTable方法
-
-    目前存放代理的有两种, 使用changeTable方法切换操作对象：
-        raw_proxy： 存放原始的代理；
-        useful_proxy： 存放检验后的代理；
-
-
-    抽象方法定义：
-        get(proxy): 返回指定proxy的信息;
-        put(proxy): 存入一个proxy信息;
-        pop(): 返回并删除一个proxy信息;
-        update(proxy): 更新指定proxy信息;
-        delete(proxy): 删除指定proxy;
-        exists(proxy): 判断指定proxy是否存在;
-        getAll(): 列表形式返回所有代理;
-        clean(): 清除所有proxy信息;
-        getNumber(): 返回proxy数据量;
-        changeTable(name): 切换操作对象 raw_proxy/useful_proxy
-
-
-        所有方法需要相应类去具体实现：
-            ssdb: SsdbClient.py
-            redis: RedisClient.py
-            mongodb: MongodbClient.py
-
-    """
-
-    __metaclass__ = Singleton
-
-    def __init__(self):
-        """
-        init
-        :return:
-        """
-        self.__initDbClient()
-
-    def __initDbClient(self):
-        """
-        init DB Client
-        :return:
-        """
-        __type = None
-        if "SSDB" == config.db_type:
-            __type = "SsdbClient"
-        elif "REDIS" == config.db_type:
-            __type = "RedisClient"
-        elif "MONGODB" == config.db_type:
-            __type = "MongodbClient"
-        else:
-            pass
-        assert __type, 'type error, Not support DB type: {}'.format(config.db_type)
-        self.client = getattr(__import__(__type), __type)(name=config.db_name,
-                                                          host=config.db_host,
-                                                          port=config.db_port,
-                                                          password=config.db_password)
-
-    def get(self, key, **kwargs):
-        return self.client.get(key, **kwargs)
-
-    def put(self, key, **kwargs):
-        return self.client.put(key, **kwargs)
-
-    def update(self, key, value, **kwargs):
-        return self.client.update(key, value, **kwargs)
-
-    def delete(self, key, **kwargs):
-        return self.client.delete(key, **kwargs)
-
-    def exists(self, key, **kwargs):
-        return self.client.exists(key, **kwargs)
-
-    def pop(self, **kwargs):
-        return self.client.pop(**kwargs)
-
-    def getAll(self):
-        return self.client.getAll()
-
-    def clear(self):
-        return self.client.clear()
-
-    def changeTable(self, name):
-        self.client.changeTable(name)
-
-    def getNumber(self):
-        return self.client.getNumber()

Dockerfile

@@ -18,6 +18,4 @@ COPY . .
 
 EXPOSE 5010
 
-WORKDIR /app/cli
-
 ENTRYPOINT [ "sh", "start.sh" ]

Manager/ProxyManager.py

@@ -1,108 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：     ProxyManager.py
-   Description :
-   Author :       JHao
-   date：          2016/12/3
--------------------------------------------------
-   Change Activity:
-                   2016/12/3:
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-import random
-
-from ProxyHelper import Proxy
-from DB.DbClient import DbClient
-from Config.ConfigGetter import config
-from Util.LogHandler import LogHandler
-from Util.utilFunction import verifyProxyFormat
-from ProxyGetter.getFreeProxy import GetFreeProxy
-
-
-class ProxyManager(object):
-    """
-    ProxyManager
-    """
-
-    def __init__(self):
-        self.db = DbClient()
-        self.raw_proxy_queue = 'raw_proxy'
-        self.log = LogHandler('proxy_manager')
-        self.useful_proxy_queue = 'useful_proxy'
-
-    def fetch(self):
-        """
-        fetch proxy into db by ProxyGetter
-        :return:
-        """
-        self.db.changeTable(self.raw_proxy_queue)
-        proxy_set = set()
-        self.log.info("ProxyFetch : start")
-        for proxyGetter in config.proxy_getter_functions:
-            self.log.info("ProxyFetch - {func}: start".format(func=proxyGetter))
-            try:
-                for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
-                    proxy = proxy.strip()
-
-                    if not proxy or not verifyProxyFormat(proxy):
-                        self.log.error('ProxyFetch - {func}: '
-                                       '{proxy} illegal'.format(func=proxyGetter, proxy=proxy.ljust(20)))
-                        continue
-                    elif proxy in proxy_set:
-                        self.log.info('ProxyFetch - {func}: '
-                                      '{proxy} exist'.format(func=proxyGetter, proxy=proxy.ljust(20)))
-                        continue
-                    else:
-                        self.log.info('ProxyFetch - {func}: '
-                                      '{proxy} success'.format(func=proxyGetter, proxy=proxy.ljust(20)))
-                        self.db.put(Proxy(proxy, source=proxyGetter))
-                        proxy_set.add(proxy)
-            except Exception as e:
-                self.log.error("ProxyFetch - {func}: error".format(func=proxyGetter))
-                self.log.error(str(e))
-
-    def get(self):
-        """
-        return a useful proxy
-        :return:
-        """
-        self.db.changeTable(self.useful_proxy_queue)
-        item_list = self.db.getAll()
-        if item_list:
-            random_choice = random.choice(item_list)
-            return Proxy.newProxyFromJson(random_choice)
-        return None
-
-    def delete(self, proxy_str):
-        """
-        delete proxy from pool
-        :param proxy_str:
-        :return:
-        """
-        self.db.changeTable(self.useful_proxy_queue)
-        self.db.delete(proxy_str)
-
-    def getAll(self):
-        """
-        get all proxy from pool as list
-        :return:
-        """
-        self.db.changeTable(self.useful_proxy_queue)
-        item_list = self.db.getAll()
-        return [Proxy.newProxyFromJson(_) for _ in item_list]
-
-    def getNumber(self):
-        self.db.changeTable(self.raw_proxy_queue)
-        total_raw_proxy = self.db.getNumber()
-        self.db.changeTable(self.useful_proxy_queue)
-        total_useful_queue = self.db.getNumber()
-        return {'raw_proxy': total_raw_proxy, 'useful_proxy': total_useful_queue}
-
-
-if __name__ == '__main__':
-    pp = ProxyManager()
-    pp.fetch()

ProxyGetter/CheckProxy.py

@@ -1,70 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     CheckProxy
-   Description :   used for check getFreeProxy.py
-   Author :        JHao
-   date：          2018/7/10
--------------------------------------------------
-   Change Activity:
-                   2018/7/10: CheckProxy
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-from .getFreeProxy import GetFreeProxy
-from Util.utilFunction import verifyProxyFormat
-
-
-from Util.LogHandler import LogHandler
-
-log = LogHandler('check_proxy', file=False)
-
-
-class CheckProxy(object):
-
-    @staticmethod
-    def checkAllGetProxyFunc():
-        """
-        检查getFreeProxy所有代理获取函数运行情况
-        Returns:
-            None
-        """
-        import inspect
-        member_list = inspect.getmembers(GetFreeProxy, predicate=inspect.isfunction)
-        proxy_count_dict = dict()
-        for func_name, func in member_list:
-            log.info(u"开始运行 {}".format(func_name))
-            try:
-                proxy_list = [_ for _ in func() if verifyProxyFormat(_)]
-                proxy_count_dict[func_name] = len(proxy_list)
-            except Exception as e:
-                log.info(u"代理获取函数 {} 运行出错!".format(func_name))
-                log.error(str(e))
-        log.info(u"所有函数运行完毕 " + "***" * 5)
-        for func_name, func in member_list:
-            log.info(u"函数 {n}, 获取到代理数: {c}".format(n=func_name, c=proxy_count_dict.get(func_name, 0)))
-
-    @staticmethod
-    def checkGetProxyFunc(func):
-        """
-        检查指定的getFreeProxy某个function运行情况
-        Args:
-            func: getFreeProxy中某个可调用方法
-
-        Returns:
-            None
-        """
-        func_name = getattr(func, '__name__', "None")
-        log.info("start running func: {}".format(func_name))
-        count = 0
-        for proxy in func():
-            if verifyProxyFormat(proxy):
-                log.info("{} fetch proxy: {}".format(func_name, proxy))
-                count += 1
-        log.info("{n} completed, fetch proxy number: {c}".format(n=func_name, c=count))
-
-
-if __name__ == '__main__':
-    CheckProxy.checkAllGetProxyFunc()
-    CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01)

ProxyHelper/ProxyUtil.py

@@ -1,40 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     ProxyHelper
-   Description :
-   Author :        JHao
-   date：          2019/8/8
--------------------------------------------------
-   Change Activity:
-                   2019/8/8:
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-from Util import validUsefulProxy
-
-from datetime import datetime
-
-
-def checkProxyUseful(proxy_obj):
-    """
-    检测代理是否可用
-    :param proxy_obj: Proxy object
-    :return: Proxy object, status
-    """
-
-    if validUsefulProxy(proxy_obj.proxy):
-        # 检测通过 更新proxy属性
-        proxy_obj.check_count += 1
-        proxy_obj.last_status = 1
-        proxy_obj.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        if proxy_obj.fail_count > 0:
-            proxy_obj.fail_count -= 1
-        return proxy_obj, True
-    else:
-        proxy_obj.check_count += 1
-        proxy_obj.last_status = 0
-        proxy_obj.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        proxy_obj.fail_count += 1
-        return proxy_obj, False

README.md

@@ -1,5 +1,5 @@
 
-爬虫IP代理池
+ProxyPool 爬虫代理IP池
 =======
 [![Build Status](https://travis-ci.org/jhao104/proxy_pool.svg?branch=master)](https://travis-ci.org/jhao104/proxy_pool)
 [![](https://img.shields.io/badge/Powered%20by-@j_hao104-green.svg)](http://www.spiderpy.cn/blog/)
@@ -17,101 +17,89 @@
                            __ / /
                           /___ /
 
-##### [介绍文档](https://github.com/jhao104/proxy_pool/blob/master/doc/introduce.md)
+### ProxyPool
+
+* 文档: [document](https://github.com/jhao104/proxy_pool/blob/master/doc/introduce.md)
 
 * 支持版本: ![](https://img.shields.io/badge/Python-2.x-green.svg) ![](https://img.shields.io/badge/Python-3.x-blue.svg)
 
 * 测试地址: http://118.24.52.95 (单机勿压, 感谢。 恶意访问关[小黑屋](https://github.com/jhao104/proxy_pool/blob/bff423dffe6e2881ee45d5b66d8a6ad682c8e4ab/doc/block_ips.md)哦)
 
-### 下载安装
+* 有时间的朋友帮忙点下末尾的[推广](https://tracking.gitads.io/?repo=proxy_pool)哦 :stuck_out_tongue_winking_eye:
+
+### 运行项目
+
+##### 下载代码:
 
-* 下载源码:
+* git clone
 
-```shell
+```bash
 git clone git@github.com:jhao104/proxy_pool.git
+```
+
+* releases
 
-或者直接到https://github.com/jhao104/proxy_pool/releases 下载zip文件
+```bash
+https://github.com/jhao104/proxy_pool/releases 下载对应zip文件
 ```
 
-* 安装依赖:
+##### 安装依赖:
 
-```shell
+```bash
 pip install -r requirements.txt
 ```
 
-* 配置Config/setting.py:
+#### 修改配置:
 
-```shell
-# Config/setting.py 为项目配置文件
 
-# 配置DB     
-DATABASES = {
-    "default": {
-        "TYPE": "SSDB",        # 目前支持SSDB或REDIS数据库
-        "HOST": "127.0.0.1",   # db host
-        "PORT": 8888,          # db port，例如SSDB通常使用8888，REDIS通常默认使用6379
-        "NAME": "proxy",       # 默认配置
-        "PASSWORD": ""         # db password
+```python
+# setting.py 为项目配置文件
 
-    }
-}
+# 配置API服务
 
+HOST = "0.0.0.0"               # IP
+PORT = 5000                    # 监听端口
 
-# 配置 ProxyGetter
 
-PROXY_GETTER = [
-    "freeProxy01",      # 这里是启用的代理抓取函数名，可在ProxyGetter/getFreeProxy.py 扩展
-    "freeProxy02",
-    ....
-]
+# 配置数据库
 
+DB_CONN = 'redis://@127.0.0.1:8888'
 
-# 配置 API服务
 
-SERVER_API = {
-    "HOST": "0.0.0.0",  # 监听ip, 0.0.0.0 监听所有IP
-    "PORT": 5010        # 监听端口
-}
-       
-# 上面配置启动后，代理池访问地址为 http://127.0.0.1:5010
+# 配置 ProxyFetcher
 
+PROXY_FETCHER = [
+    "freeProxy01",      # 这里是启用的代理抓取方法名，所有fetch方法位于fetcher/proxyFetcher.py
+    "freeProxy02",
+    # ....
+]
 ```
 
-* 启动:
+#### 启动项目:
 
-```shell
-# 如果你的依赖已经安装完成并且具备运行条件,可以在cli目录下通过ProxyPool.py启。动
-# 程序分为: schedule 调度程序 和 webserver Api服务
-
-# 首先启动调度程序
->>>python proxyPool.py schedule
+```bash
+# 如果已经具备运行条件, 可用通过proxyPool.py启动。
+# 程序分为: schedule 调度程序 和 server Api服务
 
-# 然后启动webApi服务
->>>python proxyPool.py webserver
+# 启动调度程序
+python proxyPool.py schedule
 
+# 启动webApi服务
+python proxyPool.py server
 
 ```
 
-### Docker
+### Docker运行
 
 ```bash
 docker pull jhao104/proxy_pool
 
-# 远程数据库
-docker run --env db_type=REDIS --env db_host=x.x.x.x --env db_port=6379 --env db_password=pwd_str -p 5010:5010 jhao104/proxy_pool
-
-# 宿主机上的数据库
-docker run --env db_type=REDIS --env db_host=host.docker.internal --env db_port=6379 --env db_password=pwd_str -p 5010:5010 jhao104/proxy_pool
-
+docker run --env DB_CONN=redis://:password@ip:port/db -p 5010:5010 jhao104/proxy_pool:2.1.0
 ```
 
 
 ### 使用
 
-　　启动过几分钟后就能看到抓取到的代理IP，你可以直接到数据库中查看，推荐一个[SSDB可视化工具](https://github.com/jhao104/SSDBAdmin)。
-
-　　也可以通过api访问http://127.0.0.1:5010 查看。
-
 * Api
 
 | api | method | Description | arg|
@@ -148,54 +136,54 @@ def getHtml():
             return html
         except Exception:
             retry_count -= 1
-    # 出错5次, 删除代理池中代理
+    # 删除代理池中代理
     delete_proxy(proxy)
     return None
 ```
 
 ### 扩展代理
 
-　　项目默认包含几个免费的代理获取方法，但是免费的毕竟质量不好，所以如果直接运行可能拿到的代理质量不理想。所以，提供了代理获取的扩展方法。
+　　项目默认包含几个免费的代理获取源，但是免费的毕竟质量有限，所以如果直接运行可能拿到的代理质量不理想。所以，提供了代理获取的扩展方法。
 
-　　添加一个新的代理获取方法如下:
+　　添加一个新的代理源方法如下:
 
-* 1、首先在[GetFreeProxy](https://github.com/jhao104/proxy_pool/blob/b9ccdfaada51b57cfb1bbd0c01d4258971bc8352/ProxyGetter/getFreeProxy.py#L32)类中添加你的获取代理的静态方法，
+* 1、首先在[ProxyFetcher](https://github.com/jhao104/proxy_pool/blob/1a3666283806a22ef287fba1a8efab7b94e94bac/fetcher/proxyFetcher.py#L21)类中添加自定义的获取代理的静态方法，
 该方法需要以生成器(yield)形式返回`host:ip`格式的代理，例如:
 
 ```python
 
-class GetFreeProxy(object):
+class ProxyFetcher(object):
     # ....
 
-    # 你自己的方法
+    # 自定义代理源获取方法
     @staticmethod
-    def freeProxyCustom():  # 命名不和已有重复即可
+    def freeProxyCustom1():  # 命名不和已有重复即可
 
-        # 通过某网站或者某接口或某数据库获取代理 任意你喜欢的姿势都行
-        # 假设你拿到了一个代理列表
-        proxies = ["139.129.166.68:3128", "139.129.166.61:3128", ...]
+        # 通过某网站或者某接口或某数据库获取代理
+        # 假设你已经拿到了一个代理列表
+        proxies = ["x.x.x.x:3128", "x.x.x.x:80"]
         for proxy in proxies:
             yield proxy
-        # 确保每个proxy都是 host:ip正确的格式就行
+        # 确保每个proxy都是 host:ip正确的格式返回
 ```
 
-* 2、添加好方法后，修改Config/setting.py文件中的`PROXY_GETTER`项：
+* 2、添加好方法后，修改[setting.py](https://github.com/jhao104/proxy_pool/blob/1a3666283806a22ef287fba1a8efab7b94e94bac/setting.py#L47)文件中的`PROXY_FETCHER`项：
 
-　　在`PROXY_GETTER`下添加自定义的方法的名字:
+　　在`PROXY_FETCHER`下添加自定义方法的名字:
 
-```shell
-PROXY_GETTER = [
+```python
+PROXY_FETCHER = [
     "freeProxy01",    
     "freeProxy02",
-    ....
-    "freeProxyCustom"  #  # 确保名字和你添加方法名字一致
+    # ....
+    "freeProxyCustom1"  #  # 确保名字和你添加方法名字一致
 ]
 ```
 
 
-　　`ProxySchedule`会每隔一段时间抓取一次代理，下次抓取时会自动识别调用你定义的方法。
+　　`schedule` 进程会每隔一段时间抓取一次代理，下次抓取时会自动识别调用你定义的方法。
 
-### 代理采集
+### 免费代理源
 
    目前实现的采集免费代理网站有(排名不分先后, 下面仅是对其发布的免费代理情况, 付费代理测评可以参考[这里](https://zhuanlan.zhihu.com/p/33576641)): 
    
@@ -205,7 +193,7 @@ PROXY_GETTER = [
   | 66代理   | 可用  | 更新很慢   |   *     |  否      | [地址](http://www.66ip.cn/) |
   | 西刺代理 | 可用   | 几分钟一次 |   *     | 否       | [地址](http://www.xicidaili.com)|
   | 全网代理 |  可用  | 几分钟一次 |   *     |  否      | [地址](http://www.goubanjia.com/)|
-  | 训代理 |  已关闭免费代理  | * |   *     |  否      | [地址](http://www.xdaili.cn/)|
+  | ~~训代理~~ |  已关闭免费代理  | * |   *     |  否      | [地址](http://www.xdaili.cn/)|
   | 快代理 |  可用  |几分钟一次|   *     |  否      | [地址](https://www.kuaidaili.com/)|
   | 云代理 |  可用  |几分钟一次|   *     |  否      | [地址](http://www.ip3366.net/)|
   | IP海 |  可用  |几小时一次|   *     |  否      | [地址](http://www.iphai.com/)|
@@ -218,7 +206,7 @@ PROXY_GETTER = [
 
 ### 问题反馈
 
-　　任何问题欢迎在[Issues](https://github.com/jhao104/proxy_pool/issues) 中反馈，如果没有账号可以去 我的[博客](http://www.spiderpy.cn/blog/message)中留言。
+　　任何问题欢迎在[Issues](https://github.com/jhao104/proxy_pool/issues) 中反馈，同时也可以到我的[博客](http://www.spiderpy.cn/blog/message)中留言。
 
 　　你的反馈会让此项目变得更加完美。
 
@@ -226,7 +214,7 @@ PROXY_GETTER = [
 
 　　本项目仅作为基本的通用的代理池架构，不接收特有功能(当然,不限于特别好的idea)。
 
-　　本项目依然不够完善，如果发现bug或有新的功能添加，请在[Issues](https://github.com/jhao104/proxy_pool/issues)中提交bug(或新功能)描述，在确认后提交你的代码。
+　　本项目依然不够完善，如果发现bug或有新的功能添加，请在[Issues](https://github.com/jhao104/proxy_pool/issues)中提交bug(或新功能)描述，我会尽力改进，使她更加完美。
 
 　　这里感谢以下contributor的无私奉献：
 
@@ -237,6 +225,7 @@ PROXY_GETTER = [
 
    [release notes](https://github.com/jhao104/proxy_pool/blob/master/doc/release_notes.md)
 
-### [AD](https://tracking.gitads.io/?repo=proxy_pool)
+### AD
 
-[![AD](https://images.gitads.io/proxy_pool)](https://tracking.gitads.io/?repo=proxy_pool)
+　　最后, 开源不易, 有时间的小伙伴可以点下[推广](https://tracking.gitads.io/?repo=proxy_pool)广告。
+　　[![AD](https://images.gitads.io/proxy_pool)](https://tracking.gitads.io/?repo=proxy_pool)

Schedule/ProxyScheduler.py

@@ -1,61 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     ProxyScheduler
-   Description :
-   Author :        JHao
-   date：          2019/8/5
--------------------------------------------------
-   Change Activity:
-                   2019/8/5: ProxyScheduler
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-import sys
-from apscheduler.schedulers.blocking import BlockingScheduler
-
-sys.path.append('../')
-
-from Schedule import doRawProxyCheck, doUsefulProxyCheck
-from Manager import ProxyManager
-from Util import LogHandler
-
-
-class DoFetchProxy(ProxyManager):
-    """ fetch proxy"""
-
-    def __init__(self):
-        ProxyManager.__init__(self)
-        self.log = LogHandler('fetch_proxy')
-
-    def main(self):
-        self.log.info("start fetch proxy")
-        self.fetch()
-        self.log.info("finish fetch proxy")
-
-
-def rawProxyScheduler():
-    DoFetchProxy().main()
-    doRawProxyCheck()
-
-
-def usefulProxyScheduler():
-    doUsefulProxyCheck()
-
-
-def runScheduler():
-    rawProxyScheduler()
-    usefulProxyScheduler()
-
-    scheduler_log = LogHandler("scheduler_log")
-    scheduler = BlockingScheduler(logger=scheduler_log)
-
-    scheduler.add_job(rawProxyScheduler, 'interval', minutes=5, id="raw_proxy_check", name="raw_proxy定时采集")
-    scheduler.add_job(usefulProxyScheduler, 'interval', minutes=1, id="useful_proxy_check", name="useful_proxy定时检查")
-
-    scheduler.start()
-
-
-if __name__ == '__main__':
-    runScheduler()

Schedule/RawProxyCheck.py

@@ -1,81 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     RawProxyCheck
-   Description :   check raw_proxy to useful
-   Author :        JHao
-   date：          2019/8/6
--------------------------------------------------
-   Change Activity:
-                   2019/8/6: check raw_proxy to useful
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-from threading import Thread
-
-try:
-    from Queue import Empty, Queue  # py2
-except:
-    from queue import Empty, Queue  # py3
-
-from Util import LogHandler
-from Manager import ProxyManager
-from ProxyHelper import Proxy, checkProxyUseful
-
-
-class RawProxyCheck(ProxyManager, Thread):
-    def __init__(self, queue, thread_name):
-        ProxyManager.__init__(self)
-        Thread.__init__(self, name=thread_name)
-        self.log = LogHandler('raw_proxy_check')
-        self.queue = queue
-
-    def run(self):
-        self.log.info("RawProxyCheck - {}  : start".format(self.name))
-        self.db.changeTable(self.useful_proxy_queue)
-        while True:
-            try:
-                proxy_json = self.queue.get(block=False)
-            except Empty:
-                self.log.info("RawProxyCheck - {}  : exit".format(self.name))
-                break
-
-            proxy_obj = Proxy.newProxyFromJson(proxy_json)
-
-            proxy_obj, status = checkProxyUseful(proxy_obj)
-            if status:
-                if self.db.exists(proxy_obj.proxy):
-                    self.log.info('RawProxyCheck - {}  : {} validation exists'.format(self.name,
-                                                                                      proxy_obj.proxy.ljust(20)))
-                else:
-                    self.db.put(proxy_obj)
-                    self.log.info(
-                        'RawProxyCheck - {}  : {} validation pass'.format(self.name, proxy_obj.proxy.ljust(20)))
-            else:
-                self.log.info('RawProxyCheck - {}  : {} validation fail'.format(self.name, proxy_obj.proxy.ljust(20)))
-            self.queue.task_done()
-
-
-def doRawProxyCheck():
-    proxy_queue = Queue()
-
-    pm = ProxyManager()
-    pm.db.changeTable(pm.raw_proxy_queue)
-    for _proxy in pm.db.getAll():
-        proxy_queue.put(_proxy)
-    pm.db.clear()
-
-    thread_list = list()
-    for index in range(20):
-        thread_list.append(RawProxyCheck(proxy_queue, "thread_%s" % index))
-
-    for thread in thread_list:
-        thread.start()
-
-    for thread in thread_list:
-        thread.join()
-
-
-if __name__ == '__main__':
-    doRawProxyCheck()

Schedule/UsefulProxyCheck.py

@@ -1,80 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     UsefulProxyCheck
-   Description :   check useful proxy
-   Author :        JHao
-   date：          2019/8/7
--------------------------------------------------
-   Change Activity:
-                   2019/8/7: check useful proxy
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-from threading import Thread
-
-try:
-    from Queue import Queue, Empty  # py2
-except:
-    from queue import Queue, Empty  # py3
-
-from Util import LogHandler
-from Manager import ProxyManager
-from ProxyHelper import checkProxyUseful, Proxy
-
-FAIL_COUNT = 0
-
-
-class UsefulProxyCheck(ProxyManager, Thread):
-    def __init__(self, queue, thread_name):
-        ProxyManager.__init__(self)
-        Thread.__init__(self, name=thread_name)
-
-        self.queue = queue
-        self.log = LogHandler('useful_proxy_check')
-
-    def run(self):
-        self.log.info("UsefulProxyCheck - {}  : start".format(self.name))
-        self.db.changeTable(self.useful_proxy_queue)
-        while True:
-            try:
-                proxy_str = self.queue.get(block=False)
-            except Empty:
-                self.log.info("UsefulProxyCheck - {}  : exit".format(self.name))
-                break
-
-            proxy_obj = Proxy.newProxyFromJson(proxy_str)
-            proxy_obj, status = checkProxyUseful(proxy_obj)
-            if status or proxy_obj.fail_count < FAIL_COUNT:
-                self.db.put(proxy_obj)
-                self.log.info('UsefulProxyCheck - {}  : {} validation pass'.format(self.name,
-                                                                                   proxy_obj.proxy.ljust(20)))
-            else:
-                self.log.info('UsefulProxyCheck - {}  : {} validation fail'.format(self.name,
-                                                                                   proxy_obj.proxy.ljust(20)))
-                self.db.delete(proxy_obj.proxy)
-            self.queue.task_done()
-
-
-def doUsefulProxyCheck():
-    proxy_queue = Queue()
-
-    pm = ProxyManager()
-    pm.db.changeTable(pm.useful_proxy_queue)
-    for _proxy in pm.db.getAll():
-        proxy_queue.put(_proxy)
-
-    thread_list = list()
-    for index in range(10):
-        thread_list.append(UsefulProxyCheck(proxy_queue, "thread_%s" % index))
-
-    for thread in thread_list:
-        thread.start()
-
-    for thread in thread_list:
-        thread.join()
-
-
-if __name__ == '__main__':
-    doUsefulProxyCheck()

Schedule/__init__.py

@@ -1,16 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     __init__.py.py  
-   Description :  
-   Author :       JHao
-   date：          2016/12/3
--------------------------------------------------
-   Change Activity:
-                   2016/12/3: 
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-from Schedule.RawProxyCheck import doRawProxyCheck
-from Schedule.UsefulProxyCheck import doUsefulProxyCheck

Test/testGetFreeProxy.py

@@ -1,37 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     testGetFreeProxy
-   Description :   test model ProxyGetter/getFreeProxy
-   Author :        J_hao
-   date：          2017/7/31
--------------------------------------------------
-   Change Activity:
-                   2017/7/31:function testGetFreeProxy
--------------------------------------------------
-"""
-__author__ = 'J_hao'
-
-
-from ProxyGetter.getFreeProxy import GetFreeProxy
-from Config.ConfigGetter import config
-
-
-def testGetFreeProxy():
-    """
-    test class GetFreeProxy in ProxyGetter/GetFreeProxy
-    :return:
-    """
-    proxy_getter_functions = config.proxy_getter_functions
-    for proxyGetter in proxy_getter_functions:
-        proxy_count = 0
-        for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
-            if proxy:
-                print('{func}: fetch proxy {proxy},proxy_count:{proxy_count}'.format(func=proxyGetter, proxy=proxy,
-                                                                                     proxy_count=proxy_count))
-                proxy_count += 1
-        # assert proxy_count >= 20, '{} fetch proxy fail'.format(proxyGetter)
-
-
-if __name__ == '__main__':
-    testGetFreeProxy()

Test/testWebRequest.py

@@ -1,30 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     testWebRequest
-   Description :   test class WebRequest
-   Author :        J_hao
-   date：          2017/7/31
--------------------------------------------------
-   Change Activity:
-                   2017/7/31: function testWebRequest
--------------------------------------------------
-"""
-__author__ = 'J_hao'
-
-from Util.WebRequest import WebRequest
-
-
-# noinspection PyPep8Naming
-def testWebRequest():
-    """
-    test class WebRequest in Util/WebRequest.py
-    :return:
-    """
-    wr = WebRequest()
-    request_object = wr.get('https://www.baidu.com/')
-    assert request_object.status_code == 200
-
-
-if __name__ == '__main__':
-    testWebRequest()

Util/utilFunction.py

@@ -1,105 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：     utilFunction.py
-   Description :  tool function
-   Author :       JHao
-   date：          2016/11/25
--------------------------------------------------
-   Change Activity:
-                   2016/11/25: 添加robustCrawl、verifyProxy、getHtmlTree
--------------------------------------------------
-"""
-from lxml import etree
-import requests
-
-from Util.WebRequest import WebRequest
-from .validators import validators
-from Config.ConfigGetter import config
-
-
-def robustCrawl(func):
-    def decorate(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except Exception as e:
-            pass
-            # logger.info(u"sorry, 抓取出错。错误原因:")
-            # logger.info(e)
-
-    return decorate
-
-
-def verifyProxyFormat(proxy):
-    """
-    检查代理格式
-    :param proxy:
-    :return:
-    """
-    import re
-    verify_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}"
-    _proxy = re.findall(verify_regex, proxy)
-    return True if len(_proxy) == 1 and _proxy[0] == proxy else False
-
-
-def getHtmlTree(url, **kwargs):
-    """
-    获取html树
-    :param url:
-    :param kwargs:
-    :return:
-    """
-
-    header = {'Connection': 'keep-alive',
-              'Cache-Control': 'max-age=0',
-              'Upgrade-Insecure-Requests': '1',
-              'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)',
-              'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-              'Accept-Encoding': 'gzip, deflate, sdch',
-              'Accept-Language': 'zh-CN,zh;q=0.8',
-              }
-    # TODO 取代理服务器用代理服务器访问
-    wr = WebRequest()
-    html = wr.get(url=url, header=header).content
-    return etree.HTML(html)
-
-
-def tcpConnect(proxy):
-    """
-    TCP 三次握手
-    :param proxy:
-    :return:
-    """
-    from socket import socket, AF_INET, SOCK_STREAM
-    s = socket(AF_INET, SOCK_STREAM)
-    ip, port = proxy.split(':')
-    result = s.connect_ex((ip, int(port)))
-    return True if result == 0 else False
-
-
-def validUsefulProxy(proxy):
-    """
-    检验代理是否可用
-    :param proxy:
-    :return:
-    """
-    if isinstance(proxy, bytes):
-        proxy = proxy.decode("utf8")
-    proxies = {"http": "http://{proxy}".format(proxy=proxy)}
-    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
-               'Accept': '*/*',
-               'Connection': 'keep-alive',
-               'Accept-Language': 'zh-CN,zh;q=0.8'}
-    try:
-        r = requests.head(config.verify_host, headers=headers, proxies=proxies, timeout=10, verify=False)
-        if r.status_code == 200:
-            return True
-    except Exception as e:
-        pass
-    return False
-
-    # for v_func in validators:
-    #     if not v_func(proxy):
-    #         return False
-    # return True

Util/validators.py

@@ -1,38 +0,0 @@
-import requests
-
-validators = []
-
-
-def validator(func):
-    validators.append(func)
-    return func
-
-
-@validator
-def timeOutValidator(proxy):
-    """
-    检测超时
-    :param proxy:
-    :return:
-    """
-    if isinstance(proxy, bytes):
-        proxy = proxy.decode("utf8")
-    proxies = {"http": "http://{proxy}".format(proxy=proxy)}
-    try:
-        r = requests.get('http://www.baidu.com', proxies=proxies, timeout=10, verify=False)
-        if r.status_code == 200:
-            return True
-    except Exception as e:
-        pass
-    return False
-
-
-@validator
-def customValidator(proxy):
-    """
-    自定义validator函数，校验代理是否可用
-    :param proxy:
-    :return:
-    """
-
-    return True

__init__.py

@@ -1,13 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     __init__.py  
-   Description :  
-   Author :       JHao
-   date：          2016/12/3
--------------------------------------------------
-   Change Activity:
-                   2016/12/3: 
--------------------------------------------------
-"""
-__author__ = 'JHao'

api/proxyApi.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+# !/usr/bin/env python
+"""
+-------------------------------------------------
+   File Name：     ProxyApi.py
+   Description :   WebApi
+   Author :       JHao
+   date：          2016/12/4
+-------------------------------------------------
+   Change Activity:
+                   2016/12/04: WebApi
+                   2019/08/14: 集成Gunicorn启动方式
+                   2020/06/23: 新增pop接口
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+import platform
+from werkzeug.wrappers import Response
+from flask import Flask, jsonify, request
+
+from util.six import iteritems
+from handler.proxyHandler import ProxyHandler
+from handler.configHandler import ConfigHandler
+from helper.proxy import Proxy
+
+app = Flask(__name__)
+conf = ConfigHandler()
+proxy_handler = ProxyHandler()
+
+
+class JsonResponse(Response):
+    @classmethod
+    def force_type(cls, response, environ=None):
+        if isinstance(response, (dict, list)):
+            response = jsonify(response)
+
+        return super(JsonResponse, cls).force_type(response, environ)
+
+
+app.response_class = JsonResponse
+
+api_list = {
+    'get': u'get an useful proxy',
+    'pop': u'get and delete an useful proxy',
+    # 'refresh': u'refresh proxy pool',
+    'get_all': u'get all proxy from proxy pool',
+    'delete?proxy=127.0.0.1:8080': u'delete an unable proxy',
+    'get_status': u'proxy number'
+}
+
+
+@app.route('/')
+def index():
+    return api_list
+
+
+@app.route('/get/')
+def get():
+    proxy = proxy_handler.get()
+    return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"}
+
+
+@app.route('/pop/')
+def pop():
+    proxy = proxy_handler.pop()
+    return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"}
+
+
+@app.route('/refresh/')
+def refresh():
+    # TODO refresh会有守护程序定时执行，由api直接调用性能较差，暂不使用
+    return 'success'
+
+
+@app.route('/get_all/')
+def getAll():
+    proxies = proxy_handler.getAll()
+    return jsonify([_.to_dict for _ in proxies])
+
+
+@app.route('/delete/', methods=['GET'])
+def delete():
+    proxy = request.args.get('proxy')
+    status = proxy_handler.delete(Proxy(proxy))
+    return {"code": 0, "src": status}
+
+
+@app.route('/get_status/')
+def getStatus():
+    status = proxy_handler.getCount()
+    return status
+
+
+def runFlask():
+    if platform.system() == "Windows":
+        app.run(host=conf.serverHost, port=conf.serverPort)
+    else:
+        import gunicorn.app.base
+
+        class StandaloneApplication(gunicorn.app.base.BaseApplication):
+
+            def __init__(self, app, options=None):
+                self.options = options or {}
+                self.application = app
+                super(StandaloneApplication, self).__init__()
+
+            def load_config(self):
+                _config = dict([(key, value) for key, value in iteritems(self.options)
+                                if key in self.cfg.settings and value is not None])
+                for key, value in iteritems(_config):
+                    self.cfg.set(key.lower(), value)
+
+            def load(self):
+                return self.application
+
+        _options = {
+            'bind': '%s:%s' % (conf.serverHost, conf.serverPort),
+            'workers': 4,
+            'accesslog': '-',  # log to stdout
+            'access_log_format': '%(h)s %(l)s %(t)s "%(r)s" %(s)s "%(a)s"'
+        }
+        StandaloneApplication(app, _options).run()
+
+
+if __name__ == '__main__':
+    runFlask()

config/setting.py

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     setting.py
+   Description :   配置文件
+   Author :        JHao
+   date：          2019/2/15
+-------------------------------------------------
+   Change Activity:
+                   2019/2/15:
+-------------------------------------------------
+"""
+
+BANNER = r"""
+****************************************************************
+*** ______  ********************* ______ *********** _  ********
+*** | ___ \_ ******************** | ___ \ ********* | | ********
+*** | |_/ / \__ __   __  _ __   _ | |_/ /___ * ___  | | ********
+*** |  __/|  _// _ \ \ \/ /| | | ||  __// _ \ / _ \ | | ********
+*** | |   | | | (_) | >  < \ |_| || |  | (_) | (_) || |___  ****
+*** \_|   |_|  \___/ /_/\_\ \__  |\_|   \___/ \___/ \_____/ ****
+****                       __ / /                          *****
+************************* /___ / *******************************
+*************************       ********************************
+****************************************************************
+"""
+
+SHOW_CONFIG = True   # print key config
+
+
+
+
+
+# import sys
+# from os import getenv
+# from logging import getLogger
+#
+# log = getLogger(__name__)
+#
+# BANNER = """
+# ****************************************************************
+# *** ______  ********************* ______ *********** _  ********
+# *** | ___ \_ ******************** | ___ \ ********* | | ********
+# *** | |_/ / \__ __   __  _ __   _ | |_/ /___ * ___  | | ********
+# *** |  __/|  _// _ \ \ \/ /| | | ||  __// _ \ / _ \ | | ********
+# *** | |   | | | (_) | >  < \ |_| || |  | (_) | (_) || |___  ****
+# *** \_|   |_|  \___/ /_/\_\ \__  |\_|   \___/ \___/ \_____/ ****
+# ****                       __ / /                          *****
+# ************************* /___ / *******************************
+# *************************       ********************************
+# ****************************************************************
+# """
+#
+# PY3 = sys.version_info >= (3,)
+#
+# DB_TYPE = getenv('db_type', 'SSDB').upper()
+# DB_HOST = getenv('db_host', '10.10.61.65')
+# DB_PORT = getenv('db_port', 6379)
+# DB_PASSWORD = getenv('db_password', '_@Fintell')
+#
+# """ 数据库配置 """
+# DATABASES = {
+#     "default": {
+#         "TYPE": DB_TYPE,
+#         "HOST": DB_HOST,
+#         "PORT": DB_PORT,
+#         "NAME": "proxy",
+#         "PASSWORD": DB_PASSWORD
+#     }
+# }
+#
+# # register the proxy getter function
+#
+# PROXY_GETTER = [
+#     "freeProxy01",
+#     # "freeProxy02",
+#     "freeProxy03",
+#     "freeProxy04",
+#     "freeProxy05",
+#     # "freeProxy06",
+#     "freeProxy07",
+#     # "freeProxy08",
+#     "freeProxy09",
+#     "freeProxy13",
+#     "freeProxy14",
+#     "freeProxy14",
+# ]
+#
+# """ API config http://127.0.0.1:5010 """
+# SERVER_API = {
+#     "HOST": "0.0.0.0",  # The ip specified which starting the web API
+#     "PORT": 5010  # port number to which the server listens to
+# }
+#
+# VERIFY_HOST = getenv('proxy_verify_host', 'http://www.baidu.com')
+
+# class ConfigError(BaseException):
+#     pass
+#
+#
+# def checkConfig():
+#     if DB_TYPE not in ["SSDB", "REDIS"]:
+#         raise ConfigError('db_type Do not support: %s, must SSDB/REDIS .' % DB_TYPE)
+#
+#     if type(DB_PORT) == str and not DB_PORT.isdigit():
+#         raise ConfigError('if db_port is string, it must be digit, not %s' % DB_PORT)
+#
+#     from proxyGetter import getFreeProxy
+#     illegal_getter = list(filter(lambda key: not hasattr(getFreeProxy.GetFreeProxy, key), PROXY_GETTER))
+#     if len(illegal_getter) > 0:
+#         raise ConfigError("ProxyGetter: %s does not exists" % "/".join(illegal_getter))
+#
+#
+# checkConfig()

db/dbClient.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+# !/usr/bin/env python
+"""
+-------------------------------------------------
+   File Name：    DbClient.py
+   Description :  DB工厂类
+   Author :       JHao
+   date：          2016/12/2
+-------------------------------------------------
+   Change Activity:
+                   2016/12/02:   DB工厂类
+                   2020/07/03:   取消raw_proxy储存
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+import os
+import sys
+
+from util.six import urlparse
+from util.singleton import Singleton
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+
+class DbClient(object):
+    """
+    DbClient DB工厂类 提供get/put/update/pop/delete/exists/getAll/clean/getCount/changeTable方法
+
+
+    抽象方法定义：
+        get(): 随机返回一个proxy;
+        put(proxy): 存入一个proxy;
+        pop(): 顺序返回并删除一个proxy;
+        update(proxy): 更新指定proxy信息;
+        delete(proxy): 删除指定proxy;
+        exists(proxy): 判断指定proxy是否存在;
+        getAll(): 返回所有代理;
+        clean(): 清除所有proxy信息;
+        getCount(): 返回proxy统计信息;
+        changeTable(name): 切换操作对象
+
+
+        所有方法需要相应类去具体实现：
+            ssdb: ssdbClient.py
+            redis: redisClient.py
+            mongodb: mongodbClient.py
+
+    """
+
+    __metaclass__ = Singleton
+
+    def __init__(self, db_conn):
+        """
+        init
+        :return:
+        """
+        self.db_conn = db_conn
+        self.parseDbConn(db_conn)
+        self.__initDbClient()
+
+    @classmethod
+    def parseDbConn(cls, db_conn):
+        db_conf = urlparse(db_conn)
+        cls.db_type = db_conf.scheme.upper().strip()
+        cls.db_host = db_conf.hostname
+        cls.db_port = db_conf.port
+        cls.db_user = db_conf.username
+        cls.db_pwd = db_conf.password
+        cls.db_name = db_conf.path[1:]
+        return cls
+
+    def __initDbClient(self):
+        """
+        init DB Client
+        :return:
+        """
+        __type = None
+        if "SSDB" == self.db_type:
+            __type = "ssdbClient"
+        elif "REDIS" == self.db_type:
+            __type = "redisClient"
+        elif "MONGODB" == self.db_type:
+            __type = "mongodbClient"
+        else:
+            pass
+        assert __type, 'type error, Not support DB type: {}'.format(self.db_type)
+        self.client = getattr(__import__(__type), "%sClient" % self.db_type.title())(host=self.db_host,
+                                                                                     port=self.db_port,
+                                                                                     username=self.db_user,
+                                                                                     password=self.db_pwd,
+                                                                                     db=self.db_name)
+
+    def get(self, **kwargs):
+        return self.client.get(**kwargs)
+
+    def put(self, key, **kwargs):
+        return self.client.put(key, **kwargs)
+
+    def update(self, key, value, **kwargs):
+        return self.client.update(key, value, **kwargs)
+
+    def delete(self, key, **kwargs):
+        return self.client.delete(key, **kwargs)
+
+    def exists(self, key, **kwargs):
+        return self.client.exists(key, **kwargs)
+
+    def pop(self, **kwargs):
+        return self.client.pop(**kwargs)
+
+    def getAll(self):
+        return self.client.getAll()
+
+    def clear(self):
+        return self.client.clear()
+
+    def changeTable(self, name):
+        self.client.changeTable(name)
+
+    def getCount(self):
+        return self.client.getCount()

DB/RedisClient.py → db/redisClient.py

@@ -1,56 +1,56 @@
 # -*- coding: utf-8 -*-
 """
--------------------------------------------------
-   File Name：     RedisClient
-   Description :  封装Redis相关操作
+-----------------------------------------------------
+   File Name：     redisClient.py
+   Description :   封装Redis相关操作
    Author :        JHao
    date：          2019/8/9
--------------------------------------------------
+------------------------------------------------------
    Change Activity:
-                   2019/8/9: 封装Redis相关操作
--------------------------------------------------
+                   2019/08/09: 封装Redis相关操作
+                   2020/06/23: 优化pop方法, 改用hscan命令
+------------------------------------------------------
 """
 __author__ = 'JHao'
 
-from Config.setting import PY3
-
 from redis.connection import BlockingConnectionPool
+from random import choice
 from redis import Redis
 
 
 class RedisClient(object):
     """
-    Redis client 和SSDB协议一致 数据结构一致, 但部分方法不通用
+    Redis client
 
     Redis中代理存放的结构为hash：
-        原始代理存放在name为raw_proxy的hash中, key为代理的ip:por, value为代理属性的字典;
-        验证后的代理存放在name为useful_proxy的hash中, key为代理的ip:port, value为代理属性的字典;
+    key为ip:port, value为代理属性的字典;
 
     """
 
-    def __init__(self, name, **kwargs):
+    def __init__(self, **kwargs):
         """
         init
-        :param name: hash name
         :param host: host
         :param port: port
         :param password: password
+        :param db: db
         :return:
         """
-        self.name = name
-        self.__conn = Redis(connection_pool=BlockingConnectionPool(**kwargs))
+        self.name = ""
+        kwargs.pop("username")
+        self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, **kwargs))
 
-    def get(self, proxy_str):
+    def get(self):
         """
-        从hash中获取对应的proxy, 使用前需要调用changeTable()
-        :param proxy_str: proxy str
+        返回一个代理
         :return:
         """
-        data = self.__conn.hget(name=self.name, key=proxy_str)
-        if data:
-            return data.decode('utf-8') if PY3 else data
+        proxies = self.__conn.hkeys(self.name)
+        proxy = choice(proxies) if proxies else None
+        if proxy:
+            return self.__conn.hget(self.name, proxy)
         else:
-            return None
+            return False
 
     def put(self, proxy_obj):
         """
@@ -58,16 +58,29 @@ class RedisClient(object):
         :param proxy_obj: Proxy obj
         :return:
         """
-        data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
+        data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
         return data
 
+    def pop(self):
+        """
+        弹出一个代理
+        :return: dict {proxy: value}
+        """
+        proxies = self.__conn.hkeys(self.name)
+        for proxy in proxies:
+            proxy_info = self.__conn.hget(self.name, proxy)
+            self.__conn.hdel(self.name, proxy)
+            return proxy_info
+        else:
+            return False
+
     def delete(self, proxy_str):
         """
         移除指定代理, 使用changeTable指定hash name
         :param proxy_str: proxy str
         :return:
         """
-        self.__conn.hdel(self.name, proxy_str)
+        return self.__conn.hdel(self.name, proxy_str)
 
     def exists(self, proxy_str):
         """
@@ -83,32 +96,15 @@ class RedisClient(object):
         :param proxy_obj:
         :return:
         """
-        self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
-
-    def pop(self):
-        """
-        弹出一个代理
-        :return: dict {proxy: value}
-        """
-        # proxies = self.__conn.hkeys(self.name)
-        # if proxies:
-        #     proxy = random.choice(proxies)
-        #     value = self.__conn.hget(self.name, proxy)
-        #     self.delete(proxy)
-        #     return {'proxy': proxy.decode('utf-8') if PY3 else proxy,
-        #             'value': value.decode('utf-8') if PY3 and value else value}
-        return None
+        return self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
 
     def getAll(self):
         """
-        列表形式返回所有代理, 使用changeTable指定hash name
+        字典形式返回所有代理, 使用changeTable指定hash name
         :return:
         """
         item_dict = self.__conn.hgetall(self.name)
-        if PY3:
-            return [value.decode('utf8') for key, value in item_dict.items()]
-        else:
-            return item_dict.values()
+        return item_dict
 
     def clear(self):
         """
@@ -117,7 +113,7 @@ class RedisClient(object):
         """
         return self.__conn.delete(self.name)
 
-    def getNumber(self):
+    def getCount(self):
         """
         返回代理数量
         :return:
@@ -127,7 +123,7 @@ class RedisClient(object):
     def changeTable(self, name):
         """
         切换操作对象
-        :param name: raw_proxy/useful_proxy
+        :param name:
         :return:
         """
         self.name = name

DB/SsdbClient.py → db/ssdbClient.py

@@ -2,22 +2,22 @@
 # !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     SsdbClient.py
-   Description :  封装SSDB操作
-   Author :       JHao
+   File Name：     ssdbClient.py
+   Description :   封装SSDB操作
+   Author :        JHao
    date：          2016/12/2
 -------------------------------------------------
    Change Activity:
                    2016/12/2:
                    2017/09/22: PY3中 redis-py返回的数据是bytes型
                    2017/09/27: 修改pop()方法 返回{proxy:value}字典
+                   2020/07/03: 2.1.0 优化代码结构
 -------------------------------------------------
 """
 __author__ = 'JHao'
 
-from Config.setting import PY3
-
 from redis.connection import BlockingConnectionPool
+from random import choice
 from redis import Redis
 
 
@@ -26,42 +26,54 @@ class SsdbClient(object):
     SSDB client
 
     SSDB中代理存放的结构为hash：
-        原始代理存放在name为raw_proxy的hash中, key为代理的ip:por, value为代理属性的字典;
-        验证后的代理存放在name为useful_proxy的hash中, key为代理的ip:port, value为代理属性的字典;
-
+    key为代理的ip:por, value为代理属性的字典;
     """
-    def __init__(self, name, **kwargs):
+
+    def __init__(self, **kwargs):
         """
         init
-        :param name: hash name
         :param host: host
         :param port: port
         :param password: password
         :return:
         """
-        self.name = name
-        self.__conn = Redis(connection_pool=BlockingConnectionPool(**kwargs))
+        self.name = ""
+        kwargs.pop("username")
+        self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, **kwargs))
 
-    def get(self, proxy_str):
+    def get(self):
         """
-        从hash中获取对应的proxy, 使用前需要调用changeTable()
-        :param proxy_str: proxy str
+        从hash中随机返回一个代理
         :return:
         """
-        data = self.__conn.hget(name=self.name, key=proxy_str)
-        if data:
-            return data.decode('utf-8') if PY3 else data
+        proxies = self.__conn.hkeys(self.name)
+        proxy = choice(proxies) if proxies else None
+        if proxy:
+            return self.__conn.hget(self.name, proxy)
         else:
             return None
 
     def put(self, proxy_obj):
         """
-        将代理放入hash, 使用changeTable指定hash name
+        将代理放入hash
         :param proxy_obj: Proxy obj
         :return:
         """
-        data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
-        return data
+        result = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
+        return result
+
+    def pop(self):
+        """
+        顺序弹出一个代理
+        :return: proxy
+        """
+        proxies = self.__conn.hkeys(self.name)
+        for proxy in proxies:
+            proxy_info = self.__conn.hget(self.name, proxy)
+            self.__conn.hdel(self.name, proxy)
+            return proxy_info
+        else:
+            return None
 
     def delete(self, proxy_str):
         """
@@ -85,41 +97,24 @@ class SsdbClient(object):
         :param proxy_obj:
         :return:
         """
-        self.__conn.hset(self.name,  proxy_obj.proxy, proxy_obj.info_json)
-
-    def pop(self):
-        """
-        弹出一个代理
-        :return: dict {proxy: value}
-        """
-        # proxies = self.__conn.hkeys(self.name)
-        # if proxies:
-        #     proxy = random.choice(proxies)
-        #     value = self.__conn.hget(self.name, proxy)
-        #     self.delete(proxy)
-        #     return {'proxy': proxy.decode('utf-8') if PY3 else proxy,
-        #             'value': value.decode('utf-8') if PY3 and value else value}
-        return None
+        self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
 
     def getAll(self):
         """
-        列表形式返回所有代理, 使用changeTable指定hash name
+        字典形式返回所有代理, 使用changeTable指定hash name
         :return:
         """
         item_dict = self.__conn.hgetall(self.name)
-        if PY3:
-            return [value.decode('utf8') for key, value in item_dict.items()]
-        else:
-            return item_dict.values()
+        return item_dict
 
     def clear(self):
         """
         清空所有代理, 使用changeTable指定hash name
         :return:
         """
-        return self.__conn.execute_command("hclear", self.name)
+        return self.__conn.delete(self.name)
 
-    def getNumber(self):
+    def getCount(self):
         """
         返回代理数量
         :return:
@@ -129,7 +124,7 @@ class SsdbClient(object):
     def changeTable(self, name):
         """
         切换操作对象
-        :param name: raw_proxy/useful_proxy
+        :param name:
         :return:
         """
         self.name = name

doc/release_notes.md

@@ -1,12 +1,15 @@
 ## Release Notes
 
-* 2.1.0
+* 2.1.0 (202007)
 
     1. 新增免费代理源 `西拉代理`  （2020-03-30）
     2. Fix Bug [#401](https://github.com/jhao104/proxy_pool/issues/401) [#356](https://github.com/jhao104/proxy_pool/issues/356)
-    3. 优化Docker镜像体积 (2020-06-19)
+    3. 优化Docker镜像体积; (2020-06-19)
+    4. 优化配置方式;
+    5. 优化代码结构;
+    6. 不再储存raw_proxy, 抓取后直接验证入库;
 
-* 2.0.1 
+* 2.0.1 (2020)
 
     1. 新增免费代理源 `89免费代理`;
     2. 新增免费代理源 `齐云代理` 

fetcher/CheckProxy.py

@@ -0,0 +1,70 @@
+# # -*- coding: utf-8 -*-
+# """
+# -------------------------------------------------
+#    File Name：     CheckProxy
+#    Description :   used for check getFreeProxy.py
+#    Author :        JHao
+#    date：          2018/7/10
+# -------------------------------------------------
+#    Change Activity:
+#                    2018/7/10: CheckProxy
+# -------------------------------------------------
+# """
+# __author__ = 'JHao'
+#
+# from .getFreeProxy import GetFreeProxy
+# from util.utilFunction import verifyProxyFormat
+#
+#
+# from util.LogHandler import LogHandler
+#
+# log = LogHandler('check_proxy', file=False)
+#
+#
+# class CheckProxy(object):
+#
+#     @staticmethod
+#     def checkAllGetProxyFunc():
+#         """
+#         检查getFreeProxy所有代理获取函数运行情况
+#         Returns:
+#             None
+#         """
+#         import inspect
+#         member_list = inspect.getmembers(GetFreeProxy, predicate=inspect.isfunction)
+#         proxy_count_dict = dict()
+#         for func_name, func in member_list:
+#             log.info(u"开始运行 {}".format(func_name))
+#             try:
+#                 proxy_list = [_ for _ in func() if verifyProxyFormat(_)]
+#                 proxy_count_dict[func_name] = len(proxy_list)
+#             except Exception as e:
+#                 log.info(u"代理获取函数 {} 运行出错!".format(func_name))
+#                 log.error(str(e))
+#         log.info(u"所有函数运行完毕 " + "***" * 5)
+#         for func_name, func in member_list:
+#             log.info(u"函数 {n}, 获取到代理数: {c}".format(n=func_name, c=proxy_count_dict.get(func_name, 0)))
+#
+#     @staticmethod
+#     def checkGetProxyFunc(func):
+#         """
+#         检查指定的getFreeProxy某个function运行情况
+#         Args:
+#             func: getFreeProxy中某个可调用方法
+#
+#         Returns:
+#             None
+#         """
+#         func_name = getattr(func, '__name__', "None")
+#         log.info("start running func: {}".format(func_name))
+#         count = 0
+#         for proxy in func():
+#             if verifyProxyFormat(proxy):
+#                 log.info("{} fetch proxy: {}".format(func_name, proxy))
+#                 count += 1
+#         log.info("{n} completed, fetch proxy number: {c}".format(n=func_name, c=count))
+#
+#
+# if __name__ == '__main__':
+#     CheckProxy.checkAllGetProxyFunc()
+#     CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01)

ProxyGetter/__init__.py → fetcher/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
 -------------------------------------------------
-   File Name：     __init__.py.py  
+   File Name：     __init__.py
    Description :  
    Author :       JHao
    date：          2016/11/25

ProxyGetter/getFreeProxy.py → fetcher/proxyFetcher.py

@@ -1,31 +1,24 @@
 # -*- coding: utf-8 -*-
-# !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     GetFreeProxy.py
-   Description :  抓取免费代理
-   Author :       JHao
+   File Name：     proxyFetcher
+   Description :
+   Author :        JHao
    date：          2016/11/25
 -------------------------------------------------
    Change Activity:
-                   2016/11/25:
+                   2016/11/25: proxyFetcher
 -------------------------------------------------
 """
+__author__ = 'JHao'
+
 import re
-import sys
-import requests
 from time import sleep
 
-sys.path.append('..')
-
-from Util.WebRequest import WebRequest
-from Util.utilFunction import getHtmlTree
+from util.webRequest import WebRequest
 
-# for debug to disable insecureWarning
-requests.packages.urllib3.disable_warnings()
 
-
-class GetFreeProxy(object):
+class ProxyFetcher(object):
     """
     proxy getter
     """
@@ -44,7 +37,7 @@ class GetFreeProxy(object):
         ]
         key = 'ABCDEFGHIZ'
         for url in url_list:
-            html_tree = getHtmlTree(url)
+            html_tree = WebRequest().get(url).tree
             ul_list = html_tree.xpath('//ul[@class="l2"]')
             for ul in ul_list:
                 try:
@@ -121,7 +114,7 @@ class GetFreeProxy(object):
         for each_url in url_list:
             for i in range(1, page_count + 1):
                 page_url = each_url + str(i)
-                tree = getHtmlTree(page_url)
+                tree = WebRequest().get(page_url).tree
                 proxy_list = tree.xpath('.//table[@id="ip_list"]//tr[position()>1]')
                 for proxy in proxy_list:
                     try:
@@ -136,7 +129,7 @@ class GetFreeProxy(object):
         :return:
         """
         url = "http://www.goubanjia.com/"
-        tree = getHtmlTree(url)
+        tree = WebRequest().get(url).tree
         proxy_list = tree.xpath('//td[@class="ip"]')
         # 此网站有隐藏的数字干扰，或抓取到多余的数字或.符号
         # 需要过滤掉<p style="display:none;">的内容
@@ -176,7 +169,7 @@ class GetFreeProxy(object):
             'https://www.kuaidaili.com/free/intr/'
         ]
         for url in url_list:
-            tree = getHtmlTree(url)
+            tree = WebRequest().get(url).tree
             proxy_list = tree.xpath('.//table//tr')
             sleep(1)  # 必须sleep 不然第二条请求不到数据
             for tr in proxy_list[1:]:
@@ -190,7 +183,7 @@ class GetFreeProxy(object):
         """
         urls = ['https://proxy.coderbusy.com/']
         for url in urls:
-            tree = getHtmlTree(url)
+            tree = WebRequest().get(url).tree
             proxy_list = tree.xpath('.//table//tr')
             for tr in proxy_list[1:]:
                 yield ':'.join(tr.xpath('./td/text()')[0:2])
@@ -203,9 +196,8 @@ class GetFreeProxy(object):
         """
         urls = ['http://www.ip3366.net/free/?stype=1',
                 "http://www.ip3366.net/free/?stype=2"]
-        request = WebRequest()
         for url in urls:
-            r = request.get(url, timeout=10)
+            r = WebRequest().get(url, timeout=10)
             proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
             for proxy in proxies:
                 yield ":".join(proxy)
@@ -222,9 +214,8 @@ class GetFreeProxy(object):
             'http://www.iphai.com/free/wg',
             'http://www.iphai.com/free/wp'
         ]
-        request = WebRequest()
         for url in urls:
-            r = request.get(url, timeout=10)
+            r = WebRequest().get(url, timeout=10)
             proxies = re.findall(r'<td>\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?</td>[\s\S]*?<td>\s*?(\d+)\s*?</td>',
                                  r.text)
             for proxy in proxies:
@@ -239,7 +230,7 @@ class GetFreeProxy(object):
         """
         for i in range(1, page_count + 1):
             url = 'http://ip.jiangxianli.com/?country=中国&?page={}'.format(i)
-            html_tree = getHtmlTree(url)
+            html_tree = WebRequest().get(url).tree
             for index, tr in enumerate(html_tree.xpath("//table//tr")):
                 if index == 0:
                     continue
@@ -293,10 +284,9 @@ class GetFreeProxy(object):
         :return:
         """
         base_url = 'http://www.qydaili.com/free/?action=china&page='
-        request = WebRequest()
         for page in range(1, max_page + 1):
             url = base_url + str(page)
-            r = request.get(url, timeout=10)
+            r = WebRequest().get(url, timeout=10)
             proxies = re.findall(
                 r'<td.*?>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td.*?>(\d+)</td>',
                 r.text)
@@ -312,10 +302,9 @@ class GetFreeProxy(object):
         :return:
         """
         base_url = 'http://www.89ip.cn/index_{}.html'
-        request = WebRequest()
         for page in range(1, max_page + 1):
             url = base_url.format(page)
-            r = request.get(url, timeout=10)
+            r = WebRequest().get(url, timeout=10)
             proxies = re.findall(
                 r'<td.*?>[\s\S]*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?</td>[\s\S]*?<td.*?>[\s\S]*?(\d+)[\s\S]*?</td>',
                 r.text)
@@ -328,28 +317,8 @@ class GetFreeProxy(object):
                 "http://www.xiladaili.com/gaoni/",
                 "http://www.xiladaili.com/http/",
                 "http://www.xiladaili.com/https/"]
-        request = WebRequest()
         for url in urls:
-            r = request.get(url, timeout=10)
+            r = WebRequest().get(url, timeout=10)
             ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", r.text)
             for ip in ips:
                 yield ip.strip()
-
-
-if __name__ == '__main__':
-    from CheckProxy import CheckProxy
-
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy02)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy03)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy04)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy05)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy06)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy07)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy08)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy09)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy13)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy14)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy15)
-
-    CheckProxy.checkAllGetProxyFunc()

Manager/__init__.py → handler/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
 -------------------------------------------------
-   File Name：     __init__.py.py  
+   File Name：     __init__.py
    Description :  
    Author :       JHao
    date：          2016/12/3
@@ -12,4 +12,4 @@
 """
 __author__ = 'JHao'
 
-from Manager.ProxyManager import ProxyManager
+# from handler.ProxyManager import ProxyManager

handler/configHandler.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     configHandler
+   Description :
+   Author :        JHao
+   date：          2020/6/22
+-------------------------------------------------
+   Change Activity:
+                   2020/6/22:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+import os
+import setting
+from util.six import reload_six
+from util.singleton import Singleton
+from util.lazyProperty import LazyProperty
+
+
+class ConfigHandler(object):
+    __metaclass__ = Singleton
+
+    def __init__(self):
+        pass
+
+    @LazyProperty
+    def serverHost(self):
+        return os.environ.get("HOST", setting.HOST)
+
+    @LazyProperty
+    def serverPort(self):
+        return os.environ.get("PORT", setting.PORT)
+
+    @LazyProperty
+    def dbConn(self):
+        return os.getenv("DB_CONN", setting.DB_CONN)
+
+    @LazyProperty
+    def tableName(self):
+        return os.getenv("TABLE_NAME", setting.TABLE_NAME)
+
+    @property
+    def fetchers(self):
+        reload_six(setting)
+        return setting.PROXY_FETCHER
+
+    @LazyProperty
+    def verifyUrl(self):
+        return os.getenv("VERIFY_URL", setting.VERIFY_RUL)
+
+    @LazyProperty
+    def verifyTimeout(self):
+        return os.getenv("VERIFY_TIMEOUT", setting.VERIFY_TIMEOUT)
+
+    @LazyProperty
+    def maxFailCount(self):
+        return os.getenv("MAX_FAIL_COUNT", setting.MAX_FAIL_COUNT)

Util/LogHandler.py → handler/logHandler.py

@@ -33,10 +33,8 @@ CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
 ROOT_PATH = os.path.join(CURRENT_PATH, os.pardir)
 LOG_PATH = os.path.join(ROOT_PATH, 'log')
 
-try:
+if not os.path.exists(LOG_PATH):
     os.mkdir(LOG_PATH)
-except FileExistsError:
-    pass
 
 
 class LogHandler(logging.Logger):
@@ -88,16 +86,6 @@ class LogHandler(logging.Logger):
             stream_handler.setLevel(level)
         self.addHandler(stream_handler)
 
-    def resetName(self, name):
-        """
-        reset name
-        :param name:
-        :return:
-        """
-        self.name = name
-        self.removeHandler(self.file_handler)
-        self.__setFileHandler__()
-
 
 if __name__ == '__main__':
     log = LogHandler('test')

handler/proxyHandler.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     ProxyHandler.py
+   Description :
+   Author :       JHao
+   date：          2016/12/3
+-------------------------------------------------
+   Change Activity:
+                   2016/12/3:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from helper.proxy import Proxy
+from db.dbClient import DbClient
+from handler.configHandler import ConfigHandler
+
+
+class ProxyHandler(object):
+    """ Proxy CRUD operator"""
+
+    def __init__(self):
+        self.conf = ConfigHandler()
+        self.db = DbClient(self.conf.dbConn)
+        self.db.changeTable(self.conf.tableName)
+
+    def get(self):
+        """
+        return a useful proxy
+        :return:
+        """
+        proxy = self.db.get()
+        if proxy:
+            return Proxy.createFromJson(proxy)
+        return None
+
+    def pop(self):
+        """
+        return and delete a useful proxy
+        :return:
+        """
+        proxy = self.db.pop()
+        if proxy:
+            return Proxy.createFromJson(proxy)
+        return None
+
+    def put(self, proxy):
+        """
+        put proxy into use proxy
+        :return:
+        """
+        self.db.put(proxy)
+
+    def delete(self, proxy):
+        """
+        delete useful proxy
+        :param proxy:
+        :return:
+        """
+        return self.db.delete(proxy.proxy)
+
+    def getAll(self):
+        """
+        get all proxy from pool as Proxy list
+        :return:
+        """
+        proxies_dict = self.db.getAll()
+        return [Proxy.createFromJson(value) for _, value in proxies_dict.items()]
+
+    def exists(self, proxy):
+        """
+        check proxy exists
+        :param proxy:
+        :return:
+        """
+        return self.db.exists(proxy.proxy)
+
+    def getCount(self):
+        """
+        return raw_proxy and use_proxy count
+        :return:
+        """
+        total_use_proxy = self.db.getCount()
+        return {'count': total_use_proxy}

ProxyHelper/__init__.py → helper/__init__.py

@@ -12,5 +12,5 @@
 """
 __author__ = 'JHao'
 
-from ProxyHelper.Proxy import Proxy
-from ProxyHelper.ProxyUtil import checkProxyUseful
+# from proxyHelper.Proxy import Proxy
+# from proxyHelper.ProxyUtil import checkProxyUseful

helper/check.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     check
+   Description :
+   Author :        JHao
+   date：          2019/8/6
+-------------------------------------------------
+   Change Activity:
+                   2019/08/06:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from util.six import Empty
+from threading import Thread
+from datetime import datetime
+
+from helper.proxy import Proxy
+from util.validators import validators
+from handler.logHandler import LogHandler
+from handler.proxyHandler import ProxyHandler
+from handler.configHandler import ConfigHandler
+
+
+def proxyCheck(proxy_obj):
+    """
+    检测代理是否可用
+    :param proxy_obj: Proxy object
+    :return: Proxy object, status
+    """
+
+    def __proxyCheck(proxy):
+        for func in validators:
+            if not func(proxy):
+                return False
+        return True
+
+    if __proxyCheck(proxy_obj.proxy):
+        # 检测通过 更新proxy属性
+        proxy_obj.check_count += 1
+        proxy_obj.last_status = 1
+        proxy_obj.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        if proxy_obj.fail_count > 0:
+            proxy_obj.fail_count -= 1
+        return proxy_obj
+    else:
+        proxy_obj.check_count += 1
+        proxy_obj.last_status = 0
+        proxy_obj.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        proxy_obj.fail_count += 1
+        return proxy_obj
+
+
+class Checker(Thread):
+    """
+    多线程检测代理是否可用
+    """
+
+    def __init__(self, check_type, queue, thread_name):
+        Thread.__init__(self, name=thread_name)
+        self.type = check_type
+        self.log = LogHandler("checker")
+        self.proxy_handler = ProxyHandler()
+        self.queue = queue
+        self.conf = ConfigHandler()
+
+    def run(self):
+        self.log.info("ProxyCheck - {}  : start".format(self.name))
+        while True:
+            try:
+                proxy_json = self.queue.get(block=False)
+            except Empty:
+                self.log.info("ProxyCheck - {}  : complete".format(self.name))
+                break
+
+            proxy = Proxy.createFromJson(proxy_json)
+            proxy = proxyCheck(proxy)
+            if self.type == "raw":
+                if proxy.last_status:
+                    if self.proxy_handler.exists(proxy):
+                        self.log.info('ProxyCheck - {}  : {} exists'.format(self.name, proxy.proxy.ljust(23)))
+                    else:
+                        self.log.info('ProxyCheck - {}  : {} success'.format(self.name, proxy.proxy.ljust(23)))
+                        self.proxy_handler.put(proxy)
+                else:
+                    self.log.info('ProxyCheck - {}  : {} fail'.format(self.name, proxy.proxy.ljust(23)))
+            else:
+                if proxy.last_status:
+                    self.log.info('ProxyCheck - {}  : {} pass'.format(self.name, proxy.proxy.ljust(23)))
+                    self.proxy_handler.put(proxy)
+                else:
+                    if proxy.fail_count > self.conf.maxFailCount:
+                        self.log.info('ProxyCheck - {}  : {} fail, count {} delete'.format(self.name,
+                                                                                           proxy.proxy.ljust(23),
+                                                                                           proxy.fail_count))
+                        self.proxy_handler.delete(proxy)
+                    else:
+                        self.log.info('ProxyCheck - {}  : {} fail, count {} keep'.format(self.name,
+                                                                                         proxy.proxy.ljust(23),
+                                                                                         proxy.fail_count))
+                        self.proxy_handler.put(proxy)
+            self.queue.task_done()
+
+
+def runChecker(tp, queue):
+    """
+    run Checker
+    :param tp: raw/use
+    :param queue: Proxy Queue
+    :return:
+    """
+    thread_list = list()
+    for index in range(20):
+        thread_list.append(Checker(tp, queue, "thread_%s" % str(index).zfill(2)))
+
+    for thread in thread_list:
+        thread.start()
+
+    for thread in thread_list:
+        thread.join()

helper/fetch.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     fetchScheduler
+   Description :
+   Author :        JHao
+   date：          2019/8/6
+-------------------------------------------------
+   Change Activity:
+                   2019/08/06:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from handler.logHandler import LogHandler
+from handler.proxyHandler import ProxyHandler
+from fetcher.proxyFetcher import ProxyFetcher
+from handler.configHandler import ConfigHandler
+
+
+class Fetcher(object):
+    name = "fetcher"
+
+    def __init__(self):
+        self.log = LogHandler(self.name)
+        self.conf = ConfigHandler()
+        self.proxy_handler = ProxyHandler()
+
+    def fetch(self):
+        """
+        fetch proxy into db with proxyFetcher
+        :return:
+        """
+        proxy_set = set()
+        self.log.info("ProxyFetch : start")
+        for fetch_name in self.conf.fetchers:
+            self.log.info("ProxyFetch - {func}: start".format(func=fetch_name))
+            fetcher = getattr(ProxyFetcher, fetch_name, None)
+            if not fetcher:
+                self.log.error("ProxyFetch - {func}: class method not exists!")
+                continue
+            if not callable(fetcher):
+                self.log.error("ProxyFetch - {func}: must be class method")
+                continue
+
+            try:
+                for proxy in fetcher():
+                    if proxy in proxy_set:
+                        self.log.info('ProxyFetch - %s: %s exist' % (fetch_name, proxy.ljust(23)))
+                        continue
+                    else:
+                        self.log.info('ProxyFetch - %s: %s success' % (fetch_name, proxy.ljust(23)))
+                    if proxy.strip():
+                        proxy_set.add(proxy)
+            except Exception as e:
+                self.log.error("ProxyFetch - {func}: error".format(func=fetch_name))
+                self.log.error(str(e))
+        self.log.info("ProxyFetch - all complete!")
+        return proxy_set
+
+
+def runFetcher():
+    return Fetcher().fetch()

ProxyHelper/Proxy.py → helper/proxy.py

@@ -29,7 +29,7 @@ class Proxy(object):
         self._last_time = last_time
 
     @classmethod
-    def newProxyFromJson(cls, proxy_json):
+    def createFromJson(cls, proxy_json):
         """
         根据proxy属性json创建Proxy实例
         :param proxy_json:
@@ -87,7 +87,7 @@ class Proxy(object):
         return self._last_time
 
     @property
-    def info_dict(self):
+    def to_dict(self):
         """ 属性字典 """
         return {"proxy": self._proxy,
                 "fail_count": self._fail_count,
@@ -99,9 +99,9 @@ class Proxy(object):
                 "last_time": self.last_time}
 
     @property
-    def info_json(self):
+    def to_json(self):
         """ 属性json格式 """
-        return json.dumps(self.info_dict, ensure_ascii=False)
+        return json.dumps(self.to_dict, ensure_ascii=False)
 
     # --- proxy method ---
     @fail_count.setter

helper/scheduler.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     proxyScheduler
+   Description :
+   Author :        JHao
+   date：          2019/8/5
+-------------------------------------------------
+   Change Activity:
+                   2019/8/5: proxyScheduler
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from apscheduler.schedulers.blocking import BlockingScheduler
+from apscheduler.executors.pool import ProcessPoolExecutor
+
+from util.six import Queue
+from helper.fetch import runFetcher
+from helper.check import runChecker
+from helper.proxy import Proxy
+from handler.logHandler import LogHandler
+from handler.proxyHandler import ProxyHandler
+
+
+def runProxyFetch():
+    proxy_queue = Queue()
+
+    for proxy in runFetcher():
+        proxy_queue.put(Proxy(proxy).to_json)
+
+    runChecker("raw", proxy_queue)
+
+
+def runProxyCheck():
+    proxy_queue = Queue()
+
+    for proxy in ProxyHandler().getAll():
+        proxy_queue.put(proxy.to_json)
+
+    runChecker("use", proxy_queue)
+
+
+def runScheduler():
+    runProxyFetch()
+
+    scheduler_log = LogHandler("scheduler")
+    scheduler = BlockingScheduler(logger=scheduler_log)
+
+    scheduler.add_job(runProxyFetch, 'interval', minutes=4, id="proxy_fetch", name="proxy采集")
+    scheduler.add_job(runProxyCheck, 'interval', minutes=2, id="proxy_check", name="proxy检查")
+
+    executors = {
+        'default': {'type': 'threadpool', 'max_workers': 20},
+        'processpool': ProcessPoolExecutor(max_workers=5)
+    }
+    job_defaults = {
+        'coalesce': False,
+        'max_instances': 10
+    }
+
+    scheduler.configure(executors=executors, job_defaults=job_defaults)
+
+    scheduler.start()
+
+
+if __name__ == '__main__':
+    runScheduler()

cli/proxyPool.py → proxyPool.py

@@ -2,31 +2,27 @@
 """
 -------------------------------------------------
    File Name：     proxy_pool
-   Description :
+   Description :   proxy pool 启动入口
    Author :        JHao
-   date：          2019/8/2
+   date：          2020/6/19
 -------------------------------------------------
    Change Activity:
-                   2019/8/2:
+                   2020/6/19:
 -------------------------------------------------
 """
 __author__ = 'JHao'
 
-import sys
 import click
-import platform
 
-sys.path.append('../')
-
-from Config.setting import HEADER
-from Schedule.ProxyScheduler import runScheduler
-from Api.ProxyApi import runFlask,runFlaskWithGunicorn
+from config.setting import BANNER
+from helper.scheduler import runScheduler
+from api.proxyApi import runFlask
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
 
 
 @click.group(context_settings=CONTEXT_SETTINGS)
-@click.version_option(version='2.0.0')
+@click.version_option(version='2.1.0')
 def cli():
     """ProxyPool cli工具"""
 
@@ -34,18 +30,15 @@ def cli():
 @cli.command(name="schedule")
 def schedule():
     """ 启动调度程序 """
-    click.echo(HEADER)
+    click.echo(BANNER)
     runScheduler()
 
 
-@cli.command(name="webserver")
+@cli.command(name="server")
 def schedule():
-    """ 启动web服务 """
-    click.echo(HEADER)
-    if platform.system() == "Windows":
-        runFlask()
-    else:
-        runFlaskWithGunicorn()
+    """ 启动api服务 """
+    click.echo(BANNER)
+    runFlask()
 
 
 if __name__ == '__main__':

requirements.txt

@@ -6,5 +6,4 @@ lxml==4.3.1
 PyExecJS==1.5.1
 click==7.0
 gunicorn==19.9.0
-pymongo
 redis

setting.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     setting.py
+   Description :   配置文件
+   Author :        JHao
+   date：          2019/2/15
+-------------------------------------------------
+   Change Activity:
+                   2019/2/15:
+-------------------------------------------------
+"""
+
+BANNER = r"""
+****************************************************************
+*** ______  ********************* ______ *********** _  ********
+*** | ___ \_ ******************** | ___ \ ********* | | ********
+*** | |_/ / \__ __   __  _ __   _ | |_/ /___ * ___  | | ********
+*** |  __/|  _// _ \ \ \/ /| | | ||  __// _ \ / _ \ | | ********
+*** | |   | | | (_) | >  < \ |_| || |  | (_) | (_) || |___  ****
+*** \_|   |_|  \___/ /_/\_\ \__  |\_|   \___/ \___/ \_____/ ****
+****                       __ / /                          *****
+************************* /___ / *******************************
+*************************       ********************************
+****************************************************************
+"""
+
+SHOW_CONFIG = True            # print key config
+
+# ############### server config ###############
+HOST = "0.0.0.0"
+
+PORT = 5010
+
+# ############### db config ###################
+# db connection uri
+# example:
+#      Redis: redis://:password@ip:port/db
+#      Ssdb:  ssdb://:password@ip:port
+DB_CONN = 'redis://:pwd@127.0.0.1:6379/0'
+
+# proxy table name
+TABLE_NAME = 'use_proxy'
+
+
+# ###### config the proxy fetch function ######
+PROXY_FETCHER = [
+    "freeProxy01",
+    # "freeProxy02",
+    # "freeProxy03",
+    "freeProxy04",
+    "freeProxy05",
+    # "freeProxy06",
+    "freeProxy07",
+    # "freeProxy08",
+    # "freeProxy09",
+    "freeProxy13",
+    "freeProxy14",
+    "freeProxy15",
+]
+
+# ############# proxy validator #################
+VERIFY_RUL = "http://www.baidu.com"
+
+VERIFY_TIMEOUT = 10
+
+MAX_FAIL_COUNT = 0
+
+

cli/start.sh → start.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
-python proxyPool.py webserver &
+python proxyPool.py server &
 python proxyPool.py schedule

test.py

@@ -12,7 +12,16 @@
 """
 __author__ = 'JHao'
 
-from Test import testConfig
+from test import testConfigHandler
+from test import testLogHandler
+from test import testDbClient
 
 if __name__ == '__main__':
-    testConfig.testConfig()
+    print("ConfigHandler:")
+    testConfigHandler.testConfig()
+
+    print("LogHandler:")
+    testLogHandler.testLogHandler()
+
+    print("DbClient:")
+    testDbClient.testDbClient()

Test/__init__.py → test/__init__.py

@@ -10,4 +10,4 @@
                    2019/2/15:
 -------------------------------------------------
 """
-__author__ = 'JHao'
+__author__ = 'JHao'

Test/testConfig.py → test/testConfigHandler.py

@@ -12,22 +12,27 @@
 """
 __author__ = 'J_hao'
 
-from Config.ConfigGetter import config
+from handler.configHandler import ConfigHandler
+from time import sleep
 
 
-# noinspection PyPep8Naming
 def testConfig():
     """
     :return:
     """
-    print(config.db_type)
-    print(config.db_name)
-    print(config.db_host)
-    print(config.db_port)
-    print(config.db_password)
-    assert isinstance(config.proxy_getter_functions, list)
-    print(config.proxy_getter_functions)
+    conf = ConfigHandler()
+    print(conf.dbConn)
+    print(conf.serverPort)
+    print(conf.serverHost)
+    print(conf.tableName)
+    assert isinstance(conf.fetchers, list)
+    print(conf.fetchers)
+
+    for _ in range(10):
+        print(conf.fetchers)
+        sleep(5)
 
 
 if __name__ == '__main__':
     testConfig()
+

test/testDbClient.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testDbClient
+   Description :
+   Author :        JHao
+   date：          2020/6/23
+-------------------------------------------------
+   Change Activity:
+                   2020/6/23:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from db.dbClient import DbClient
+
+
+def testDbClient():
+    #  ############### ssdb ###############
+    ssdb_uri = "ssdb://:password@127.0.0.1:8888"
+    s = DbClient.parseDbConn(ssdb_uri)
+    assert s.db_type == "SSDB"
+    assert s.db_pwd == "password"
+    assert s.db_host == "127.0.0.1"
+    assert s.db_port == 8888
+
+    #  ############### redis ###############
+    redis_uri = "redis://:password@127.0.0.1:6379/1"
+    r = DbClient.parseDbConn(redis_uri)
+    assert r.db_type == "REDIS"
+    assert r.db_pwd == "password"
+    assert r.db_host == "127.0.0.1"
+    assert r.db_port == 6379
+    assert r.db_name == "1"
+    print("DbClient ok!")
+
+
+if __name__ == '__main__':
+    testDbClient()

Test/testLogHandler.py → test/testLogHandler.py

@@ -12,23 +12,13 @@
 """
 __author__ = 'J_hao'
 
-from Util.LogHandler import LogHandler
+from handler.logHandler import LogHandler
 
 
-# noinspection PyPep8Naming
 def testLogHandler():
-    """
-    test function LogHandler  in Util/LogHandler
-    :return:
-    """
     log = LogHandler('test')
-    log.info('this is a log from test')
-
-    log.resetName(name='test1')
-    log.info('this is a log from test1')
-
-    log.resetName(name='test2')
-    log.info('this is a log from test2')
+    log.info('this is info')
+    log.error('this is error')
 
 
 if __name__ == '__main__':

Test/testProxyClass.py → test/testProxyClass.py

@@ -13,21 +13,22 @@
 __author__ = 'JHao'
 
 import json
-from ProxyHelper import Proxy
+from helper.proxy import Proxy
 
 
 def testProxyClass():
     proxy = Proxy("127.0.0.1:8080")
 
-    print(proxy.info_dict)
+    print(proxy.to_json)
 
     proxy.source = "test"
 
-    proxy_str = json.dumps(proxy.info_dict, ensure_ascii=False)
+    proxy_str = json.dumps(proxy.to_dict, ensure_ascii=False)
 
     print(proxy_str)
 
-    print(Proxy.newProxyFromJson(proxy_str).info_dict)
+    print(Proxy.createFromJson(proxy_str).to_dict)
 
 
-testProxyClass()
+if __name__ == '__main__':
+    testProxyClass()

test/testProxyFetcher.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testProxyFetcher
+   Description :
+   Author :        JHao
+   date：          2020/6/23
+-------------------------------------------------
+   Change Activity:
+                   2020/6/23:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from fetcher.proxyFetcher import ProxyFetcher
+from handler.configHandler import ConfigHandler
+
+
+def testProxyFetcher():
+    conf = ConfigHandler()
+    proxy_getter_functions = conf.fetchers
+    for proxyGetter in proxy_getter_functions:
+        proxy_count = 0
+        for proxy in getattr(ProxyFetcher, proxyGetter.strip())():
+            if proxy:
+                print('{func}: fetch proxy {proxy},proxy_count:{proxy_count}'.format(func=proxyGetter, proxy=proxy,
+                                                                                     proxy_count=proxy_count))
+                proxy_count += 1
+
+
+if __name__ == '__main__':
+    testProxyFetcher()

test/testRedisClient.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testRedisClient
+   Description :
+   Author :        JHao
+   date：          2020/6/23
+-------------------------------------------------
+   Change Activity:
+                   2020/6/23:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+
+def testRedisClient():
+    from db.dbClient import DbClient
+    from helper.proxy import Proxy
+
+    uri = "redis://:pwd@127.0.0.1:6379"
+    db = DbClient(uri)
+    db.changeTable("use_proxy")
+    proxy = Proxy.createFromJson(
+        '{"proxy": "27.38.96.101:9797", "fail_count": 0, "region": "", "type": "",'
+        ' "source": "freeProxy03", "check_count": 0, "last_status": "", "last_time": ""}')
+
+    print("put: ", db.put(proxy))
+
+    print("get: ", db.get())
+
+    print("exists: ", db.exists("27.38.96.101:9797"))
+
+    print("exists: ", db.exists("27.38.96.101:8888"))
+
+    print("pop: ", db.pop())
+
+    print("getAll: ", db.getAll())
+
+    print("getCount", db.getCount())
+
+
+if __name__ == '__main__':
+    testRedisClient()

test/testSsdbClient.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testSsdbClient
+   Description :
+   Author :        JHao
+   date：          2020/7/3
+-------------------------------------------------
+   Change Activity:
+                   2020/7/3:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+
+def testSsdbClient():
+    from db.dbClient import DbClient
+    from helper.proxy import Proxy
+
+    uri = "ssdb://@127.0.0.1:8888"
+    db = DbClient(uri)
+    db.changeTable("use_proxy")
+    proxy = Proxy.createFromJson(
+        '{"proxy": "27.38.96.101:9797", "fail_count": 0, "region": "", "type": "",'
+        ' "source": "freeProxy03", "check_count": 0, "last_status": "", "last_time": ""}')
+
+    print("put: ", db.put(proxy))
+
+    print("get: ", db.get())
+
+    print("exists: ", db.exists("27.38.96.101:9797"))
+
+    print("exists: ", db.exists("27.38.96.101:8888"))
+
+    print("getAll: ", db.getAll())
+
+    # print("pop: ", db.pop())
+
+    print("clear: ", db.clear())
+
+    print("getCount", db.getCount())
+
+
+if __name__ == '__main__':
+    testSsdbClient()

Util/__init__.py → util/__init__.py

@@ -11,6 +11,6 @@
 -------------------------------------------------
 """
 
-from Util.utilFunction import validUsefulProxy
-from Util.LogHandler import LogHandler
-from Util.utilClass import Singleton
+# from util.utilFunction import validUsefulProxy
+# from util.LogHandler import LogHandler
+# from util.utilClass import Singleton

Util/utilClass.py → util/lazyProperty.py

@@ -1,14 +1,13 @@
 # -*- coding: utf-8 -*-
-# !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     utilClass.py  
-   Description :  tool class
-   Author :       JHao
+   File Name：     lazyProperty
+   Description :
+   Author :        JHao
    date：          2016/12/3
 -------------------------------------------------
    Change Activity:
-                   2016/12/3: Class LazyProperty
+                   2016/12/3:
 -------------------------------------------------
 """
 __author__ = 'JHao'
@@ -30,16 +29,3 @@ class LazyProperty(object):
             value = self.func(instance)
             setattr(instance, self.func.__name__, value)
             return value
-
-
-class Singleton(type):
-    """
-    Singleton Metaclass
-    """
-
-    _inst = {}
-
-    def __call__(cls, *args, **kwargs):
-        if cls not in cls._inst:
-            cls._inst[cls] = super(Singleton, cls).__call__(*args)
-        return cls._inst[cls]

util/singleton.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     singleton
+   Description :
+   Author :        JHao
+   date：          2016/12/3
+-------------------------------------------------
+   Change Activity:
+                   2016/12/3:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+
+class Singleton(type):
+    """
+    Singleton Metaclass
+    """
+
+    _inst = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._inst:
+            cls._inst[cls] = super(Singleton, cls).__call__(*args)
+        return cls._inst[cls]

util/six.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     six
+   Description :
+   Author :        JHao
+   date：          2020/6/22
+-------------------------------------------------
+   Change Activity:
+                   2020/6/22:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+import sys
+
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    def iteritems(d, **kw):
+        return iter(d.items(**kw))
+else:
+    def iteritems(d, **kw):
+        return d.iteritems(**kw)
+
+
+if PY3:
+    from urllib.parse import urlparse
+else:
+    from urlparse import urlparse
+
+if PY3:
+    from imp import reload as reload_six
+else:
+    reload_six = reload
+
+if PY3:
+    from queue import Empty, Queue
+else:
+    from Queue import Empty, Queue

util/validators.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+
+import requests
+from re import findall
+from handler.configHandler import ConfigHandler
+
+conf = ConfigHandler()
+validators = []
+
+
+def validator(func):
+    validators.append(func)
+    return func
+
+
+@validator
+def formatValidator(proxy):
+    """
+    检查代理格式
+    :param proxy:
+    :return:
+    """
+    verify_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}"
+    _proxy = findall(verify_regex, proxy)
+    return True if len(_proxy) == 1 and _proxy[0] == proxy else False
+
+
+@validator
+def timeOutValidator(proxy):
+    """
+    检测超时
+    :param proxy:
+    :return:
+    """
+
+    proxies = {"http": "http://{proxy}".format(proxy=proxy), "https": "https://{proxy}".format(proxy=proxy)}
+    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
+               'Accept': '*/*',
+               'Connection': 'keep-alive',
+               'Accept-Language': 'zh-CN,zh;q=0.8'}
+    try:
+        r = requests.head(conf.verifyUrl, headers=headers, proxies=proxies, timeout=conf.verifyTimeout, verify=False)
+        if r.status_code == 200:
+            return True
+    except Exception as e:
+        pass
+    return False
+
+
+@validator
+def customValidator(proxy):
+    """
+    自定义validator函数，校验代理是否可用
+    :param proxy:
+    :return:
+    """
+
+    return True

Util/WebRequest.py → util/webRequest.py

@@ -13,14 +13,22 @@
 __author__ = 'J_hao'
 
 from requests.models import Response
+from lxml import etree
 import requests
 import random
 import time
 
+from handler.logHandler import LogHandler
+
+requests.packages.urllib3.disable_warnings()
+
 
 class WebRequest(object):
+    name = "web_request"
+
     def __init__(self, *args, **kwargs):
-        pass
+        self.log = LogHandler(self.name, file=False)
+        self.response = Response()
 
     @property
     def user_agent(self):
@@ -51,18 +59,14 @@ class WebRequest(object):
                 'Connection': 'keep-alive',
                 'Accept-Language': 'zh-CN,zh;q=0.8'}
 
-    def get(self, url, header=None, retry_time=5, timeout=30,
-            retry_flag=list(), retry_interval=5, *args, **kwargs):
+    def get(self, url, header=None, retry_time=3, retry_interval=5, timeout=5, *args, **kwargs):
         """
         get method
         :param url: target url
         :param header: headers
-        :param retry_time: retry time when network error
+        :param retry_time: retry time
+        :param retry_interval: retry interval
         :param timeout: network timeout
-        :param retry_flag: if retry_flag in content. do retry
-        :param retry_interval: retry interval(second)
-        :param args:
-        :param kwargs:
         :return:
         """
         headers = self.header
@@ -70,16 +74,23 @@ class WebRequest(object):
             headers.update(header)
         while True:
             try:
-                html = requests.get(url, headers=headers, timeout=timeout, **kwargs)
-                if any(f in html.content for f in retry_flag):
-                    raise Exception
-                return html
+                self.response = requests.get(url, headers=headers, timeout=timeout, *args, **kwargs)
+                return self
             except Exception as e:
-                print(e)
+                self.log.error("requests: %s error: %s" % (url, str(e)))
                 retry_time -= 1
                 if retry_time <= 0:
-                    # 多次请求失败
                     resp = Response()
                     resp.status_code = 200
-                    return resp
+                    return self
+                self.log.info("retry %s second after" % retry_interval)
                 time.sleep(retry_interval)
+
+    @property
+    def tree(self):
+        return etree.HTML(self.response.content)
+
+    @property
+    def text(self):
+        return self.response.text
+