[update] struct

Config/ConfigGetter.py

@@ -1,75 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     ConfigGetter
-   Description :   读取配置
-   Author :        JHao
-   date：          2019/2/15
--------------------------------------------------
-   Change Activity:
-                   2019/2/15:
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-
-from Util.utilClass import LazyProperty
-from Config.setting import *
-
-
-class ConfigGetter(object):
-    """
-    get config
-    """
-
-    def __init__(self):
-        pass
-
-    @LazyProperty
-    def db_type(self):
-        return DATABASES.get("default", {}).get("TYPE", "SSDB")
-
-    @LazyProperty
-    def db_name(self):
-        return DATABASES.get("default", {}).get("NAME", "proxy")
-
-    @LazyProperty
-    def db_host(self):
-        return DATABASES.get("default", {}).get("HOST", "127.0.0.1")
-
-    @LazyProperty
-    def db_port(self):
-        return DATABASES.get("default", {}).get("PORT", 8888)
-
-    @LazyProperty
-    def db_password(self):
-        return DATABASES.get("default", {}).get("PASSWORD", "")
-
-    @LazyProperty
-    def proxy_getter_functions(self):
-        return PROXY_GETTER
-
-    @LazyProperty
-    def host_ip(self):
-        return SERVER_API.get("HOST", "127.0.0.1")
-
-    @LazyProperty
-    def host_port(self):
-        return SERVER_API.get("PORT", 5010)
-
-    @LazyProperty
-    def verify_host(self):
-        return VERIFY_HOST
-
-
-config = ConfigGetter()
-
-if __name__ == '__main__':
-    print(config.db_type)
-    print(config.db_name)
-    print(config.db_host)
-    print(config.db_port)
-    print(config.proxy_getter_functions)
-    print(config.host_ip)
-    print(config.host_port)
-    print(config.db_password)

Config/__init__.py

@@ -1,12 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     __init__
-   Description :
-   Author :        JHao
-   date：          2019/2/15
--------------------------------------------------
-   Change Activity:
-                   2019/2/15:
--------------------------------------------------
-"""

Config/setting.py

@@ -1,95 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     setting.py
-   Description :   配置文件
-   Author :        JHao
-   date：          2019/2/15
--------------------------------------------------
-   Change Activity:
-                   2019/2/15:
--------------------------------------------------
-"""
-import sys
-from os import getenv
-from logging import getLogger
-
-log = getLogger(__name__)
-
-HEADER = """
-****************************************************************
-*** ______  ********************* ______ *********** _  ********
-*** | ___ \_ ******************** | ___ \ ********* | | ********
-*** | |_/ / \__ __   __  _ __   _ | |_/ /___ * ___  | | ********
-*** |  __/|  _// _ \ \ \/ /| | | ||  __// _ \ / _ \ | | ********
-*** | |   | | | (_) | >  < \ |_| || |  | (_) | (_) || |___  ****
-*** \_|   |_|  \___/ /_/\_\ \__  |\_|   \___/ \___/ \_____/ ****
-****                       __ / /                          *****
-************************* /___ / *******************************
-*************************       ********************************
-****************************************************************
-"""
-
-PY3 = sys.version_info >= (3,)
-
-DB_TYPE = getenv('db_type', 'SSDB').upper()
-DB_HOST = getenv('db_host', '127.0.0.1')
-DB_PORT = getenv('db_port', 8888)
-DB_PASSWORD = getenv('db_password', '')
-
-
-""" 数据库配置 """
-DATABASES = {
-    "default": {
-        "TYPE": DB_TYPE,
-        "HOST": DB_HOST,
-        "PORT": DB_PORT,
-        "NAME": "proxy",
-        "PASSWORD": DB_PASSWORD
-    }
-}
-
-# register the proxy getter function
-
-PROXY_GETTER = [
-    "freeProxy01",
-    # "freeProxy02",
-    "freeProxy03",
-    "freeProxy04",
-    "freeProxy05",
-    # "freeProxy06",
-    "freeProxy07",
-    # "freeProxy08",
-    "freeProxy09",
-    "freeProxy13",
-    "freeProxy14",
-    "freeProxy14",
-]
-
-""" API config http://127.0.0.1:5010 """
-SERVER_API = {
-    "HOST": "0.0.0.0",  # The ip specified which starting the web API
-    "PORT": 5010  # port number to which the server listens to
-}
-
-VERIFY_HOST = getenv('proxy_verify_host', 'http://www.baidu.com')
-
-
-class ConfigError(BaseException):
-    pass
-
-
-def checkConfig():
-    if DB_TYPE not in ["SSDB", "REDIS"]:
-        raise ConfigError('db_type Do not support: %s, must SSDB/REDIS .' % DB_TYPE)
-
-    if type(DB_PORT) == str and not DB_PORT.isdigit():
-        raise ConfigError('if db_port is string, it must be digit, not %s' % DB_PORT)
-
-    from ProxyGetter import getFreeProxy
-    illegal_getter = list(filter(lambda key: not hasattr(getFreeProxy.GetFreeProxy, key), PROXY_GETTER))
-    if len(illegal_getter) > 0:
-        raise ConfigError("ProxyGetter: %s does not exists" % "/".join(illegal_getter))
-
-
-checkConfig()

DB/DbClient.py

@@ -16,72 +16,86 @@ __author__ = 'JHao'
 import os
 import sys
 
-from Config.ConfigGetter import config
-from Util import Singleton
+from util.six import urlparse
+from util.singleton import Singleton
 
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 
 
 class DbClient(object):
     """
-    DbClient DB工厂类 提供get/put/update/pop/delete/exists/getAll/clean/getNumber/changeTable方法
+    DbClient DB工厂类 提供get/put/update/pop/delete/exists/getAll/clean/getCount/changeTable方法
 
     目前存放代理的有两种, 使用changeTable方法切换操作对象：
-        raw_proxy： 存放原始的代理；
-        useful_proxy： 存放检验后的代理；
+        raw_proxy： 存放Fetch的原始代理；
+        use_proxy： 存放校验后的代理信息；
 
 
     抽象方法定义：
-        get(proxy): 返回指定proxy的信息;
-        put(proxy): 存入一个proxy信息;
-        pop(): 返回并删除一个proxy信息;
+        get(): 返回一个proxy;
+        put(proxy): 存入一个proxy;
+        pop(): 返回并删除一个proxy;
         update(proxy): 更新指定proxy信息;
         delete(proxy): 删除指定proxy;
         exists(proxy): 判断指定proxy是否存在;
-        getAll(): 列表形式返回所有代理;
+        getAll(): 返回所有代理;
         clean(): 清除所有proxy信息;
-        getNumber(): 返回proxy数据量;
-        changeTable(name): 切换操作对象 raw_proxy/useful_proxy
+        getCount(): 返回proxy统计信息;
+        changeTable(name): 切换操作对象 raw_proxy/use_proxy
 
 
         所有方法需要相应类去具体实现：
-            ssdb: SsdbClient.py
-            redis: RedisClient.py
-            mongodb: MongodbClient.py
+            ssdb: ssdbClient.py
+            redis: redisClient.py
+            mongodb: mongodbClient.py
 
     """
 
     __metaclass__ = Singleton
 
-    def __init__(self):
+    def __init__(self, db_conn):
         """
         init
         :return:
         """
+        self.db_conn = db_conn
+        self.parseDbConn(db_conn)
         self.__initDbClient()
 
+    @classmethod
+    def parseDbConn(cls, db_conn):
+        db_conf = urlparse(db_conn)
+        cls.db_type = db_conf.scheme.upper().strip()
+        cls.db_host = db_conf.hostname
+        cls.db_port = db_conf.port
+        cls.db_user = db_conf.username
+        cls.db_pwd = db_conf.password
+        cls.db_name = db_conf.path[1:]
+        return cls
+
     def __initDbClient(self):
         """
         init DB Client
         :return:
         """
         __type = None
-        if "SSDB" == config.db_type:
-            __type = "SsdbClient"
-        elif "REDIS" == config.db_type:
-            __type = "RedisClient"
-        elif "MONGODB" == config.db_type:
-            __type = "MongodbClient"
+        if "SSDB" == self.db_type:
+            __type = "ssdbClient"
+        elif "REDIS" == self.db_type:
+            __type = "redisClient"
+        elif "MONGODB" == self.db_type:
+            __type = "mongodbClient"
         else:
             pass
-        assert __type, 'type error, Not support DB type: {}'.format(config.db_type)
-        self.client = getattr(__import__(__type), __type)(name=config.db_name,
-                                                          host=config.db_host,
-                                                          port=config.db_port,
-                                                          password=config.db_password)
+        assert __type, 'type error, Not support DB type: {}'.format(self.db_type)
+        self.client = getattr(__import__(__type), "%sClient" % self.db_type.title())(host=self.db_host,
+                                                                                     port=self.db_port,
+                                                                                     username=self.db_user,
+                                                                                     password=self.db_pwd,
+                                                                                     db=self.db_name)
 
-    def get(self, key, **kwargs):
-        return self.client.get(key, **kwargs)
+    def get(self, **kwargs):
+        return self.client.get(**kwargs)
 
     def put(self, key, **kwargs):
         return self.client.put(key, **kwargs)
@@ -107,5 +121,5 @@ class DbClient(object):
     def changeTable(self, name):
         self.client.changeTable(name)
 
-    def getNumber(self):
-        return self.client.getNumber()
+    def getCount(self):
+        return self.client.getCount()

DB/SsdbClient.py

@@ -15,7 +15,7 @@
 """
 __author__ = 'JHao'
 
-from Config.setting import PY3
+from config.setting import PY3
 
 from redis.connection import BlockingConnectionPool
 from redis import Redis

proxyDB/redisClient.py → DB/redisClient.py

@@ -14,6 +14,7 @@
 __author__ = 'JHao'
 
 from redis.connection import BlockingConnectionPool
+from random import choice
 from redis import Redis
 
 
@@ -40,14 +41,17 @@ class RedisClient(object):
         kwargs.pop("username")
         self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, **kwargs))
 
-    def get(self, proxy_str):
+    def get(self):
         """
-        从hash中获取对应的proxy, 使用前需要调用changeTable()
-        :param proxy_str: proxy str
+        返回一个代理
         :return:
         """
-        data = self.__conn.hget(name=self.name, key=proxy_str)
-        return data
+        proxies = self.__conn.hkeys(self.name)
+        proxy = choice(proxies) if proxies else None
+        if proxy:
+            return self.__conn.hget(self.name, proxy)
+        else:
+            return False
 
     def put(self, proxy_obj):
         """
@@ -58,13 +62,26 @@ class RedisClient(object):
         data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
         return data
 
+    def pop(self):
+        """
+        弹出一个代理
+        :return: dict {proxy: value}
+        """
+        proxies = self.__conn.hkeys(self.name)
+        for proxy in proxies:
+            proxy_info = self.__conn.hget(self.name, proxy)
+            self.__conn.hdel(self.name, proxy)
+            return proxy_info
+        else:
+            return False
+
     def delete(self, proxy_str):
         """
         移除指定代理, 使用changeTable指定hash name
         :param proxy_str: proxy str
         :return:
         """
-        self.__conn.hdel(self.name, proxy_str)
+        return self.__conn.hdel(self.name, proxy_str)
 
     def exists(self, proxy_str):
         """
@@ -80,19 +97,7 @@ class RedisClient(object):
         :param proxy_obj:
         :return:
         """
-        self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
-
-    def pop(self):
-        """
-        弹出一个代理
-        :return: dict {proxy: value}
-        """
-        cursor, data_dict = self.__conn.hscan(self.name, count=1)
-        for key, value in data_dict.items():
-            self.__conn.hdel(self.name, key)
-            return value
-        else:
-            return False
+        return self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
 
     def getAll(self):
         """

Manager/ProxyManager.py

@@ -1,108 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：     ProxyManager.py
-   Description :
-   Author :       JHao
-   date：          2016/12/3
--------------------------------------------------
-   Change Activity:
-                   2016/12/3:
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-import random
-
-from ProxyHelper import Proxy
-from DB.DbClient import DbClient
-from Config.ConfigGetter import config
-from Util.LogHandler import LogHandler
-from Util.utilFunction import verifyProxyFormat
-from ProxyGetter.getFreeProxy import GetFreeProxy
-
-
-class ProxyManager(object):
-    """
-    ProxyManager
-    """
-
-    def __init__(self):
-        self.db = DbClient()
-        self.raw_proxy_queue = 'raw_proxy'
-        self.log = LogHandler('proxy_manager')
-        self.useful_proxy_queue = 'useful_proxy'
-
-    def fetch(self):
-        """
-        fetch proxy into db by ProxyGetter
-        :return:
-        """
-        self.db.changeTable(self.raw_proxy_queue)
-        proxy_set = set()
-        self.log.info("ProxyFetch : start")
-        for proxyGetter in config.proxy_getter_functions:
-            self.log.info("ProxyFetch - {func}: start".format(func=proxyGetter))
-            try:
-                for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
-                    proxy = proxy.strip()
-
-                    if not proxy or not verifyProxyFormat(proxy):
-                        self.log.error('ProxyFetch - {func}: '
-                                       '{proxy} illegal'.format(func=proxyGetter, proxy=proxy.ljust(20)))
-                        continue
-                    elif proxy in proxy_set:
-                        self.log.info('ProxyFetch - {func}: '
-                                      '{proxy} exist'.format(func=proxyGetter, proxy=proxy.ljust(20)))
-                        continue
-                    else:
-                        self.log.info('ProxyFetch - {func}: '
-                                      '{proxy} success'.format(func=proxyGetter, proxy=proxy.ljust(20)))
-                        self.db.put(Proxy(proxy, source=proxyGetter))
-                        proxy_set.add(proxy)
-            except Exception as e:
-                self.log.error("ProxyFetch - {func}: error".format(func=proxyGetter))
-                self.log.error(str(e))
-
-    def get(self):
-        """
-        return a useful proxy
-        :return:
-        """
-        self.db.changeTable(self.useful_proxy_queue)
-        item_list = self.db.getAll()
-        if item_list:
-            random_choice = random.choice(item_list)
-            return Proxy.newProxyFromJson(random_choice)
-        return None
-
-    def delete(self, proxy_str):
-        """
-        delete proxy from pool
-        :param proxy_str:
-        :return:
-        """
-        self.db.changeTable(self.useful_proxy_queue)
-        self.db.delete(proxy_str)
-
-    def getAll(self):
-        """
-        get all proxy from pool as list
-        :return:
-        """
-        self.db.changeTable(self.useful_proxy_queue)
-        item_list = self.db.getAll()
-        return [Proxy.newProxyFromJson(_) for _ in item_list]
-
-    def getNumber(self):
-        self.db.changeTable(self.raw_proxy_queue)
-        total_raw_proxy = self.db.getNumber()
-        self.db.changeTable(self.useful_proxy_queue)
-        total_useful_queue = self.db.getNumber()
-        return {'raw_proxy': total_raw_proxy, 'useful_proxy': total_useful_queue}
-
-
-if __name__ == '__main__':
-    pp = ProxyManager()
-    pp.fetch()

ProxyGetter/CheckProxy.py

@@ -1,70 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     CheckProxy
-   Description :   used for check getFreeProxy.py
-   Author :        JHao
-   date：          2018/7/10
--------------------------------------------------
-   Change Activity:
-                   2018/7/10: CheckProxy
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-from .getFreeProxy import GetFreeProxy
-from Util.utilFunction import verifyProxyFormat
-
-
-from Util.LogHandler import LogHandler
-
-log = LogHandler('check_proxy', file=False)
-
-
-class CheckProxy(object):
-
-    @staticmethod
-    def checkAllGetProxyFunc():
-        """
-        检查getFreeProxy所有代理获取函数运行情况
-        Returns:
-            None
-        """
-        import inspect
-        member_list = inspect.getmembers(GetFreeProxy, predicate=inspect.isfunction)
-        proxy_count_dict = dict()
-        for func_name, func in member_list:
-            log.info(u"开始运行 {}".format(func_name))
-            try:
-                proxy_list = [_ for _ in func() if verifyProxyFormat(_)]
-                proxy_count_dict[func_name] = len(proxy_list)
-            except Exception as e:
-                log.info(u"代理获取函数 {} 运行出错!".format(func_name))
-                log.error(str(e))
-        log.info(u"所有函数运行完毕 " + "***" * 5)
-        for func_name, func in member_list:
-            log.info(u"函数 {n}, 获取到代理数: {c}".format(n=func_name, c=proxy_count_dict.get(func_name, 0)))
-
-    @staticmethod
-    def checkGetProxyFunc(func):
-        """
-        检查指定的getFreeProxy某个function运行情况
-        Args:
-            func: getFreeProxy中某个可调用方法
-
-        Returns:
-            None
-        """
-        func_name = getattr(func, '__name__', "None")
-        log.info("start running func: {}".format(func_name))
-        count = 0
-        for proxy in func():
-            if verifyProxyFormat(proxy):
-                log.info("{} fetch proxy: {}".format(func_name, proxy))
-                count += 1
-        log.info("{n} completed, fetch proxy number: {c}".format(n=func_name, c=count))
-
-
-if __name__ == '__main__':
-    CheckProxy.checkAllGetProxyFunc()
-    CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01)

ProxyGetter/getFreeProxy.py

@@ -1,355 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：     GetFreeProxy.py
-   Description :  抓取免费代理
-   Author :       JHao
-   date：          2016/11/25
--------------------------------------------------
-   Change Activity:
-                   2016/11/25:
--------------------------------------------------
-"""
-import re
-import sys
-import requests
-from time import sleep
-
-sys.path.append('..')
-
-from Util.WebRequest import WebRequest
-from Util.utilFunction import getHtmlTree
-
-# for debug to disable insecureWarning
-requests.packages.urllib3.disable_warnings()
-
-
-class GetFreeProxy(object):
-    """
-    proxy getter
-    """
-
-    @staticmethod
-    def freeProxy01():
-        """
-        无忧代理 http://www.data5u.com/
-        几乎没有能用的
-        :return:
-        """
-        url_list = [
-            'http://www.data5u.com/',
-            'http://www.data5u.com/free/gngn/index.shtml',
-            'http://www.data5u.com/free/gnpt/index.shtml'
-        ]
-        key = 'ABCDEFGHIZ'
-        for url in url_list:
-            html_tree = getHtmlTree(url)
-            ul_list = html_tree.xpath('//ul[@class="l2"]')
-            for ul in ul_list:
-                try:
-                    ip = ul.xpath('./span[1]/li/text()')[0]
-                    classnames = ul.xpath('./span[2]/li/attribute::class')[0]
-                    classname = classnames.split(' ')[1]
-                    port_sum = 0
-                    for c in classname:
-                        port_sum *= 10
-                        port_sum += key.index(c)
-                    port = port_sum >> 3
-                    yield '{}:{}'.format(ip, port)
-                except Exception as e:
-                    print(e)
-
-    @staticmethod
-    def freeProxy02(count=20):
-        """
-        代理66 http://www.66ip.cn/
-        :param count: 提取数量
-        :return:
-        """
-        urls = [
-            "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=",
-            "http://www.66ip.cn/nmtq.php?getnum={}&isp=0&anonymoustype=0&s"
-            "tart=&ports=&export=&ipaddress=&area=0&proxytype=2&api=66ip"
-        ]
-
-        try:
-            import execjs
-            import requests
-
-            headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
-                       'Accept': '*/*',
-                       'Connection': 'keep-alive',
-                       'Accept-Language': 'zh-CN,zh;q=0.8'}
-            session = requests.session()
-            src = session.get("http://www.66ip.cn/", headers=headers).text
-            src = src.split("</script>")[0] + '}'
-            src = src.replace("<script>", "function test() {")
-            src = src.replace("while(z++)try{eval(", ';var num=10;while(z++)try{var tmp=')
-            src = src.replace(");break}", ";num--;if(tmp.search('cookie') != -1 | num<0){return tmp}}")
-            ctx = execjs.compile(src)
-            src = ctx.call("test")
-            src = src[src.find("document.cookie="): src.find("};if((")]
-            src = src.replace("document.cookie=", "")
-            src = "function test() {var window={}; return %s }" % src
-            cookie = execjs.compile(src).call('test')
-            js_cookie = cookie.split(";")[0].split("=")[-1]
-        except Exception as e:
-            print(e)
-            return
-
-        for url in urls:
-            try:
-                html = session.get(url.format(count), cookies={"__jsl_clearance": js_cookie}, headers=headers).text
-                ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", html)
-                for ip in ips:
-                    yield ip.strip()
-            except Exception as e:
-                print(e)
-                pass
-
-    @staticmethod
-    def freeProxy03(page_count=1):
-        """
-        西刺代理 http://www.xicidaili.com
-        :return:
-        """
-        url_list = [
-            'http://www.xicidaili.com/nn/',  # 高匿
-            'http://www.xicidaili.com/nt/',  # 透明
-        ]
-        for each_url in url_list:
-            for i in range(1, page_count + 1):
-                page_url = each_url + str(i)
-                tree = getHtmlTree(page_url)
-                proxy_list = tree.xpath('.//table[@id="ip_list"]//tr[position()>1]')
-                for proxy in proxy_list:
-                    try:
-                        yield ':'.join(proxy.xpath('./td/text()')[0:2])
-                    except Exception as e:
-                        pass
-
-    @staticmethod
-    def freeProxy04():
-        """
-        guobanjia http://www.goubanjia.com/
-        :return:
-        """
-        url = "http://www.goubanjia.com/"
-        tree = getHtmlTree(url)
-        proxy_list = tree.xpath('//td[@class="ip"]')
-        # 此网站有隐藏的数字干扰，或抓取到多余的数字或.符号
-        # 需要过滤掉<p style="display:none;">的内容
-        xpath_str = """.//*[not(contains(@style, 'display: none'))
-                                        and not(contains(@style, 'display:none'))
-                                        and not(contains(@class, 'port'))
-                                        ]/text()
-                                """
-        for each_proxy in proxy_list:
-            try:
-                # :符号裸放在td下，其他放在div span p中，先分割找出ip，再找port
-                ip_addr = ''.join(each_proxy.xpath(xpath_str))
-
-                # HTML中的port是随机数，真正的端口编码在class后面的字母中。
-                # 比如这个：
-                # <span class="port CFACE">9054</span>
-                # CFACE解码后对应的是3128。
-                port = 0
-                for _ in each_proxy.xpath(".//span[contains(@class, 'port')]"
-                                          "/attribute::class")[0]. \
-                        replace("port ", ""):
-                    port *= 10
-                    port += (ord(_) - ord('A'))
-                port /= 8
-
-                yield '{}:{}'.format(ip_addr, int(port))
-            except Exception as e:
-                pass
-
-    @staticmethod
-    def freeProxy05():
-        """
-        快代理 https://www.kuaidaili.com
-        """
-        url_list = [
-            'https://www.kuaidaili.com/free/inha/',
-            'https://www.kuaidaili.com/free/intr/'
-        ]
-        for url in url_list:
-            tree = getHtmlTree(url)
-            proxy_list = tree.xpath('.//table//tr')
-            sleep(1)  # 必须sleep 不然第二条请求不到数据
-            for tr in proxy_list[1:]:
-                yield ':'.join(tr.xpath('./td/text()')[0:2])
-
-    @staticmethod
-    def freeProxy06():
-        """
-        码农代理 https://proxy.coderbusy.com/
-        :return:
-        """
-        urls = ['https://proxy.coderbusy.com/']
-        for url in urls:
-            tree = getHtmlTree(url)
-            proxy_list = tree.xpath('.//table//tr')
-            for tr in proxy_list[1:]:
-                yield ':'.join(tr.xpath('./td/text()')[0:2])
-
-    @staticmethod
-    def freeProxy07():
-        """
-        云代理 http://www.ip3366.net/free/
-        :return:
-        """
-        urls = ['http://www.ip3366.net/free/?stype=1',
-                "http://www.ip3366.net/free/?stype=2"]
-        request = WebRequest()
-        for url in urls:
-            r = request.get(url, timeout=10)
-            proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
-            for proxy in proxies:
-                yield ":".join(proxy)
-
-    @staticmethod
-    def freeProxy08():
-        """
-        IP海 http://www.iphai.com/free/ng
-        :return:
-        """
-        urls = [
-            'http://www.iphai.com/free/ng',
-            'http://www.iphai.com/free/np',
-            'http://www.iphai.com/free/wg',
-            'http://www.iphai.com/free/wp'
-        ]
-        request = WebRequest()
-        for url in urls:
-            r = request.get(url, timeout=10)
-            proxies = re.findall(r'<td>\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?</td>[\s\S]*?<td>\s*?(\d+)\s*?</td>',
-                                 r.text)
-            for proxy in proxies:
-                yield ":".join(proxy)
-
-    @staticmethod
-    def freeProxy09(page_count=1):
-        """
-        http://ip.jiangxianli.com/?page=
-        免费代理库
-        :return:
-        """
-        for i in range(1, page_count + 1):
-            url = 'http://ip.jiangxianli.com/?country=中国&?page={}'.format(i)
-            html_tree = getHtmlTree(url)
-            for index, tr in enumerate(html_tree.xpath("//table//tr")):
-                if index == 0:
-                    continue
-                yield ":".join(tr.xpath("./td/text()")[0:2]).strip()
-
-    # @staticmethod
-    # def freeProxy10():
-    #     """
-    #     墙外网站 cn-proxy
-    #     :return:
-    #     """
-    #     urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218']
-    #     request = WebRequest()
-    #     for url in urls:
-    #         r = request.get(url, timeout=10)
-    #         proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)
-    #         for proxy in proxies:
-    #             yield ':'.join(proxy)
-
-    # @staticmethod
-    # def freeProxy11():
-    #     """
-    #     https://proxy-list.org/english/index.php
-    #     :return:
-    #     """
-    #     urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)]
-    #     request = WebRequest()
-    #     import base64
-    #     for url in urls:
-    #         r = request.get(url, timeout=10)
-    #         proxies = re.findall(r"Proxy\('(.*?)'\)", r.text)
-    #         for proxy in proxies:
-    #             yield base64.b64decode(proxy).decode()
-
-    # @staticmethod
-    # def freeProxy12():
-    #     urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1']
-    #     request = WebRequest()
-    #     for url in urls:
-    #         r = request.get(url, timeout=10)
-    #         proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
-    #         for proxy in proxies:
-    #             yield ':'.join(proxy)
-
-    @staticmethod
-    def freeProxy13(max_page=2):
-        """
-        http://www.qydaili.com/free/?action=china&page=1
-        齐云代理
-        :param max_page:
-        :return:
-        """
-        base_url = 'http://www.qydaili.com/free/?action=china&page='
-        request = WebRequest()
-        for page in range(1, max_page + 1):
-            url = base_url + str(page)
-            r = request.get(url, timeout=10)
-            proxies = re.findall(
-                r'<td.*?>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td.*?>(\d+)</td>',
-                r.text)
-            for proxy in proxies:
-                yield ':'.join(proxy)
-
-    @staticmethod
-    def freeProxy14(max_page=2):
-        """
-        http://www.89ip.cn/index.html
-        89免费代理
-        :param max_page:
-        :return:
-        """
-        base_url = 'http://www.89ip.cn/index_{}.html'
-        request = WebRequest()
-        for page in range(1, max_page + 1):
-            url = base_url.format(page)
-            r = request.get(url, timeout=10)
-            proxies = re.findall(
-                r'<td.*?>[\s\S]*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?</td>[\s\S]*?<td.*?>[\s\S]*?(\d+)[\s\S]*?</td>',
-                r.text)
-            for proxy in proxies:
-                yield ':'.join(proxy)
-
-    @staticmethod
-    def freeProxy15():
-        urls = ['http://www.xiladaili.com/putong/',
-                "http://www.xiladaili.com/gaoni/",
-                "http://www.xiladaili.com/http/",
-                "http://www.xiladaili.com/https/"]
-        request = WebRequest()
-        for url in urls:
-            r = request.get(url, timeout=10)
-            ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", r.text)
-            for ip in ips:
-                yield ip.strip()
-
-
-if __name__ == '__main__':
-    from CheckProxy import CheckProxy
-
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy02)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy03)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy04)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy05)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy06)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy07)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy08)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy09)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy13)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy14)
-    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy15)
-
-    CheckProxy.checkAllGetProxyFunc()

Schedule/RawProxyCheck.py

@@ -1,81 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     RawProxyCheck
-   Description :   check raw_proxy to useful
-   Author :        JHao
-   date：          2019/8/6
--------------------------------------------------
-   Change Activity:
-                   2019/8/6: check raw_proxy to useful
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-from threading import Thread
-
-try:
-    from Queue import Empty, Queue  # py2
-except:
-    from queue import Empty, Queue  # py3
-
-from Util import LogHandler
-from Manager import ProxyManager
-from ProxyHelper import Proxy, checkProxyUseful
-
-
-class RawProxyCheck(ProxyManager, Thread):
-    def __init__(self, queue, thread_name):
-        ProxyManager.__init__(self)
-        Thread.__init__(self, name=thread_name)
-        self.log = LogHandler('raw_proxy_check')
-        self.queue = queue
-
-    def run(self):
-        self.log.info("RawProxyCheck - {}  : start".format(self.name))
-        self.db.changeTable(self.useful_proxy_queue)
-        while True:
-            try:
-                proxy_json = self.queue.get(block=False)
-            except Empty:
-                self.log.info("RawProxyCheck - {}  : exit".format(self.name))
-                break
-
-            proxy_obj = Proxy.newProxyFromJson(proxy_json)
-
-            proxy_obj, status = checkProxyUseful(proxy_obj)
-            if status:
-                if self.db.exists(proxy_obj.proxy):
-                    self.log.info('RawProxyCheck - {}  : {} validation exists'.format(self.name,
-                                                                                      proxy_obj.proxy.ljust(20)))
-                else:
-                    self.db.put(proxy_obj)
-                    self.log.info(
-                        'RawProxyCheck - {}  : {} validation pass'.format(self.name, proxy_obj.proxy.ljust(20)))
-            else:
-                self.log.info('RawProxyCheck - {}  : {} validation fail'.format(self.name, proxy_obj.proxy.ljust(20)))
-            self.queue.task_done()
-
-
-def doRawProxyCheck():
-    proxy_queue = Queue()
-
-    pm = ProxyManager()
-    pm.db.changeTable(pm.raw_proxy_queue)
-    for _proxy in pm.db.getAll():
-        proxy_queue.put(_proxy)
-    pm.db.clear()
-
-    thread_list = list()
-    for index in range(20):
-        thread_list.append(RawProxyCheck(proxy_queue, "thread_%s" % index))
-
-    for thread in thread_list:
-        thread.start()
-
-    for thread in thread_list:
-        thread.join()
-
-
-if __name__ == '__main__':
-    doRawProxyCheck()

Schedule/UsefulProxyCheck.py

@@ -1,80 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     UsefulProxyCheck
-   Description :   check useful proxy
-   Author :        JHao
-   date：          2019/8/7
--------------------------------------------------
-   Change Activity:
-                   2019/8/7: check useful proxy
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-from threading import Thread
-
-try:
-    from Queue import Queue, Empty  # py2
-except:
-    from queue import Queue, Empty  # py3
-
-from Util import LogHandler
-from Manager import ProxyManager
-from ProxyHelper import checkProxyUseful, Proxy
-
-FAIL_COUNT = 0
-
-
-class UsefulProxyCheck(ProxyManager, Thread):
-    def __init__(self, queue, thread_name):
-        ProxyManager.__init__(self)
-        Thread.__init__(self, name=thread_name)
-
-        self.queue = queue
-        self.log = LogHandler('useful_proxy_check')
-
-    def run(self):
-        self.log.info("UsefulProxyCheck - {}  : start".format(self.name))
-        self.db.changeTable(self.useful_proxy_queue)
-        while True:
-            try:
-                proxy_str = self.queue.get(block=False)
-            except Empty:
-                self.log.info("UsefulProxyCheck - {}  : exit".format(self.name))
-                break
-
-            proxy_obj = Proxy.newProxyFromJson(proxy_str)
-            proxy_obj, status = checkProxyUseful(proxy_obj)
-            if status or proxy_obj.fail_count < FAIL_COUNT:
-                self.db.put(proxy_obj)
-                self.log.info('UsefulProxyCheck - {}  : {} validation pass'.format(self.name,
-                                                                                   proxy_obj.proxy.ljust(20)))
-            else:
-                self.log.info('UsefulProxyCheck - {}  : {} validation fail'.format(self.name,
-                                                                                   proxy_obj.proxy.ljust(20)))
-                self.db.delete(proxy_obj.proxy)
-            self.queue.task_done()
-
-
-def doUsefulProxyCheck():
-    proxy_queue = Queue()
-
-    pm = ProxyManager()
-    pm.db.changeTable(pm.useful_proxy_queue)
-    for _proxy in pm.db.getAll():
-        proxy_queue.put(_proxy)
-
-    thread_list = list()
-    for index in range(10):
-        thread_list.append(UsefulProxyCheck(proxy_queue, "thread_%s" % index))
-
-    for thread in thread_list:
-        thread.start()
-
-    for thread in thread_list:
-        thread.join()
-
-
-if __name__ == '__main__':
-    doUsefulProxyCheck()

Test/testConfig.py → Test/testConfigHandler.py

@@ -12,22 +12,28 @@
 """
 __author__ = 'J_hao'
 
-from Config.ConfigGetter import config
+from handler.configHandler import ConfigHandler
+from time import sleep
 
 
-# noinspection PyPep8Naming
 def testConfig():
     """
     :return:
     """
-    print(config.db_type)
-    print(config.db_name)
-    print(config.db_host)
-    print(config.db_port)
-    print(config.db_password)
-    assert isinstance(config.proxy_getter_functions, list)
-    print(config.proxy_getter_functions)
+    conf = ConfigHandler()
+    print(conf.dbConn)
+    print(conf.serverPort)
+    print(conf.serverHost)
+    print(conf.useProxy)
+    print(conf.rawProxy)
+    assert isinstance(conf.fetchers, list)
+    print(conf.fetchers)
+
+    for _ in range(10):
+        print(conf.fetchers)
+        sleep(5)
 
 
 if __name__ == '__main__':
     testConfig()
+

Test/testDbClient.py

@@ -12,7 +12,7 @@
 """
 __author__ = 'JHao'
 
-from proxyDB.DbClient import DbClient
+from db.DbClient import DbClient
 
 if __name__ == '__main__':
     #  ############### ssdb ###############

Test/testGetFreeProxy.py

@@ -13,8 +13,8 @@
 __author__ = 'J_hao'
 
 
-from ProxyGetter.getFreeProxy import GetFreeProxy
-from Config.ConfigGetter import config
+from fetcher.getFreeProxy import GetFreeProxy
+from config.ConfigGetter import config
 
 
 def testGetFreeProxy():

Test/testLogHandler.py

@@ -12,7 +12,7 @@
 """
 __author__ = 'J_hao'
 
-from Util.LogHandler import LogHandler
+from util.LogHandler import LogHandler
 
 
 # noinspection PyPep8Naming

Test/testProxyClass.py

@@ -13,7 +13,7 @@
 __author__ = 'JHao'
 
 import json
-from ProxyHelper import Proxy
+from helper import Proxy
 
 
 def testProxyClass():

Test/testProxyFetcher.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testProxyFetcher
+   Description :
+   Author :        JHao
+   date：          2020/6/23
+-------------------------------------------------
+   Change Activity:
+                   2020/6/23:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from fetcher.proxyFetcher import ProxyFetcher
+
+
+def test():
+    # for _ in ProxyFetcher.freeProxy01():
+    #     print(_)
+    #
+    # for _ in ProxyFetcher.freeProxy02():
+    #     print(_)
+    #
+    # for _ in ProxyFetcher.freeProxy03():
+    #     print(_)
+    #
+    # for _ in ProxyFetcher.freeProxy04():
+    #     print(_)
+    #
+    # for _ in ProxyFetcher.freeProxy05():
+    #     print(_)
+    #
+    # for _ in ProxyFetcher.freeProxy06():
+    #     print(_)
+    #
+    # for _ in ProxyFetcher.freeProxy07():
+    #     print(_)
+    #
+    # for _ in ProxyFetcher.freeProxy08():
+    #     print(_)
+    #
+    # for _ in ProxyFetcher.freeProxy09():
+    #     print(_)
+
+    # for _ in ProxyFetcher.freeProxy13():
+    #     print(_)
+
+    # for _ in ProxyFetcher.freeProxy14():
+    #     print(_)
+
+    for _ in ProxyFetcher.freeProxy15():
+        print(_)
+
+
+if __name__ == '__main__':
+    test()

Test/testRedisClient.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testRedisClient
+   Description :
+   Author :        JHao
+   date：          2020/6/23
+-------------------------------------------------
+   Change Activity:
+                   2020/6/23:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+
+def main():
+    from db.DbClient import DbClient
+    from helper.proxy import Proxy
+
+    db = DbClient("redis://:_@Fintell@10.10.61.65:6379")
+    db.changeTable("raw_proxy")
+    proxy = Proxy.newProxyFromJson(
+        '{"proxy": "27.38.96.101:9797", "fail_count": 0, "region": "", "type": "",'
+        ' "source": "freeProxy03", "check_count": 0, "last_status": "", "last_time": ""}')
+
+    print("put: ", db.put(proxy))
+
+    print("get: ", db.get())
+
+    print("exists: ", db.exists("27.38.96.101:9797"))
+
+    print("exists: ", db.exists("27.38.96.101:8888"))
+
+    print("pop: ", db.pop())
+
+    print("getAll: ", db.getAll())
+
+    print("getCount", db.getCount())
+
+
+if __name__ == '__main__':
+    main()

Test/testWebRequest.py

@@ -12,7 +12,7 @@
 """
 __author__ = 'J_hao'
 
-from Util.WebRequest import WebRequest
+from util.WebRequest import WebRequest
 
 
 # noinspection PyPep8Naming

Util/LogHandler.py

@@ -11,7 +11,7 @@
                    2017/9/21: 屏幕输出/文件输出 可选(默认屏幕和文件均输出)
 -------------------------------------------------
 """
-__author__ = 'JHao'
+# __author__ = 'JHao'
 
 import os
 
@@ -19,15 +19,15 @@ import logging
 
 from logging.handlers import TimedRotatingFileHandler
 
-# 日志级别
-CRITICAL = 50
-FATAL = CRITICAL
-ERROR = 40
-WARNING = 30
-WARN = WARNING
-INFO = 20
-DEBUG = 10
-NOTSET = 0
+# # 日志级别
+# CRITICAL = 50
+# FATAL = CRITICAL
+# ERROR = 40
+# WARNING = 30
+# WARN = WARNING
+# INFO = 20
+# DEBUG = 10
+# NOTSET = 0
 
 CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
 ROOT_PATH = os.path.join(CURRENT_PATH, os.pardir)
@@ -44,59 +44,59 @@ class LogHandler(logging.Logger):
     LogHandler
     """
 
-    def __init__(self, name, level=DEBUG, stream=True, file=True):
-        self.name = name
-        self.level = level
-        logging.Logger.__init__(self, self.name, level=level)
-        if stream:
-            self.__setStreamHandler__()
-        if file:
-            self.__setFileHandler__()
+    # def __init__(self, name, level=DEBUG, stream=True, file=True):
+    #     self.name = name
+    #     self.level = level
+    #     logging.Logger.__init__(self, self.name, level=level)
+        # if stream:
+        #     self.__setStreamHandler__()
+        # if file:
+        #     self.__setFileHandler__()
 
-    def __setFileHandler__(self, level=None):
-        """
-        set file handler
-        :param level:
-        :return:
-        """
-        file_name = os.path.join(LOG_PATH, '{name}.log'.format(name=self.name))
-        # 设置日志回滚, 保存在log目录, 一天保存一个文件, 保留15天
-        file_handler = TimedRotatingFileHandler(filename=file_name, when='D', interval=1, backupCount=15)
-        file_handler.suffix = '%Y%m%d.log'
-        if not level:
-            file_handler.setLevel(self.level)
-        else:
-            file_handler.setLevel(level)
-        formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')
-
-        file_handler.setFormatter(formatter)
-        self.file_handler = file_handler
-        self.addHandler(file_handler)
-
-    def __setStreamHandler__(self, level=None):
-        """
-        set stream handler
-        :param level:
-        :return:
-        """
-        stream_handler = logging.StreamHandler()
-        formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')
-        stream_handler.setFormatter(formatter)
-        if not level:
-            stream_handler.setLevel(self.level)
-        else:
-            stream_handler.setLevel(level)
-        self.addHandler(stream_handler)
-
-    def resetName(self, name):
-        """
-        reset name
-        :param name:
-        :return:
-        """
-        self.name = name
-        self.removeHandler(self.file_handler)
-        self.__setFileHandler__()
+    # def __setFileHandler__(self, level=None):
+    #     """
+    #     set file handler
+    #     :param level:
+    #     :return:
+    #     """
+    #     file_name = os.path.join(LOG_PATH, '{name}.log'.format(name=self.name))
+    #     # 设置日志回滚, 保存在log目录, 一天保存一个文件, 保留15天
+    #     file_handler = TimedRotatingFileHandler(filename=file_name, when='D', interval=1, backupCount=15)
+    #     file_handler.suffix = '%Y%m%d.log'
+    #     if not level:
+    #         file_handler.setLevel(self.level)
+    #     else:
+    #         file_handler.setLevel(level)
+    #     formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')
+    #
+    #     file_handler.setFormatter(formatter)
+    #     self.file_handler = file_handler
+    #     self.addHandler(file_handler)
+    #
+    # def __setStreamHandler__(self, level=None):
+    #     """
+    #     set stream handler
+    #     :param level:
+    #     :return:
+    #     """
+    #     stream_handler = logging.StreamHandler()
+    #     formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')
+    #     stream_handler.setFormatter(formatter)
+    #     if not level:
+    #         stream_handler.setLevel(self.level)
+    #     else:
+    #         stream_handler.setLevel(level)
+    #     self.addHandler(stream_handler)
+    #
+    # def resetName(self, name):
+    #     """
+    #     reset name
+    #     :param name:
+    #     :return:
+    #     """
+    #     self.name = name
+    #     self.removeHandler(self.file_handler)
+    #     self.__setFileHandler__()
 
 
 if __name__ == '__main__':

Util/__init__.py

@@ -11,6 +11,6 @@
 -------------------------------------------------
 """
 
-from Util.utilFunction import validUsefulProxy
-from Util.LogHandler import LogHandler
-from Util.utilClass import Singleton
+# from util.utilFunction import validUsefulProxy
+# from util.LogHandler import LogHandler
+# from util.utilClass import Singleton

Util/six.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     six
+   Description :
+   Author :        JHao
+   date：          2020/6/22
+-------------------------------------------------
+   Change Activity:
+                   2020/6/22:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+import sys
+
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    def iteritems(d, **kw):
+        return iter(d.items(**kw))
+else:
+    def iteritems(d, **kw):
+        return d.iteritems(**kw)
+
+
+if PY3:
+    from urllib.parse import urlparse
+else:
+    from urlparse import urlparse
+
+if PY3:
+    from imp import reload as reload_six
+else:
+    reload_six = reload
+
+if PY3:
+    from queue import Empty, Queue
+else:
+    from Queue import Empty, Queue

Util/utilFunction.py

@@ -14,9 +14,9 @@
 from lxml import etree
 import requests
 
-from Util.WebRequest import WebRequest
+from util.WebRequest import WebRequest
 from .validators import validators
-from Config.ConfigGetter import config
+from config.ConfigGetter import config
 
 
 def robustCrawl(func):

Util/validators.py

@@ -1,5 +1,10 @@
+# -*- coding: utf-8 -*-
+
 import requests
+from re import findall
+from handler.configHandler import ConfigHandler
 
+conf = ConfigHandler()
 validators = []
 
 
@@ -8,6 +13,18 @@ def validator(func):
     return func
 
 
+@validator
+def formatValidator(proxy):
+    """
+    检查代理格式
+    :param proxy:
+    :return:
+    """
+    verify_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}"
+    _proxy = findall(verify_regex, proxy)
+    return True if len(_proxy) == 1 and _proxy[0] == proxy else False
+
+
 @validator
 def timeOutValidator(proxy):
     """
@@ -15,11 +32,14 @@ def timeOutValidator(proxy):
     :param proxy:
     :return:
     """
-    if isinstance(proxy, bytes):
-        proxy = proxy.decode("utf8")
-    proxies = {"http": "http://{proxy}".format(proxy=proxy)}
+
+    proxies = {"http": "http://{proxy}".format(proxy=proxy), "https": "https://{proxy}".format(proxy=proxy)}
+    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
+               'Accept': '*/*',
+               'Connection': 'keep-alive',
+               'Accept-Language': 'zh-CN,zh;q=0.8'}
     try:
-        r = requests.get('http://www.baidu.com', proxies=proxies, timeout=10, verify=False)
+        r = requests.head(conf.verifyUrl, headers=headers, proxies=proxies, timeout=conf.verifyTimeout, verify=False)
         if r.status_code == 200:
             return True
     except Exception as e:

Util/webRequest.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     WebRequest
+   Description :   Network Requests Class
+   Author :        J_hao
+   date：          2017/7/31
+-------------------------------------------------
+   Change Activity:
+                   2017/7/31:
+-------------------------------------------------
+"""
+__author__ = 'J_hao'
+
+from requests.models import Response
+from lxml import etree
+import requests
+import random
+import time
+
+from handler.logHandler import LogHandler
+
+requests.packages.urllib3.disable_warnings()
+
+
+class WebRequest(object):
+    name = "web_request"
+
+    def __init__(self, *args, **kwargs):
+        self.log = LogHandler(self.name, file=False)
+        self.response = Response()
+
+    @property
+    def user_agent(self):
+        """
+        return an User-Agent at random
+        :return:
+        """
+        ua_list = [
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71',
+            'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
+            'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
+        ]
+        return random.choice(ua_list)
+
+    @property
+    def header(self):
+        """
+        basic header
+        :return:
+        """
+        return {'User-Agent': self.user_agent,
+                'Accept': '*/*',
+                'Connection': 'keep-alive',
+                'Accept-Language': 'zh-CN,zh;q=0.8'}
+
+    def get(self, url, header=None, retry_time=5, retry_interval=5, timeout=30, *args, **kwargs):
+        """
+        get method
+        :param url: target url
+        :param header: headers
+        :param retry_time: retry time
+        :param retry_interval: retry interval
+        :param timeout: network timeout
+        :return:
+        """
+        headers = self.header
+        if header and isinstance(header, dict):
+            headers.update(header)
+        while True:
+            try:
+                self.response = requests.get(url, headers=headers, timeout=timeout, *args, **kwargs)
+                return self
+            except Exception as e:
+                self.log.error("requests: %s error: %s" % (url, str(e)))
+                retry_time -= 1
+                if retry_time <= 0:
+                    resp = Response()
+                    resp.status_code = 200
+                    return self
+                self.log.info("retry %s second after" % retry_interval)
+                time.sleep(retry_interval)
+
+    @property
+    def tree(self):
+        return etree.HTML(self.response.content)
+
+    @property
+    def text(self):
+        return self.response.text
+

cli/proxyPool.py

@@ -1,52 +1,52 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     proxy_pool
-   Description :
-   Author :        JHao
-   date：          2019/8/2
--------------------------------------------------
-   Change Activity:
-                   2019/8/2:
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-import sys
-import click
-import platform
-
-sys.path.append('../')
-
-from Config.setting import HEADER
-from Schedule.ProxyScheduler import runScheduler
-from Api.ProxyApi import runFlask,runFlaskWithGunicorn
-
-CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
-
-
-@click.group(context_settings=CONTEXT_SETTINGS)
-@click.version_option(version='2.0.0')
-def cli():
-    """ProxyPool cli工具"""
-
-
-@cli.command(name="schedule")
-def schedule():
-    """ 启动调度程序 """
-    click.echo(HEADER)
-    runScheduler()
-
-
-@cli.command(name="webserver")
-def schedule():
-    """ 启动web服务 """
-    click.echo(HEADER)
-    if platform.system() == "Windows":
-        runFlask()
-    else:
-        runFlaskWithGunicorn()
-
-
-if __name__ == '__main__':
-    cli()
+# # -*- coding: utf-8 -*-
+# """
+# -------------------------------------------------
+#    File Name：     proxy_pool
+#    Description :
+#    Author :        JHao
+#    date：          2019/8/2
+# -------------------------------------------------
+#    Change Activity:
+#                    2019/8/2:
+# -------------------------------------------------
+# """
+# __author__ = 'JHao'
+#
+# import sys
+# import click
+# import platform
+#
+# sys.path.append('../')
+#
+# from Config.setting import HEADER
+# from Schedule.ProxyScheduler import runScheduler
+# from Api.ProxyApi import runFlask,runFlaskWithGunicorn
+#
+# CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
+#
+#
+# @click.group(context_settings=CONTEXT_SETTINGS)
+# @click.version_option(version='2.0.0')
+# def cli():
+#     """ProxyPool cli工具"""
+#
+#
+# @cli.command(name="schedule")
+# def schedule():
+#     """ 启动调度程序 """
+#     click.echo(HEADER)
+#     runScheduler()
+#
+#
+# @cli.command(name="webserver")
+# def schedule():
+#     """ 启动web服务 """
+#     click.echo(HEADER)
+#     if platform.system() == "Windows":
+#         runFlask()
+#     else:
+#         runFlaskWithGunicorn()
+#
+#
+# if __name__ == '__main__':
+#     cli()

config/ConfigGetter.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     ConfigGetter
+   Description :   读取配置
+   Author :        JHao
+   date：          2019/2/15
+-------------------------------------------------
+   Change Activity:
+                   2019/2/15:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+
+from util.utilClass import LazyProperty
+from config.setting import *
+
+
+class ConfigGetter(object):
+    """
+    get config
+    """
+
+    def __init__(self):
+        pass
+
+    # @LazyProperty
+    # def db_type(self):
+    #     return DATABASES.get("default", {}).get("TYPE", "SSDB")
+    #
+    # @LazyProperty
+    # def db_name(self):
+    #     return DATABASES.get("default", {}).get("NAME", "proxy")
+    #
+    # @LazyProperty
+    # def db_host(self):
+    #     return DATABASES.get("default", {}).get("HOST", "127.0.0.1")
+    #
+    # @LazyProperty
+    # def db_port(self):
+    #     return DATABASES.get("default", {}).get("PORT", 8888)
+    #
+    # @LazyProperty
+    # def db_password(self):
+    #     return DATABASES.get("default", {}).get("PASSWORD", "")
+    #
+    # @LazyProperty
+    # def proxy_getter_functions(self):
+    #     return PROXY_GETTER
+    #
+    # @LazyProperty
+    # def host_ip(self):
+    #     return SERVER_API.get("HOST", "127.0.0.1")
+    #
+    # @LazyProperty
+    # def host_port(self):
+    #     return SERVER_API.get("PORT", 5010)
+    #
+    # @LazyProperty
+    # def verify_host(self):
+    #     return VERIFY_HOST
+
+
+config = ConfigGetter()
+
+if __name__ == '__main__':
+    print(config.db_type)
+    print(config.db_name)
+    print(config.db_host)
+    print(config.db_port)
+    print(config.proxy_getter_functions)
+    print(config.host_ip)
+    print(config.host_port)
+    print(config.db_password)

config/setting.py

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     setting.py
+   Description :   配置文件
+   Author :        JHao
+   date：          2019/2/15
+-------------------------------------------------
+   Change Activity:
+                   2019/2/15:
+-------------------------------------------------
+"""
+
+BANNER = r"""
+****************************************************************
+*** ______  ********************* ______ *********** _  ********
+*** | ___ \_ ******************** | ___ \ ********* | | ********
+*** | |_/ / \__ __   __  _ __   _ | |_/ /___ * ___  | | ********
+*** |  __/|  _// _ \ \ \/ /| | | ||  __// _ \ / _ \ | | ********
+*** | |   | | | (_) | >  < \ |_| || |  | (_) | (_) || |___  ****
+*** \_|   |_|  \___/ /_/\_\ \__  |\_|   \___/ \___/ \_____/ ****
+****                       __ / /                          *****
+************************* /___ / *******************************
+*************************       ********************************
+****************************************************************
+"""
+
+SHOW_CONFIG = True   # print key config
+
+
+
+
+
+# import sys
+# from os import getenv
+# from logging import getLogger
+#
+# log = getLogger(__name__)
+#
+# BANNER = """
+# ****************************************************************
+# *** ______  ********************* ______ *********** _  ********
+# *** | ___ \_ ******************** | ___ \ ********* | | ********
+# *** | |_/ / \__ __   __  _ __   _ | |_/ /___ * ___  | | ********
+# *** |  __/|  _// _ \ \ \/ /| | | ||  __// _ \ / _ \ | | ********
+# *** | |   | | | (_) | >  < \ |_| || |  | (_) | (_) || |___  ****
+# *** \_|   |_|  \___/ /_/\_\ \__  |\_|   \___/ \___/ \_____/ ****
+# ****                       __ / /                          *****
+# ************************* /___ / *******************************
+# *************************       ********************************
+# ****************************************************************
+# """
+#
+# PY3 = sys.version_info >= (3,)
+#
+# DB_TYPE = getenv('db_type', 'SSDB').upper()
+# DB_HOST = getenv('db_host', '10.10.61.65')
+# DB_PORT = getenv('db_port', 6379)
+# DB_PASSWORD = getenv('db_password', '_@Fintell')
+#
+# """ 数据库配置 """
+# DATABASES = {
+#     "default": {
+#         "TYPE": DB_TYPE,
+#         "HOST": DB_HOST,
+#         "PORT": DB_PORT,
+#         "NAME": "proxy",
+#         "PASSWORD": DB_PASSWORD
+#     }
+# }
+#
+# # register the proxy getter function
+#
+# PROXY_GETTER = [
+#     "freeProxy01",
+#     # "freeProxy02",
+#     "freeProxy03",
+#     "freeProxy04",
+#     "freeProxy05",
+#     # "freeProxy06",
+#     "freeProxy07",
+#     # "freeProxy08",
+#     "freeProxy09",
+#     "freeProxy13",
+#     "freeProxy14",
+#     "freeProxy14",
+# ]
+#
+# """ API config http://127.0.0.1:5010 """
+# SERVER_API = {
+#     "HOST": "0.0.0.0",  # The ip specified which starting the web API
+#     "PORT": 5010  # port number to which the server listens to
+# }
+#
+# VERIFY_HOST = getenv('proxy_verify_host', 'http://www.baidu.com')
+
+# class ConfigError(BaseException):
+#     pass
+#
+#
+# def checkConfig():
+#     if DB_TYPE not in ["SSDB", "REDIS"]:
+#         raise ConfigError('db_type Do not support: %s, must SSDB/REDIS .' % DB_TYPE)
+#
+#     if type(DB_PORT) == str and not DB_PORT.isdigit():
+#         raise ConfigError('if db_port is string, it must be digit, not %s' % DB_PORT)
+#
+#     from proxyGetter import getFreeProxy
+#     illegal_getter = list(filter(lambda key: not hasattr(getFreeProxy.GetFreeProxy, key), PROXY_GETTER))
+#     if len(illegal_getter) > 0:
+#         raise ConfigError("ProxyGetter: %s does not exists" % "/".join(illegal_getter))
+#
+#
+# checkConfig()

db/SsdbClient.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+# !/usr/bin/env python
+"""
+-------------------------------------------------
+   File Name：     SsdbClient.py
+   Description :  封装SSDB操作
+   Author :       JHao
+   date：          2016/12/2
+-------------------------------------------------
+   Change Activity:
+                   2016/12/2:
+                   2017/09/22: PY3中 redis-py返回的数据是bytes型
+                   2017/09/27: 修改pop()方法 返回{proxy:value}字典
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from Config.setting import PY3
+
+from redis.connection import BlockingConnectionPool
+from redis import Redis
+
+
+class SsdbClient(object):
+    """
+    SSDB client
+
+    SSDB中代理存放的结构为hash：
+        原始代理存放在name为raw_proxy的hash中, key为代理的ip:por, value为代理属性的字典;
+        验证后的代理存放在name为useful_proxy的hash中, key为代理的ip:port, value为代理属性的字典;
+
+    """
+    def __init__(self, name, **kwargs):
+        """
+        init
+        :param name: hash name
+        :param host: host
+        :param port: port
+        :param password: password
+        :return:
+        """
+        self.name = name
+        self.__conn = Redis(connection_pool=BlockingConnectionPool(**kwargs))
+
+    def get(self, proxy_str):
+        """
+        从hash中获取对应的proxy, 使用前需要调用changeTable()
+        :param proxy_str: proxy str
+        :return:
+        """
+        data = self.__conn.hget(name=self.name, key=proxy_str)
+        if data:
+            return data.decode('utf-8') if PY3 else data
+        else:
+            return None
+
+    def put(self, proxy_obj):
+        """
+        将代理放入hash, 使用changeTable指定hash name
+        :param proxy_obj: Proxy obj
+        :return:
+        """
+        data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
+        return data
+
+    def delete(self, proxy_str):
+        """
+        移除指定代理, 使用changeTable指定hash name
+        :param proxy_str: proxy str
+        :return:
+        """
+        self.__conn.hdel(self.name, proxy_str)
+
+    def exists(self, proxy_str):
+        """
+        判断指定代理是否存在, 使用changeTable指定hash name
+        :param proxy_str: proxy str
+        :return:
+        """
+        return self.__conn.hexists(self.name, proxy_str)
+
+    def update(self, proxy_obj):
+        """
+        更新 proxy 属性
+        :param proxy_obj:
+        :return:
+        """
+        self.__conn.hset(self.name,  proxy_obj.proxy, proxy_obj.info_json)
+
+    def pop(self):
+        """
+        弹出一个代理
+        :return: dict {proxy: value}
+        """
+        # proxies = self.__conn.hkeys(self.name)
+        # if proxies:
+        #     proxy = random.choice(proxies)
+        #     value = self.__conn.hget(self.name, proxy)
+        #     self.delete(proxy)
+        #     return {'proxy': proxy.decode('utf-8') if PY3 else proxy,
+        #             'value': value.decode('utf-8') if PY3 and value else value}
+        return None
+
+    def getAll(self):
+        """
+        列表形式返回所有代理, 使用changeTable指定hash name
+        :return:
+        """
+        item_dict = self.__conn.hgetall(self.name)
+        if PY3:
+            return [value.decode('utf8') for key, value in item_dict.items()]
+        else:
+            return item_dict.values()
+
+    def clear(self):
+        """
+        清空所有代理, 使用changeTable指定hash name
+        :return:
+        """
+        return self.__conn.execute_command("hclear", self.name)
+
+    def getNumber(self):
+        """
+        返回代理数量
+        :return:
+        """
+        return self.__conn.hlen(self.name)
+
+    def changeTable(self, name):
+        """
+        切换操作对象
+        :param name: raw_proxy/useful_proxy
+        :return:
+        """
+        self.name = name

fetcher/CheckProxy.py

@@ -0,0 +1,70 @@
+# # -*- coding: utf-8 -*-
+# """
+# -------------------------------------------------
+#    File Name：     CheckProxy
+#    Description :   used for check getFreeProxy.py
+#    Author :        JHao
+#    date：          2018/7/10
+# -------------------------------------------------
+#    Change Activity:
+#                    2018/7/10: CheckProxy
+# -------------------------------------------------
+# """
+# __author__ = 'JHao'
+#
+# from .getFreeProxy import GetFreeProxy
+# from util.utilFunction import verifyProxyFormat
+#
+#
+# from util.LogHandler import LogHandler
+#
+# log = LogHandler('check_proxy', file=False)
+#
+#
+# class CheckProxy(object):
+#
+#     @staticmethod
+#     def checkAllGetProxyFunc():
+#         """
+#         检查getFreeProxy所有代理获取函数运行情况
+#         Returns:
+#             None
+#         """
+#         import inspect
+#         member_list = inspect.getmembers(GetFreeProxy, predicate=inspect.isfunction)
+#         proxy_count_dict = dict()
+#         for func_name, func in member_list:
+#             log.info(u"开始运行 {}".format(func_name))
+#             try:
+#                 proxy_list = [_ for _ in func() if verifyProxyFormat(_)]
+#                 proxy_count_dict[func_name] = len(proxy_list)
+#             except Exception as e:
+#                 log.info(u"代理获取函数 {} 运行出错!".format(func_name))
+#                 log.error(str(e))
+#         log.info(u"所有函数运行完毕 " + "***" * 5)
+#         for func_name, func in member_list:
+#             log.info(u"函数 {n}, 获取到代理数: {c}".format(n=func_name, c=proxy_count_dict.get(func_name, 0)))
+#
+#     @staticmethod
+#     def checkGetProxyFunc(func):
+#         """
+#         检查指定的getFreeProxy某个function运行情况
+#         Args:
+#             func: getFreeProxy中某个可调用方法
+#
+#         Returns:
+#             None
+#         """
+#         func_name = getattr(func, '__name__', "None")
+#         log.info("start running func: {}".format(func_name))
+#         count = 0
+#         for proxy in func():
+#             if verifyProxyFormat(proxy):
+#                 log.info("{} fetch proxy: {}".format(func_name, proxy))
+#                 count += 1
+#         log.info("{n} completed, fetch proxy number: {c}".format(n=func_name, c=count))
+#
+#
+# if __name__ == '__main__':
+#     CheckProxy.checkAllGetProxyFunc()
+#     CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01)

ProxyGetter/__init__.py → fetcher/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
 -------------------------------------------------
-   File Name：     __init__.py.py  
+   File Name：     __init__.py
    Description :  
    Author :       JHao
    date：          2016/11/25

fetcher/getFreeProxy.py

@@ -0,0 +1,355 @@
+# # -*- coding: utf-8 -*-
+# # !/usr/bin/env python
+# """
+# -------------------------------------------------
+#    File Name：     GetFreeProxy.py
+#    Description :  抓取免费代理
+#    Author :       JHao
+#    date：          2016/11/25
+# -------------------------------------------------
+#    Change Activity:
+#                    2016/11/25:
+# -------------------------------------------------
+# """
+# import re
+# import sys
+# import requests
+# from time import sleep
+#
+# sys.path.append('..')
+#
+# from util.WebRequest import WebRequest
+# from util.utilFunction import getHtmlTree
+#
+# # for debug to disable insecureWarning
+# requests.packages.urllib3.disable_warnings()
+#
+#
+# class GetFreeProxy(object):
+#     """
+#     proxy getter
+#     """
+#
+#     @staticmethod
+#     def freeProxy01():
+#         """
+#         无忧代理 http://www.data5u.com/
+#         几乎没有能用的
+#         :return:
+#         """
+#         url_list = [
+#             'http://www.data5u.com/',
+#             'http://www.data5u.com/free/gngn/index.shtml',
+#             'http://www.data5u.com/free/gnpt/index.shtml'
+#         ]
+#         key = 'ABCDEFGHIZ'
+#         for url in url_list:
+#             html_tree = getHtmlTree(url)
+#             ul_list = html_tree.xpath('//ul[@class="l2"]')
+#             for ul in ul_list:
+#                 try:
+#                     ip = ul.xpath('./span[1]/li/text()')[0]
+#                     classnames = ul.xpath('./span[2]/li/attribute::class')[0]
+#                     classname = classnames.split(' ')[1]
+#                     port_sum = 0
+#                     for c in classname:
+#                         port_sum *= 10
+#                         port_sum += key.index(c)
+#                     port = port_sum >> 3
+#                     yield '{}:{}'.format(ip, port)
+#                 except Exception as e:
+#                     print(e)
+#
+#     @staticmethod
+#     def freeProxy02(count=20):
+#         """
+#         代理66 http://www.66ip.cn/
+#         :param count: 提取数量
+#         :return:
+#         """
+#         urls = [
+#             "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=",
+#             "http://www.66ip.cn/nmtq.php?getnum={}&isp=0&anonymoustype=0&s"
+#             "tart=&ports=&export=&ipaddress=&area=0&proxytype=2&api=66ip"
+#         ]
+#
+#         try:
+#             import execjs
+#             import requests
+#
+#             headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
+#                        'Accept': '*/*',
+#                        'Connection': 'keep-alive',
+#                        'Accept-Language': 'zh-CN,zh;q=0.8'}
+#             session = requests.session()
+#             src = session.get("http://www.66ip.cn/", headers=headers).text
+#             src = src.split("</script>")[0] + '}'
+#             src = src.replace("<script>", "function test() {")
+#             src = src.replace("while(z++)try{eval(", ';var num=10;while(z++)try{var tmp=')
+#             src = src.replace(");break}", ";num--;if(tmp.search('cookie') != -1 | num<0){return tmp}}")
+#             ctx = execjs.compile(src)
+#             src = ctx.call("test")
+#             src = src[src.find("document.cookie="): src.find("};if((")]
+#             src = src.replace("document.cookie=", "")
+#             src = "function test() {var window={}; return %s }" % src
+#             cookie = execjs.compile(src).call('test')
+#             js_cookie = cookie.split(";")[0].split("=")[-1]
+#         except Exception as e:
+#             print(e)
+#             return
+#
+#         for url in urls:
+#             try:
+#                 html = session.get(url.format(count), cookies={"__jsl_clearance": js_cookie}, headers=headers).text
+#                 ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", html)
+#                 for ip in ips:
+#                     yield ip.strip()
+#             except Exception as e:
+#                 print(e)
+#                 pass
+#
+#     @staticmethod
+#     def freeProxy03(page_count=1):
+#         """
+#         西刺代理 http://www.xicidaili.com
+#         :return:
+#         """
+#         url_list = [
+#             'http://www.xicidaili.com/nn/',  # 高匿
+#             'http://www.xicidaili.com/nt/',  # 透明
+#         ]
+#         for each_url in url_list:
+#             for i in range(1, page_count + 1):
+#                 page_url = each_url + str(i)
+#                 tree = getHtmlTree(page_url)
+#                 proxy_list = tree.xpath('.//table[@id="ip_list"]//tr[position()>1]')
+#                 for proxy in proxy_list:
+#                     try:
+#                         yield ':'.join(proxy.xpath('./td/text()')[0:2])
+#                     except Exception as e:
+#                         pass
+#
+#     @staticmethod
+#     def freeProxy04():
+#         """
+#         guobanjia http://www.goubanjia.com/
+#         :return:
+#         """
+#         url = "http://www.goubanjia.com/"
+#         tree = getHtmlTree(url)
+#         proxy_list = tree.xpath('//td[@class="ip"]')
+#         # 此网站有隐藏的数字干扰，或抓取到多余的数字或.符号
+#         # 需要过滤掉<p style="display:none;">的内容
+#         xpath_str = """.//*[not(contains(@style, 'display: none'))
+#                                         and not(contains(@style, 'display:none'))
+#                                         and not(contains(@class, 'port'))
+#                                         ]/text()
+#                                 """
+#         for each_proxy in proxy_list:
+#             try:
+#                 # :符号裸放在td下，其他放在div span p中，先分割找出ip，再找port
+#                 ip_addr = ''.join(each_proxy.xpath(xpath_str))
+#
+#                 # HTML中的port是随机数，真正的端口编码在class后面的字母中。
+#                 # 比如这个：
+#                 # <span class="port CFACE">9054</span>
+#                 # CFACE解码后对应的是3128。
+#                 port = 0
+#                 for _ in each_proxy.xpath(".//span[contains(@class, 'port')]"
+#                                           "/attribute::class")[0]. \
+#                         replace("port ", ""):
+#                     port *= 10
+#                     port += (ord(_) - ord('A'))
+#                 port /= 8
+#
+#                 yield '{}:{}'.format(ip_addr, int(port))
+#             except Exception as e:
+#                 pass
+#
+#     @staticmethod
+#     def freeProxy05():
+#         """
+#         快代理 https://www.kuaidaili.com
+#         """
+#         url_list = [
+#             'https://www.kuaidaili.com/free/inha/',
+#             'https://www.kuaidaili.com/free/intr/'
+#         ]
+#         for url in url_list:
+#             tree = getHtmlTree(url)
+#             proxy_list = tree.xpath('.//table//tr')
+#             sleep(1)  # 必须sleep 不然第二条请求不到数据
+#             for tr in proxy_list[1:]:
+#                 yield ':'.join(tr.xpath('./td/text()')[0:2])
+#
+#     @staticmethod
+#     def freeProxy06():
+#         """
+#         码农代理 https://proxy.coderbusy.com/
+#         :return:
+#         """
+#         urls = ['https://proxy.coderbusy.com/']
+#         for url in urls:
+#             tree = getHtmlTree(url)
+#             proxy_list = tree.xpath('.//table//tr')
+#             for tr in proxy_list[1:]:
+#                 yield ':'.join(tr.xpath('./td/text()')[0:2])
+#
+#     @staticmethod
+#     def freeProxy07():
+#         """
+#         云代理 http://www.ip3366.net/free/
+#         :return:
+#         """
+#         urls = ['http://www.ip3366.net/free/?stype=1',
+#                 "http://www.ip3366.net/free/?stype=2"]
+#         request = WebRequest()
+#         for url in urls:
+#             r = request.get(url, timeout=10)
+#             proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
+#             for proxy in proxies:
+#                 yield ":".join(proxy)
+#
+#     @staticmethod
+#     def freeProxy08():
+#         """
+#         IP海 http://www.iphai.com/free/ng
+#         :return:
+#         """
+#         urls = [
+#             'http://www.iphai.com/free/ng',
+#             'http://www.iphai.com/free/np',
+#             'http://www.iphai.com/free/wg',
+#             'http://www.iphai.com/free/wp'
+#         ]
+#         request = WebRequest()
+#         for url in urls:
+#             r = request.get(url, timeout=10)
+#             proxies = re.findall(r'<td>\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?</td>[\s\S]*?<td>\s*?(\d+)\s*?</td>',
+#                                  r.text)
+#             for proxy in proxies:
+#                 yield ":".join(proxy)
+#
+#     @staticmethod
+#     def freeProxy09(page_count=1):
+#         """
+#         http://ip.jiangxianli.com/?page=
+#         免费代理库
+#         :return:
+#         """
+#         for i in range(1, page_count + 1):
+#             url = 'http://ip.jiangxianli.com/?country=中国&?page={}'.format(i)
+#             html_tree = getHtmlTree(url)
+#             for index, tr in enumerate(html_tree.xpath("//table//tr")):
+#                 if index == 0:
+#                     continue
+#                 yield ":".join(tr.xpath("./td/text()")[0:2]).strip()
+#
+#     # @staticmethod
+#     # def freeProxy10():
+#     #     """
+#     #     墙外网站 cn-proxy
+#     #     :return:
+#     #     """
+#     #     urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218']
+#     #     request = WebRequest()
+#     #     for url in urls:
+#     #         r = request.get(url, timeout=10)
+#     #         proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)
+#     #         for proxy in proxies:
+#     #             yield ':'.join(proxy)
+#
+#     # @staticmethod
+#     # def freeProxy11():
+#     #     """
+#     #     https://proxy-list.org/english/index.php
+#     #     :return:
+#     #     """
+#     #     urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)]
+#     #     request = WebRequest()
+#     #     import base64
+#     #     for url in urls:
+#     #         r = request.get(url, timeout=10)
+#     #         proxies = re.findall(r"Proxy\('(.*?)'\)", r.text)
+#     #         for proxy in proxies:
+#     #             yield base64.b64decode(proxy).decode()
+#
+#     # @staticmethod
+#     # def freeProxy12():
+#     #     urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1']
+#     #     request = WebRequest()
+#     #     for url in urls:
+#     #         r = request.get(url, timeout=10)
+#     #         proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
+#     #         for proxy in proxies:
+#     #             yield ':'.join(proxy)
+#
+#     @staticmethod
+#     def freeProxy13(max_page=2):
+#         """
+#         http://www.qydaili.com/free/?action=china&page=1
+#         齐云代理
+#         :param max_page:
+#         :return:
+#         """
+#         base_url = 'http://www.qydaili.com/free/?action=china&page='
+#         request = WebRequest()
+#         for page in range(1, max_page + 1):
+#             url = base_url + str(page)
+#             r = request.get(url, timeout=10)
+#             proxies = re.findall(
+#                 r'<td.*?>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td.*?>(\d+)</td>',
+#                 r.text)
+#             for proxy in proxies:
+#                 yield ':'.join(proxy)
+#
+#     @staticmethod
+#     def freeProxy14(max_page=2):
+#         """
+#         http://www.89ip.cn/index.html
+#         89免费代理
+#         :param max_page:
+#         :return:
+#         """
+#         base_url = 'http://www.89ip.cn/index_{}.html'
+#         request = WebRequest()
+#         for page in range(1, max_page + 1):
+#             url = base_url.format(page)
+#             r = request.get(url, timeout=10)
+#             proxies = re.findall(
+#                 r'<td.*?>[\s\S]*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?</td>[\s\S]*?<td.*?>[\s\S]*?(\d+)[\s\S]*?</td>',
+#                 r.text)
+#             for proxy in proxies:
+#                 yield ':'.join(proxy)
+#
+#     @staticmethod
+#     def freeProxy15():
+#         urls = ['http://www.xiladaili.com/putong/',
+#                 "http://www.xiladaili.com/gaoni/",
+#                 "http://www.xiladaili.com/http/",
+#                 "http://www.xiladaili.com/https/"]
+#         request = WebRequest()
+#         for url in urls:
+#             r = request.get(url, timeout=10)
+#             ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", r.text)
+#             for ip in ips:
+#                 yield ip.strip()
+#
+#
+# if __name__ == '__main__':
+#     from CheckProxy import CheckProxy
+#
+#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01)
+#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy02)
+#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy03)
+#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy04)
+#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy05)
+#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy06)
+#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy07)
+#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy08)
+#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy09)
+#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy13)
+#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy14)
+#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy15)
+#
+#     CheckProxy.checkAllGetProxyFunc()

handler/ProxyManager.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+# !/usr/bin/env python
+"""
+-------------------------------------------------
+   File Name：     ProxyManager.py
+   Description :
+   Author :       JHao
+   date：          2016/12/3
+-------------------------------------------------
+   Change Activity:
+                   2016/12/3:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+import random
+
+from helper import Proxy
+from db.DbClient import DbClient
+from config.ConfigGetter import config
+from util.LogHandler import LogHandler
+from util.utilFunction import verifyProxyFormat
+from fetcher.getFreeProxy import GetFreeProxy
+
+
+class ProxyManager(object):
+    pass
+#     """
+#     ProxyManager
+#     """
+#
+#     def __init__(self):
+#         self.db = DbClient()
+#         self.raw_proxy_queue = 'raw_proxy'
+#         self.log = LogHandler('proxy_manager')
+#         self.useful_proxy_queue = 'useful_proxy'
+#
+#     def fetch(self):
+#         """
+#         fetch proxy into db by ProxyGetter
+#         :return:
+#         """
+#         self.db.changeTable(self.raw_proxy_queue)
+#         proxy_set = set()
+#         self.log.info("ProxyFetch : start")
+#         for proxyGetter in config.proxy_getter_functions:
+#             self.log.info("ProxyFetch - {func}: start".format(func=proxyGetter))
+#             try:
+#                 for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
+#                     proxy = proxy.strip()
+#
+#                     if not proxy or not verifyProxyFormat(proxy):
+#                         self.log.error('ProxyFetch - {func}: '
+#                                        '{proxy} illegal'.format(func=proxyGetter, proxy=proxy.ljust(20)))
+#                         continue
+#                     elif proxy in proxy_set:
+#                         self.log.info('ProxyFetch - {func}: '
+#                                       '{proxy} exist'.format(func=proxyGetter, proxy=proxy.ljust(20)))
+#                         continue
+#                     else:
+#                         self.log.info('ProxyFetch - {func}: '
+#                                       '{proxy} success'.format(func=proxyGetter, proxy=proxy.ljust(20)))
+#                         self.db.put(Proxy(proxy, source=proxyGetter))
+#                         proxy_set.add(proxy)
+#             except Exception as e:
+#                 self.log.error("ProxyFetch - {func}: error".format(func=proxyGetter))
+#                 self.log.error(str(e))
+#
+#     def get(self):
+#         """
+#         return a useful proxy
+#         :return:
+#         """
+#         self.db.changeTable(self.useful_proxy_queue)
+#         item_list = self.db.getAll()
+#         if item_list:
+#             random_choice = random.choice(item_list)
+#             return Proxy.newProxyFromJson(random_choice)
+#         return None
+#
+#     def delete(self, proxy_str):
+#         """
+#         delete proxy from pool
+#         :param proxy_str:
+#         :return:
+#         """
+#         self.db.changeTable(self.useful_proxy_queue)
+#         self.db.delete(proxy_str)
+#
+#     def getAll(self):
+#         """
+#         get all proxy from pool as list
+#         :return:
+#         """
+#         self.db.changeTable(self.useful_proxy_queue)
+#         item_list = self.db.getAll()
+#         return [Proxy.newProxyFromJson(_) for _ in item_list]
+#
+#     def getNumber(self):
+#         self.db.changeTable(self.raw_proxy_queue)
+#         total_raw_proxy = self.db.getNumber()
+#         self.db.changeTable(self.useful_proxy_queue)
+#         total_useful_queue = self.db.getNumber()
+#         return {'raw_proxy': total_raw_proxy, 'useful_proxy': total_useful_queue}
+
+
+# if __name__ == '__main__':
+#     pp = ProxyManager()
+#     pp.fetch()

Manager/__init__.py → handler/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
 -------------------------------------------------
-   File Name：     __init__.py.py  
+   File Name：     __init__.py
    Description :  
    Author :       JHao
    date：          2016/12/3
@@ -12,4 +12,4 @@
 """
 __author__ = 'JHao'
 
-from Manager.ProxyManager import ProxyManager
+# from handler.ProxyManager import ProxyManager

handler/configHandler.py

@@ -14,6 +14,7 @@ __author__ = 'JHao'
 
 import os
 import setting
+from util.six import reload_six
 from util.singleton import Singleton
 from util.lazyProperty import LazyProperty
 
@@ -35,3 +36,24 @@ class ConfigHandler(object):
     @LazyProperty
     def dbConn(self):
         return os.getenv("DB_CONN", setting.DB_CONN)
+
+    @LazyProperty
+    def rawProxy(self):
+        return setting.RAW_PROXY
+
+    @LazyProperty
+    def useProxy(self):
+        return setting.USE_PROXY
+
+    @property
+    def fetchers(self):
+        reload_six(setting)
+        return setting.PROXY_FETCHER
+
+    @LazyProperty
+    def verifyUrl(self):
+        return os.getenv("VERIFY_URL", setting.VERIFY_RUL)
+
+    @LazyProperty
+    def verifyTimeout(self):
+        return os.getenv("VERIFY_TIMEOUT", setting.VERIFY_TIMEOUT)

handler/proxyHandler.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     ProxyHandler.py
+   Description :
+   Author :       JHao
+   date：          2016/12/3
+-------------------------------------------------
+   Change Activity:
+                   2016/12/3:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from helper.proxy import Proxy
+from db.dbClient import DbClient
+from handler.configHandler import ConfigHandler
+
+
+class ProxyHandler(object):
+    """ Proxy CRUD operator"""
+
+    def __init__(self):
+        self.conf = ConfigHandler()
+        self.db = DbClient(self.conf.dbConn)
+
+    def get(self):
+        """
+        return a useful proxy
+        :return:
+        """
+        self.db.changeTable(self.conf.useProxy)
+        proxy = self.db.get()
+        if proxy:
+            return Proxy.newProxyFromJson(proxy)
+        return None
+
+    def pop(self):
+        """
+        return and delete a useful proxy
+        :return:
+        """
+        self.db.changeTable(self.conf.useProxy)
+        proxy = self.db.pop()
+        if proxy:
+            return Proxy.newProxyFromJson(proxy)
+        return None
+
+    def delete(self, proxy_str):
+        """
+        delete useful proxy
+        :param proxy_str:
+        :return:
+        """
+        self.db.changeTable(self.conf.useProxy)
+        return self.db.delete(proxy_str)
+
+    def getAll(self):
+        """
+        get all proxy from pool as Proxy list
+        :return:
+        """
+        self.db.changeTable(self.conf.useProxy)
+        proxies_dict = self.db.getAll()
+        return [Proxy.newProxyFromJson(value) for _, value in proxies_dict.items()]
+
+    def exists(self, proxy_str):
+        """
+        check proxy exists
+        :param proxy_str:
+        :return:
+        """
+        self.db.changeTable(self.conf.useProxy)
+        return self.db.exists(proxy_str)
+
+    def getCount(self):
+        """
+        return raw_proxy and use_proxy count
+        :return:
+        """
+        self.db.changeTable(self.conf.useProxy)
+        total_raw_proxy = self.db.getCount()
+        self.db.changeTable(self.conf.rawProxy)
+        total_use_proxy = self.db.getCount()
+        return {'raw_proxy': total_raw_proxy, 'use_proxy': total_use_proxy}

ProxyHelper/ProxyUtil.py → helper/ProxyUtil.py

@@ -12,7 +12,7 @@
 """
 __author__ = 'JHao'
 
-from Util import validUsefulProxy
+from util import validUsefulProxy
 
 from datetime import datetime
 

ProxyHelper/__init__.py → helper/__init__.py

@@ -12,5 +12,5 @@
 """
 __author__ = 'JHao'
 
-from ProxyHelper.Proxy import Proxy
-from ProxyHelper.ProxyUtil import checkProxyUseful
+# from proxyHelper.Proxy import Proxy
+# from proxyHelper.ProxyUtil import checkProxyUseful

helper/proxyHelper.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     proxyHelper
+   Description :
+   Author :        JHao
+   date：          2020/6/24
+-------------------------------------------------
+   Change Activity:
+                   2020/6/24:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from util.validators import validators
+
+from datetime import datetime
+
+
+def proxyCheck(proxy_obj):
+    """
+    检测代理是否可用
+    :param proxy_obj: Proxy object
+    :return: Proxy object, status
+    """
+
+    def __proxyCheck(proxy):
+        for func in validators:
+            if not func(proxy):
+                return False
+        return True
+
+    if __proxyCheck(proxy_obj.proxy):
+        # 检测通过 更新proxy属性
+        proxy_obj.check_count += 1
+        proxy_obj.last_status = 1
+        proxy_obj.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        if proxy_obj.fail_count > 0:
+            proxy_obj.fail_count -= 1
+        return proxy_obj
+    else:
+        proxy_obj.check_count += 1
+        proxy_obj.last_status = 0
+        proxy_obj.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        proxy_obj.fail_count += 1
+        return proxy_obj

proxyPool.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     proxy_pool
+   Description :   proxy pool 启动入口
+   Author :        JHao
+   date：          2020/6/19
+-------------------------------------------------
+   Change Activity:
+                   2020/6/19:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+import click
+
+from util import six
+from config.setting import BANNER
+
+# from Schedule.ProxyScheduler import runScheduler
+from api.proxyApi import runFlask
+
+CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
+
+
+@click.group(context_settings=CONTEXT_SETTINGS)
+@click.version_option(version='2.1.0')
+def cli():
+    """ProxyPool cli工具"""
+
+
+@cli.command(name="schedule")
+def schedule():
+    """ 启动调度程序 """
+    click.echo(BANNER)
+    # runScheduler()
+
+
+@cli.command(name="server")
+def schedule():
+    """ 启动api服务 """
+    # click.echo(BANNER)
+    # runFlask()
+    from test import testProxyFetcher
+    testProxyFetcher.test()
+
+
+if __name__ == '__main__':
+    cli()

schedule/RawProxyCheck.py

@@ -0,0 +1,81 @@
+# # -*- coding: utf-8 -*-
+# """
+# -------------------------------------------------
+#    File Name：     RawProxyCheck
+#    Description :   check raw_proxy to useful
+#    Author :        JHao
+#    date：          2019/8/6
+# -------------------------------------------------
+#    Change Activity:
+#                    2019/8/6: check raw_proxy to useful
+# -------------------------------------------------
+# """
+# __author__ = 'JHao'
+#
+# from threading import Thread
+#
+# try:
+#     from Queue import Empty, Queue  # py2
+# except:
+#     from queue import Empty, Queue  # py3
+#
+# from util import LogHandler
+# from handler import ProxyManager
+# from helper import Proxy, checkProxyUseful
+#
+#
+# class RawProxyCheck(ProxyManager, Thread):
+#     def __init__(self, queue, thread_name):
+#         ProxyManager.__init__(self)
+#         Thread.__init__(self, name=thread_name)
+#         self.log = LogHandler('raw_proxy_check')
+#         self.queue = queue
+#
+#     def run(self):
+#         self.log.info("RawProxyCheck - {}  : start".format(self.name))
+#         self.db.changeTable(self.useful_proxy_queue)
+#         while True:
+#             try:
+#                 proxy_json = self.queue.get(block=False)
+#             except Empty:
+#                 self.log.info("RawProxyCheck - {}  : exit".format(self.name))
+#                 break
+#
+#             proxy_obj = Proxy.newProxyFromJson(proxy_json)
+#
+#             proxy_obj, status = checkProxyUseful(proxy_obj)
+#             if status:
+#                 if self.db.exists(proxy_obj.proxy):
+#                     self.log.info('RawProxyCheck - {}  : {} validation exists'.format(self.name,
+#                                                                                       proxy_obj.proxy.ljust(20)))
+#                 else:
+#                     self.db.put(proxy_obj)
+#                     self.log.info(
+#                         'RawProxyCheck - {}  : {} validation pass'.format(self.name, proxy_obj.proxy.ljust(20)))
+#             else:
+#                 self.log.info('RawProxyCheck - {}  : {} validation fail'.format(self.name, proxy_obj.proxy.ljust(20)))
+#             self.queue.task_done()
+#
+#
+# def doRawProxyCheck():
+#     proxy_queue = Queue()
+#
+#     pm = ProxyManager()
+#     pm.db.changeTable(pm.raw_proxy_queue)
+#     for _proxy in pm.db.getAll():
+#         proxy_queue.put(_proxy)
+#     pm.db.clear()
+#
+#     thread_list = list()
+#     for index in range(20):
+#         thread_list.append(RawProxyCheck(proxy_queue, "thread_%s" % index))
+#
+#     for thread in thread_list:
+#         thread.start()
+#
+#     for thread in thread_list:
+#         thread.join()
+#
+#
+# if __name__ == '__main__':
+#     doRawProxyCheck()

schedule/UsefulProxyCheck.py

@@ -0,0 +1,80 @@
+# # -*- coding: utf-8 -*-
+# """
+# -------------------------------------------------
+#    File Name：     UsefulProxyCheck
+#    Description :   check useful proxy
+#    Author :        JHao
+#    date：          2019/8/7
+# -------------------------------------------------
+#    Change Activity:
+#                    2019/8/7: check useful proxy
+# -------------------------------------------------
+# """
+# __author__ = 'JHao'
+#
+# from threading import Thread
+#
+# try:
+#     from Queue import Queue, Empty  # py2
+# except:
+#     from queue import Queue, Empty  # py3
+#
+# from util import LogHandler
+# from handler import ProxyManager
+# from helper import checkProxyUseful, Proxy
+#
+# FAIL_COUNT = 0
+#
+#
+# class UsefulProxyCheck(ProxyManager, Thread):
+#     def __init__(self, queue, thread_name):
+#         ProxyManager.__init__(self)
+#         Thread.__init__(self, name=thread_name)
+#
+#         self.queue = queue
+#         self.log = LogHandler('useful_proxy_check')
+#
+#     def run(self):
+#         self.log.info("UsefulProxyCheck - {}  : start".format(self.name))
+#         self.db.changeTable(self.useful_proxy_queue)
+#         while True:
+#             try:
+#                 proxy_str = self.queue.get(block=False)
+#             except Empty:
+#                 self.log.info("UsefulProxyCheck - {}  : exit".format(self.name))
+#                 break
+#
+#             proxy_obj = Proxy.newProxyFromJson(proxy_str)
+#             proxy_obj, status = checkProxyUseful(proxy_obj)
+#             if status or proxy_obj.fail_count < FAIL_COUNT:
+#                 self.db.put(proxy_obj)
+#                 self.log.info('UsefulProxyCheck - {}  : {} validation pass'.format(self.name,
+#                                                                                    proxy_obj.proxy.ljust(20)))
+#             else:
+#                 self.log.info('UsefulProxyCheck - {}  : {} validation fail'.format(self.name,
+#                                                                                    proxy_obj.proxy.ljust(20)))
+#                 self.db.delete(proxy_obj.proxy)
+#             self.queue.task_done()
+#
+#
+# def doUsefulProxyCheck():
+#     proxy_queue = Queue()
+#
+#     pm = ProxyManager()
+#     pm.db.changeTable(pm.useful_proxy_queue)
+#     for _proxy in pm.db.getAll():
+#         proxy_queue.put(_proxy)
+#
+#     thread_list = list()
+#     for index in range(10):
+#         thread_list.append(UsefulProxyCheck(proxy_queue, "thread_%s" % index))
+#
+#     for thread in thread_list:
+#         thread.start()
+#
+#     for thread in thread_list:
+#         thread.join()
+#
+#
+# if __name__ == '__main__':
+#     doUsefulProxyCheck()

schedule/check.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     check
+   Description :
+   Author :        JHao
+   date：          2019/8/6
+-------------------------------------------------
+   Change Activity:
+                   2019/8/6:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from threading import Thread
+from datetime import datetime
+from util.six import Queue, Empty
+
+from helper.proxy import Proxy
+from util.validators import validators
+from handler.logHandler import LogHandler
+from handler.proxyHandler import ProxyHandler
+
+
+def proxyCheck(proxy_obj):
+    """
+    检测代理是否可用
+    :param proxy_obj: Proxy object
+    :return: Proxy object, status
+    """
+
+    def __proxyCheck(proxy):
+        for func in validators:
+            if not func(proxy):
+                return False
+        return True
+
+    if __proxyCheck(proxy_obj.proxy):
+        # 检测通过 更新proxy属性
+        proxy_obj.check_count += 1
+        proxy_obj.last_status = 1
+        proxy_obj.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        if proxy_obj.fail_count > 0:
+            proxy_obj.fail_count -= 1
+        return proxy_obj
+    else:
+        proxy_obj.check_count += 1
+        proxy_obj.last_status = 0
+        proxy_obj.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        proxy_obj.fail_count += 1
+        return proxy_obj
+
+
+class ProxyCheck(Thread):
+
+    def __init__(self, check_type, queue, thread_name):
+        Thread.__init__(self, name=thread_name)
+        self.type = check_type
+        self.log = LogHandler(self.name)
+        self.proxy_handler = ProxyHandler()
+        self.queue = queue
+
+    def run(self):
+        self.log.info("ProxyCheck - {}  : start".format(self.name))
+        while True:
+            try:
+                proxy_json = self.queue.get(block=False)
+            except Empty:
+                self.log.info("ProxyCheck - {}  : exit".format(self.name))
+                break
+
+            proxy = Proxy.newProxyFromJson(proxy_json)
+            proxy = proxyCheck(proxy)
+            if self.type == "raw":
+                if proxy.last_status:
+                    if self.proxy_handler.exists(proxy_obj.proxy):
+                        self.log.info('RawProxyCheck - {}  : {} validation exists'.format(self.name,
+                                                                                          proxy_obj.proxy.ljust(20)))
+                else:
+                    self.db.put(proxy_obj)
+                    self.log.info(
+                        'RawProxyCheck - {}  : {} validation pass'.format(self.name, proxy_obj.proxy.ljust(20)))
+            else:
+                self.log.info('RawProxyCheck - {}  : {} validation fail'.format(self.name, proxy_obj.proxy.ljust(20)))
+            self.queue.task_done()

schedule/fetch.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     fetchScheduler
+   Description :
+   Author :        JHao
+   date：          2019/8/6
+-------------------------------------------------
+   Change Activity:
+                   2019/08/06:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+from handler.logHandler import LogHandler
+from handler.proxyHandler import ProxyHandler
+from fetcher.proxyFetcher import ProxyFetcher
+from handler.configHandler import ConfigHandler
+
+
+class Fetcher(object):
+    name = "fetcher"
+
+    def __init__(self):
+        self.log = LogHandler(self.name)
+        self.conf = ConfigHandler()
+        self.proxy_handler = ProxyHandler()
+
+    def fetch(self):
+        """
+        fetch proxy into db with proxyFetcher
+        :return:
+        """
+        proxy_set = set()
+        self.log.info("ProxyFetcher : start")
+        for fetch_name in self.conf.fetchers:
+            self.log.info("ProxyFetcher - {func}: start".format(func=fetch_name))
+            fetcher = getattr(ProxyFetcher, fetch_name, None)
+            if not fetcher:
+                self.log.error("ProxyFetcher - {func}: class method not exists!")
+                continue
+            if not callable(fetcher):
+                self.log.error("ProxyFetcher - {func}: must be class method")
+                continue
+
+            try:
+                for proxy in fetcher():
+                    proxy = proxy.strip()
+                    if not proxy or not verifyProxyFormat(proxy):
+                        self.log.error('ProxyFetch - {func}: '
+                                       '{proxy} illegal'.format(func=proxyGetter, proxy=proxy.ljust(20)))
+                        continue
+                    elif proxy in proxy_set:
+                        self.log.info('ProxyFetch - {func}: '
+                                      '{proxy} exist'.format(func=proxyGetter, proxy=proxy.ljust(20)))
+                        continue
+                    else:
+                        self.log.info('ProxyFetch - {func}: '
+                                      '{proxy} success'.format(func=proxyGetter, proxy=proxy.ljust(20)))
+                        self.db.put(Proxy(proxy, source=proxyGetter))
+                        proxy_set.add(proxy)
+            except Exception as e:
+                self.log.error("ProxyFetch - {func}: error".format(func=proxyGetter))
+                self.log.error(str(e))
+
+
+if __name__ == '__main__':
+    a = callable(getattr(ProxyFetcher, 'freeProxy01'))
+    pass

Schedule/ProxyScheduler.py → schedule/proxyScheduler.py

@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
 """
 -------------------------------------------------
-   File Name：     ProxyScheduler
+   File Name：     proxyScheduler
    Description :
    Author :        JHao
    date：          2019/8/5
 -------------------------------------------------
    Change Activity:
-                   2019/8/5: ProxyScheduler
+                   2019/8/5: proxyScheduler
 -------------------------------------------------
 """
 __author__ = 'JHao'
@@ -17,9 +17,9 @@ from apscheduler.schedulers.blocking import BlockingScheduler
 
 sys.path.append('../')
 
-from Schedule import doRawProxyCheck, doUsefulProxyCheck
-from Manager import ProxyManager
-from Util import LogHandler
+from schedule import doRawProxyCheck, doUsefulProxyCheck
+from handler import ProxyManager
+from util import LogHandler
 
 
 class DoFetchProxy(ProxyManager):

setting.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     setting.py
+   Description :   配置文件
+   Author :        JHao
+   date：          2019/2/15
+-------------------------------------------------
+   Change Activity:
+                   2019/2/15:
+-------------------------------------------------
+"""
+
+BANNER = r"""
+****************************************************************
+*** ______  ********************* ______ *********** _  ********
+*** | ___ \_ ******************** | ___ \ ********* | | ********
+*** | |_/ / \__ __   __  _ __   _ | |_/ /___ * ___  | | ********
+*** |  __/|  _// _ \ \ \/ /| | | ||  __// _ \ / _ \ | | ********
+*** | |   | | | (_) | >  < \ |_| || |  | (_) | (_) || |___  ****
+*** \_|   |_|  \___/ /_/\_\ \__  |\_|   \___/ \___/ \_____/ ****
+****                       __ / /                          *****
+************************* /___ / *******************************
+*************************       ********************************
+****************************************************************
+"""
+
+SHOW_CONFIG = True            # print key config
+
+# ############### server config ###############
+HOST = "0.0.0.0"
+PORT = 5000
+
+# ############### db config ###################
+# db connection uri
+# example:
+#      Redis: redis://:password@ip:port/db
+#      Ssdb:  ssdb://:password@ip:port
+DB_CONN = 'redis://:_@Fintell@10.10.61.65:6379'
+
+# raw proxy db
+RAW_PROXY = 'raw_proxy'
+
+# useful proxy db
+USE_PROXY = 'use_proxy'
+
+
+# ###### config the proxy fetch function ######
+PROXY_FETCHER = [
+    "freeProxy01",
+    # "freeProxy02",
+    "freeProxy03",
+    "freeProxy04",
+    "freeProxy05",
+    # "freeProxy06",
+    "freeProxy07",
+    # "freeProxy08",
+    "freeProxy09",
+    "freeProxy13",
+    "freeProxy14",
+    "freeProxy15",
+]
+
+# ############# proxy validator #################
+VERIFY_RUL = "http://www.baidu.com"
+VERIFY_TIMEOUT = 10
+
+

test.py

@@ -12,7 +12,7 @@
 """
 __author__ = 'JHao'
 
-from Test import testConfig
+from test import testConfig
 
 if __name__ == '__main__':
     testConfig.testConfig()

test/testGetFreeProxy.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testGetFreeProxy
+   Description :   test model ProxyGetter/getFreeProxy
+   Author :        J_hao
+   date：          2017/7/31
+-------------------------------------------------
+   Change Activity:
+                   2017/7/31:function testGetFreeProxy
+-------------------------------------------------
+"""
+__author__ = 'J_hao'
+
+
+from ProxyGetter.getFreeProxy import GetFreeProxy
+from Config.ConfigGetter import config
+
+
+def testGetFreeProxy():
+    """
+    test class GetFreeProxy in ProxyGetter/GetFreeProxy
+    :return:
+    """
+    proxy_getter_functions = config.proxy_getter_functions
+    for proxyGetter in proxy_getter_functions:
+        proxy_count = 0
+        for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
+            if proxy:
+                print('{func}: fetch proxy {proxy},proxy_count:{proxy_count}'.format(func=proxyGetter, proxy=proxy,
+                                                                                     proxy_count=proxy_count))
+                proxy_count += 1
+        # assert proxy_count >= 20, '{} fetch proxy fail'.format(proxyGetter)
+
+
+if __name__ == '__main__':
+    testGetFreeProxy()

test/testLogHandler.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testLogHandler
+   Description :
+   Author :        J_hao
+   date：          2017/8/2
+-------------------------------------------------
+   Change Activity:
+                   2017/8/2:
+-------------------------------------------------
+"""
+__author__ = 'J_hao'
+
+from Util.LogHandler import LogHandler
+
+
+# noinspection PyPep8Naming
+def testLogHandler():
+    """
+    test function LogHandler  in Util/LogHandler
+    :return:
+    """
+    log = LogHandler('test')
+    log.info('this is a log from test')
+
+    log.resetName(name='test1')
+    log.info('this is a log from test1')
+
+    log.resetName(name='test2')
+    log.info('this is a log from test2')
+
+
+if __name__ == '__main__':
+    testLogHandler()

test/testProxyClass.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testProxyClass
+   Description :
+   Author :        JHao
+   date：          2019/8/8
+-------------------------------------------------
+   Change Activity:
+                   2019/8/8:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+import json
+from ProxyHelper import Proxy
+
+
+def testProxyClass():
+    proxy = Proxy("127.0.0.1:8080")
+
+    print(proxy.info_dict)
+
+    proxy.source = "test"
+
+    proxy_str = json.dumps(proxy.info_dict, ensure_ascii=False)
+
+    print(proxy_str)
+
+    print(Proxy.newProxyFromJson(proxy_str).info_dict)
+
+
+testProxyClass()

test/testWebRequest.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testWebRequest
+   Description :   test class WebRequest
+   Author :        J_hao
+   date：          2017/7/31
+-------------------------------------------------
+   Change Activity:
+                   2017/7/31: function testWebRequest
+-------------------------------------------------
+"""
+__author__ = 'J_hao'
+
+from Util.WebRequest import WebRequest
+
+
+# noinspection PyPep8Naming
+def testWebRequest():
+    """
+    test class WebRequest in Util/WebRequest.py
+    :return:
+    """
+    wr = WebRequest()
+    request_object = wr.get('https://www.baidu.com/')
+    assert request_object.status_code == 200
+
+
+if __name__ == '__main__':
+    testWebRequest()

util/LogHandler.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     LogHandler.py
+   Description :  日志操作模块
+   Author :       JHao
+   date：          2017/3/6
+-------------------------------------------------
+   Change Activity:
+                   2017/3/6: log handler
+                   2017/9/21: 屏幕输出/文件输出 可选(默认屏幕和文件均输出)
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+import os
+
+import logging
+
+from logging.handlers import TimedRotatingFileHandler
+
+# 日志级别
+CRITICAL = 50
+FATAL = CRITICAL
+ERROR = 40
+WARNING = 30
+WARN = WARNING
+INFO = 20
+DEBUG = 10
+NOTSET = 0
+
+CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
+ROOT_PATH = os.path.join(CURRENT_PATH, os.pardir)
+LOG_PATH = os.path.join(ROOT_PATH, 'log')
+
+try:
+    os.mkdir(LOG_PATH)
+except FileExistsError:
+    pass
+
+
+class LogHandler(logging.Logger):
+    """
+    LogHandler
+    """
+
+    def __init__(self, name, level=DEBUG, stream=True, file=True):
+        self.name = name
+        self.level = level
+        logging.Logger.__init__(self, self.name, level=level)
+        if stream:
+            self.__setStreamHandler__()
+        if file:
+            self.__setFileHandler__()
+
+    def __setFileHandler__(self, level=None):
+        """
+        set file handler
+        :param level:
+        :return:
+        """
+        file_name = os.path.join(LOG_PATH, '{name}.log'.format(name=self.name))
+        # 设置日志回滚, 保存在log目录, 一天保存一个文件, 保留15天
+        file_handler = TimedRotatingFileHandler(filename=file_name, when='D', interval=1, backupCount=15)
+        file_handler.suffix = '%Y%m%d.log'
+        if not level:
+            file_handler.setLevel(self.level)
+        else:
+            file_handler.setLevel(level)
+        formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')
+
+        file_handler.setFormatter(formatter)
+        self.file_handler = file_handler
+        self.addHandler(file_handler)
+
+    def __setStreamHandler__(self, level=None):
+        """
+        set stream handler
+        :param level:
+        :return:
+        """
+        stream_handler = logging.StreamHandler()
+        formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')
+        stream_handler.setFormatter(formatter)
+        if not level:
+            stream_handler.setLevel(self.level)
+        else:
+            stream_handler.setLevel(level)
+        self.addHandler(stream_handler)
+
+    def resetName(self, name):
+        """
+        reset name
+        :param name:
+        :return:
+        """
+        self.name = name
+        self.removeHandler(self.file_handler)
+        self.__setFileHandler__()
+
+
+if __name__ == '__main__':
+    log = LogHandler('test')
+    log.info('this is a test msg')

util/__init__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     __init__.py.py  
+   Description :  
+   Author :       JHao
+   date：          2016/11/25
+-------------------------------------------------
+   Change Activity:
+                   2016/11/25: 
+-------------------------------------------------
+"""
+
+from Util.utilFunction import validUsefulProxy
+from Util.LogHandler import LogHandler
+from Util.utilClass import Singleton

util/utilFunction.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+# !/usr/bin/env python
+"""
+-------------------------------------------------
+   File Name：     utilFunction.py
+   Description :  tool function
+   Author :       JHao
+   date：          2016/11/25
+-------------------------------------------------
+   Change Activity:
+                   2016/11/25: 添加robustCrawl、verifyProxy、getHtmlTree
+-------------------------------------------------
+"""
+from lxml import etree
+import requests
+
+from Util.WebRequest import WebRequest
+from .validators import validators
+from Config.ConfigGetter import config
+
+
+def robustCrawl(func):
+    def decorate(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            pass
+            # logger.info(u"sorry, 抓取出错。错误原因:")
+            # logger.info(e)
+
+    return decorate
+
+
+def verifyProxyFormat(proxy):
+    """
+    检查代理格式
+    :param proxy:
+    :return:
+    """
+    import re
+    verify_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}"
+    _proxy = re.findall(verify_regex, proxy)
+    return True if len(_proxy) == 1 and _proxy[0] == proxy else False
+
+
+def getHtmlTree(url, **kwargs):
+    """
+    获取html树
+    :param url:
+    :param kwargs:
+    :return:
+    """
+
+    header = {'Connection': 'keep-alive',
+              'Cache-Control': 'max-age=0',
+              'Upgrade-Insecure-Requests': '1',
+              'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)',
+              'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+              'Accept-Encoding': 'gzip, deflate, sdch',
+              'Accept-Language': 'zh-CN,zh;q=0.8',
+              }
+    # TODO 取代理服务器用代理服务器访问
+    wr = WebRequest()
+    html = wr.get(url=url, header=header).content
+    return etree.HTML(html)
+
+
+def tcpConnect(proxy):
+    """
+    TCP 三次握手
+    :param proxy:
+    :return:
+    """
+    from socket import socket, AF_INET, SOCK_STREAM
+    s = socket(AF_INET, SOCK_STREAM)
+    ip, port = proxy.split(':')
+    result = s.connect_ex((ip, int(port)))
+    return True if result == 0 else False
+
+
+def validUsefulProxy(proxy):
+    """
+    检验代理是否可用
+    :param proxy:
+    :return:
+    """
+    if isinstance(proxy, bytes):
+        proxy = proxy.decode("utf8")
+    proxies = {"http": "http://{proxy}".format(proxy=proxy)}
+    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
+               'Accept': '*/*',
+               'Connection': 'keep-alive',
+               'Accept-Language': 'zh-CN,zh;q=0.8'}
+    try:
+        r = requests.head(config.verify_host, headers=headers, proxies=proxies, timeout=10, verify=False)
+        if r.status_code == 200:
+            return True
+    except Exception as e:
+        pass
+    return False
+
+    # for v_func in validators:
+    #     if not v_func(proxy):
+    #         return False
+    # return True

util/validators.py

@@ -0,0 +1,38 @@
+import requests
+
+validators = []
+
+
+def validator(func):
+    validators.append(func)
+    return func
+
+
+@validator
+def timeOutValidator(proxy):
+    """
+    检测超时
+    :param proxy:
+    :return:
+    """
+    if isinstance(proxy, bytes):
+        proxy = proxy.decode("utf8")
+    proxies = {"http": "http://{proxy}".format(proxy=proxy)}
+    try:
+        r = requests.get('http://www.baidu.com', proxies=proxies, timeout=10, verify=False)
+        if r.status_code == 200:
+            return True
+    except Exception as e:
+        pass
+    return False
+
+
+@validator
+def customValidator(proxy):
+    """
+    自定义validator函数，校验代理是否可用
+    :param proxy:
+    :return:
+    """
+
+    return True