[update] 重新整理项目结构

Util/LogHandler.py

@@ -1,104 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     LogHandler.py
-   Description :  日志操作模块
-   Author :       JHao
-   date：          2017/3/6
--------------------------------------------------
-   Change Activity:
-                   2017/3/6: log handler
-                   2017/9/21: 屏幕输出/文件输出 可选(默认屏幕和文件均输出)
--------------------------------------------------
-"""
-# __author__ = 'JHao'
-
-import os
-
-import logging
-
-from logging.handlers import TimedRotatingFileHandler
-
-# # 日志级别
-# CRITICAL = 50
-# FATAL = CRITICAL
-# ERROR = 40
-# WARNING = 30
-# WARN = WARNING
-# INFO = 20
-# DEBUG = 10
-# NOTSET = 0
-
-CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
-ROOT_PATH = os.path.join(CURRENT_PATH, os.pardir)
-LOG_PATH = os.path.join(ROOT_PATH, 'log')
-
-try:
-    os.mkdir(LOG_PATH)
-except FileExistsError:
-    pass
-
-
-class LogHandler(logging.Logger):
-    """
-    LogHandler
-    """
-
-    # def __init__(self, name, level=DEBUG, stream=True, file=True):
-    #     self.name = name
-    #     self.level = level
-    #     logging.Logger.__init__(self, self.name, level=level)
-        # if stream:
-        #     self.__setStreamHandler__()
-        # if file:
-        #     self.__setFileHandler__()
-
-    # def __setFileHandler__(self, level=None):
-    #     """
-    #     set file handler
-    #     :param level:
-    #     :return:
-    #     """
-    #     file_name = os.path.join(LOG_PATH, '{name}.log'.format(name=self.name))
-    #     # 设置日志回滚, 保存在log目录, 一天保存一个文件, 保留15天
-    #     file_handler = TimedRotatingFileHandler(filename=file_name, when='D', interval=1, backupCount=15)
-    #     file_handler.suffix = '%Y%m%d.log'
-    #     if not level:
-    #         file_handler.setLevel(self.level)
-    #     else:
-    #         file_handler.setLevel(level)
-    #     formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')
-    #
-    #     file_handler.setFormatter(formatter)
-    #     self.file_handler = file_handler
-    #     self.addHandler(file_handler)
-    #
-    # def __setStreamHandler__(self, level=None):
-    #     """
-    #     set stream handler
-    #     :param level:
-    #     :return:
-    #     """
-    #     stream_handler = logging.StreamHandler()
-    #     formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')
-    #     stream_handler.setFormatter(formatter)
-    #     if not level:
-    #         stream_handler.setLevel(self.level)
-    #     else:
-    #         stream_handler.setLevel(level)
-    #     self.addHandler(stream_handler)
-    #
-    # def resetName(self, name):
-    #     """
-    #     reset name
-    #     :param name:
-    #     :return:
-    #     """
-    #     self.name = name
-    #     self.removeHandler(self.file_handler)
-    #     self.__setFileHandler__()
-
-
-if __name__ == '__main__':
-    log = LogHandler('test')
-    log.info('this is a test msg')

Util/utilFunction.py

@@ -1,105 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：     utilFunction.py
-   Description :  tool function
-   Author :       JHao
-   date：          2016/11/25
--------------------------------------------------
-   Change Activity:
-                   2016/11/25: 添加robustCrawl、verifyProxy、getHtmlTree
--------------------------------------------------
-"""
-from lxml import etree
-import requests
-
-from util.WebRequest import WebRequest
-from .validators import validators
-from config.ConfigGetter import config
-
-
-def robustCrawl(func):
-    def decorate(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except Exception as e:
-            pass
-            # logger.info(u"sorry, 抓取出错。错误原因:")
-            # logger.info(e)
-
-    return decorate
-
-
-def verifyProxyFormat(proxy):
-    """
-    检查代理格式
-    :param proxy:
-    :return:
-    """
-    import re
-    verify_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}"
-    _proxy = re.findall(verify_regex, proxy)
-    return True if len(_proxy) == 1 and _proxy[0] == proxy else False
-
-
-def getHtmlTree(url, **kwargs):
-    """
-    获取html树
-    :param url:
-    :param kwargs:
-    :return:
-    """
-
-    header = {'Connection': 'keep-alive',
-              'Cache-Control': 'max-age=0',
-              'Upgrade-Insecure-Requests': '1',
-              'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)',
-              'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-              'Accept-Encoding': 'gzip, deflate, sdch',
-              'Accept-Language': 'zh-CN,zh;q=0.8',
-              }
-    # TODO 取代理服务器用代理服务器访问
-    wr = WebRequest()
-    html = wr.get(url=url, header=header).content
-    return etree.HTML(html)
-
-
-def tcpConnect(proxy):
-    """
-    TCP 三次握手
-    :param proxy:
-    :return:
-    """
-    from socket import socket, AF_INET, SOCK_STREAM
-    s = socket(AF_INET, SOCK_STREAM)
-    ip, port = proxy.split(':')
-    result = s.connect_ex((ip, int(port)))
-    return True if result == 0 else False
-
-
-def validUsefulProxy(proxy):
-    """
-    检验代理是否可用
-    :param proxy:
-    :return:
-    """
-    if isinstance(proxy, bytes):
-        proxy = proxy.decode("utf8")
-    proxies = {"http": "http://{proxy}".format(proxy=proxy)}
-    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
-               'Accept': '*/*',
-               'Connection': 'keep-alive',
-               'Accept-Language': 'zh-CN,zh;q=0.8'}
-    try:
-        r = requests.head(config.verify_host, headers=headers, proxies=proxies, timeout=10, verify=False)
-        if r.status_code == 200:
-            return True
-    except Exception as e:
-        pass
-    return False
-
-    # for v_func in validators:
-    #     if not v_func(proxy):
-    #         return False
-    # return True

__init__.py

@@ -1,13 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     __init__.py  
-   Description :  
-   Author :       JHao
-   date：          2016/12/3
--------------------------------------------------
-   Change Activity:
-                   2016/12/3: 
--------------------------------------------------
-"""
-__author__ = 'JHao'

cli/proxyPool.py

@@ -1,52 +0,0 @@
-# # -*- coding: utf-8 -*-
-# """
-# -------------------------------------------------
-#    File Name：     proxy_pool
-#    Description :
-#    Author :        JHao
-#    date：          2019/8/2
-# -------------------------------------------------
-#    Change Activity:
-#                    2019/8/2:
-# -------------------------------------------------
-# """
-# __author__ = 'JHao'
-#
-# import sys
-# import click
-# import platform
-#
-# sys.path.append('../')
-#
-# from Config.setting import HEADER
-# from Schedule.ProxyScheduler import runScheduler
-# from Api.ProxyApi import runFlask,runFlaskWithGunicorn
-#
-# CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
-#
-#
-# @click.group(context_settings=CONTEXT_SETTINGS)
-# @click.version_option(version='2.0.0')
-# def cli():
-#     """ProxyPool cli工具"""
-#
-#
-# @cli.command(name="schedule")
-# def schedule():
-#     """ 启动调度程序 """
-#     click.echo(HEADER)
-#     runScheduler()
-#
-#
-# @cli.command(name="webserver")
-# def schedule():
-#     """ 启动web服务 """
-#     click.echo(HEADER)
-#     if platform.system() == "Windows":
-#         runFlask()
-#     else:
-#         runFlaskWithGunicorn()
-#
-#
-# if __name__ == '__main__':
-#     cli()

config/ConfigGetter.py

@@ -1,75 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     ConfigGetter
-   Description :   读取配置
-   Author :        JHao
-   date：          2019/2/15
--------------------------------------------------
-   Change Activity:
-                   2019/2/15:
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-
-from util.utilClass import LazyProperty
-from config.setting import *
-
-
-class ConfigGetter(object):
-    """
-    get config
-    """
-
-    def __init__(self):
-        pass
-
-    # @LazyProperty
-    # def db_type(self):
-    #     return DATABASES.get("default", {}).get("TYPE", "SSDB")
-    #
-    # @LazyProperty
-    # def db_name(self):
-    #     return DATABASES.get("default", {}).get("NAME", "proxy")
-    #
-    # @LazyProperty
-    # def db_host(self):
-    #     return DATABASES.get("default", {}).get("HOST", "127.0.0.1")
-    #
-    # @LazyProperty
-    # def db_port(self):
-    #     return DATABASES.get("default", {}).get("PORT", 8888)
-    #
-    # @LazyProperty
-    # def db_password(self):
-    #     return DATABASES.get("default", {}).get("PASSWORD", "")
-    #
-    # @LazyProperty
-    # def proxy_getter_functions(self):
-    #     return PROXY_GETTER
-    #
-    # @LazyProperty
-    # def host_ip(self):
-    #     return SERVER_API.get("HOST", "127.0.0.1")
-    #
-    # @LazyProperty
-    # def host_port(self):
-    #     return SERVER_API.get("PORT", 5010)
-    #
-    # @LazyProperty
-    # def verify_host(self):
-    #     return VERIFY_HOST
-
-
-config = ConfigGetter()
-
-if __name__ == '__main__':
-    print(config.db_type)
-    print(config.db_name)
-    print(config.db_host)
-    print(config.db_port)
-    print(config.proxy_getter_functions)
-    print(config.host_ip)
-    print(config.host_port)
-    print(config.db_password)

fetcher/getFreeProxy.py

@@ -1,355 +0,0 @@
-# # -*- coding: utf-8 -*-
-# # !/usr/bin/env python
-# """
-# -------------------------------------------------
-#    File Name：     GetFreeProxy.py
-#    Description :  抓取免费代理
-#    Author :       JHao
-#    date：          2016/11/25
-# -------------------------------------------------
-#    Change Activity:
-#                    2016/11/25:
-# -------------------------------------------------
-# """
-# import re
-# import sys
-# import requests
-# from time import sleep
-#
-# sys.path.append('..')
-#
-# from util.WebRequest import WebRequest
-# from util.utilFunction import getHtmlTree
-#
-# # for debug to disable insecureWarning
-# requests.packages.urllib3.disable_warnings()
-#
-#
-# class GetFreeProxy(object):
-#     """
-#     proxy getter
-#     """
-#
-#     @staticmethod
-#     def freeProxy01():
-#         """
-#         无忧代理 http://www.data5u.com/
-#         几乎没有能用的
-#         :return:
-#         """
-#         url_list = [
-#             'http://www.data5u.com/',
-#             'http://www.data5u.com/free/gngn/index.shtml',
-#             'http://www.data5u.com/free/gnpt/index.shtml'
-#         ]
-#         key = 'ABCDEFGHIZ'
-#         for url in url_list:
-#             html_tree = getHtmlTree(url)
-#             ul_list = html_tree.xpath('//ul[@class="l2"]')
-#             for ul in ul_list:
-#                 try:
-#                     ip = ul.xpath('./span[1]/li/text()')[0]
-#                     classnames = ul.xpath('./span[2]/li/attribute::class')[0]
-#                     classname = classnames.split(' ')[1]
-#                     port_sum = 0
-#                     for c in classname:
-#                         port_sum *= 10
-#                         port_sum += key.index(c)
-#                     port = port_sum >> 3
-#                     yield '{}:{}'.format(ip, port)
-#                 except Exception as e:
-#                     print(e)
-#
-#     @staticmethod
-#     def freeProxy02(count=20):
-#         """
-#         代理66 http://www.66ip.cn/
-#         :param count: 提取数量
-#         :return:
-#         """
-#         urls = [
-#             "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=",
-#             "http://www.66ip.cn/nmtq.php?getnum={}&isp=0&anonymoustype=0&s"
-#             "tart=&ports=&export=&ipaddress=&area=0&proxytype=2&api=66ip"
-#         ]
-#
-#         try:
-#             import execjs
-#             import requests
-#
-#             headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
-#                        'Accept': '*/*',
-#                        'Connection': 'keep-alive',
-#                        'Accept-Language': 'zh-CN,zh;q=0.8'}
-#             session = requests.session()
-#             src = session.get("http://www.66ip.cn/", headers=headers).text
-#             src = src.split("</script>")[0] + '}'
-#             src = src.replace("<script>", "function test() {")
-#             src = src.replace("while(z++)try{eval(", ';var num=10;while(z++)try{var tmp=')
-#             src = src.replace(");break}", ";num--;if(tmp.search('cookie') != -1 | num<0){return tmp}}")
-#             ctx = execjs.compile(src)
-#             src = ctx.call("test")
-#             src = src[src.find("document.cookie="): src.find("};if((")]
-#             src = src.replace("document.cookie=", "")
-#             src = "function test() {var window={}; return %s }" % src
-#             cookie = execjs.compile(src).call('test')
-#             js_cookie = cookie.split(";")[0].split("=")[-1]
-#         except Exception as e:
-#             print(e)
-#             return
-#
-#         for url in urls:
-#             try:
-#                 html = session.get(url.format(count), cookies={"__jsl_clearance": js_cookie}, headers=headers).text
-#                 ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", html)
-#                 for ip in ips:
-#                     yield ip.strip()
-#             except Exception as e:
-#                 print(e)
-#                 pass
-#
-#     @staticmethod
-#     def freeProxy03(page_count=1):
-#         """
-#         西刺代理 http://www.xicidaili.com
-#         :return:
-#         """
-#         url_list = [
-#             'http://www.xicidaili.com/nn/',  # 高匿
-#             'http://www.xicidaili.com/nt/',  # 透明
-#         ]
-#         for each_url in url_list:
-#             for i in range(1, page_count + 1):
-#                 page_url = each_url + str(i)
-#                 tree = getHtmlTree(page_url)
-#                 proxy_list = tree.xpath('.//table[@id="ip_list"]//tr[position()>1]')
-#                 for proxy in proxy_list:
-#                     try:
-#                         yield ':'.join(proxy.xpath('./td/text()')[0:2])
-#                     except Exception as e:
-#                         pass
-#
-#     @staticmethod
-#     def freeProxy04():
-#         """
-#         guobanjia http://www.goubanjia.com/
-#         :return:
-#         """
-#         url = "http://www.goubanjia.com/"
-#         tree = getHtmlTree(url)
-#         proxy_list = tree.xpath('//td[@class="ip"]')
-#         # 此网站有隐藏的数字干扰，或抓取到多余的数字或.符号
-#         # 需要过滤掉<p style="display:none;">的内容
-#         xpath_str = """.//*[not(contains(@style, 'display: none'))
-#                                         and not(contains(@style, 'display:none'))
-#                                         and not(contains(@class, 'port'))
-#                                         ]/text()
-#                                 """
-#         for each_proxy in proxy_list:
-#             try:
-#                 # :符号裸放在td下，其他放在div span p中，先分割找出ip，再找port
-#                 ip_addr = ''.join(each_proxy.xpath(xpath_str))
-#
-#                 # HTML中的port是随机数，真正的端口编码在class后面的字母中。
-#                 # 比如这个：
-#                 # <span class="port CFACE">9054</span>
-#                 # CFACE解码后对应的是3128。
-#                 port = 0
-#                 for _ in each_proxy.xpath(".//span[contains(@class, 'port')]"
-#                                           "/attribute::class")[0]. \
-#                         replace("port ", ""):
-#                     port *= 10
-#                     port += (ord(_) - ord('A'))
-#                 port /= 8
-#
-#                 yield '{}:{}'.format(ip_addr, int(port))
-#             except Exception as e:
-#                 pass
-#
-#     @staticmethod
-#     def freeProxy05():
-#         """
-#         快代理 https://www.kuaidaili.com
-#         """
-#         url_list = [
-#             'https://www.kuaidaili.com/free/inha/',
-#             'https://www.kuaidaili.com/free/intr/'
-#         ]
-#         for url in url_list:
-#             tree = getHtmlTree(url)
-#             proxy_list = tree.xpath('.//table//tr')
-#             sleep(1)  # 必须sleep 不然第二条请求不到数据
-#             for tr in proxy_list[1:]:
-#                 yield ':'.join(tr.xpath('./td/text()')[0:2])
-#
-#     @staticmethod
-#     def freeProxy06():
-#         """
-#         码农代理 https://proxy.coderbusy.com/
-#         :return:
-#         """
-#         urls = ['https://proxy.coderbusy.com/']
-#         for url in urls:
-#             tree = getHtmlTree(url)
-#             proxy_list = tree.xpath('.//table//tr')
-#             for tr in proxy_list[1:]:
-#                 yield ':'.join(tr.xpath('./td/text()')[0:2])
-#
-#     @staticmethod
-#     def freeProxy07():
-#         """
-#         云代理 http://www.ip3366.net/free/
-#         :return:
-#         """
-#         urls = ['http://www.ip3366.net/free/?stype=1',
-#                 "http://www.ip3366.net/free/?stype=2"]
-#         request = WebRequest()
-#         for url in urls:
-#             r = request.get(url, timeout=10)
-#             proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
-#             for proxy in proxies:
-#                 yield ":".join(proxy)
-#
-#     @staticmethod
-#     def freeProxy08():
-#         """
-#         IP海 http://www.iphai.com/free/ng
-#         :return:
-#         """
-#         urls = [
-#             'http://www.iphai.com/free/ng',
-#             'http://www.iphai.com/free/np',
-#             'http://www.iphai.com/free/wg',
-#             'http://www.iphai.com/free/wp'
-#         ]
-#         request = WebRequest()
-#         for url in urls:
-#             r = request.get(url, timeout=10)
-#             proxies = re.findall(r'<td>\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?</td>[\s\S]*?<td>\s*?(\d+)\s*?</td>',
-#                                  r.text)
-#             for proxy in proxies:
-#                 yield ":".join(proxy)
-#
-#     @staticmethod
-#     def freeProxy09(page_count=1):
-#         """
-#         http://ip.jiangxianli.com/?page=
-#         免费代理库
-#         :return:
-#         """
-#         for i in range(1, page_count + 1):
-#             url = 'http://ip.jiangxianli.com/?country=中国&?page={}'.format(i)
-#             html_tree = getHtmlTree(url)
-#             for index, tr in enumerate(html_tree.xpath("//table//tr")):
-#                 if index == 0:
-#                     continue
-#                 yield ":".join(tr.xpath("./td/text()")[0:2]).strip()
-#
-#     # @staticmethod
-#     # def freeProxy10():
-#     #     """
-#     #     墙外网站 cn-proxy
-#     #     :return:
-#     #     """
-#     #     urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218']
-#     #     request = WebRequest()
-#     #     for url in urls:
-#     #         r = request.get(url, timeout=10)
-#     #         proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)
-#     #         for proxy in proxies:
-#     #             yield ':'.join(proxy)
-#
-#     # @staticmethod
-#     # def freeProxy11():
-#     #     """
-#     #     https://proxy-list.org/english/index.php
-#     #     :return:
-#     #     """
-#     #     urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)]
-#     #     request = WebRequest()
-#     #     import base64
-#     #     for url in urls:
-#     #         r = request.get(url, timeout=10)
-#     #         proxies = re.findall(r"Proxy\('(.*?)'\)", r.text)
-#     #         for proxy in proxies:
-#     #             yield base64.b64decode(proxy).decode()
-#
-#     # @staticmethod
-#     # def freeProxy12():
-#     #     urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1']
-#     #     request = WebRequest()
-#     #     for url in urls:
-#     #         r = request.get(url, timeout=10)
-#     #         proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
-#     #         for proxy in proxies:
-#     #             yield ':'.join(proxy)
-#
-#     @staticmethod
-#     def freeProxy13(max_page=2):
-#         """
-#         http://www.qydaili.com/free/?action=china&page=1
-#         齐云代理
-#         :param max_page:
-#         :return:
-#         """
-#         base_url = 'http://www.qydaili.com/free/?action=china&page='
-#         request = WebRequest()
-#         for page in range(1, max_page + 1):
-#             url = base_url + str(page)
-#             r = request.get(url, timeout=10)
-#             proxies = re.findall(
-#                 r'<td.*?>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td.*?>(\d+)</td>',
-#                 r.text)
-#             for proxy in proxies:
-#                 yield ':'.join(proxy)
-#
-#     @staticmethod
-#     def freeProxy14(max_page=2):
-#         """
-#         http://www.89ip.cn/index.html
-#         89免费代理
-#         :param max_page:
-#         :return:
-#         """
-#         base_url = 'http://www.89ip.cn/index_{}.html'
-#         request = WebRequest()
-#         for page in range(1, max_page + 1):
-#             url = base_url.format(page)
-#             r = request.get(url, timeout=10)
-#             proxies = re.findall(
-#                 r'<td.*?>[\s\S]*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?</td>[\s\S]*?<td.*?>[\s\S]*?(\d+)[\s\S]*?</td>',
-#                 r.text)
-#             for proxy in proxies:
-#                 yield ':'.join(proxy)
-#
-#     @staticmethod
-#     def freeProxy15():
-#         urls = ['http://www.xiladaili.com/putong/',
-#                 "http://www.xiladaili.com/gaoni/",
-#                 "http://www.xiladaili.com/http/",
-#                 "http://www.xiladaili.com/https/"]
-#         request = WebRequest()
-#         for url in urls:
-#             r = request.get(url, timeout=10)
-#             ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", r.text)
-#             for ip in ips:
-#                 yield ip.strip()
-#
-#
-# if __name__ == '__main__':
-#     from CheckProxy import CheckProxy
-#
-#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01)
-#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy02)
-#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy03)
-#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy04)
-#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy05)
-#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy06)
-#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy07)
-#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy08)
-#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy09)
-#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy13)
-#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy14)
-#     # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy15)
-#
-#     CheckProxy.checkAllGetProxyFunc()

handler/ProxyManager.py

@@ -1,109 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：     ProxyManager.py
-   Description :
-   Author :       JHao
-   date：          2016/12/3
--------------------------------------------------
-   Change Activity:
-                   2016/12/3:
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-import random
-
-from helper import Proxy
-from db.DbClient import DbClient
-from config.ConfigGetter import config
-from util.LogHandler import LogHandler
-from util.utilFunction import verifyProxyFormat
-from fetcher.getFreeProxy import GetFreeProxy
-
-
-class ProxyManager(object):
-    pass
-#     """
-#     ProxyManager
-#     """
-#
-#     def __init__(self):
-#         self.db = DbClient()
-#         self.raw_proxy_queue = 'raw_proxy'
-#         self.log = LogHandler('proxy_manager')
-#         self.useful_proxy_queue = 'useful_proxy'
-#
-#     def fetch(self):
-#         """
-#         fetch proxy into db by ProxyGetter
-#         :return:
-#         """
-#         self.db.changeTable(self.raw_proxy_queue)
-#         proxy_set = set()
-#         self.log.info("ProxyFetch : start")
-#         for proxyGetter in config.proxy_getter_functions:
-#             self.log.info("ProxyFetch - {func}: start".format(func=proxyGetter))
-#             try:
-#                 for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
-#                     proxy = proxy.strip()
-#
-#                     if not proxy or not verifyProxyFormat(proxy):
-#                         self.log.error('ProxyFetch - {func}: '
-#                                        '{proxy} illegal'.format(func=proxyGetter, proxy=proxy.ljust(20)))
-#                         continue
-#                     elif proxy in proxy_set:
-#                         self.log.info('ProxyFetch - {func}: '
-#                                       '{proxy} exist'.format(func=proxyGetter, proxy=proxy.ljust(20)))
-#                         continue
-#                     else:
-#                         self.log.info('ProxyFetch - {func}: '
-#                                       '{proxy} success'.format(func=proxyGetter, proxy=proxy.ljust(20)))
-#                         self.db.put(Proxy(proxy, source=proxyGetter))
-#                         proxy_set.add(proxy)
-#             except Exception as e:
-#                 self.log.error("ProxyFetch - {func}: error".format(func=proxyGetter))
-#                 self.log.error(str(e))
-#
-#     def get(self):
-#         """
-#         return a useful proxy
-#         :return:
-#         """
-#         self.db.changeTable(self.useful_proxy_queue)
-#         item_list = self.db.getAll()
-#         if item_list:
-#             random_choice = random.choice(item_list)
-#             return Proxy.newProxyFromJson(random_choice)
-#         return None
-#
-#     def delete(self, proxy_str):
-#         """
-#         delete proxy from pool
-#         :param proxy_str:
-#         :return:
-#         """
-#         self.db.changeTable(self.useful_proxy_queue)
-#         self.db.delete(proxy_str)
-#
-#     def getAll(self):
-#         """
-#         get all proxy from pool as list
-#         :return:
-#         """
-#         self.db.changeTable(self.useful_proxy_queue)
-#         item_list = self.db.getAll()
-#         return [Proxy.newProxyFromJson(_) for _ in item_list]
-#
-#     def getNumber(self):
-#         self.db.changeTable(self.raw_proxy_queue)
-#         total_raw_proxy = self.db.getNumber()
-#         self.db.changeTable(self.useful_proxy_queue)
-#         total_useful_queue = self.db.getNumber()
-#         return {'raw_proxy': total_raw_proxy, 'useful_proxy': total_useful_queue}
-
-
-# if __name__ == '__main__':
-#     pp = ProxyManager()
-#     pp.fetch()

proxyPool.py

@@ -15,7 +15,6 @@ __author__ = 'JHao'
 import click
 
 from config.setting import BANNER
-
 from helper.scheduler import runScheduler
 from api.proxyApi import runFlask
 
@@ -38,10 +37,8 @@ def schedule():
 @cli.command(name="server")
 def schedule():
     """ 启动api服务 """
-    # click.echo(BANNER)
-    # runFlask()
-    from test import testProxyFetcher
-    testProxyFetcher.test()
+    click.echo(BANNER)
+    runFlask()
 
 
 if __name__ == '__main__':

schedule/RawProxyCheck.py

@@ -1,81 +0,0 @@
-# # -*- coding: utf-8 -*-
-# """
-# -------------------------------------------------
-#    File Name：     RawProxyCheck
-#    Description :   check raw_proxy to useful
-#    Author :        JHao
-#    date：          2019/8/6
-# -------------------------------------------------
-#    Change Activity:
-#                    2019/8/6: check raw_proxy to useful
-# -------------------------------------------------
-# """
-# __author__ = 'JHao'
-#
-# from threading import Thread
-#
-# try:
-#     from Queue import Empty, Queue  # py2
-# except:
-#     from queue import Empty, Queue  # py3
-#
-# from util import LogHandler
-# from handler import ProxyManager
-# from helper import Proxy, checkProxyUseful
-#
-#
-# class RawProxyCheck(ProxyManager, Thread):
-#     def __init__(self, queue, thread_name):
-#         ProxyManager.__init__(self)
-#         Thread.__init__(self, name=thread_name)
-#         self.log = LogHandler('raw_proxy_check')
-#         self.queue = queue
-#
-#     def run(self):
-#         self.log.info("RawProxyCheck - {}  : start".format(self.name))
-#         self.db.changeTable(self.useful_proxy_queue)
-#         while True:
-#             try:
-#                 proxy_json = self.queue.get(block=False)
-#             except Empty:
-#                 self.log.info("RawProxyCheck - {}  : exit".format(self.name))
-#                 break
-#
-#             proxy_obj = Proxy.newProxyFromJson(proxy_json)
-#
-#             proxy_obj, status = checkProxyUseful(proxy_obj)
-#             if status:
-#                 if self.db.exists(proxy_obj.proxy):
-#                     self.log.info('RawProxyCheck - {}  : {} validation exists'.format(self.name,
-#                                                                                       proxy_obj.proxy.ljust(20)))
-#                 else:
-#                     self.db.put(proxy_obj)
-#                     self.log.info(
-#                         'RawProxyCheck - {}  : {} validation pass'.format(self.name, proxy_obj.proxy.ljust(20)))
-#             else:
-#                 self.log.info('RawProxyCheck - {}  : {} validation fail'.format(self.name, proxy_obj.proxy.ljust(20)))
-#             self.queue.task_done()
-#
-#
-# def doRawProxyCheck():
-#     proxy_queue = Queue()
-#
-#     pm = ProxyManager()
-#     pm.db.changeTable(pm.raw_proxy_queue)
-#     for _proxy in pm.db.getAll():
-#         proxy_queue.put(_proxy)
-#     pm.db.clear()
-#
-#     thread_list = list()
-#     for index in range(20):
-#         thread_list.append(RawProxyCheck(proxy_queue, "thread_%s" % index))
-#
-#     for thread in thread_list:
-#         thread.start()
-#
-#     for thread in thread_list:
-#         thread.join()
-#
-#
-# if __name__ == '__main__':
-#     doRawProxyCheck()

schedule/UsefulProxyCheck.py

@@ -1,80 +0,0 @@
-# # -*- coding: utf-8 -*-
-# """
-# -------------------------------------------------
-#    File Name：     UsefulProxyCheck
-#    Description :   check useful proxy
-#    Author :        JHao
-#    date：          2019/8/7
-# -------------------------------------------------
-#    Change Activity:
-#                    2019/8/7: check useful proxy
-# -------------------------------------------------
-# """
-# __author__ = 'JHao'
-#
-# from threading import Thread
-#
-# try:
-#     from Queue import Queue, Empty  # py2
-# except:
-#     from queue import Queue, Empty  # py3
-#
-# from util import LogHandler
-# from handler import ProxyManager
-# from helper import checkProxyUseful, Proxy
-#
-# FAIL_COUNT = 0
-#
-#
-# class UsefulProxyCheck(ProxyManager, Thread):
-#     def __init__(self, queue, thread_name):
-#         ProxyManager.__init__(self)
-#         Thread.__init__(self, name=thread_name)
-#
-#         self.queue = queue
-#         self.log = LogHandler('useful_proxy_check')
-#
-#     def run(self):
-#         self.log.info("UsefulProxyCheck - {}  : start".format(self.name))
-#         self.db.changeTable(self.useful_proxy_queue)
-#         while True:
-#             try:
-#                 proxy_str = self.queue.get(block=False)
-#             except Empty:
-#                 self.log.info("UsefulProxyCheck - {}  : exit".format(self.name))
-#                 break
-#
-#             proxy_obj = Proxy.newProxyFromJson(proxy_str)
-#             proxy_obj, status = checkProxyUseful(proxy_obj)
-#             if status or proxy_obj.fail_count < FAIL_COUNT:
-#                 self.db.put(proxy_obj)
-#                 self.log.info('UsefulProxyCheck - {}  : {} validation pass'.format(self.name,
-#                                                                                    proxy_obj.proxy.ljust(20)))
-#             else:
-#                 self.log.info('UsefulProxyCheck - {}  : {} validation fail'.format(self.name,
-#                                                                                    proxy_obj.proxy.ljust(20)))
-#                 self.db.delete(proxy_obj.proxy)
-#             self.queue.task_done()
-#
-#
-# def doUsefulProxyCheck():
-#     proxy_queue = Queue()
-#
-#     pm = ProxyManager()
-#     pm.db.changeTable(pm.useful_proxy_queue)
-#     for _proxy in pm.db.getAll():
-#         proxy_queue.put(_proxy)
-#
-#     thread_list = list()
-#     for index in range(10):
-#         thread_list.append(UsefulProxyCheck(proxy_queue, "thread_%s" % index))
-#
-#     for thread in thread_list:
-#         thread.start()
-#
-#     for thread in thread_list:
-#         thread.join()
-#
-#
-# if __name__ == '__main__':
-#     doUsefulProxyCheck()

schedule/__init__.py

@@ -1,13 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     __init__.py.py  
-   Description :  
-   Author :       JHao
-   date：          2016/12/3
--------------------------------------------------
-   Change Activity:
-                   2016/12/3: 
--------------------------------------------------
-"""
-__author__ = 'JHao'

cli/start.sh → start.sh

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
-python proxyPool.py webserver &
+python proxyPool.py server &
 python proxyPool.py schedule

test.py

@@ -12,7 +12,7 @@
 """
 __author__ = 'JHao'
 
-from test import testConfig
+from test import testConfigHandler
 
 if __name__ == '__main__':
-    testConfig.testConfig()
+    testConfigHandler.testConfig()

util/LogHandler.py

@@ -1,104 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     LogHandler.py
-   Description :  日志操作模块
-   Author :       JHao
-   date：          2017/3/6
--------------------------------------------------
-   Change Activity:
-                   2017/3/6: log handler
-                   2017/9/21: 屏幕输出/文件输出 可选(默认屏幕和文件均输出)
--------------------------------------------------
-"""
-# __author__ = 'JHao'
-
-import os
-
-import logging
-
-from logging.handlers import TimedRotatingFileHandler
-
-# # 日志级别
-# CRITICAL = 50
-# FATAL = CRITICAL
-# ERROR = 40
-# WARNING = 30
-# WARN = WARNING
-# INFO = 20
-# DEBUG = 10
-# NOTSET = 0
-
-CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
-ROOT_PATH = os.path.join(CURRENT_PATH, os.pardir)
-LOG_PATH = os.path.join(ROOT_PATH, 'log')
-
-try:
-    os.mkdir(LOG_PATH)
-except FileExistsError:
-    pass
-
-
-class LogHandler(logging.Logger):
-    """
-    LogHandler
-    """
-
-    # def __init__(self, name, level=DEBUG, stream=True, file=True):
-    #     self.name = name
-    #     self.level = level
-    #     logging.Logger.__init__(self, self.name, level=level)
-        # if stream:
-        #     self.__setStreamHandler__()
-        # if file:
-        #     self.__setFileHandler__()
-
-    # def __setFileHandler__(self, level=None):
-    #     """
-    #     set file handler
-    #     :param level:
-    #     :return:
-    #     """
-    #     file_name = os.path.join(LOG_PATH, '{name}.log'.format(name=self.name))
-    #     # 设置日志回滚, 保存在log目录, 一天保存一个文件, 保留15天
-    #     file_handler = TimedRotatingFileHandler(filename=file_name, when='D', interval=1, backupCount=15)
-    #     file_handler.suffix = '%Y%m%d.log'
-    #     if not level:
-    #         file_handler.setLevel(self.level)
-    #     else:
-    #         file_handler.setLevel(level)
-    #     formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')
-    #
-    #     file_handler.setFormatter(formatter)
-    #     self.file_handler = file_handler
-    #     self.addHandler(file_handler)
-    #
-    # def __setStreamHandler__(self, level=None):
-    #     """
-    #     set stream handler
-    #     :param level:
-    #     :return:
-    #     """
-    #     stream_handler = logging.StreamHandler()
-    #     formatter = logging.Formatter('%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s')
-    #     stream_handler.setFormatter(formatter)
-    #     if not level:
-    #         stream_handler.setLevel(self.level)
-    #     else:
-    #         stream_handler.setLevel(level)
-    #     self.addHandler(stream_handler)
-    #
-    # def resetName(self, name):
-    #     """
-    #     reset name
-    #     :param name:
-    #     :return:
-    #     """
-    #     self.name = name
-    #     self.removeHandler(self.file_handler)
-    #     self.__setFileHandler__()
-
-
-if __name__ == '__main__':
-    log = LogHandler('test')
-    log.info('this is a test msg')

util/utilClass.py

@@ -1,45 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：     utilClass.py  
-   Description :  tool class
-   Author :       JHao
-   date：          2016/12/3
--------------------------------------------------
-   Change Activity:
-                   2016/12/3: Class LazyProperty
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-
-class LazyProperty(object):
-    """
-    LazyProperty
-    explain: http://www.spiderpy.cn/blog/5/
-    """
-
-    def __init__(self, func):
-        self.func = func
-
-    def __get__(self, instance, owner):
-        if instance is None:
-            return self
-        else:
-            value = self.func(instance)
-            setattr(instance, self.func.__name__, value)
-            return value
-
-
-class Singleton(type):
-    """
-    Singleton Metaclass
-    """
-
-    _inst = {}
-
-    def __call__(cls, *args, **kwargs):
-        if cls not in cls._inst:
-            cls._inst[cls] = super(Singleton, cls).__call__(*args)
-        return cls._inst[cls]

util/utilFunction.py

@@ -1,105 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：     utilFunction.py
-   Description :  tool function
-   Author :       JHao
-   date：          2016/11/25
--------------------------------------------------
-   Change Activity:
-                   2016/11/25: 添加robustCrawl、verifyProxy、getHtmlTree
--------------------------------------------------
-"""
-from lxml import etree
-import requests
-
-from util.WebRequest import WebRequest
-from .validators import validators
-from config.ConfigGetter import config
-
-
-def robustCrawl(func):
-    def decorate(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except Exception as e:
-            pass
-            # logger.info(u"sorry, 抓取出错。错误原因:")
-            # logger.info(e)
-
-    return decorate
-
-
-def verifyProxyFormat(proxy):
-    """
-    检查代理格式
-    :param proxy:
-    :return:
-    """
-    import re
-    verify_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}"
-    _proxy = re.findall(verify_regex, proxy)
-    return True if len(_proxy) == 1 and _proxy[0] == proxy else False
-
-
-def getHtmlTree(url, **kwargs):
-    """
-    获取html树
-    :param url:
-    :param kwargs:
-    :return:
-    """
-
-    header = {'Connection': 'keep-alive',
-              'Cache-Control': 'max-age=0',
-              'Upgrade-Insecure-Requests': '1',
-              'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)',
-              'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-              'Accept-Encoding': 'gzip, deflate, sdch',
-              'Accept-Language': 'zh-CN,zh;q=0.8',
-              }
-    # TODO 取代理服务器用代理服务器访问
-    wr = WebRequest()
-    html = wr.get(url=url, header=header).content
-    return etree.HTML(html)
-
-
-def tcpConnect(proxy):
-    """
-    TCP 三次握手
-    :param proxy:
-    :return:
-    """
-    from socket import socket, AF_INET, SOCK_STREAM
-    s = socket(AF_INET, SOCK_STREAM)
-    ip, port = proxy.split(':')
-    result = s.connect_ex((ip, int(port)))
-    return True if result == 0 else False
-
-
-def validUsefulProxy(proxy):
-    """
-    检验代理是否可用
-    :param proxy:
-    :return:
-    """
-    if isinstance(proxy, bytes):
-        proxy = proxy.decode("utf8")
-    proxies = {"http": "http://{proxy}".format(proxy=proxy)}
-    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
-               'Accept': '*/*',
-               'Connection': 'keep-alive',
-               'Accept-Language': 'zh-CN,zh;q=0.8'}
-    try:
-        r = requests.head(config.verify_host, headers=headers, proxies=proxies, timeout=10, verify=False)
-        if r.status_code == 200:
-            return True
-    except Exception as e:
-        pass
-    return False
-
-    # for v_func in validators:
-    #     if not v_func(proxy):
-    #         return False
-    # return True