1

Util/singleton.py

@@ -1,26 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     singleton
-   Description :
-   Author :        JHao
-   date：          2016/12/3
--------------------------------------------------
-   Change Activity:
-                   2016/12/3:
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-
-class Singleton(type):
-    """
-    Singleton Metaclass
-    """
-
-    _inst = {}
-
-    def __call__(cls, *args, **kwargs):
-        if cls not in cls._inst:
-            cls._inst[cls] = super(Singleton, cls).__call__(*args)
-        return cls._inst[cls]

Util/validators.py

@@ -1,58 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import requests
-from re import findall
-from handler.configHandler import ConfigHandler
-
-conf = ConfigHandler()
-validators = []
-
-
-def validator(func):
-    validators.append(func)
-    return func
-
-
-@validator
-def formatValidator(proxy):
-    """
-    检查代理格式
-    :param proxy:
-    :return:
-    """
-    verify_regex = r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}"
-    _proxy = findall(verify_regex, proxy)
-    return True if len(_proxy) == 1 and _proxy[0] == proxy else False
-
-
-@validator
-def timeOutValidator(proxy):
-    """
-    检测超时
-    :param proxy:
-    :return:
-    """
-
-    proxies = {"http": "http://{proxy}".format(proxy=proxy), "https": "https://{proxy}".format(proxy=proxy)}
-    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
-               'Accept': '*/*',
-               'Connection': 'keep-alive',
-               'Accept-Language': 'zh-CN,zh;q=0.8'}
-    try:
-        r = requests.head(conf.verifyUrl, headers=headers, proxies=proxies, timeout=conf.verifyTimeout, verify=False)
-        if r.status_code == 200:
-            return True
-    except Exception as e:
-        pass
-    return False
-
-
-@validator
-def customValidator(proxy):
-    """
-    自定义validator函数，校验代理是否可用
-    :param proxy:
-    :return:
-    """
-
-    return True

Util/webRequest.py

@@ -1,96 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     WebRequest
-   Description :   Network Requests Class
-   Author :        J_hao
-   date：          2017/7/31
--------------------------------------------------
-   Change Activity:
-                   2017/7/31:
--------------------------------------------------
-"""
-__author__ = 'J_hao'
-
-from requests.models import Response
-from lxml import etree
-import requests
-import random
-import time
-
-from handler.logHandler import LogHandler
-
-requests.packages.urllib3.disable_warnings()
-
-
-class WebRequest(object):
-    name = "web_request"
-
-    def __init__(self, *args, **kwargs):
-        self.log = LogHandler(self.name, file=False)
-        self.response = Response()
-
-    @property
-    def user_agent(self):
-        """
-        return an User-Agent at random
-        :return:
-        """
-        ua_list = [
-            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101',
-            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122',
-            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71',
-            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95',
-            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71',
-            'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
-            'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
-            'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
-        ]
-        return random.choice(ua_list)
-
-    @property
-    def header(self):
-        """
-        basic header
-        :return:
-        """
-        return {'User-Agent': self.user_agent,
-                'Accept': '*/*',
-                'Connection': 'keep-alive',
-                'Accept-Language': 'zh-CN,zh;q=0.8'}
-
-    def get(self, url, header=None, retry_time=5, retry_interval=5, timeout=10, *args, **kwargs):
-        """
-        get method
-        :param url: target url
-        :param header: headers
-        :param retry_time: retry time
-        :param retry_interval: retry interval
-        :param timeout: network timeout
-        :return:
-        """
-        headers = self.header
-        if header and isinstance(header, dict):
-            headers.update(header)
-        while True:
-            try:
-                self.response = requests.get(url, headers=headers, timeout=timeout, *args, **kwargs)
-                return self
-            except Exception as e:
-                self.log.error("requests: %s error: %s" % (url, str(e)))
-                retry_time -= 1
-                if retry_time <= 0:
-                    resp = Response()
-                    resp.status_code = 200
-                    return self
-                self.log.info("retry %s second after" % retry_interval)
-                time.sleep(retry_interval)
-
-    @property
-    def tree(self):
-        return etree.HTML(self.response.content)
-
-    @property
-    def text(self):
-        return self.response.text
-