restruct :lollipop:

DB/SsdbClient.py

@@ -1,130 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：     ssdbClient.py
-   Description :   封装SSDB操作
-   Author :        JHao
-   date：          2016/12/2
--------------------------------------------------
-   Change Activity:
-                   2016/12/2:
-                   2017/09/22: PY3中 redis-py返回的数据是bytes型
-                   2017/09/27: 修改pop()方法 返回{proxy:value}字典
-                   2020/07/03: 2.1.0 优化代码结构
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-from redis.connection import BlockingConnectionPool
-from random import choice
-from redis import Redis
-
-
-class SsdbClient(object):
-    """
-    SSDB client
-
-    SSDB中代理存放的结构为hash：
-    key为代理的ip:por, value为代理属性的字典;
-    """
-
-    def __init__(self, **kwargs):
-        """
-        init
-        :param host: host
-        :param port: port
-        :param password: password
-        :return:
-        """
-        self.name = ""
-        kwargs.pop("username")
-        self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, **kwargs))
-
-    def get(self):
-        """
-        从hash中随机返回一个代理
-        :return:
-        """
-        proxies = self.__conn.hkeys(self.name)
-        proxy = choice(proxies) if proxies else None
-        if proxy:
-            return self.__conn.hget(self.name, proxy)
-        else:
-            return None
-
-    def put(self, proxy_obj):
-        """
-        将代理放入hash
-        :param proxy_obj: Proxy obj
-        :return:
-        """
-        result = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
-        return result
-
-    def pop(self):
-        """
-        顺序弹出一个代理
-        :return: proxy
-        """
-        proxies = self.__conn.hkeys(self.name)
-        for proxy in proxies:
-            proxy_info = self.__conn.hget(self.name, proxy)
-            self.__conn.hdel(self.name, proxy)
-            return proxy_info
-        else:
-            return None
-
-    def delete(self, proxy_str):
-        """
-        移除指定代理, 使用changeTable指定hash name
-        :param proxy_str: proxy str
-        :return:
-        """
-        self.__conn.hdel(self.name, proxy_str)
-
-    def exists(self, proxy_str):
-        """
-        判断指定代理是否存在, 使用changeTable指定hash name
-        :param proxy_str: proxy str
-        :return:
-        """
-        return self.__conn.hexists(self.name, proxy_str)
-
-    def update(self, proxy_obj):
-        """
-        更新 proxy 属性
-        :param proxy_obj:
-        :return:
-        """
-        self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
-
-    def getAll(self):
-        """
-        字典形式返回所有代理, 使用changeTable指定hash name
-        :return:
-        """
-        item_dict = self.__conn.hgetall(self.name)
-        return item_dict
-
-    def clear(self):
-        """
-        清空所有代理, 使用changeTable指定hash name
-        :return:
-        """
-        return self.__conn.delete(self.name)
-
-    def getCount(self):
-        """
-        返回代理数量
-        :return:
-        """
-        return self.__conn.hlen(self.name)
-
-    def changeTable(self, name):
-        """
-        切换操作对象
-        :param name:
-        :return:
-        """
-        self.name = name

DB/ssdbClient.py

@@ -1,130 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：     ssdbClient.py
-   Description :   封装SSDB操作
-   Author :        JHao
-   date：          2016/12/2
--------------------------------------------------
-   Change Activity:
-                   2016/12/2:
-                   2017/09/22: PY3中 redis-py返回的数据是bytes型
-                   2017/09/27: 修改pop()方法 返回{proxy:value}字典
-                   2020/07/03: 2.1.0 优化代码结构
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-from redis.connection import BlockingConnectionPool
-from random import choice
-from redis import Redis
-
-
-class SsdbClient(object):
-    """
-    SSDB client
-
-    SSDB中代理存放的结构为hash：
-    key为代理的ip:por, value为代理属性的字典;
-    """
-
-    def __init__(self, **kwargs):
-        """
-        init
-        :param host: host
-        :param port: port
-        :param password: password
-        :return:
-        """
-        self.name = ""
-        kwargs.pop("username")
-        self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, **kwargs))
-
-    def get(self):
-        """
-        从hash中随机返回一个代理
-        :return:
-        """
-        proxies = self.__conn.hkeys(self.name)
-        proxy = choice(proxies) if proxies else None
-        if proxy:
-            return self.__conn.hget(self.name, proxy)
-        else:
-            return None
-
-    def put(self, proxy_obj):
-        """
-        将代理放入hash
-        :param proxy_obj: Proxy obj
-        :return:
-        """
-        result = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
-        return result
-
-    def pop(self):
-        """
-        顺序弹出一个代理
-        :return: proxy
-        """
-        proxies = self.__conn.hkeys(self.name)
-        for proxy in proxies:
-            proxy_info = self.__conn.hget(self.name, proxy)
-            self.__conn.hdel(self.name, proxy)
-            return proxy_info
-        else:
-            return None
-
-    def delete(self, proxy_str):
-        """
-        移除指定代理, 使用changeTable指定hash name
-        :param proxy_str: proxy str
-        :return:
-        """
-        self.__conn.hdel(self.name, proxy_str)
-
-    def exists(self, proxy_str):
-        """
-        判断指定代理是否存在, 使用changeTable指定hash name
-        :param proxy_str: proxy str
-        :return:
-        """
-        return self.__conn.hexists(self.name, proxy_str)
-
-    def update(self, proxy_obj):
-        """
-        更新 proxy 属性
-        :param proxy_obj:
-        :return:
-        """
-        self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
-
-    def getAll(self):
-        """
-        字典形式返回所有代理, 使用changeTable指定hash name
-        :return:
-        """
-        item_dict = self.__conn.hgetall(self.name)
-        return item_dict
-
-    def clear(self):
-        """
-        清空所有代理, 使用changeTable指定hash name
-        :return:
-        """
-        return self.__conn.delete(self.name)
-
-    def getCount(self):
-        """
-        返回代理数量
-        :return:
-        """
-        return self.__conn.hlen(self.name)
-
-    def changeTable(self, name):
-        """
-        切换操作对象
-        :param name:
-        :return:
-        """
-        self.name = name

Test/testDbClient.py

@@ -1,35 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     testDbClient
-   Description :
-   Author :        JHao
-   date：          2020/6/23
--------------------------------------------------
-   Change Activity:
-                   2020/6/23:
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-from db.dbClient import DbClient
-
-if __name__ == '__main__':
-    #  ############### ssdb ###############
-    ssdb_uri = "ssdb://:password@127.0.0.1:8888"
-    s = DbClient.parseDbConn(ssdb_uri)
-    assert s.db_type == "SSDB"
-    assert s.db_pwd == "password"
-    assert s.db_host == "127.0.0.1"
-    assert s.db_port == 8888
-
-    #  ############### redis ###############
-    redis_uri = "redis://:password@127.0.0.1:6379/1"
-    r = DbClient.parseDbConn(redis_uri)
-    assert r.db_type == "REDIS"
-    assert r.db_pwd == "password"
-    assert r.db_host == "127.0.0.1"
-    assert r.db_port == 6379
-    assert r.db_name == "1"
-
-    print("DbClient ok!")

Test/testGetFreeProxy.py

@@ -1,37 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     testGetFreeProxy
-   Description :   test model ProxyGetter/getFreeProxy
-   Author :        J_hao
-   date：          2017/7/31
--------------------------------------------------
-   Change Activity:
-                   2017/7/31:function testGetFreeProxy
--------------------------------------------------
-"""
-__author__ = 'J_hao'
-
-
-from fetcher.getFreeProxy import GetFreeProxy
-from config.ConfigGetter import config
-
-
-def testGetFreeProxy():
-    """
-    test class GetFreeProxy in ProxyGetter/GetFreeProxy
-    :return:
-    """
-    proxy_getter_functions = config.proxy_getter_functions
-    for proxyGetter in proxy_getter_functions:
-        proxy_count = 0
-        for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
-            if proxy:
-                print('{func}: fetch proxy {proxy},proxy_count:{proxy_count}'.format(func=proxyGetter, proxy=proxy,
-                                                                                     proxy_count=proxy_count))
-                proxy_count += 1
-        # assert proxy_count >= 20, '{} fetch proxy fail'.format(proxyGetter)
-
-
-if __name__ == '__main__':
-    testGetFreeProxy()

Test/testLogHandler.py

@@ -1,35 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     testLogHandler
-   Description :
-   Author :        J_hao
-   date：          2017/8/2
--------------------------------------------------
-   Change Activity:
-                   2017/8/2:
--------------------------------------------------
-"""
-__author__ = 'J_hao'
-
-from util.LogHandler import LogHandler
-
-
-# noinspection PyPep8Naming
-def testLogHandler():
-    """
-    test function LogHandler  in Util/LogHandler
-    :return:
-    """
-    log = LogHandler('test')
-    log.info('this is a log from test')
-
-    log.resetName(name='test1')
-    log.info('this is a log from test1')
-
-    log.resetName(name='test2')
-    log.info('this is a log from test2')
-
-
-if __name__ == '__main__':
-    testLogHandler()

Test/testProxyClass.py

@@ -1,33 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     testProxyClass
-   Description :
-   Author :        JHao
-   date：          2019/8/8
--------------------------------------------------
-   Change Activity:
-                   2019/8/8:
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-import json
-from helper import Proxy
-
-
-def testProxyClass():
-    proxy = Proxy("127.0.0.1:8080")
-
-    print(proxy.info_dict)
-
-    proxy.source = "test"
-
-    proxy_str = json.dumps(proxy.info_dict, ensure_ascii=False)
-
-    print(proxy_str)
-
-    print(Proxy.newProxyFromJson(proxy_str).info_dict)
-
-
-testProxyClass()

Test/testWebRequest.py

@@ -1,30 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
--------------------------------------------------
-   File Name：     testWebRequest
-   Description :   test class WebRequest
-   Author :        J_hao
-   date：          2017/7/31
--------------------------------------------------
-   Change Activity:
-                   2017/7/31: function testWebRequest
--------------------------------------------------
-"""
-__author__ = 'J_hao'
-
-from util.WebRequest import WebRequest
-
-
-# noinspection PyPep8Naming
-def testWebRequest():
-    """
-    test class WebRequest in Util/WebRequest.py
-    :return:
-    """
-    wr = WebRequest()
-    request_object = wr.get('https://www.baidu.com/')
-    assert request_object.status_code == 200
-
-
-if __name__ == '__main__':
-    testWebRequest()