proxy_pool/proxyDB/redisClient.py

# -*- coding: utf-8 -*-
"""
-----------------------------------------------------
   File Name：     redisClient.py
   Description :   封装Redis相关操作
   Author :        JHao
   date：          2019/8/9
------------------------------------------------------
   Change Activity:
                   2019/08/09: 封装Redis相关操作
                   2020/06/23: 优化pop方法, 改用hscan命令
------------------------------------------------------
"""
__author__ = 'JHao'

from redis.connection import BlockingConnectionPool
from redis import Redis


class RedisClient(object):
    """
    Redis client

    Redis中代理存放的结构为hash：
        原始代理存放在name为raw_proxy的hash中, key为ip:port, value为代理属性的字典;
        验证后的代理存放在name为use_proxy的hash中, key为代理的ip:port, value为代理属性的字典;

    """

    def __init__(self, **kwargs):
        """
        init
        :param host: host
        :param port: port
        :param password: password
        :param db: db
        :return:
        """
        self.name = ""
        kwargs.pop("username")
        self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, **kwargs))

    def get(self, proxy_str):
        """
        从hash中获取对应的proxy, 使用前需要调用changeTable()
        :param proxy_str: proxy str
        :return:
        """
        data = self.__conn.hget(name=self.name, key=proxy_str)
        return data

    def put(self, proxy_obj):
        """
        将代理放入hash, 使用changeTable指定hash name
        :param proxy_obj: Proxy obj
        :return:
        """
        data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
        return data

    def delete(self, proxy_str):
        """
        移除指定代理, 使用changeTable指定hash name
        :param proxy_str: proxy str
        :return:
        """
        self.__conn.hdel(self.name, proxy_str)

    def exists(self, proxy_str):
        """
        判断指定代理是否存在, 使用changeTable指定hash name
        :param proxy_str: proxy str
        :return:
        """
        return self.__conn.hexists(self.name, proxy_str)

    def update(self, proxy_obj):
        """
        更新 proxy 属性
        :param proxy_obj:
        :return:
        """
        self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)

    def pop(self):
        """
        弹出一个代理
        :return: dict {proxy: value}
        """
        cursor, data_dict = self.__conn.hscan(self.name, count=1)
        for key, value in data_dict.items():
            self.__conn.hdel(self.name, key)
            return value
        else:
            return False

    def getAll(self):
        """
        字典形式返回所有代理, 使用changeTable指定hash name
        :return:
        """
        item_dict = self.__conn.hgetall(self.name)
        return item_dict

    def clear(self):
        """
        清空所有代理, 使用changeTable指定hash name
        :return:
        """
        return self.__conn.delete(self.name)

    def getCount(self):
        """
        返回代理数量
        :return:
        """
        return self.__conn.hlen(self.name)

    def changeTable(self, name):
        """
        切换操作对象
        :param name: raw_proxy/use_proxy
        :return:
        """
        self.name = name