[update] redisClient.py

proxyDB/RedisClient.py → proxyDB/redisClient.py

@@ -1,44 +1,44 @@
 # -*- coding: utf-8 -*-
 """
--------------------------------------------------
-   File Name：     RedisClient
-   Description :  封装Redis相关操作
+-----------------------------------------------------
+   File Name：     redisClient.py
+   Description :   封装Redis相关操作
    Author :        JHao
    date：          2019/8/9
--------------------------------------------------
+------------------------------------------------------
    Change Activity:
-                   2019/8/9: 封装Redis相关操作
--------------------------------------------------
+                   2019/08/09: 封装Redis相关操作
+                   2020/06/23: 优化pop方法, 改用hscan命令
+------------------------------------------------------
 """
 __author__ = 'JHao'
 
-from config.setting import PY3
-
 from redis.connection import BlockingConnectionPool
 from redis import Redis
 
 
 class RedisClient(object):
     """
-    Redis client 和SSDB协议一致 数据结构一致, 但部分方法不通用
+    Redis client
 
     Redis中代理存放的结构为hash：
-        原始代理存放在name为raw_proxy的hash中, key为代理的ip:por, value为代理属性的字典;
-        验证后的代理存放在name为useful_proxy的hash中, key为代理的ip:port, value为代理属性的字典;
+        原始代理存放在name为raw_proxy的hash中, key为ip:port, value为代理属性的字典;
+        验证后的代理存放在name为use_proxy的hash中, key为代理的ip:port, value为代理属性的字典;
 
     """
 
-    def __init__(self, name, **kwargs):
+    def __init__(self, **kwargs):
         """
         init
-        :param name: hash name
         :param host: host
         :param port: port
         :param password: password
+        :param db: db
         :return:
         """
-        self.name = name
-        self.__conn = Redis(connection_pool=BlockingConnectionPool(**kwargs))
+        self.name = ""
+        kwargs.pop("username")
+        self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, **kwargs))
 
     def get(self, proxy_str):
         """
@@ -47,10 +47,7 @@ class RedisClient(object):
         :return:
         """
         data = self.__conn.hget(name=self.name, key=proxy_str)
-        if data:
-            return data.decode('utf-8') if PY3 else data
-        else:
-            return None
+        return data
 
     def put(self, proxy_obj):
         """
@@ -90,25 +87,20 @@ class RedisClient(object):
         弹出一个代理
         :return: dict {proxy: value}
         """
-        # proxies = self.__conn.hkeys(self.name)
-        # if proxies:
-        #     proxy = random.choice(proxies)
-        #     value = self.__conn.hget(self.name, proxy)
-        #     self.delete(proxy)
-        #     return {'proxy': proxy.decode('utf-8') if PY3 else proxy,
-        #             'value': value.decode('utf-8') if PY3 and value else value}
-        return None
+        cursor, data_dict = self.__conn.hscan(self.name, count=1)
+        for key, value in data_dict.items():
+            self.__conn.hdel(self.name, key)
+            return value
+        else:
+            return False
 
     def getAll(self):
         """
-        列表形式返回所有代理, 使用changeTable指定hash name
+        字典形式返回所有代理, 使用changeTable指定hash name
         :return:
         """
         item_dict = self.__conn.hgetall(self.name)
-        if PY3:
-            return [value.decode('utf8') for key, value in item_dict.items()]
-        else:
-            return item_dict.values()
+        return item_dict
 
     def clear(self):
         """
@@ -117,7 +109,7 @@ class RedisClient(object):
         """
         return self.__conn.delete(self.name)
 
-    def getNumber(self):
+    def getCount(self):
         """
         返回代理数量
         :return:
@@ -127,7 +119,7 @@ class RedisClient(object):
     def changeTable(self, name):
         """
         切换操作对象
-        :param name: raw_proxy/useful_proxy
+        :param name: raw_proxy/use_proxy
         :return:
         """
         self.name = name