py

db/ssdbClient.py

@@ -11,13 +11,13 @@
                    2016/12/2:
                    2017/09/22: PY3中 redis-py返回的数据是bytes型
                    2017/09/27: 修改pop()方法 返回{proxy:value}字典
+                   2020/07/03: 2.1.0 优化代码结构
 -------------------------------------------------
 """
 __author__ = 'JHao'
 
-from config.setting import PY3
-
 from redis.connection import BlockingConnectionPool
+from random import choice
 from redis import Redis
 
 
@@ -26,42 +26,54 @@ class SsdbClient(object):
     SSDB client
 
     SSDB中代理存放的结构为hash：
-        原始代理存放在name为raw_proxy的hash中, key为代理的ip:por, value为代理属性的字典;
-        验证后的代理存放在name为useful_proxy的hash中, key为代理的ip:port, value为代理属性的字典;
-
+    key为代理的ip:por, value为代理属性的字典;
     """
-    def __init__(self, name, **kwargs):
+
+    def __init__(self, **kwargs):
         """
         init
-        :param name: hash name
         :param host: host
         :param port: port
         :param password: password
         :return:
         """
-        self.name = name
-        self.__conn = Redis(connection_pool=BlockingConnectionPool(**kwargs))
+        self.name = ""
+        kwargs.pop("username")
+        self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, **kwargs))
 
-    def get(self, proxy_str):
+    def get(self):
         """
-        从hash中获取对应的proxy, 使用前需要调用changeTable()
-        :param proxy_str: proxy str
+        从hash中随机返回一个代理
         :return:
         """
-        data = self.__conn.hget(name=self.name, key=proxy_str)
-        if data:
-            return data.decode('utf-8') if PY3 else data
+        proxies = self.__conn.hkeys(self.name)
+        proxy = choice(proxies) if proxies else None
+        if proxy:
+            return self.__conn.hget(self.name, proxy)
         else:
             return None
 
     def put(self, proxy_obj):
         """
-        将代理放入hash, 使用changeTable指定hash name
+        将代理放入hash
         :param proxy_obj: Proxy obj
         :return:
         """
-        data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
-        return data
+        result = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
+        return result
+
+    def pop(self):
+        """
+        顺序弹出一个代理
+        :return: proxy
+        """
+        proxies = self.__conn.hkeys(self.name)
+        for proxy in proxies:
+            proxy_info = self.__conn.hget(self.name, proxy)
+            self.__conn.hdel(self.name, proxy)
+            return proxy_info
+        else:
+            return None
 
     def delete(self, proxy_str):
         """
@@ -85,41 +97,24 @@ class SsdbClient(object):
         :param proxy_obj:
         :return:
         """
-        self.__conn.hset(self.name,  proxy_obj.proxy, proxy_obj.info_json)
-
-    def pop(self):
-        """
-        弹出一个代理
-        :return: dict {proxy: value}
-        """
-        # proxies = self.__conn.hkeys(self.name)
-        # if proxies:
-        #     proxy = random.choice(proxies)
-        #     value = self.__conn.hget(self.name, proxy)
-        #     self.delete(proxy)
-        #     return {'proxy': proxy.decode('utf-8') if PY3 else proxy,
-        #             'value': value.decode('utf-8') if PY3 and value else value}
-        return None
+        self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
 
     def getAll(self):
         """
-        列表形式返回所有代理, 使用changeTable指定hash name
+        字典形式返回所有代理, 使用changeTable指定hash name
         :return:
         """
         item_dict = self.__conn.hgetall(self.name)
-        if PY3:
-            return [value.decode('utf8') for key, value in item_dict.items()]
-        else:
-            return item_dict.values()
+        return item_dict
 
     def clear(self):
         """
         清空所有代理, 使用changeTable指定hash name
         :return:
         """
-        return self.__conn.execute_command("hclear", self.name)
+        return self.__conn.delete(self.name)
 
-    def getNumber(self):
+    def getCount(self):
         """
         返回代理数量
         :return:
@@ -129,7 +124,7 @@ class SsdbClient(object):
     def changeTable(self, name):
         """
         切换操作对象
-        :param name: raw_proxy/useful_proxy
+        :param name:
         :return:
         """
         self.name = name