[update] SsdbClient.py :smile_cat:

DB/DbClient.py

@@ -29,9 +29,9 @@ class DbClient(object):
 
 
     抽象方法定义：
-        get(): 返回一个proxy;
+        get(): 随机返回一个proxy;
         put(proxy): 存入一个proxy;
-        pop(): 返回并删除一个proxy;
+        pop(): 顺序返回并删除一个proxy;
         update(proxy): 更新指定proxy信息;
         delete(proxy): 删除指定proxy;
         exists(proxy): 判断指定proxy是否存在;

DB/SsdbClient.py

@@ -11,13 +11,13 @@
                    2016/12/2:
                    2017/09/22: PY3中 redis-py返回的数据是bytes型
                    2017/09/27: 修改pop()方法 返回{proxy:value}字典
+                   2020/07/03: 2.1.0 优化代码结构
 -------------------------------------------------
 """
 __author__ = 'JHao'
 
-from config.setting import PY3
-
 from redis.connection import BlockingConnectionPool
+from random import choice
 from redis import Redis
 
 
@@ -26,42 +26,54 @@ class SsdbClient(object):
     SSDB client
 
     SSDB中代理存放的结构为hash：
-        原始代理存放在name为raw_proxy的hash中, key为代理的ip:por, value为代理属性的字典;
-        验证后的代理存放在name为useful_proxy的hash中, key为代理的ip:port, value为代理属性的字典;
-
+    key为代理的ip:por, value为代理属性的字典;
     """
-    def __init__(self, name, **kwargs):
+
+    def __init__(self, **kwargs):
         """
         init
-        :param name: hash name
         :param host: host
         :param port: port
         :param password: password
         :return:
         """
-        self.name = name
-        self.__conn = Redis(connection_pool=BlockingConnectionPool(**kwargs))
+        self.name = ""
+        kwargs.pop("username")
+        self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, **kwargs))
 
-    def get(self, proxy_str):
+    def get(self):
         """
-        从hash中获取对应的proxy, 使用前需要调用changeTable()
-        :param proxy_str: proxy str
+        从hash中随机返回一个代理
         :return:
         """
-        data = self.__conn.hget(name=self.name, key=proxy_str)
-        if data:
-            return data.decode('utf-8') if PY3 else data
+        proxies = self.__conn.hkeys(self.name)
+        proxy = choice(proxies) if proxies else None
+        if proxy:
+            return self.__conn.hget(self.name, proxy)
         else:
             return None
 
     def put(self, proxy_obj):
         """
-        将代理放入hash, 使用changeTable指定hash name
+        将代理放入hash
         :param proxy_obj: Proxy obj
         :return:
         """
-        data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
-        return data
+        result = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
+        return result
+
+    def pop(self):
+        """
+        顺序弹出一个代理
+        :return: proxy
+        """
+        proxies = self.__conn.hkeys(self.name)
+        for proxy in proxies:
+            proxy_info = self.__conn.hget(self.name, proxy)
+            self.__conn.hdel(self.name, proxy)
+            return proxy_info
+        else:
+            return None
 
     def delete(self, proxy_str):
         """
@@ -85,41 +97,24 @@ class SsdbClient(object):
         :param proxy_obj:
         :return:
         """
-        self.__conn.hset(self.name,  proxy_obj.proxy, proxy_obj.info_json)
-
-    def pop(self):
-        """
-        弹出一个代理
-        :return: dict {proxy: value}
-        """
-        # proxies = self.__conn.hkeys(self.name)
-        # if proxies:
-        #     proxy = random.choice(proxies)
-        #     value = self.__conn.hget(self.name, proxy)
-        #     self.delete(proxy)
-        #     return {'proxy': proxy.decode('utf-8') if PY3 else proxy,
-        #             'value': value.decode('utf-8') if PY3 and value else value}
-        return None
+        self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.to_json)
 
     def getAll(self):
         """
-        列表形式返回所有代理, 使用changeTable指定hash name
+        字典形式返回所有代理, 使用changeTable指定hash name
         :return:
         """
         item_dict = self.__conn.hgetall(self.name)
-        if PY3:
-            return [value.decode('utf8') for key, value in item_dict.items()]
-        else:
-            return item_dict.values()
+        return item_dict
 
     def clear(self):
         """
         清空所有代理, 使用changeTable指定hash name
         :return:
         """
-        return self.__conn.execute_command("hclear", self.name)
+        return self.__conn.delete(self.name)
 
-    def getNumber(self):
+    def getCount(self):
         """
         返回代理数量
         :return:
@@ -129,7 +124,7 @@ class SsdbClient(object):
     def changeTable(self, name):
         """
         切换操作对象
-        :param name: raw_proxy/useful_proxy
+        :param name:
         :return:
         """
         self.name = name

Test/testRedisClient.py

@@ -13,11 +13,12 @@
 __author__ = 'JHao'
 
 
-def main():
+def testRedisClient():
     from db.dbClient import DbClient
     from helper.proxy import Proxy
 
-    db = DbClient("redis://:_@Fintell@10.10.61.65:6379")
+    uri = "redis://:_@Fintell@10.10.61.65:6379"
+    db = DbClient(uri)
     db.changeTable("use_proxy")
     proxy = Proxy.createFromJson(
         '{"proxy": "27.38.96.101:9797", "fail_count": 0, "region": "", "type": "",'
@@ -39,4 +40,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    testRedisClient()

Test/testSsdbClient.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testSsdbClient
+   Description :
+   Author :        JHao
+   date：          2020/7/3
+-------------------------------------------------
+   Change Activity:
+                   2020/7/3:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+
+def testSsdbClient():
+    from db.dbClient import DbClient
+    from helper.proxy import Proxy
+
+    uri = "ssdb://@120.79.78.193:8888"
+    db = DbClient(uri)
+    db.changeTable("use_proxy")
+    proxy = Proxy.createFromJson(
+        '{"proxy": "27.38.96.101:9797", "fail_count": 0, "region": "", "type": "",'
+        ' "source": "freeProxy03", "check_count": 0, "last_status": "", "last_time": ""}')
+
+    print("put: ", db.put(proxy))
+
+    print("get: ", db.get())
+
+    print("exists: ", db.exists("27.38.96.101:9797"))
+
+    print("exists: ", db.exists("27.38.96.101:8888"))
+
+    print("getAll: ", db.getAll())
+
+    print("pop: ", db.pop())
+
+    print("getCount", db.getCount())
+
+
+if __name__ == '__main__':
+    testSsdbClient()