[update] 修改ssdb module repo

Api/ProxyApi.py

@@ -54,7 +54,7 @@ def refresh():
 @app.route('/get_all/')
 def getAll():
     proxies = ProxyManager().getAll()
-    return jsonify(list(proxies))
+    return jsonify([proxy.decode('utf8') for proxy in proxies])
 
 
 @app.route('/delete/', methods=['GET'])

DB/SsdbClient.py

@@ -14,8 +14,10 @@
 """
 __author__ = 'JHao'
 
-from ssdb.connection import BlockingConnectionPool
-from ssdb import SSDB
+from Util import EnvUtil
+
+from redis.connection import BlockingConnectionPool
+from redis import Redis
 import random
 import json
 
@@ -39,7 +41,7 @@ class SsdbClient(object):
         :return:
         """
         self.name = name
-        self.__conn = SSDB(connection_pool=BlockingConnectionPool(host=host, port=port))
+        self.__conn = Redis(connection_pool=BlockingConnectionPool(host=host, port=port))
 
     def get(self):
         """
@@ -48,8 +50,9 @@ class SsdbClient(object):
         从useful_proxy_queue随机获取一个可用代理, 使用前需要调用changeTable("useful_proxy_queue")
         :return:
         """
-        values = self.__conn.hgetall(name=self.name)
-        return random.choice(values.keys()) if values else None
+        values = self.__conn.hkeys(name=self.name)
+        keys = list(values) if EnvUtil.PY3 else values
+        return random.choice(keys) if values else None
 
     def put(self, key):
         """
@@ -59,9 +62,8 @@ class SsdbClient(object):
         :param key:
         :return:
         """
-        key = json.dump(key, ensure_ascii=False).encode('utf-8') if isinstance(key, (dict, list)) else key
-        return self.__conn.hincr(self.name, key, 1)
-        # return self.__conn.hset(self.name, value, None)
+        key = json.dump(key, ensure_ascii=False) if isinstance(key, (dict, list)) else key
+        return self.__conn.hincrby(self.name, key, 1)
 
     def getvalue(self, key):
         value = self.__conn.hget(self.name, key)
@@ -88,17 +90,18 @@ class SsdbClient(object):
         self.__conn.hdel(self.name, key)
 
     def inckey(self, key, value):
-        self.__conn.hincr(self.name, key, value)
+        self.__conn.hincrby(self.name, key, value)
 
     def getAll(self):
-        return self.__conn.hgetall(self.name).keys()
+        keys = self.__conn.hkeys(self.name)
+        return list(keys) if EnvUtil.PY3 else keys
 
     def get_status(self):
         """
         Return the number of elements in hash ``name``
         :return:
         """
-        return self.__conn.hsize(self.name)
+        return self.__conn.hlen(self.name)
 
     def changeTable(self, name):
         self.name = name

Schedule/ProxyValidSchedule.py

@@ -52,7 +52,7 @@ class ProxyValidSchedule(ProxyManager):
                     # 计数器小于-5删除该代理
                     self.db.delete(each_proxy)
             self.log.info('validProxy_a running normal')
-            sleep(60 * 5)
+            sleep(60 * 1)
 
     def main(self):
         self.__validProxy()

Util/utilFunction.py

@@ -74,6 +74,8 @@ def validUsefulProxy(proxy):
     :param proxy:
     :return:
     """
+    if isinstance(proxy, bytes):
+        proxy = proxy.decode('utf8')
     proxies = {"https": "https://{proxy}".format(proxy=proxy)}
     try:
         # 超过40秒的代理就不要了

requirements.txt

@@ -3,7 +3,6 @@ Flask==0.11.1
 requests==2.11.0
 lxml==3.7.1
 
-ssdb==0.0.3
 pymongo==3.2.2
 redis==2.10.5