Merge pull request #38 from halleywj/master

修改redis的部分操作，增加代理计数器属性

.gitignore

@@ -1,3 +1,4 @@
 .idea/
 *.pyc
-test/
+*.log
+test/

Config.ini

@@ -1,12 +1,12 @@
 [DB]
 type = SSDB
 host = localhost
-port = 8889
+port = 8888
 name = proxy
 
 [ProxyGetter]
 ;register the proxy getter function
-;freeProxyFirst  = 1
+freeProxyFirst  = 1
 freeProxySecond = 1
 freeProxyThird  = 1
 freeProxyFourth = 1

DB/DbClient.py

@@ -2,13 +2,13 @@
 # !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     DbClient.py  
+   File Name：     DbClient.py
    Description :  DB工厂类
    Author :       JHao
    date：          2016/12/2
 -------------------------------------------------
    Change Activity:
-                   2016/12/2: 
+                   2016/12/2:
 -------------------------------------------------
 """
 __author__ = 'JHao'
@@ -33,13 +33,15 @@ class DbClient(object):
     抽象方法定义：
         get: 随机返回一个代理；
         put: 放回一个代理；
+        getvalue: 返回代理属性（一个计数器）；
+        inckey: 修改代理属性计数器的值;
         delete: 删除指定代理；
         getAll: 返回所有代理；
-        changeTable: 切换 table or collection or hash
+        changeTable: 切换 table or collection or hash;
 
         所有方法需要相应类去具体实现：
             SSDB：SsdbClient.py
-            REDIS:RedisClient.py    只是对redis set的操作，不一定好复用
+            REDIS:RedisClient.py
 
     """
 
@@ -73,14 +75,20 @@ class DbClient(object):
     def get(self, **kwargs):
         return self.client.get(**kwargs)
 
-    def put(self, value, **kwargs):
-        return self.client.put(value, **kwargs)
+    def put(self, key, **kwargs):
+        return self.client.put(key, **kwargs)
+
+    def getvalue(self, key, **kwargs):
+        return self.client.getvalue(key, **kwargs)
 
     def pop(self, **kwargs):
         return self.client.pop(**kwargs)
 
-    def delete(self, value, **kwargs):
-        return self.client.delete(value, **kwargs)
+    def inckey(self, key, value, **kwargs):
+        return self.client.inckey(key, value, **kwargs)
+
+    def delete(self, key, **kwargs):
+        return self.client.delete(key, **kwargs)
 
     def getAll(self):
         return self.client.getAll()

DB/RedisClient.py

@@ -7,7 +7,7 @@ self.name为Redis中的一个key
 '''
 
 import json
-
+import random
 import redis
 
 
@@ -32,37 +32,54 @@ class RedisClient(object):
         get random result
         :return:
         """
-        return self.__conn.srandmember(name=self.name).decode('utf-8')         #redis return bytes
+        key = self.__conn.hgetall(name=self.name)
+        return random.choice(key.keys()) if key else None
+        # return self.__conn.srandmember(name=self.name)
 
-    def put(self, value):
+    def put(self, key):
         """
         put an  item
         :param value:
         :return:
         """
-        value = json.dumps(value) if isinstance(value, (dict, list)) else value
-        return self.__conn.sadd(self.name, value)
+        key = json.dumps(key) if isinstance(key, (dict, list)) else key
+        return self.__conn.hincrby(self.name, key, 1)
+        # return self.__conn.sadd(self.name, value)
+
+    def getvalue(self, key):
+        value = self.__conn.hget(self.name, key)
+        return value if value else None
 
     def pop(self):
         """
         pop an item
         :return:
         """
-        return self.__conn.spop(self.name).decode('utf-8')                    #redis return bytes
+        key = self.get()
+        if key:
+            self.__conn.hdel(self.name, key)
+        return key
+        # return self.__conn.spop(self.name)
 
-    def delete(self, value):
+    def delete(self, key):
         """
         delete an item
         :param key:
         :return:
         """
-        self.__conn.srem(self.name, value)
+        self.__conn.hdel(self.name, key)
+        # self.__conn.srem(self.name, value)
+
+    def inckey(self, key, value):
+        self.__conn.hincrby(self.name, key, value)
 
     def getAll(self):
-        return self.__conn.smembers(self.name)
+        return self.__conn.hgetall(self.name).keys()
+        # return self.__conn.smembers(self.name)
 
     def get_status(self):
-        return self.__conn.scard(self.name)
+        return self.__conn.hlen(self.name)
+        # return self.__conn.scard(self.name)
 
     def changeTable(self, name):
         self.name = name
@@ -85,4 +102,4 @@ if __name__ == '__main__':
     # redis_con.put('132.112.43.221:8888')
     # redis_con.changeTable('proxy')
     print(redis_con.get_status())
-    print(redis_con.getAll())
+    print(redis_con.getAll())

DB/SsdbClient.py

@@ -2,13 +2,13 @@
 # !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     SsdbClient.py  
+   File Name：     SsdbClient.py
    Description :  封装SSDB操作
    Author :       JHao
    date：          2016/12/2
 -------------------------------------------------
    Change Activity:
-                   2016/12/2: 
+                   2016/12/2:
                    2017/04/26: 添加get_status方法获取hash长度
 -------------------------------------------------
 """
@@ -51,16 +51,21 @@ class SsdbClient(object):
         values = self.__conn.hgetall(name=self.name)
         return random.choice(values.keys()) if values else None
 
-    def put(self, value):
+    def put(self, key):
         """
         put an  item
 
         将代理放入hash, 使用changeTable指定hash name
-        :param value:
+        :param key:
         :return:
         """
-        value = json.dump(value, ensure_ascii=False).encode('utf-8') if isinstance(value, (dict, list)) else value
-        return self.__conn.hset(self.name, value, None)
+        key = json.dump(key, ensure_ascii=False).encode('utf-8') if isinstance(key, (dict, list)) else key
+        return self.__conn.hincr(self.name, key, 1)
+        # return self.__conn.hset(self.name, value, None)
+
+    def getvalue(self, key):
+        value = self.__conn.hget(self.name, key)
+        return value if value else None
 
     def pop(self):
         """
@@ -82,6 +87,9 @@ class SsdbClient(object):
         """
         self.__conn.hdel(self.name, key)
 
+    def inckey(self, key, value):
+        self.__conn.hincr(self.name, key, value)
+
     def getAll(self):
         return self.__conn.hgetall(self.name).keys()
 

Schedule/ProxyRefreshSchedule.py

@@ -2,7 +2,7 @@
 # !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     ProxyRefreshSchedule.py  
+   File Name：     ProxyRefreshSchedule.py
    Description :  代理定时刷新
    Author :       JHao
    date：          2016/12/4
@@ -88,7 +88,7 @@ def main(process_num=30):
 def run():
     # main()
     sched = BlockingScheduler()
-    sched.add_job(main, 'interval', minutes=10)
+    sched.add_job(main, 'interval', minutes=5)
     sched.start()
 
 

Schedule/ProxyValidSchedule.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 """
 -------------------------------------------------
-   File Name：     ProxyValidSchedule.py  
+   File Name：     ProxyValidSchedule.py
    Description :  验证useful_proxy_queue中的代理,将不可用的移出
    Author :       JHao
    date：          2017/3/31
@@ -38,10 +38,18 @@ class ProxyValidSchedule(ProxyManager):
                     each_proxy = each_proxy.decode('utf-8')
 
                 if validUsefulProxy(each_proxy):
+                    # 成功计数器加1
+                    self.db.inckey(each_proxy, 1)
                     self.log.debug('validProxy_b: {} validation pass'.format(each_proxy))
                 else:
-                    self.db.delete(each_proxy)
+                    # 失败计数器减一
+                    self.db.inckey(each_proxy, -1)
+                    # self.db.delete(each_proxy)
                     self.log.info('validProxy_b: {} validation fail'.format(each_proxy))
+                value = self.db.getvalue(each_proxy)
+                if value and value < -5:
+                    # 计数器小于-5删除该代理
+                    self.db.delete(each_proxy)
         self.log.info('validProxy_a running normal')
 
     def main(self):

Util/utilFunction.py

@@ -2,7 +2,7 @@
 # !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     utilFunction.py  
+   File Name：     utilFunction.py
    Description :  tool function
    Author :       JHao
    date：          2016/11/25
@@ -85,7 +85,7 @@ def validUsefulProxy(proxy):
     proxies = {"https": "https://{proxy}".format(proxy=proxy)}
     try:
         # 超过20秒的代理就不要了
-        r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=20, verify=False)
+        r = requests.get('https://www.baidu.com', proxies=proxies, timeout=40, verify=False)
         if r.status_code == 200:
             logger.debug('%s is ok' % proxy)
             return True