Merge pull request #6 from bernieyangmh/master

Resdie客户端

DB/RedisClient.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+# !/usr/bin/env python
+
+'''
+self.name为Redis中的一个key
+'''
+
+import random
+import json
+import redis
+
+class ReidsClient(object):
+    """
+    Reids client
+    """
+
+    def __init__(self, name, host, port):
+        """
+        init
+        :param name:
+        :param host:
+        :param port:
+        :return:
+        """
+        self.name = name
+        self.__conn = redis.Redis(host=host, port=port, db=0)
+
+    def get(self):
+        """
+        get an item
+        :return:
+        """
+        values = self.__conn.smembers(name=self.name)
+
+        return random.choice(list(values)) if values else None
+
+    def put(self, value):
+        """
+        put an  item
+        :param value:
+        :return:
+        """
+        value = json.dump(value, ensure_ascii=False).encode('utf-8') if isinstance(value, (dict, list)) else value
+        return self.__conn.sadd(self.name, value)
+
+    def pop(self):
+        """
+        pop an item
+        :return:
+        """
+        value = self.get()
+        if value:
+            self.__conn.spop(self.name, value)
+        return value
+
+    def delete(self, value):
+        """
+        delete an item
+        :param key:
+        :return:
+        """
+        self.__conn.srem(self.name, value)
+
+    def getAll(self):
+        return self.__conn.smembers(self.name)
+
+

ProxyGetter/getFreeProxy.py

@@ -97,9 +97,12 @@ class GetFreeProxy(object):
         """
         url = "http://www.goubanjia.com/free/gngn/index.shtml"
         tree = getHtmlTree(url)
-        proxy_list = tree.xpath('.//td[@class="ip"]')
-        for proxy in proxy_list:
-            yield ''.join(proxy.xpath('.//text()'))
+        # 现在每天最多放15个（一页）
+        for i in xrange(15):
+            d = tree.xpath('.//table[@class="table"]/tbody/tr[{}]/td'.format(i + 1))[0]
+            o = d.xpath('.//span/text() | .//div/text()')
+            a = ''.join(o[:-1]) + ':' + o[-1]
+            yield ''.join(o[:-1]) + ':' + o[-1]
 
 
 if __name__ == '__main__':