Jelajahi Sumber

redis在py3中返回bytes类型,增加两处返回时的解码操作

bobobo80 8 tahun lalu
induk
melakukan
f7923896db
2 mengubah file dengan 28 tambahan dan 9 penghapusan
  1. 8 7
      Config.ini
  2. 20 2
      ProxyGetter/getFreeProxy.py

+ 8 - 7
Config.ini

@@ -1,15 +1,16 @@
 [DB]
 ;Configure the database information
 ;type: SSDB/REDIS
-type = SSDB
+type = REDIS
 host = localhost
-port = 8888
+port = 6379
 name = proxy
 
 [ProxyGetter]
 ;register the proxy getter function
-freeProxyFirst  = 1
-freeProxySecond = 1
-freeProxyThird  = 1
-freeProxyFourth = 1
-freeProxyFifth  = 1
+;freeProxyFirst  = 1
+;freeProxySecond = 1
+;freeProxyThird  = 1
+;freeProxyFourth = 1
+;freeProxyFifth  = 1
+freeProxySixth  = 1

+ 20 - 2
ProxyGetter/getFreeProxy.py

@@ -13,6 +13,8 @@
 """
 import re
 import requests
+import sys
+sys.path.append('../')
 
 try:
     from importlib import reload  # py3 实际不会实用,只是为了不显示语法错误
@@ -115,6 +117,22 @@ class GetFreeProxy(object):
             for each_proxy in proxy_list:
                 yield ''.join(each_proxy.xpath('.//text()'))
 
+    @staticmethod
+    @robustCrawl
+    def freeProxySixth():
+        """
+        抓取讯代理免费proxy http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10
+        :return:
+        """
+        url = 'http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10'
+        request = WebRequest()
+        try:
+            res = request.get(url).json()
+            for row in res['RESULT']['rows']:
+                yield '{}:{}'.format(row['ip'], row['port'])
+        except Exception as e:
+            pass
+
 
 if __name__ == '__main__':
     gg = GetFreeProxy()
@@ -127,8 +145,8 @@ if __name__ == '__main__':
     # for e in gg.freeProxyThird():
     #     print e
 
-    for e in gg.freeProxyFourth():
-        print e
+    for e in gg.freeProxySixth():
+        print(e)
 
         # for e in gg.freeProxyFifth():
         #     print(e)