Merge pull request #3 from bobobo80/xdaili

增加讯代理免费代理

Config.ini

@@ -1,20 +1,16 @@
 [DB]
 ;Configure the database information
-;type: SSDB/REDIS/MONGODB if use redis, only modify the host port，the type should be SSDB
+;type: SSDB/REDIS
 type = SSDB
 host = localhost
-port = 8888
+port = 6379
 name = proxy
 
 [ProxyGetter]
 ;register the proxy getter function
-freeProxyFirst  = 1
-freeProxySecond = 1
-freeProxyThird  = 1
-freeProxyFourth = 1
-freeProxyFifth  = 1
-
-[HOST]
-; API接口配置 http://127.0.0.1:5051
-ip = 0.0.0.0
-port = 5010
+;freeProxyFirst  = 1
+;freeProxySecond = 1
+;freeProxyThird  = 1
+;freeProxyFourth = 1
+;freeProxyFifth  = 1
+freeProxySixth = 1

ProxyGetter/getFreeProxy.py

@@ -13,6 +13,8 @@
 """
 import re
 import requests
+import sys
+sys.path.append('../')
 
 try:
     from importlib import reload  # py3 实际不会实用，只是为了不显示语法错误
@@ -127,6 +129,22 @@ class GetFreeProxy(object):
                 port = each_proxy.xpath(".//span[contains(@class, 'port')]/text()")[0]
                 yield '{}:{}'.format(ip_addr, port)
 
+    @staticmethod
+    @robustCrawl
+    def freeProxySixth():
+        """
+        抓取讯代理免费proxy http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10
+        :return:
+        """
+        url = 'http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10'
+        request = WebRequest()
+        try:
+            res = request.get(url).json()
+            for row in res['RESULT']['rows']:
+                yield '{}:{}'.format(row['ip'], row['port'])
+        except Exception as e:
+            pass
+
 
 if __name__ == '__main__':
     gg = GetFreeProxy()
@@ -139,8 +157,8 @@ if __name__ == '__main__':
     # for e in gg.freeProxyThird():
     #     print e
 
-    # for e in gg.freeProxyFourth():
-    #     print(e)
+    for e in gg.freeProxySixth():
+        print(e)
 
-    for e in gg.freeProxyFifth():
-         print(e)
+        # for e in gg.freeProxyFifth():
+        #     print(e)