Merge pull request #127 from forvendettaw/master

修改了

Config.ini

@@ -3,8 +3,8 @@
 ;type: SSDB/REDIS/MONGODB if use redis, only modify the host port，the type should be SSDB
 type = SSDB
 host = 127.0.0.1
-;port = 6379
-port = 8888
+port = 6379
+;port = 8888
 name = proxy
 
 [ProxyGetter]
@@ -13,15 +13,15 @@ freeProxyFirst  = 1
 freeProxySecond = 1
 ;freeProxyThird  = 1
 freeProxyFourth = 1
-;freeProxyFifth  = 1
+freeProxyFifth  = 1
 freeProxySixth = 1
 freeProxySeventh = 1
-
+freeProxyEight = 1
 ;foreign website, outside the wall
 ;freeProxyWallFirst = 1
 ;freeProxyWallSecond = 1
 
 [HOST]
-; API接口配置 http://127.0.0.1:5051
+; API接口配置 http://127.0.0.1:5010
 ip = 0.0.0.0
 port = 5010

ProxyGetter/getFreeProxy.py

@@ -30,18 +30,16 @@ from Util.WebRequest import WebRequest
 requests.packages.urllib3.disable_warnings()
 
 """
-66ip.cn
-data5u.com
-ip181.com
-xicidaili.com
-goubanjia.com
-xdaili.cn
-kuaidaili.com
-cn-proxy.com
-www.mimiip.com
-proxy-list.org
-cz88.net
-ip181.com
+    66ip.cn
+    data5u.com
+    ip181.com
+    xicidaili.com
+    goubanjia.com
+    xdaili.cn
+    kuaidaili.com
+    cn-proxy.com
+    proxy-list.org
+    www.mimiip.com
 """
 
 
@@ -124,30 +122,27 @@ class GetFreeProxy(object):
     @staticmethod
     def freeProxyFifth():
         """
-        抓取guobanjia http://www.goubanjia.com/free/gngn/index.shtml
+        抓取guobanjia http://www.goubanjia.com/
         :return:
         """
-        url = "http://www.goubanjia.com/free/gngn/index{page}.shtml"
-        for page in range(1, 10):
-            page_url = url.format(page=page)
-            tree = getHtmlTree(page_url)
-            proxy_list = tree.xpath('//td[@class="ip"]')
-            # 此网站有隐藏的数字干扰，或抓取到多余的数字或.符号
-            # 需要过滤掉<p style="display:none;">的内容
-            xpath_str = """.//*[not(contains(@style, 'display: none'))
-                                and not(contains(@style, 'display:none'))
-                                and not(contains(@class, 'port'))
-                                ]/text()
-                        """
-            for each_proxy in proxy_list:
-                try:
-                    # :符号裸放在td下，其他放在div span p中，先分割找出ip，再找port
-                    ip_addr = ''.join(each_proxy.xpath(xpath_str))
-                    port = each_proxy.xpath(
-                        ".//span[contains(@class, 'port')]/text()")[0]
-                    yield '{}:{}'.format(ip_addr, port)
-                except Exception as e:
-                    pass
+        url = "http://www.goubanjia.com/"
+        tree = getHtmlTree(url)
+        proxy_list = tree.xpath('//td[@class="ip"]')
+        # 此网站有隐藏的数字干扰，或抓取到多余的数字或.符号
+        # 需要过滤掉<p style="display:none;">的内容
+        xpath_str = """.//*[not(contains(@style, 'display: none'))
+                                        and not(contains(@style, 'display:none'))
+                                        and not(contains(@class, 'port'))
+                                        ]/text()
+                                """
+        for each_proxy in proxy_list:
+            try:
+                # :符号裸放在td下，其他放在div span p中，先分割找出ip，再找port
+                ip_addr = ''.join(each_proxy.xpath(xpath_str))
+                port = each_proxy.xpath(".//span[contains(@class, 'port')]/text()")[0]
+                yield '{}:{}'.format(ip_addr, port)
+            except Exception as e:
+                pass
 
     @staticmethod
     def freeProxySixth():
@@ -177,6 +172,23 @@ class GetFreeProxy(object):
             for tr in proxy_list[1:]:
                 yield ':'.join(tr.xpath('./td/text()')[0:2])
 
+    @staticmethod
+    def freeProxyEight():
+        """
+        秘密代理IP网站http://www.mimiip.com
+        """
+        url_gngao = ['http://www.mimiip.com/gngao/%s'%n for n in range(1, 10)]     #国内高匿
+        url_gnpu =  ['http://www.mimiip.com/gnpu/%s' %n for n in range(1, 10)]     #国内普匿
+        url_gntou = ['http://www.mimiip.com/gntou/%s' % n for n in range(1, 10)]   #国内透明
+        url_list = url_gngao + url_gnpu + url_gntou
+
+        request = WebRequest()
+        for url in url_list:
+            r = requests.get(url)
+            proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W].*<td>(\d+)</td>', r.text)
+            for proxy in proxies:
+                yield ':'.join(proxy)
+
     @staticmethod
     def freeProxyWallFirst():
         """
@@ -187,15 +199,13 @@ class GetFreeProxy(object):
         request = WebRequest()
         for url in urls:
             r = request.get(url)
-            proxies = re.findall(
-                r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)
+            proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)
             for proxy in proxies:
                 yield ':'.join(proxy)
 
     @staticmethod
     def freeProxyWallSecond():
-        urls = ['https://proxy-list.org/english/index.php?p=%s' %
-                n for n in range(1, 10)]
+        urls = ['https://proxy-list.org/english/index.php?p=%s'%n for n in range(1, 10)]
         request = WebRequest()
         import base64
         for url in urls:
@@ -227,11 +237,9 @@ if __name__ == '__main__':
     #
     # for e in gg.freeProxySeventh():
     #     print(e)
-
-    #
-    #
+    # for e in gg.freeProxyEight():
+    #     print(e)
     # for e in gg.freeProxyWallFirst():
     #     print(e)
-    #
-    for e in gg.freeProxyWallSecond():
-        print(e)
+    # for e in gg.freeProxyWallSecond():
+    #     print(e)