增加爬取秘密代理IP网站http:/www.mimiip.com

ProxyGetter/getFreeProxy.py

@@ -30,18 +30,16 @@ from Util.WebRequest import WebRequest
 requests.packages.urllib3.disable_warnings()
 
 """
-66ip.cn
-data5u.com
-ip181.com
-xicidaili.com
-goubanjia.com
-xdaili.cn
-kuaidaili.com
-cn-proxy.com
-www.mimiip.com
-proxy-list.org
-cz88.net
-ip181.com
+    66ip.cn
+    data5u.com
+    ip181.com
+    xicidaili.com
+    goubanjia.com
+    xdaili.cn
+    kuaidaili.com
+    cn-proxy.com
+    proxy-list.org
+    www.mimiip.com
 """
 
 
@@ -120,34 +118,6 @@ class GetFreeProxy(object):
                     yield ':'.join(proxy.xpath('./td/text()')[0:2])
                 except Exception as e:
                     pass
-    # 网站改版
-    # @staticmethod
-    # def freeProxyFifth():
-    #     """
-    #     抓取guobanjia http://www.goubanjia.com/free/gngn/index.shtml
-    #     :return:
-    #     """
-    #     url = "http://www.goubanjia.com/free/gngn/index{page}.shtml"
-    #     for page in range(1, 10):
-    #         page_url = url.format(page=page)
-    #         tree = getHtmlTree(page_url)
-    #         proxy_list = tree.xpath('//td[@class="ip"]')
-    #         # 此网站有隐藏的数字干扰，或抓取到多余的数字或.符号
-    #         # 需要过滤掉<p style="display:none;">的内容
-    #         xpath_str = """.//*[not(contains(@style, 'display: none'))
-    #                             and not(contains(@style, 'display:none'))
-    #                             and not(contains(@class, 'port'))
-    #                             ]/text()
-    #                     """
-    #         for each_proxy in proxy_list:
-    #             try:
-    #                 # :符号裸放在td下，其他放在div span p中，先分割找出ip，再找port
-    #                 ip_addr = ''.join(each_proxy.xpath(xpath_str))
-    #                 port = each_proxy.xpath(
-    #                     ".//span[contains(@class, 'port')]/text()")[0]
-    #                 yield '{}:{}'.format(ip_addr, port)
-    #             except Exception as e:
-    #                 pass
 
     @staticmethod
     def freeProxyFifth():
@@ -202,6 +172,23 @@ class GetFreeProxy(object):
             for tr in proxy_list[1:]:
                 yield ':'.join(tr.xpath('./td/text()')[0:2])
 
+    @staticmethod
+    def freeProxyEight():
+        """
+        秘密代理IP网站http://www.mimiip.com
+        """
+        url_gngao = ['http://www.mimiip.com/gngao/%s'%n for n in range(1, 10)]     #国内高匿
+        url_gnpu =  ['http://www.mimiip.com/gnpu/%s' %n for n in range(1, 10)]     #国内普匿
+        url_gntou = ['http://www.mimiip.com/gntou/%s' % n for n in range(1, 10)]   #国内透明
+        url_list = url_gngao + url_gnpu + url_gntou
+
+        request = WebRequest()
+        for url in url_list:
+            r = requests.get(url)
+            proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W].*<td>(\d+)</td>', r.text)
+            for proxy in proxies:
+                yield ':'.join(proxy)
+
     @staticmethod
     def freeProxyWallFirst():
         """
@@ -212,15 +199,13 @@ class GetFreeProxy(object):
         request = WebRequest()
         for url in urls:
             r = request.get(url)
-            proxies = re.findall(
-                r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)
+            proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)
             for proxy in proxies:
                 yield ':'.join(proxy)
 
     @staticmethod
     def freeProxyWallSecond():
-        urls = ['https://proxy-list.org/english/index.php?p=%s' %
-                n for n in range(1, 10)]
+        urls = ['https://proxy-list.org/english/index.php?p=%s'%n for n in range(1, 10)]
         request = WebRequest()
         import base64
         for url in urls:
@@ -244,17 +229,17 @@ if __name__ == '__main__':
     # for e in gg.freeProxyFourth():
     #     print(e)
     #
-    for e in gg.freeProxyFifth():
-        print(e)
+    # for e in gg.freeProxyFifth():
+    #     print(e)
     #
     # for e in gg.freeProxySixth():
     #     print(e)
     #
     # for e in gg.freeProxySeventh():
     #     print(e)
-    #
+    # for e in gg.freeProxyEight():
+    #     print(e)
     # for e in gg.freeProxyWallFirst():
     #     print(e)
-    #
     # for e in gg.freeProxyWallSecond():
     #     print(e)