[fix] fix proxy getter

ProxyGetter/getFreeProxy.py

@@ -24,6 +24,15 @@ from Util.utilFunction import robustCrawl, getHtmlTree
 # for debug to disable insecureWarning
 requests.packages.urllib3.disable_warnings()
 
+HEADER = {'Connection': 'keep-alive',
+          'Cache-Control': 'max-age=0',
+          'Upgrade-Insecure-Requests': '1',
+          'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)',
+          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+          'Accept-Encoding': 'gzip, deflate, sdch',
+          'Accept-Language': 'zh-CN,zh;q=0.8',
+          }
+
 
 class GetFreeProxy(object):
     """
@@ -43,23 +52,13 @@ class GetFreeProxy(object):
         """
         url_list = ('http://www.kuaidaili.com/proxylist/{page}/'.format(page=page) for page in range(1, page + 1))
         # 页数不用太多， 后面的全是历史IP， 可用性不高
-        header = {
-            'Host': 'www.kuaidaili.com',
-            'Connection': 'keep-alive',
-            'Cache-Control': 'max-age=0',
-            'Upgrade-Insecure-Requests': '1',
-            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-            'Accept-Encoding': 'gzip, deflate, sdch',
-            'Accept-Language': 'zh-CN,zh;q=0.8',
-        }
 
         for url in url_list:
-            tree = getHtmlTree(url, header=header)
+            print url
+            tree = getHtmlTree(url)
             proxy_list = tree.xpath('.//div[@id="index_free_list"]//tbody/tr')
             for proxy in proxy_list:
                 yield ':'.join(proxy.xpath('./td/text()')[0:2])
-        print 'finish kuaidaili fetching proxy ip'
 
     @staticmethod
     @robustCrawl
@@ -71,10 +70,9 @@ class GetFreeProxy(object):
         """
         url = "http://m.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format(
             proxy_number)
-        html = requests.get(url).content
+        html = requests.get(url, headers=HEADER).content
         for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
             yield proxy
-        print 'finish 66ip fetching proxy ip'
 
     @staticmethod
     @robustCrawl
@@ -85,29 +83,14 @@ class GetFreeProxy(object):
         :return:
         """
         url = "http://www.youdaili.net/Daili/http/"
-        header = {
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-            'Accept-Encoding': 'gzip, deflate, sdch',
-            'Accept-Language': 'zh-CN,zh;q=0.8',
-            'Cache-Control': 'max-age=0',
-            'Connection': 'keep-alive',
-            # 'Cookie': 'Hm_lvt_f8bdd88d72441a9ad0f8c82db3113a84=1487640273; Hm_lpvt_f8bdd88d72441a9ad0f8c82db3113a84=1487640642',
-            'Host': 'www.youdaili.net',
-            # 'If-Modified-Since': 'Tue, 21 Feb 2017 00:40:52 GMT',
-            'If-None-Match': "5c67-548ffa1d8ca68-gzip",
-            'Upgrade-Insecure-Requests': '1',
-            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-        }
-
-        tree = getHtmlTree(url, header=header)
+        tree = getHtmlTree(url)
         page_url_list = tree.xpath('.//div[@class="chunlist"]/ul/li/p/a/@href')[0:days]
         for page_url in page_url_list:
-            html = requests.get(page_url, headers=header).content
+            html = requests.get(page_url, headers=HEADER).content
             # print html
             proxy_list = re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html)
             for proxy in proxy_list:
                 yield proxy
-        print 'finish youdaili fetching proxy ip'
 
     @staticmethod
     @robustCrawl
@@ -116,11 +99,14 @@ class GetFreeProxy(object):
         抓取西刺代理 http://api.xicidaili.com/free2016.txt
         :return:
         """
-        url = "http://api.xicidaili.com/free2016.txt"
-        html = requests.get(url).content
-        for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
-            yield proxy
-        print 'finish xici fetching proxy ip'
+        url_list = ['http://www.xicidaili.com/nn',  # 高匿
+                    'http://www.xicidaili.com/nt',  # 透明
+                    ]
+        for each_url in url_list:
+            tree = getHtmlTree(each_url)
+            proxy_list = tree.xpath('.//table[@id="ip_list"]//tr')
+            for proxy in proxy_list:
+                yield ':'.join(proxy.xpath('./td/text()')[0:2])
 
     @staticmethod
     @robustCrawl
@@ -129,45 +115,27 @@ class GetFreeProxy(object):
         抓取guobanjia http://www.goubanjia.com/free/gngn/index.shtml
         :return:
         """
-        url = "http://www.goubanjia.com/free/gngn/index.shtml"
-        header = {
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-            'Accept-Encoding': 'gzip, deflate, sdch',
-            'Accept-Language': 'zh-CN,zh;q=0.8',
-            'Cache-Control': 'max-age=0',
-            'Connection': 'keep-alive',
-            # Cookie:auth=49b87d158a2b9e02589295bb9b74e8cf; JSESSIONID=BE57CA28BB0D8DDC119A542A75216B30; CNZZDATA1253707717=2116078297-1487635832-%7C1487641386; Hm_lvt_2e4ebee39b2c69a3920a396b87bbb8cc=1487641109; Hm_lpvt_2e4ebee39b2c69a3920a396b87bbb8cc=1487641408
-            'Host': 'www.goubanjia.com',
-            'Upgrade-Insecure-Requests': '1',
-            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-        }
-
-        tree = getHtmlTree(url, header=header)
-        # 现在每天最多放15个（一页）
-        for i in xrange(15):
-            d = tree.xpath('//*[@id="list"]/table/tbody/tr[{}]/td'.format(i + 1))[0]
-            # print d
-
-            o = d.xpath('.//span/text() | .//div/text()')
-            # print o
-
-            yield ''.join(o[:-1]) + ':' + o[-1]
-        print 'finish guobanjia fetching proxy ip'
-
+        url = "http://www.goubanjia.com/free/gngn/index{page}.shtml"
+        for page in range(1, 10):
+            page_url = url.format(page=page)
+            tree = getHtmlTree(page_url)
+            proxy_list = tree.xpath('//td[@class="ip"]')
+            for each_proxy in proxy_list:
+                yield ''.join(each_proxy.xpath('.//text()'))
 
 if __name__ == '__main__':
     gg = GetFreeProxy()
     # for e in gg.freeProxyFirst():
     #     print e
-    #
+
     # for e in gg.freeProxySecond():
     #     print e
-    #
+
     # for e in gg.freeProxyThird():
     #     print e
-
+    #
     # for e in gg.freeProxyFourth():
     #     print e
-    #
+
     for e in gg.freeProxyFifth():
         print e