[update]搜索censys.io

ProxyGetter/getFreeProxy.py

@@ -3,41 +3,23 @@
 """
 -------------------------------------------------
    File Name：     GetFreeProxy.py
-   Description :  抓取免费代理
+   Description :  通过关键字扫描censys.io中的疑似ip
    Author :       JHao
    date：          2016/11/25
 -------------------------------------------------
    Change Activity:
-                   2016/11/25: 
-                   这一部分考虑用scrapy框架代替
+                   2017/06/15: 通过关键字扫描censys.io中的疑似ip
 -------------------------------------------------
 """
-import re
-import requests
-
-try:
-    from importlib import reload   #py3 实际不会实用，只是为了不显示语法错误
-except:
-    import sys     # py2
-    reload(sys)
-    sys.setdefaultencoding('utf-8')
-
-
-
 
-from Util.utilFunction import robustCrawl, getHtmlTree, getHTMLText
-
-# for debug to disable insecureWarning
-requests.packages.urllib3.disable_warnings()
+import json
+import requests
+import threading
 
-HEADER = {'Connection': 'keep-alive',
-          'Cache-Control': 'max-age=0',
-          'Upgrade-Insecure-Requests': '1',
-          'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)',
-          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-          'Accept-Encoding': 'gzip, deflate, sdch',
-          'Accept-Language': 'zh-CN,zh;q=0.8',
-          }
+API_URL = "https://www.censys.io/api/v1/search/"
+UID = "45bbe9db-87c9-4256-b1f0-0509037f1e84"
+SECRET = "DqtxZX43liWHZPY0gNLkorptTCIaAgyu"
+KEY_WORD = ['Squid', 'CCProxy', 'Tinyproxy', 'Wingate', 'Pound', 'Proxy', 'Mikrotik']
 
 
 class GetFreeProxy(object):
@@ -49,99 +31,49 @@ class GetFreeProxy(object):
         pass
 
     @staticmethod
-    @robustCrawl    #decoration print error if exception happen
-    def freeProxyFirst(page=10):
-        """
-        抓取快代理IP http://www.kuaidaili.com/
-        :param page: 翻页数
-        :return:
-        """
-        url_list = ('http://www.kuaidaili.com/proxylist/{page}/'.format(page=page) for page in range(1, page + 1))
-        # 页数不用太多， 后面的全是历史IP， 可用性不高
-
-        for url in url_list:
-            tree = getHtmlTree(url)
-            proxy_list = tree.xpath('.//div[@id="index_free_list"]//tbody/tr')
-            for proxy in proxy_list:
-                yield ':'.join(proxy.xpath('./td/text()')[0:2])
-
-    @staticmethod
-    @robustCrawl
-    def freeProxySecond(proxy_number=100):
-        """
-        抓取代理66 http://www.66ip.cn/
-        :param proxy_number: 代理数量
-        :return:
-        """
-        url = "http://m.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format(
-            proxy_number)
+    def scanner_ip():
+        for key in KEY_WORD:
+            print('Search Key: {key}'.format(key=key))
+            t = ThreadScanner(key)
+            t.start()
+
+
+class ThreadScanner(threading.Thread):
+    def __init__(self, key):
+        super(ThreadScanner, self).__init__()
+        self.key = key
+        self.query = {'query': self.key, 'page': 1, 'fields': ['ip']}
+
+    def run(self):
+        self.scanner_ip()
+
+    def get_total_page(self):
+        res = requests.post(API_URL + 'ipv4', data=json.dumps(self.query),
+                           auth=(UID, SECRET), timeout=30)
+        res_result = res.json()
+        total_page = res_result.get('metadata').get('pages', 400)
+        return total_page
+
+    def scanner_ip(self):
+        total_page = self.get_total_page()
+        for page in range(1, total_page):
+            self.query.update({'page': page})
+            try:
+                res = requests.post(API_URL + 'ipv4', data=json.dumps(self.query),
+                                    auth=(UID, SECRET), timeout=30)
+                res_result = res.json()
+                total_page = res_result.get('metadata').get('pages')
+                ip_results = res_result.get('results')
+            except Exception as e:
+                ip_results = list()
+            for each_ip in ip_results:
+                ip = each_ip.get('ip')
+                print(ip)
+            if page >= total_page:
+                break
+            print('Key {k} page: {p}'.format(k=self.key, p=page))
 
-        html = getHTMLText(url, headers=HEADER)
-        for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
-            yield proxy
-
-    @staticmethod
-    @robustCrawl
-    def freeProxyThird(days=1):
-        """
-        抓取有代理 http://www.youdaili.net/Daili/http/
-        :param days:
-        :return:
-        """
-        url = "http://www.youdaili.net/Daili/http/"
-        tree = getHtmlTree(url)
-        page_url_list = tree.xpath('.//div[@class="chunlist"]/ul/li/p/a/@href')[0:days]
-        for page_url in page_url_list:
-            html = requests.get(page_url, headers=HEADER).content
-            # print html
-            proxy_list = re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html)
-            for proxy in proxy_list:
-                yield proxy
-
-    @staticmethod
-    @robustCrawl
-    def freeProxyFourth():
-        """
-        抓取西刺代理 http://api.xicidaili.com/free2016.txt
-        :return:
-        """
-        url_list = ['http://www.xicidaili.com/nn',  # 高匿
-                    'http://www.xicidaili.com/nt',  # 透明
-                    ]
-        for each_url in url_list:
-            tree = getHtmlTree(each_url)
-            proxy_list = tree.xpath('.//table[@id="ip_list"]//tr')
-            for proxy in proxy_list:
-                yield ':'.join(proxy.xpath('./td/text()')[0:2])
-
-    @staticmethod
-    @robustCrawl
-    def freeProxyFifth():
-        """
-        抓取guobanjia http://www.goubanjia.com/free/gngn/index.shtml
-        :return:
-        """
-        url = "http://www.goubanjia.com/free/gngn/index{page}.shtml"
-        for page in range(1, 10):
-            page_url = url.format(page=page)
-            tree = getHtmlTree(page_url)
-            proxy_list = tree.xpath('//td[@class="ip"]')
-            for each_proxy in proxy_list:
-                yield ''.join(each_proxy.xpath('.//text()'))
 
 if __name__ == '__main__':
-    gg = GetFreeProxy()
-    # for e in gg.freeProxyFirst():
-    #     print e
-
-    # for e in gg.freeProxySecond():
-    #     print e
-
-    # for e in gg.freeProxyThird():
-    #     print e
-    #
-    # for e in gg.freeProxyFourth():
-    #     print e
-
-    for e in gg.freeProxyFifth():
-        print(e)
+    g = GetFreeProxy()
+    g.scanner_ip()