[update]搜索censys.io

ProxyGetter/getFreeProxy.py

@@ -12,13 +12,22 @@
 -------------------------------------------------
 """
 
-import json
+from lxml import etree
 import requests
 import threading
 
-API_URL = "https://www.censys.io/api/v1/search/"
-UID = "45bbe9db-87c9-4256-b1f0-0509037f1e84"
-SECRET = "DqtxZX43liWHZPY0gNLkorptTCIaAgyu"
+API_URL = "https://www.censys.io/ipv4/_search?q={k}&page={p}"
+header = {
+    'Host': 'www.censys.io',
+    'Connection': 'keep-alive',
+    'Accept': '*/*',
+    'X-Requested-With': 'XMLHttpRequest',
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
+    'Referer': 'https://www.censys.io/',
+    'Accept-Encoding': 'gzip, deflate, sdch, br',
+    'Accept-Language': 'zh-CN,zh;q=0.8'
+}
+
 KEY_WORD = ['Squid', 'CCProxy', 'Tinyproxy', 'Wingate', 'Pound', 'Proxy', 'Mikrotik']
 
 
@@ -30,12 +39,26 @@ class GetFreeProxy(object):
     def __init__(self):
         pass
 
-    @staticmethod
-    def scanner_ip():
+    def scanner_ip(self):
+        """
+        根据关键字搜索ip
+        :return:
+        """
         for key in KEY_WORD:
-            print('Search Key: {key}'.format(key=key))
-            t = ThreadScanner(key)
-            t.start()
+            for page in range(1, 200):
+                url = API_URL.format(k=key, p=page)
+                try:
+                    res = requests.get(url, headers=header, timeout=30, )
+                    if res.status_code != 200:
+                        break
+                    tree = etree.HTML(res.content)
+                    ip_list_el = tree.xpath('//span[@class="ip"]/a/text()')
+                    ip_list = [each.strip() for each in ip_list_el if each.strip()]
+                    for each in ip_list:
+                        yield each
+                except Exception as e:
+                    print(e)
+                print('Key {k} page: {p}'.format(k=key, p=page))
 
 
 class ThreadScanner(threading.Thread):
@@ -47,33 +70,24 @@ class ThreadScanner(threading.Thread):
     def run(self):
         self.scanner_ip()
 
-    def get_total_page(self):
-        res = requests.post(API_URL + 'ipv4', data=json.dumps(self.query),
-                           auth=(UID, SECRET), timeout=30)
-        res_result = res.json()
-        total_page = res_result.get('metadata').get('pages', 400)
-        return total_page
-
     def scanner_ip(self):
-        total_page = self.get_total_page()
-        for page in range(1, total_page):
-            self.query.update({'page': page})
+        for page in range(1, 200):
+            url = API_URL.format(k=self.key, p=page)
             try:
-                res = requests.post(API_URL + 'ipv4', data=json.dumps(self.query),
-                                    auth=(UID, SECRET), timeout=30)
-                res_result = res.json()
-                total_page = res_result.get('metadata').get('pages')
-                ip_results = res_result.get('results')
+                res = requests.get(url, headers=header, timeout=30, proxies={'https': 'https://106.75.87.49:53100'})
+                if res.status_code == 429:
+                    break
+                tree = etree.HTML(res.content)
+                ip_list_el = tree.xpath('//span[@class="ip"]/a/text()')
+                ip_list = [each.strip() for each in ip_list_el if each.strip()]
+                for each in ip_list:
+                    print(each)
             except Exception as e:
-                ip_results = list()
-            for each_ip in ip_results:
-                ip = each_ip.get('ip')
-                print(ip)
-            if page >= total_page:
-                break
+                print(e)
             print('Key {k} page: {p}'.format(k=self.key, p=page))
 
 
 if __name__ == '__main__':
     g = GetFreeProxy()
-    g.scanner_ip()
+    for each in g.scanner_ip():
+        print(each)