Merge branch 'master' into Censys_Scanner

# Conflicts:
#	ProxyGetter/getFreeProxy.py

.gitignore

@@ -1,4 +1,3 @@
 .idea/
 *.pyc
 *.log
-test/

Config.ini

@@ -1,4 +1,6 @@
 [DB]
+;Configure the database information
+;type: SSDB/REDIS
 type = SSDB
 host = localhost
 port = 8888
@@ -11,4 +13,3 @@ freeProxySecond = 1
 freeProxyThird  = 1
 freeProxyFourth = 1
 freeProxyFifth  = 1
-

ProxyGetter/getFreeProxy.py

@@ -3,32 +3,30 @@
 """
 -------------------------------------------------
    File Name：     GetFreeProxy.py
-   Description :  通过关键字扫描censys.io中的疑似ip
+   Description :  抓取免费代理
    Author :       JHao
    date：          2016/11/25
 -------------------------------------------------
    Change Activity:
-                   2017/06/15: 通过关键字扫描censys.io中的疑似ip
+                   2016/11/25:
 -------------------------------------------------
 """
-
-from lxml import etree
+import re
 import requests
-import threading
 
-API_URL = "https://www.censys.io/ipv4/_search?q={k}&page={p}"
-header = {
-    'Host': 'www.censys.io',
-    'Connection': 'keep-alive',
-    'Accept': '*/*',
-    'X-Requested-With': 'XMLHttpRequest',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
-    'Referer': 'https://www.censys.io/',
-    'Accept-Encoding': 'gzip, deflate, sdch, br',
-    'Accept-Language': 'zh-CN,zh;q=0.8'
-}
+try:
+    from importlib import reload  # py3 实际不会实用，只是为了不显示语法错误
+except:
+    import sys  # py2
+
+    reload(sys)
+    sys.setdefaultencoding('utf-8')
+
+from Util.utilFunction import robustCrawl, getHtmlTree
+from Util.WebRequest import WebRequest
 
-KEY_WORD = ['Squid', 'CCProxy', 'Tinyproxy', 'Wingate', 'Pound', 'Proxy', 'Mikrotik']
+# for debug to disable insecureWarning
+requests.packages.urllib3.disable_warnings()
 
 
 class GetFreeProxy(object):
@@ -39,55 +37,98 @@ class GetFreeProxy(object):
     def __init__(self):
         pass
 
-    def scanner_ip(self):
+    @staticmethod
+    @robustCrawl  # decoration print error if exception happen
+    def freeProxyFirst(page=10):
+        """
+        抓取无忧代理 http://www.data5u.com/
+        :param page: 页数
+        :return:
+        """
+        url_list = ['http://www.data5u.com/',
+                    'http://www.data5u.com/free/',
+                    'http://www.data5u.com/free/gngn/index.shtml',
+                    'http://www.data5u.com/free/gnpt/index.shtml']
+        for url in url_list:
+            html_tree = getHtmlTree(url)
+            ul_list = html_tree.xpath('//ul[@class="l2"]')
+            for ul in ul_list:
+                yield ':'.join(ul.xpath('.//li/text()')[0:2])
+
+    @staticmethod
+    @robustCrawl
+    def freeProxySecond(proxy_number=100):
         """
-        根据关键字搜索ip
+        抓取代理66 http://www.66ip.cn/
+        :param proxy_number: 代理数量
         :return:
         """
-        for key in KEY_WORD:
-            for page in range(1, 200):
-                url = API_URL.format(k=key, p=page)
-                try:
-                    res = requests.get(url, headers=header, timeout=30, )
-                    if res.status_code != 200:
-                        break
-                    tree = etree.HTML(res.content)
-                    ip_list_el = tree.xpath('//span[@class="ip"]/a/text()')
-                    ip_list = [each.strip() for each in ip_list_el if each.strip()]
-                    for each in ip_list:
-                        yield each
-                except Exception as e:
-                    print(e)
-                print('Key {k} page: {p}'.format(k=key, p=page))
-
-
-class ThreadScanner(threading.Thread):
-    def __init__(self, key):
-        super(ThreadScanner, self).__init__()
-        self.key = key
-        self.query = {'query': self.key, 'page': 1, 'fields': ['ip']}
-
-    def run(self):
-        self.scanner_ip()
-
-    def scanner_ip(self):
-        for page in range(1, 200):
-            url = API_URL.format(k=self.key, p=page)
-            try:
-                res = requests.get(url, headers=header, timeout=30, proxies={'https': 'https://106.75.87.49:53100'})
-                if res.status_code == 429:
-                    break
-                tree = etree.HTML(res.content)
-                ip_list_el = tree.xpath('//span[@class="ip"]/a/text()')
-                ip_list = [each.strip() for each in ip_list_el if each.strip()]
-                for each in ip_list:
-                    print(each)
-            except Exception as e:
-                print(e)
-            print('Key {k} page: {p}'.format(k=self.key, p=page))
+        url = "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format(
+                proxy_number)
+        request = WebRequest()
+        html = request.get(url).content
+        for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
+            yield proxy
+
+    @staticmethod
+    @robustCrawl
+    def freeProxyThird(days=1):
+        """
+        抓取ip181 http://www.ip181.com/
+        :param days:
+        :return:
+        """
+        url = 'http://www.ip181.com/'
+        html_tree = getHtmlTree(url)
+        tr_list = html_tree.xpath('//tr')[1:]
+        for tr in tr_list:
+            yield ':'.join(tr.xpath('./td/text()')[0:2])
+
+    @staticmethod
+    @robustCrawl
+    def freeProxyFourth():
+        """
+        抓取西刺代理 http://api.xicidaili.com/free2016.txt
+        :return:
+        """
+        url_list = ['http://www.xicidaili.com/nn',  # 高匿
+                    'http://www.xicidaili.com/nt',  # 透明
+                    ]
+        for each_url in url_list:
+            tree = getHtmlTree(each_url)
+            proxy_list = tree.xpath('.//table[@id="ip_list"]//tr')
+            for proxy in proxy_list:
+                yield ':'.join(proxy.xpath('./td/text()')[0:2])
+
+    @staticmethod
+    @robustCrawl
+    def freeProxyFifth():
+        """
+        抓取guobanjia http://www.goubanjia.com/free/gngn/index.shtml
+        :return:
+        """
+        url = "http://www.goubanjia.com/free/gngn/index{page}.shtml"
+        for page in range(1, 10):
+            page_url = url.format(page=page)
+            tree = getHtmlTree(page_url)
+            proxy_list = tree.xpath('//td[@class="ip"]')
+            for each_proxy in proxy_list:
+                yield ''.join(each_proxy.xpath('.//text()'))
 
 
 if __name__ == '__main__':
-    g = GetFreeProxy()
-    for each in g.scanner_ip():
-        print(each)
+    gg = GetFreeProxy()
+    # for e in gg.freeProxyFirst():
+    #     print e
+
+    # for e in gg.freeProxySecond():
+    #     print e
+
+    # for e in gg.freeProxyThird():
+    #     print e
+
+    for e in gg.freeProxyFourth():
+        print e
+
+        # for e in gg.freeProxyFifth():
+        #     print(e)

README.md

@@ -12,6 +12,7 @@
 * 代理IP从何而来？
 
 　　刚自学爬虫的时候没有代理IP就去西刺、快代理之类有免费代理的网站去爬，还是有个别代理能用。当然，如果你有更好的代理接口也可以自己接入。
+　　
 　　免费代理的采集也很简单，无非就是：访问页面页面 —> 正则/xpath提取 —> 保存
 
 * 如何保证代理质量？
@@ -20,7 +21,7 @@
 
 * 采集回来的代理如何存储？
 
-　　这里不得不推荐一个高性能支持多种数据结构的NoSQL数据库[SSDB](http://ssdb.io/docs/zh_cn/)，用于代理Redis。支持队列、hash、set、k-v对，支持T级别数据。是做分布式爬虫很好中间存储工具。
+　　这里不得不推荐一个高性能支持多种数据结构的NoSQL数据库[SSDB](http://ssdb.io/docs/zh_cn/)，用于替代Redis。支持队列、hash、set、k-v对，支持T级别数据。是做分布式爬虫很好中间存储工具。
 
 * 如何让爬虫更简单的使用这些代理？
 

Schedule/ProxyValidSchedule.py

@@ -47,7 +47,7 @@ class ProxyValidSchedule(ProxyManager):
                     # self.db.delete(each_proxy)
                     self.log.info('validProxy_b: {} validation fail'.format(each_proxy))
                 value = self.db.getvalue(each_proxy)
-                if value and value < -5:
+                if value and int(value) < -5:
                     # 计数器小于-5删除该代理
                     self.db.delete(each_proxy)
         self.log.info('validProxy_a running normal')

Test/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     __init__.py
+   Description :   
+   Author :        J_hao
+   date：          2017/7/31
+-------------------------------------------------
+   Change Activity:
+                   2017/7/31:
+-------------------------------------------------
+"""
+__author__ = 'J_hao'

Test/testGetConfig.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testGetConfig
+   Description :   test all function in GetConfig.py
+   Author :        J_hao
+   date：          2017/7/31
+-------------------------------------------------
+   Change Activity:
+                   2017/7/31:
+-------------------------------------------------
+"""
+__author__ = 'J_hao'
+
+from Util.GetConfig import GetConfig
+
+
+# noinspection PyPep8Naming
+def testGetConfig():
+    """
+    test class GetConfig in Util/GetConfig
+    :return:
+    """
+    gg = GetConfig()
+    print(gg.db_type)
+    print(gg.db_name)
+    print(gg.db_host)
+    print(gg.db_port)
+    assert isinstance(gg.proxy_getter_functions, list)
+    print(gg.proxy_getter_functions)
+
+if __name__ == '__main__':
+    testGetConfig()

Test/testGetFreeProxy.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testGetFreeProxy
+   Description :   test model ProxyGetter/getFreeProxy
+   Author :        J_hao
+   date：          2017/7/31
+-------------------------------------------------
+   Change Activity:
+                   2017/7/31:function testGetFreeProxy
+-------------------------------------------------
+"""
+__author__ = 'J_hao'
+
+from ProxyGetter.getFreeProxy import GetFreeProxy
+from Util.GetConfig import GetConfig
+
+
+# noinspection PyPep8Naming
+def testGetFreeProxy():
+    """
+    test class GetFreeProxy in ProxyGetter/GetFreeProxy
+    :return:
+    """
+    gc = GetConfig()
+    proxy_getter_functions = gc.proxy_getter_functions
+    for proxyGetter in proxy_getter_functions:
+        proxy_count = 0
+        for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
+            if proxy:
+                print('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy))
+                proxy_count += 1
+        assert proxy_count >= 20, '{} fetch proxy fail'.format(proxyGetter)
+
+
+if __name__ == '__main__':
+    testGetFreeProxy()

Test/testWebRequest.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     testWebRequest
+   Description :   test class WebRequest
+   Author :        J_hao
+   date：          2017/7/31
+-------------------------------------------------
+   Change Activity:
+                   2017/7/31: function testWebRequest
+-------------------------------------------------
+"""
+__author__ = 'J_hao'
+
+from Util.WebRequest import WebRequest
+
+
+# noinspection PyPep8Naming
+def testWebRequest():
+    """
+    test class WebRequest in Util/WebRequest.py
+    :return:
+    """
+    wr = WebRequest()
+    request_object = wr.get('https://www.baidu.com/')
+    assert request_object.status_code == 200
+
+
+if __name__ == '__main__':
+    testWebRequest()

Util/WebRequest.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     WebRequest
+   Description :   Network Requests Class
+   Author :        J_hao
+   date：          2017/7/31
+-------------------------------------------------
+   Change Activity:
+                   2017/7/31:
+-------------------------------------------------
+"""
+__author__ = 'J_hao'
+
+import requests
+import random
+import time
+
+
+class WebRequest(object):
+    def __init__(self, *args, **kwargs):
+        pass
+
+    @property
+    def user_agent(self):
+        """
+        return an User-Agent at random
+        :return:
+        """
+        ua_list = [
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71',
+            'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
+            'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
+            ]
+        return random.choice(ua_list)
+
+    @property
+    def header(self):
+        """
+        basic header
+        :return:
+        """
+        return {'User-Agent': self.user_agent,
+                'Accept': '*/*',
+                'Connection': 'keep-alive',
+                'Accept-Language': 'zh-CN,zh;q=0.8'}
+
+    def get(self, url, header=None, retry_time=5, timeout=30,
+            retry_flag=list(), retry_interval=5, *args, **kwargs):
+        """
+        get method
+        :param url: target url
+        :param header: headers
+        :param retry_time: retry time when network error
+        :param timeout: network timeout
+        :param retry_flag: if retry_flag in content. do retry
+        :param retry_interval: retry interval(second)
+        :param args:
+        :param kwargs:
+        :return:
+        """
+        headers = self.header
+        if header and isinstance(header, dict):
+            headers.update(header)
+        while True:
+            try:
+                html = requests.get(url, headers=headers, timeout=timeout)
+                if filter(lambda key: key in html.content, retry_flag):
+                    raise Exception
+                else:
+                    return html
+            except Exception as e:
+                print(e)
+                retry_time -= 1
+                if retry_time <= 0:
+                    return
+                time.sleep(retry_interval)

Util/utilFunction.py

@@ -12,25 +12,14 @@
 -------------------------------------------------
 """
 import requests
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
-
-requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+from lxml import etree
 
 from Util.LogHandler import LogHandler
+from Util.WebRequest import WebRequest
 
 logger = LogHandler(__name__)
 
 
-def getHTMLText(url, headers={'user': 'Mozilla/5.0'}):
-    try:
-        response = requests.get(url, headers=headers, timeout=10)
-        response.raise_for_status()
-        response.encoding = response.apparent_encoding
-        return response.text
-    except:
-        return response.status_code
-
-
 # noinspection PyPep8Naming
 def robustCrawl(func):
     def decorate(*args, **kwargs):
@@ -43,7 +32,8 @@ def robustCrawl(func):
     return decorate
 
 
-def verifyProxy(proxy):
+# noinspection PyPep8Naming
+def verifyProxyFormat(proxy):
     """
     检查代理格式
     :param proxy:
@@ -54,6 +44,7 @@ def verifyProxy(proxy):
     return True if re.findall(verify_regex, proxy) else False
 
 
+# noinspection PyPep8Naming
 def getHtmlTree(url, **kwargs):
     """
     获取html树
@@ -61,8 +52,7 @@ def getHtmlTree(url, **kwargs):
     :param kwargs:
     :return:
     """
-    import requests
-    from lxml import etree
+
     header = {'Connection': 'keep-alive',
               'Cache-Control': 'max-age=0',
               'Upgrade-Insecure-Requests': '1',
@@ -72,19 +62,21 @@ def getHtmlTree(url, **kwargs):
               'Accept-Language': 'zh-CN,zh;q=0.8',
               }
     # TODO 取代理服务器用代理服务器访问
-    html = requests.get(url=url, headers=header, timeout=30).content
+    wr = WebRequest()
+    html = wr.get(url=url, header=header).content
     return etree.HTML(html)
 
 
+# noinspection PyPep8Naming
 def validUsefulProxy(proxy):
     """
-    检验代理可以性
+    检验代理是否可用
     :param proxy:
     :return:
     """
     proxies = {"https": "https://{proxy}".format(proxy=proxy)}
     try:
-        # 超过20秒的代理就不要了
+        # 超过40秒的代理就不要了
         r = requests.get('https://www.baidu.com', proxies=proxies, timeout=40, verify=False)
         if r.status_code == 200:
             logger.debug('%s is ok' % proxy)