添加两个代理网站

Config.ini

@@ -17,9 +17,10 @@ freeProxyFifth  = 1
 freeProxySixth = 1
 freeProxySeventh = 1
 freeProxyEight = 1
+freeProxyNinth = 1
 ;foreign website, outside the wall
-;freeProxyWallFirst = 1
-;freeProxyWallSecond = 1
+freeProxyWallFirst = 1
+freeProxyWallSecond = 1
 
 [HOST]
 ; API接口配置 http://127.0.0.1:5010

ProxyGetter/getFreeProxy.py

@@ -21,7 +21,7 @@ except:
     reload(sys)
     sys.setdefaultencoding('utf-8')
 
-sys.path.append('../')
+sys.path.append('..')
 
 from Util.utilFunction import robustCrawl, getHtmlTree
 from Util.WebRequest import WebRequest
@@ -32,14 +32,13 @@ requests.packages.urllib3.disable_warnings()
 """
     66ip.cn
     data5u.com
-    ip181.com
     xicidaili.com
     goubanjia.com
     xdaili.cn
     kuaidaili.com
     cn-proxy.com
     proxy-list.org
-    www.mimiip.com
+    www.mimiip.com to do
 """
 
 
@@ -55,13 +54,15 @@ class GetFreeProxy(object):
     def freeProxyFirst(page=10):
         """
         抓取无忧代理 http://www.data5u.com/
+        几乎没有能用的
         :param page: 页数
         :return:
         """
-        url_list = ['http://www.data5u.com/',
-                    'http://www.data5u.com/free/',
-                    'http://www.data5u.com/free/gngn/index.shtml',
-                    'http://www.data5u.com/free/gnpt/index.shtml']
+        url_list = [
+            'http://www.data5u.com/',
+            'http://www.data5u.com/free/gngn/index.shtml',
+            'http://www.data5u.com/free/gnpt/index.shtml'
+        ]
         for url in url_list:
             html_tree = getHtmlTree(url)
             ul_list = html_tree.xpath('//ul[@class="l2"]')
@@ -69,10 +70,10 @@ class GetFreeProxy(object):
                 try:
                     yield ':'.join(ul.xpath('.//li/text()')[0:2])
                 except Exception as e:
-                    pass
+                    print(e)
 
     @staticmethod
-    def freeProxySecond(proxy_number=100):
+    def deprecatedFreeProxySecond(proxy_number=100):
         """
         抓取代理66 http://www.66ip.cn/
         :param proxy_number: 代理数量
@@ -85,6 +86,29 @@ class GetFreeProxy(object):
         for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
             yield proxy
 
+    @staticmethod
+    def freeProxySecond(area=33):
+        """
+        修改抓取代理66 http://www.66ip.cn/
+        :param page:抓取代理页数，page=1北京代理页，page=2上海代理页......
+        :return:
+        """
+        if area > 33:
+            page = 33
+        for area_index in range(1, area + 1):
+            page_count = 5
+            for i in range(1, page_count + 1):
+                url = "http://www.66ip.cn/areaindex_{}/{}.html".format(area_index, i)
+                html_tree = getHtmlTree(url)
+                tr_list = html_tree.xpath("//*[@id='footer']/div/table/tr[position()>1]")
+                if len(tr_list) == 0:
+                    continue
+                for tr in tr_list:
+                    yield tr.xpath("./td[1]/text()")[0] + ":" + tr.xpath("./td[2]/text()")[0]
+                break
+
+    '''
+    不能用了
     @staticmethod
     def freeProxyThird(days=1):
         """
@@ -100,24 +124,28 @@ class GetFreeProxy(object):
                 yield ':'.join(tr.xpath('./td/text()')[0:2])
         except Exception as e:
             pass
+    '''
 
     @staticmethod
-    def freeProxyFourth():
+    def freeProxyFourth(page_count=2):
         """
         抓取西刺代理 http://api.xicidaili.com/free2016.txt
         :return:
         """
-        url_list = ['http://www.xicidaili.com/nn',  # 高匿
-                    'http://www.xicidaili.com/nt',  # 透明
-                    ]
+        url_list = [
+            'http://www.xicidaili.com/nn/',  # 高匿
+            'http://www.xicidaili.com/nt/',  # 透明
+        ]
         for each_url in url_list:
-            tree = getHtmlTree(each_url)
-            proxy_list = tree.xpath('.//table[@id="ip_list"]//tr')
-            for proxy in proxy_list:
-                try:
-                    yield ':'.join(proxy.xpath('./td/text()')[0:2])
-                except Exception as e:
-                    pass
+            for i in range(1, page_count + 1):
+                page_url = each_url + str(i)
+                tree = getHtmlTree(page_url)
+                proxy_list = tree.xpath('.//table[@id="ip_list"]//tr[position()>1]')
+                for proxy in proxy_list:
+                    try:
+                        yield ':'.join(proxy.xpath('./td/text()')[0:2])
+                    except Exception as e:
+                        pass
 
     @staticmethod
     def freeProxyFifth():
@@ -164,13 +192,17 @@ class GetFreeProxy(object):
         """
         快代理免费https://www.kuaidaili.com/free/inha/1/
         """
-        url = 'https://www.kuaidaili.com/free/inha/{page}/'
-        for page in range(1, 10):
-            page_url = url.format(page=page)
-            tree = getHtmlTree(page_url)
-            proxy_list = tree.xpath('.//table//tr')
-            for tr in proxy_list[1:]:
-                yield ':'.join(tr.xpath('./td/text()')[0:2])
+        url_list = [
+            'https://www.kuaidaili.com/free/inha/{page}/',
+            'https://www.kuaidaili.com/free/intr/{page}/'
+        ]
+        for url in url_list:
+            for page in range(1, 5):
+                page_url = url.format(page=page)
+                tree = getHtmlTree(page_url)
+                proxy_list = tree.xpath('.//table//tr')
+                for tr in proxy_list[1:]:
+                    yield ':'.join(tr.xpath('./td/text()')[0:2])
 
     @staticmethod
     def freeProxyEight():
@@ -184,15 +216,33 @@ class GetFreeProxy(object):
 
         request = WebRequest()
         for url in url_list:
-            r = request.get(url)
+            r = request.get(url, use_proxy=True)
             proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W].*<td>(\d+)</td>', r.text)
             for proxy in proxies:
                 yield ':'.join(proxy)
 
+
+    @staticmethod
+    def freeProxyNinth():
+        """
+        coderBusy
+        https://proxy.coderbusy.com/
+        :return:
+        """
+        urls = ['https://proxy.coderbusy.com/classical/country/cn.aspx?page=1']
+        request = WebRequest()
+        for url in urls:
+            r = request.get(url)
+            proxies = re.findall('data-ip="(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})".+?>(\d+)</td>', r.text)
+            for proxy in proxies:
+                yield ':'.join(proxy)
+
+
     @staticmethod
     def freeProxyWallFirst():
         """
         墙外网站 cn-proxy
+        并没有被墙
         :return:
         """
         urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218']
@@ -205,6 +255,10 @@ class GetFreeProxy(object):
 
     @staticmethod
     def freeProxyWallSecond():
+        '''
+        并没有被墙
+        :return:
+        '''
         urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)]
         request = WebRequest()
         import base64
@@ -215,34 +269,101 @@ class GetFreeProxy(object):
                 yield base64.b64decode(proxy).decode()
 
 
+    @staticmethod
+    def freeProxyWallThird():
+        urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1']
+        request = WebRequest()
+        for url in urls:
+            r = request.get(url)
+            proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
+            for proxy in proxies:
+                yield ':'.join(proxy)
+
+
+import threading
+
+lock = threading.Lock()
+success = 0
+total = 0
+
+
+def test_once(proxy):
+    ip_port = proxy.split(":")
+    ip = ip_port[0]
+    port = ip_port[1]
+    import requests
+
+    req_url = "http://www.baidu.com"
+    proxies = {
+        "http": "http://%s:%s" % (ip, port),
+        "https": "https://%s:%s" % (ip, port)
+    }
+
+    global total
+
+    try:
+        response = requests.get(req_url, proxies=proxies, timeout=4)
+        if response.status_code != 200:
+            print("unknow error, status code:" + str(response.status_code))
+            lock.acquire()
+            total += 1
+            lock.release()
+            return 0
+        print("success")
+        global success
+        lock.acquire()
+        success += 1
+        total += 1
+        lock.release()
+        return 1
+    except requests.exceptions.Timeout:
+        print("timeout")
+    except requests.exceptions.ConnectionError:
+        print("poxy unusable")
+    except Exception:
+        print("request error")
+
+    lock.acquire()
+    total += 1
+    lock.release()
+    return 0
+
+
+def test_batch(iterator):
+    global success
+    global total
+
+    for proxy in iterator:
+        t = threading.Thread(target=test_once, args=(proxy,))
+        t.start()
+        t.join()
+
+    print("success:" + str(success) + "\ttotal:" + str(total))
+
+
 if __name__ == '__main__':
     gg = GetFreeProxy()
-    # for e in gg.freeProxyFirst():
-    #     print(e)
-    #
-    # for e in gg.freeProxySecond():
-    #     print(e)
-    #
-    # for e in gg.freeProxyThird():
-    #     print(e)
-    #
-    # for e in gg.freeProxyFourth():
-    #     print(e)
-    #
-    # for e in gg.freeProxyFifth():
-    #     print(e)
-    #
-    # for e in gg.freeProxySixth():
-    #     print(e)
-    #
-    # for e in gg.freeProxySeventh():
-    #     print(e)
-    #
-    # for e in gg.freeProxyEight():
-    #     print(e)
-    #
-    # for e in gg.freeProxyWallFirst():
-    #     print(e)
-    #
-    # for e in gg.freeProxyWallSecond():
-    #     print(e)
+
+    # test_batch(gg.freeProxyFirst())
+
+    # test_batch(gg.freeProxySecond())
+
+    # test_batch(gg.freeProxyFourth())
+
+    # test_batch(gg.freeProxyFifth())
+
+    # test_batch(gg.freeProxySixth())
+
+    # test_batch(gg.freeProxySeventh())
+
+    # to do
+    test_batch(gg.freeProxyEight())
+    # gg.freeProxyEight()
+
+    # test_batch(gg.freeProxyNinth())
+
+    # test_batch(gg.freeProxyWallFirst())
+
+    # test_batch(gg.freeProxyWallSecond())
+
+    # test_batch(gg.freeProxyWallThird())

Util/WebRequest.py

@@ -15,6 +15,7 @@ __author__ = 'J_hao'
 import requests
 import random
 import time
+from requests.models import Response
 
 
 class WebRequest(object):
@@ -51,7 +52,7 @@ class WebRequest(object):
                 'Accept-Language': 'zh-CN,zh;q=0.8'}
 
     def get(self, url, header=None, retry_time=5, timeout=30,
-            retry_flag=list(), retry_interval=5, *args, **kwargs):
+            retry_flag=list(), retry_interval=5, use_proxy=False, *args, **kwargs):
         """
         get method
         :param url: target url
@@ -60,6 +61,7 @@ class WebRequest(object):
         :param timeout: network timeout
         :param retry_flag: if retry_flag in content. do retry
         :param retry_interval: retry interval(second)
+        :param use_proxy: 是否使用代理
         :param args:
         :param kwargs:
         :return:
@@ -69,7 +71,16 @@ class WebRequest(object):
             headers.update(header)
         while True:
             try:
-                html = requests.get(url, headers=headers, timeout=timeout)
+                if use_proxy:
+                    proxy_url = "http://127.0.0.1:5010/get"
+                    ip_proxy = requests.get(proxy_url).text
+                    proxies = {
+                        "http": "http://" + ip_proxy,
+                        "https": "https://" + ip_proxy
+                    }
+                    html = requests.get(url, headers=headers, timeout=timeout, proxies=proxies)
+                else:
+                    html = requests.get(url, headers=headers, timeout=timeout)
                 if any(f in html.content for f in retry_flag):
                     raise Exception
                 return html
@@ -78,5 +89,8 @@ class WebRequest(object):
                 retry_time -= 1
                 if retry_time <= 0:
                     # 多次请求失败时，返回百度页面
-                    return requests.get("https://www.baidu.com/")
+                    resp = Response()
+                    resp.status_code = 200
+                    return resp
                 time.sleep(retry_interval)
+

requirements.txt

@@ -1,9 +1,9 @@
 APScheduler==3.2.0
-Flask==0.11.1
-requests==2.11.0
-lxml==3.7.1
+Flask
+requests
+lxml
 
-pymongo==3.2.2
-redis==2.10.5
+pymongo
+redis