1 optimized redis api

2 modified DBClient
3 fix getFreeProxy
4 add get_status api in ProxyApi

Api/ProxyApi.py

@@ -50,7 +50,7 @@ def refresh():
 @app.route('/get_all/')
 def getAll():
     proxys = ProxyManager().getAll()
-    return jsonify(proxys)
+    return jsonify(list(proxys))
 
 
 @app.route('/delete/', methods=['GET'])
@@ -59,6 +59,12 @@ def delete():
     ProxyManager().delete(proxy)
     return 'success'
 
+@app.route('/get_status/')
+def get_status():
+    status = ProxyManager().get_status()
+    return jsonify(status)
+
+
 
 if __name__ == '__main__':
-    app.run()
+    app.run(host='0.0.0.0', port=65432)

Config.ini

@@ -1,7 +1,7 @@
 [DB]
-type = SSDB
-host = security
-port = 8888
+type = REDIS
+host = localhost
+port = 6379
 name = proxy
 
 [ProxyGetter]
@@ -9,6 +9,6 @@ name = proxy
 freeProxyFirst  = 1
 freeProxySecond = 1
 freeProxyThird  = 1
-freeProxyFourth = 1
-freeProxyFifth  = 1
+;freeProxyFourth = 1
+;freeProxyFifth  = 1
 

DB/DbClient.py

@@ -15,6 +15,7 @@ __author__ = 'JHao'
 
 import os
 import sys
+
 from Util.GetConfig import GetConfig
 from Util.utilClass import Singleton
 
@@ -59,6 +60,8 @@ class DbClient(object):
         __type = None
         if "SSDB" == self.config.db_type:
             __type = "SsdbClient"
+        elif "REDIS" == self.config.db_type:
+            __type = "RedisClient"
         else:
             pass
         assert __type, 'type error, Not support DB type: {}'.format(self.config.db_type)
@@ -84,6 +87,9 @@ class DbClient(object):
     def changeTable(self, name):
         self.client.changeTable(name)
 
+    def get_status(self):
+        return self.client.get_status()
+
 
 if __name__ == "__main__":
     account = DbClient()

DB/RedisClient.py

@@ -5,11 +5,12 @@
 self.name为Redis中的一个key
 '''
 
-import random
 import json
+
 import redis
 
-class ReidsClient(object):
+
+class RedisClient(object):
     """
     Reids client
     """
@@ -27,12 +28,10 @@ class ReidsClient(object):
 
     def get(self):
         """
-        get an item
+        get random result
         :return:
         """
-        values = self.__conn.smembers(name=self.name)
-
-        return random.choice(list(values)) if values else None
+        return self.__conn.srandmember(name=self.name)
 
     def put(self, value):
         """
@@ -50,7 +49,7 @@ class ReidsClient(object):
         """
         value = self.get()
         if value:
-            self.__conn.spop(self.name, value)
+            self.__conn.spop(self.name)
         return value
 
     def delete(self, value):
@@ -64,4 +63,26 @@ class ReidsClient(object):
     def getAll(self):
         return self.__conn.smembers(self.name)
 
+    def get_status(self):
+        return self.__conn.scard(self.name)
+
+    def changeTable(self, name):
+        self.name = name
+
+
+if __name__ == '__main__':
+    redis_con = RedisClient('proxy', 'localhost', 6379)
+    # redis_con.put('abc')
+    # redis_con.put('123')
+    # redis_con.put('123.115.235.221:8800')
+    # print redis_con.getAll()
+    # redis_con.delete('abc')
+    # print redis_con.getAll()
+    # redis_con.pop()
+    # print redis_con.getAll()
+    redis_con.changeTable('raw_proxy')
 
+    redis_con.put('132.112.43.221:8888')
+    # redis_con.changeTable('proxy')
+    print redis_con.get_status()
+    print redis_con.getAll()

Manager/ProxyManager.py

@@ -51,7 +51,8 @@ class ProxyManager(object):
         :return:
         """
         self.db.changeTable(self.useful_proxy_queue)
-        return self.db.pop()
+        return self.db.get()
+        # return self.db.pop()
 
     def delete(self, proxy):
         """
@@ -70,7 +71,14 @@ class ProxyManager(object):
         self.db.changeTable(self.useful_proxy_queue)
         return self.db.getAll()
 
+    def get_status(self):
+        self.db.changeTable(self.raw_proxy_queue)
+        quan_raw_proxy = self.db.get_status()
+        self.db.changeTable(self.useful_proxy_queue)
+        quan_useful_queue = self.db.get_status()
+        return {'raw_proxy': quan_raw_proxy, 'useful_proxy_queue': quan_useful_queue}
 
 if __name__ == '__main__':
     pp = ProxyManager()
-    pp.refresh()
+    # pp.refresh()
+    print pp.get_status()

ProxyGetter/getFreeProxy.py

@@ -13,6 +13,7 @@
 """
 import re
 import sys
+
 import requests
 
 reload(sys)
@@ -23,6 +24,7 @@ from Util.utilFunction import robustCrawl, getHtmlTree
 # for debug to disable insecureWarning
 requests.packages.urllib3.disable_warnings()
 
+
 class GetFreeProxy(object):
     """
     proxy getter
@@ -41,8 +43,19 @@ class GetFreeProxy(object):
         """
         url_list = ('http://www.kuaidaili.com/proxylist/{page}/'.format(page=page) for page in range(1, page + 1))
         # 页数不用太多， 后面的全是历史IP， 可用性不高
+        header = {
+            'Host': 'www.kuaidaili.com',
+            'Connection': 'keep-alive',
+            'Cache-Control': 'max-age=0',
+            'Upgrade-Insecure-Requests': '1',
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Encoding': 'gzip, deflate, sdch',
+            'Accept-Language': 'zh-CN,zh;q=0.8',
+        }
+
         for url in url_list:
-            tree = getHtmlTree(url)
+            tree = getHtmlTree(url, header=header)
             proxy_list = tree.xpath('.//div[@id="index_free_list"]//tbody/tr')
             for proxy in proxy_list:
                 yield ':'.join(proxy.xpath('./td/text()')[0:2])
@@ -57,7 +70,7 @@ class GetFreeProxy(object):
         :return:
         """
         url = "http://m.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format(
-                proxy_number)
+            proxy_number)
         html = requests.get(url).content
         for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
             yield proxy
@@ -72,10 +85,25 @@ class GetFreeProxy(object):
         :return:
         """
         url = "http://www.youdaili.net/Daili/http/"
-        tree = getHtmlTree(url)
-        page_url_list = tree.xpath('.//div[@class="chunlist"]/ul//a/@href')[0:days]
+        header = {
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Encoding': 'gzip, deflate, sdch',
+            'Accept-Language': 'zh-CN,zh;q=0.8',
+            'Cache-Control': 'max-age=0',
+            'Connection': 'keep-alive',
+            # 'Cookie': 'Hm_lvt_f8bdd88d72441a9ad0f8c82db3113a84=1487640273; Hm_lpvt_f8bdd88d72441a9ad0f8c82db3113a84=1487640642',
+            'Host': 'www.youdaili.net',
+            # 'If-Modified-Since': 'Tue, 21 Feb 2017 00:40:52 GMT',
+            'If-None-Match': "5c67-548ffa1d8ca68-gzip",
+            'Upgrade-Insecure-Requests': '1',
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
+        }
+
+        tree = getHtmlTree(url, header=header)
+        page_url_list = tree.xpath('.//div[@class="chunlist"]/ul/li/p/a/@href')[0:days]
         for page_url in page_url_list:
-            html = requests.get(page_url).content
+            html = requests.get(page_url, headers=header).content
+            # print html
             proxy_list = re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html)
             for proxy in proxy_list:
                 yield proxy
@@ -102,7 +130,19 @@ class GetFreeProxy(object):
         :return:
         """
         url = "http://www.goubanjia.com/free/gngn/index.shtml"
-        tree = getHtmlTree(url)
+        header = {
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Encoding': 'gzip, deflate, sdch',
+            'Accept-Language': 'zh-CN,zh;q=0.8',
+            'Cache-Control': 'max-age=0',
+            'Connection': 'keep-alive',
+            # Cookie:auth=49b87d158a2b9e02589295bb9b74e8cf; JSESSIONID=BE57CA28BB0D8DDC119A542A75216B30; CNZZDATA1253707717=2116078297-1487635832-%7C1487641386; Hm_lvt_2e4ebee39b2c69a3920a396b87bbb8cc=1487641109; Hm_lpvt_2e4ebee39b2c69a3920a396b87bbb8cc=1487641408
+            'Host': 'www.goubanjia.com',
+            'Upgrade-Insecure-Requests': '1',
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
+        }
+
+        tree = getHtmlTree(url, header=header)
         # 现在每天最多放15个（一页）
         for i in xrange(15):
             d = tree.xpath('.//table[@class="table"]/tbody/tr[{}]/td'.format(i + 1))[0]
@@ -113,5 +153,17 @@ class GetFreeProxy(object):
 
 if __name__ == '__main__':
     gg = GetFreeProxy()
-    for e in gg.freeProxyFifth():
+    for e in gg.freeProxyFirst():
         print e
+
+    for e in gg.freeProxySecond():
+        print e
+
+    for e in gg.freeProxyThird():
+        print e
+
+    # for e in gg.freeProxyFourth():
+    #     print e
+    #
+    # for e in gg.freeProxyFifth():
+    #     print e

Schedule/ProxyRefreshSchedule.py

@@ -11,19 +11,19 @@
                    2016/12/4: 代理定时刷新
 -------------------------------------------------
 """
-__author__ = 'JHao'
-
-import sys
-import time
+import logging
+import random
 from multiprocessing import Process
 
 import requests
 from apscheduler.schedulers.blocking import BlockingScheduler
-
+import sys
 sys.path.append('../')
-
+from DB.DbClient import DbClient
 from Manager.ProxyManager import ProxyManager
 
+__author__ = 'JHao'
+
 
 class ProxyRefreshSchedule(ProxyManager):
     """
@@ -33,68 +33,115 @@ class ProxyRefreshSchedule(ProxyManager):
     def __init__(self):
         ProxyManager.__init__(self)
 
-    def validProxy(self):
+    def valid_proxy(self):
         self.db.changeTable(self.raw_proxy_queue)
         raw_proxy = self.db.pop()
+        print '[*]check raw proxy {} ...'.format(raw_proxy)
         while raw_proxy:
             proxies = {"http": "http://{proxy}".format(proxy=raw_proxy),
                        "https": "https://{proxy}".format(proxy=raw_proxy)}
             try:
-                r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=50, verify=False)
+                # 超过30秒的代理就不要了
+                r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=30, verify=False)
                 if r.status_code == 200:
                     self.db.changeTable(self.useful_proxy_queue)
                     self.db.put(raw_proxy)
-            except Exception as e:
-                # print e
+            except Exception, e:
+                print e
                 pass
             self.db.changeTable(self.raw_proxy_queue)
             raw_proxy = self.db.pop()
             print 'validate a  proxy'
 
+    def validate_useful_proxy(self, proxy_list):
+        self.db.changeTable(self.useful_proxy_queue)
+        for proxy in proxy_list:
+            print '[*]validating proxy : {} ...({} remained)'.format(proxy, len(proxy_list))
+            proxies = {"http": "http://{proxy}".format(proxy=proxy),
+                       "https": "https://{proxy}".format(proxy=proxy)}
+            try:
+                r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=30, verify=False)
+                if r.status_code == 200:
+                    continue
+            except Exception, e:
+                self.db.delete(proxy)
+                print '[-]delete proxy {}'.format(proxy)
+
+
+def refresh_pool():
+    pp = ProxyRefreshSchedule()
+    pp.valid_proxy()
+
 
-def refreshPool():
+def validate_user_proxy(proxy_list):
     pp = ProxyRefreshSchedule()
-    pp.validProxy()
+    pp.validate_useful_proxy(proxy_list)
 
 
-def main(process_num=100):
+def main(process_num=10):
     p = ProxyRefreshSchedule()
     p.refresh()
     pl = []
     for num in range(process_num):
-        P = Process(target=refreshPool, args=())
-        P.daemon = True
-        pl.append(P)
+        proc = Process(target=refresh_pool, args=())
+        proc.daemon = True
+        pl.append(proc)
 
     for num in range(process_num):
         pl[num].start()
 
-    print 'all proc start'
+    print 'All raw_proxy_crawler sub-processes start.'
 
     for num in range(process_num):
         pl[num].join()
 
 
-#
-#     print '{time}: refresh complete!'.format(time=time.ctime())
+def main_check(process_num=10):
+    db = DbClient()
+    useful_proxy_queue = 'useful_proxy_queue'
+    db.changeTable(useful_proxy_queue)
+    proxy_list = db.getAll()
 
+    uncheck_list = [list() for i in xrange(process_num)]
+    for proxy in proxy_list:
+        uncheck_list[random.randint(0, process_num - 1)].append(proxy)
 
-def main(process_num=100):
-    p = ProxyRefreshSchedule()
-    p.refresh()
+    pl = []
     for num in range(process_num):
-        P = Process(target=refreshPool, args=())
-        P.daemon = True
-        P.start()
-        P.join()
+        proc = Process(target=validate_user_proxy, args=(uncheck_list[num],))
+        proc.daemon = True
+        pl.append(proc)
 
+    for num in range(process_num):
+        pl[num].start()
 
-print '{time}: refresh complete!'.format(time=time.ctime())
+    print 'All proxy validator sub-processes start.'
+
+    for num in range(process_num):
+        pl[num].join()
+
+
+# def main(process_num=100):
+#     p = ProxyRefreshSchedule()
+#     p.refresh()
+#     for num in range(process_num):
+#         P = Process(target=refreshPool, args=())
+#         P.daemon = True
+#         P.start()
+#         P.join()
+# print '{time}: refresh complete!'.format(time=time.ctime())
 
 if __name__ == '__main__':
-    # pp = ProxyRefreshSchedule()
-    # pp.main()
-    main()
+    log = logging.getLogger('apscheduler')
+    log.setLevel(logging.INFO)  # DEBUG
+
+    fmt = logging.Formatter('%(levelname)s:%(name)s:%(message)s')
+    h = logging.StreamHandler()
+    h.setFormatter(fmt)
+    log.addHandler(h)
+
+    # main()
     sched = BlockingScheduler()
-    sched.add_job(main, 'interval', seconds=2)
+    # sched.add_job(main, 'interval', seconds=10)
+    sched.add_job(main_check, 'interval', seconds=15)
     sched.start()

Util/utilFunction.py

@@ -36,7 +36,7 @@ def verifyProxy(proxy):
     return True if re.findall(verify_regex, proxy) else False
 
 
-def getHtmlTree(url, **kwargs):
+def getHtmlTree(url, header=None, **kwargs):
     """
     获取html树
     :param url:
@@ -45,5 +45,5 @@ def getHtmlTree(url, **kwargs):
     """
     import requests
     from lxml import etree
-    html = requests.get(url=url).content
+    html = requests.get(url=url, headers=header).content
     return etree.HTML(html)