Merge pull request #20 from StSphinx/dev_#11

Dev #11

.gitignore

@@ -1,2 +1,3 @@
 .idea/
-*.pyc
+*.pyc
+test/

Api/ProxyApi.py

@@ -13,9 +13,10 @@
 """
 __author__ = 'JHao'
 
-from flask import Flask, jsonify, request
 import sys
 
+from flask import Flask, jsonify, request
+
 sys.path.append('../')
 
 from Manager.ProxyManager import ProxyManager
@@ -50,7 +51,7 @@ def refresh():
 @app.route('/get_all/')
 def getAll():
     proxys = ProxyManager().getAll()
-    return jsonify(proxys)
+    return jsonify(list(proxys))
 
 
 @app.route('/delete/', methods=['GET'])
@@ -60,5 +61,11 @@ def delete():
     return 'success'
 
 
+@app.route('/get_status/')
+def get_status():
+    status = ProxyManager().get_status()
+    return jsonify(status)
+
+
 if __name__ == '__main__':
-    app.run()
+    app.run(host='0.0.0.0', port=65432)

Api/__init__.py

@@ -10,4 +10,5 @@
                    2016/12/3: 
 -------------------------------------------------
 """
-__author__ = 'JHao'
+__author__ = 'JHao'
+

Config.ini

@@ -1,7 +1,7 @@
 [DB]
-type = SSDB
-host = security
-port = 57888
+type = REDIS
+host = localhost
+port = 6379
 name = proxy
 
 [ProxyGetter]
@@ -9,6 +9,6 @@ name = proxy
 freeProxyFirst  = 1
 freeProxySecond = 1
 freeProxyThird  = 1
-freeProxyFourth = 1
+;freeProxyFourth = 1
 freeProxyFifth  = 1
 

DB/DbClient.py

@@ -15,6 +15,7 @@ __author__ = 'JHao'
 
 import os
 import sys
+
 from Util.GetConfig import GetConfig
 from Util.utilClass import Singleton
 
@@ -59,6 +60,8 @@ class DbClient(object):
         __type = None
         if "SSDB" == self.config.db_type:
             __type = "SsdbClient"
+        elif "REDIS" == self.config.db_type:
+            __type = "RedisClient"
         else:
             pass
         assert __type, 'type error, Not support DB type: {}'.format(self.config.db_type)
@@ -84,6 +87,9 @@ class DbClient(object):
     def changeTable(self, name):
         self.client.changeTable(name)
 
+    def get_status(self):
+        return self.client.get_status()
+
 
 if __name__ == "__main__":
     account = DbClient()

DB/RedisClient.py

@@ -5,11 +5,12 @@
 self.name为Redis中的一个key
 '''
 
-import random
 import json
+
 import redis
 
-class ReidsClient(object):
+
+class RedisClient(object):
     """
     Reids client
     """
@@ -27,12 +28,10 @@ class ReidsClient(object):
 
     def get(self):
         """
-        get an item
+        get random result
         :return:
         """
-        values = self.__conn.smembers(name=self.name)
-
-        return random.choice(list(values)) if values else None
+        return self.__conn.srandmember(name=self.name)
 
     def put(self, value):
         """
@@ -50,7 +49,7 @@ class ReidsClient(object):
         """
         value = self.get()
         if value:
-            self.__conn.spop(self.name, value)
+            self.__conn.spop(self.name)
         return value
 
     def delete(self, value):
@@ -64,4 +63,26 @@ class ReidsClient(object):
     def getAll(self):
         return self.__conn.smembers(self.name)
 
+    def get_status(self):
+        return self.__conn.scard(self.name)
+
+    def changeTable(self, name):
+        self.name = name
+
+
+if __name__ == '__main__':
+    redis_con = RedisClient('proxy', 'localhost', 6379)
+    # redis_con.put('abc')
+    # redis_con.put('123')
+    # redis_con.put('123.115.235.221:8800')
+    # print redis_con.getAll()
+    # redis_con.delete('abc')
+    # print redis_con.getAll()
+    # redis_con.pop()
+    # print redis_con.getAll()
+    redis_con.changeTable('raw_proxy')
 
+    redis_con.put('132.112.43.221:8888')
+    # redis_con.changeTable('proxy')
+    print redis_con.get_status()
+    print redis_con.getAll()

Manager/ProxyManager.py

@@ -51,7 +51,8 @@ class ProxyManager(object):
         :return:
         """
         self.db.changeTable(self.useful_proxy_queue)
-        return self.db.pop()
+        return self.db.get()
+        # return self.db.pop()
 
     def delete(self, proxy):
         """
@@ -70,7 +71,14 @@ class ProxyManager(object):
         self.db.changeTable(self.useful_proxy_queue)
         return self.db.getAll()
 
+    def get_status(self):
+        self.db.changeTable(self.raw_proxy_queue)
+        quan_raw_proxy = self.db.get_status()
+        self.db.changeTable(self.useful_proxy_queue)
+        quan_useful_queue = self.db.get_status()
+        return {'raw_proxy': quan_raw_proxy, 'useful_proxy_queue': quan_useful_queue}
 
 if __name__ == '__main__':
     pp = ProxyManager()
-    pp.refresh()
+    # pp.refresh()
+    print pp.get_status()

ProxyGetter/getFreeProxy.py

@@ -13,6 +13,7 @@
 """
 import re
 import sys
+
 import requests
 
 reload(sys)
@@ -20,6 +21,9 @@ sys.setdefaultencoding('utf-8')
 
 from Util.utilFunction import robustCrawl, getHtmlTree
 
+# for debug to disable insecureWarning
+requests.packages.urllib3.disable_warnings()
+
 
 class GetFreeProxy(object):
     """
@@ -39,11 +43,23 @@ class GetFreeProxy(object):
         """
         url_list = ('http://www.kuaidaili.com/proxylist/{page}/'.format(page=page) for page in range(1, page + 1))
         # 页数不用太多， 后面的全是历史IP， 可用性不高
+        header = {
+            'Host': 'www.kuaidaili.com',
+            'Connection': 'keep-alive',
+            'Cache-Control': 'max-age=0',
+            'Upgrade-Insecure-Requests': '1',
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Encoding': 'gzip, deflate, sdch',
+            'Accept-Language': 'zh-CN,zh;q=0.8',
+        }
+
         for url in url_list:
-            tree = getHtmlTree(url)
+            tree = getHtmlTree(url, header=header)
             proxy_list = tree.xpath('.//div[@id="index_free_list"]//tbody/tr')
             for proxy in proxy_list:
                 yield ':'.join(proxy.xpath('./td/text()')[0:2])
+        print 'finish kuaidaili fetching proxy ip'
 
     @staticmethod
     @robustCrawl
@@ -54,10 +70,11 @@ class GetFreeProxy(object):
         :return:
         """
         url = "http://m.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format(
-                proxy_number)
+            proxy_number)
         html = requests.get(url).content
         for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
             yield proxy
+        print 'finish 66ip fetching proxy ip'
 
     @staticmethod
     @robustCrawl
@@ -68,13 +85,29 @@ class GetFreeProxy(object):
         :return:
         """
         url = "http://www.youdaili.net/Daili/http/"
-        tree = getHtmlTree(url)
-        page_url_list = tree.xpath('.//div[@class="chunlist"]/ul//a/@href')[0:days]
+        header = {
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Encoding': 'gzip, deflate, sdch',
+            'Accept-Language': 'zh-CN,zh;q=0.8',
+            'Cache-Control': 'max-age=0',
+            'Connection': 'keep-alive',
+            # 'Cookie': 'Hm_lvt_f8bdd88d72441a9ad0f8c82db3113a84=1487640273; Hm_lpvt_f8bdd88d72441a9ad0f8c82db3113a84=1487640642',
+            'Host': 'www.youdaili.net',
+            # 'If-Modified-Since': 'Tue, 21 Feb 2017 00:40:52 GMT',
+            'If-None-Match': "5c67-548ffa1d8ca68-gzip",
+            'Upgrade-Insecure-Requests': '1',
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
+        }
+
+        tree = getHtmlTree(url, header=header)
+        page_url_list = tree.xpath('.//div[@class="chunlist"]/ul/li/p/a/@href')[0:days]
         for page_url in page_url_list:
-            html = requests.get(page_url).content
+            html = requests.get(page_url, headers=header).content
+            # print html
             proxy_list = re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html)
             for proxy in proxy_list:
                 yield proxy
+        print 'finish youdaili fetching proxy ip'
 
     @staticmethod
     @robustCrawl
@@ -87,6 +120,7 @@ class GetFreeProxy(object):
         html = requests.get(url).content
         for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
             yield proxy
+        print 'finish xici fetching proxy ip'
 
     @staticmethod
     @robustCrawl
@@ -96,15 +130,44 @@ class GetFreeProxy(object):
         :return:
         """
         url = "http://www.goubanjia.com/free/gngn/index.shtml"
-        tree = getHtmlTree(url)
+        header = {
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+            'Accept-Encoding': 'gzip, deflate, sdch',
+            'Accept-Language': 'zh-CN,zh;q=0.8',
+            'Cache-Control': 'max-age=0',
+            'Connection': 'keep-alive',
+            # Cookie:auth=49b87d158a2b9e02589295bb9b74e8cf; JSESSIONID=BE57CA28BB0D8DDC119A542A75216B30; CNZZDATA1253707717=2116078297-1487635832-%7C1487641386; Hm_lvt_2e4ebee39b2c69a3920a396b87bbb8cc=1487641109; Hm_lpvt_2e4ebee39b2c69a3920a396b87bbb8cc=1487641408
+            'Host': 'www.goubanjia.com',
+            'Upgrade-Insecure-Requests': '1',
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
+        }
+
+        tree = getHtmlTree(url, header=header)
         # 现在每天最多放15个（一页）
         for i in xrange(15):
-            d = tree.xpath('.//table[@class="table"]/tbody/tr[{}]/td'.format(i + 1))[0]
+            d = tree.xpath('//*[@id="list"]/table/tbody/tr[{}]/td'.format(i + 1))[0]
+            # print d
+
             o = d.xpath('.//span/text() | .//div/text()')
+            # print o
+
             yield ''.join(o[:-1]) + ':' + o[-1]
+        print 'finish guobanjia fetching proxy ip'
 
 
 if __name__ == '__main__':
     gg = GetFreeProxy()
+    # for e in gg.freeProxyFirst():
+    #     print e
+    #
+    # for e in gg.freeProxySecond():
+    #     print e
+    #
+    # for e in gg.freeProxyThird():
+    #     print e
+
+    # for e in gg.freeProxyFourth():
+    #     print e
+    #
     for e in gg.freeProxyFifth():
         print e

Schedule/ProxyRefreshSchedule.py

@@ -11,18 +11,24 @@
                    2016/12/4: 代理定时刷新
 -------------------------------------------------
 """
-__author__ = 'JHao'
-
-from apscheduler.schedulers.blocking import BlockingScheduler
+import logging
+import os
+import random
+import sys
+import time
 from multiprocessing import Process
+
 import requests
-import time
-import sys
+from apscheduler.schedulers.blocking import BlockingScheduler
 
 sys.path.append('../')
-
+from DB.DbClient import DbClient
 from Manager.ProxyManager import ProxyManager
 
+__author__ = 'JHao'
+
+log = logging.getLogger('apscheduler')
+
 
 class ProxyRefreshSchedule(ProxyManager):
     """
@@ -32,43 +38,148 @@ class ProxyRefreshSchedule(ProxyManager):
     def __init__(self):
         ProxyManager.__init__(self)
 
-    def validProxy(self):
+    def valid_proxy(self):
+        logger_raw = logging.getLogger('apscheduler.raw_check-{}'.format(os.getpid()))
+
+        fmt = logging.Formatter('%(asctime)s - %(levelname)s : %(name)s : %(message)s')
+        fh = logging.FileHandler(filename='../log/raw_proxy_log.txt')
+        fh.setFormatter(fmt=fmt)
+        logger_raw.addHandler(fh)
+
         self.db.changeTable(self.raw_proxy_queue)
         raw_proxy = self.db.pop()
         while raw_proxy:
+            logger_raw.debug('[*] check raw proxy {} ...'.format(raw_proxy))
+            # print '[*] check raw proxy {} ...'.format(raw_proxy)
             proxies = {"http": "http://{proxy}".format(proxy=raw_proxy),
                        "https": "https://{proxy}".format(proxy=raw_proxy)}
             try:
-                r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=50, verify=False)
+                # 超过30秒的代理就不要了
+                r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=30, verify=False)
                 if r.status_code == 200:
                     self.db.changeTable(self.useful_proxy_queue)
                     self.db.put(raw_proxy)
-            except Exception as e:
-                # print e
+                    logger_raw.debug('[+] raw proxy {} succeed validating...'.format(raw_proxy))
+                    # print '[+] raw proxy {} succeed validating...'.format(raw_proxy)
+            except Exception, e:
+                print e
                 pass
             self.db.changeTable(self.raw_proxy_queue)
             raw_proxy = self.db.pop()
+            logger_raw.debug('[-] raw proxy {} invalid'.format(raw_proxy))
+            # print '[-] raw proxy {} invalid'.format(raw_proxy)
+
+    def validate_useful_proxy(self, proxy_list):
+        logger_avail = logging.getLogger('apscheduler.avail_check-{}'.format(os.getpid()))
+
+        fmt = logging.Formatter('%(asctime)s - %(levelname)s : %(name)s : %(message)s')
+        fh = logging.FileHandler(filename='../log/available_proxy_log.txt')
+        fh.setFormatter(fmt=fmt)
+        logger_avail.addHandler(fh)
+
+        len_proxy = len(proxy_list)
+
+        self.db.changeTable(self.useful_proxy_queue)
+        while proxy_list:
+            proxy = proxy_list.pop()
+            logger_avail.debug('[*] check available proxy : {} ...({} remained)'.format(proxy, len(proxy_list)))
+            # print '[*] check available proxy : {} ...({} remained)'.format(proxy, len(proxy_list))
+            proxies = {"http": "http://{proxy}".format(proxy=proxy),
+                       "https": "https://{proxy}".format(proxy=proxy)}
+            try:
+                r = requests.get('https://www.baidu.com/', proxies=proxies, timeout=30, verify=False)
+                if r.status_code == 200:
+                    logger_avail.debug('[+] proxy {} is still available'.format(proxy))
+                    continue
+            except Exception, e:
+                self.db.delete(proxy)
+                logger_avail.debug('[-] A checked proxy {} has been removed from useful_proxy_queue'.format(proxy))
+                # print '[-] A checked proxy {} has been removed from useful_proxy_queue'.format(proxy)
+        logger_avail.debug('Process {} finished checking {} proxies.'.format(os.getpid(), len_proxy))
+
+
+def refresh_pool():
+    pp = ProxyRefreshSchedule()
+    pp.valid_proxy()
 
 
-def refreshPool():
+def validate_user_proxy(proxy_list):
     pp = ProxyRefreshSchedule()
-    pp.validProxy()
+    pp.validate_useful_proxy(proxy_list)
 
 
-def main(process_num=100):
+def main(process_num=10):
     p = ProxyRefreshSchedule()
     p.refresh()
+    pl = []
+    for num in range(process_num):
+        proc = Process(target=refresh_pool, args=())
+        # proc.daemon = True
+        pl.append(proc)
+
+    for num in range(process_num):
+        pl[num].start()
+
+    for num in range(process_num):
+        pl[num].join()
+
+    log.debug('Process main completed.')
 
+
+def main_check(process_num=10):
+    db = DbClient()
+    useful_proxy_queue = 'useful_proxy_queue'
+    db.changeTable(useful_proxy_queue)
+    proxy_list = db.getAll()
+
+    uncheck_list = [list() for i in xrange(process_num)]
+    for proxy in proxy_list:
+        uncheck_list[random.randint(0, process_num - 1)].append(proxy)
+
+    pl = []
     for num in range(process_num):
-        P = Process(target=refreshPool, args=())
-        P.start()
-    print '{time}: refresh complete!'.format(time=time.ctime())
+        proc = Process(target=validate_user_proxy, args=(uncheck_list[num],))
+        # proc.daemon = True
+        pl.append(proc)
+
+    for num in range(process_num):
+        pl[num].start()
+
+    for num in range(process_num):
+        pl[num].join()
+
+    log.debug('Process main_check completed.')
+
+
+to_time = time.time()
+
+
+def test():
+    print 'start test', time.time() - to_time
+    time.sleep(20)
+    print 'end test', time.time() - to_time
+
+
+def test2():
+    print 'start test2', time.time() - to_time
+    time.sleep(40)
+    print 'end test2', time.time() - to_time
 
 
 if __name__ == '__main__':
-    # pp = ProxyRefreshSchedule()
-    # pp.main()
-    main()
+    log.setLevel(logging.DEBUG)  # DEBUG
+
+    fmt = logging.Formatter('%(asctime)s - %(levelname)s : %(name)s : %(message)s')
+    h = logging.StreamHandler()
+    h.setFormatter(fmt)
+    log.addHandler(h)
+
+    fh = logging.FileHandler(filename='../log/log.txt')
+    fh.setFormatter(fmt=fmt)
+    log.addHandler(fh)
+
+    # main()
     sched = BlockingScheduler()
-    sched.add_job(main, 'interval', minutes=20)
+    sched.add_job(main, 'interval', minutes=10)
+    sched.add_job(main_check, 'interval', minutes=15)
     sched.start()

Util/utilFunction.py

@@ -36,7 +36,7 @@ def verifyProxy(proxy):
     return True if re.findall(verify_regex, proxy) else False
 
 
-def getHtmlTree(url, **kwargs):
+def getHtmlTree(url, header=None, **kwargs):
     """
     获取html树
     :param url:
@@ -45,5 +45,5 @@ def getHtmlTree(url, **kwargs):
     """
     import requests
     from lxml import etree
-    html = requests.get(url=url).content
+    html = requests.get(url=url, headers=header).content
     return etree.HTML(html)