Merge pull request #191 from YeClimEric/master

fix issues #190

Api/ProxyApi.py

@@ -2,13 +2,13 @@
 # !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     ProxyApi.py  
-   Description :  
+   File Name：     ProxyApi.py
+   Description :
    Author :       JHao
    date：          2016/12/4
 -------------------------------------------------
    Change Activity:
-                   2016/12/4: 
+                   2016/12/4:
 -------------------------------------------------
 """
 __author__ = 'JHao'
@@ -26,7 +26,6 @@ app = Flask(__name__)
 
 
 class JsonResponse(Response):
-
     @classmethod
     def force_type(cls, response, environ=None):
         if isinstance(response, (dict, list)):
@@ -86,7 +85,7 @@ def getStatus():
 
 def run():
     config = GetConfig()
-    app.run(host=config.host_ip, port=config.host_port)
+    app.run(host=config.host_ip, port=config.host_port, threaded=False, processes=config.processes)
 
 
 if __name__ == '__main__':

Config.ini

@@ -9,11 +9,11 @@ name = proxy
 
 [ProxyGetter]
 ;register the proxy getter function
-freeProxyFirst  = 1
+freeProxyFirst = 1
 freeProxySecond = 1
 ;freeProxyThird  = 1
 freeProxyFourth = 1
-freeProxyFifth  = 1
+freeProxyFifth = 1
 freeProxySixth = 1
 freeProxySeventh = 1
 freeProxyEight = 1
@@ -26,7 +26,9 @@ freeProxyWallFirst = 1
 freeProxyWallSecond = 1
 freeProxyWallThird = 1
 
-[HOST]
+[API]
 ; API接口配置 http://127.0.0.1:5010
 ip = 0.0.0.0
 port = 5010
+; flask多进程处理请求
+processes = 10

Dockerfile

@@ -3,28 +3,25 @@ WORKDIR /usr/src/app
 COPY . .
 ENV DEBIAN_FRONTEND noninteractive
 ENV TZ Asia/Shanghai
-RUN pip install --no-cache-dir -r requirements.txt && \
-	apt-get update && \
-	apt-get install -y --force-yes git make gcc g++ autoconf && apt-get clean && \
-	git clone --depth 1 https://github.com/ideawu/ssdb.git ssdb && \
-	cd ssdb && make && make install && cp ssdb-server /usr/bin && \
-	apt-get remove -y --force-yes git make gcc g++ autoconf && \
-	apt-get autoremove -y && \
-	rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
-	cp ssdb.conf /etc && cd .. && yes | rm -r ssdb && \
-	mkdir -p /var/lib/ssdb && \
-	sed \
-		-e 's@home.*@home /var/lib@' \
-		-e 's/loglevel.*/loglevel info/' \
-		-e 's@work_dir = .*@work_dir = /var/lib/ssdb@' \
-		-e 's@pidfile = .*@pidfile = /run/ssdb.pid@' \
-		-e 's@level:.*@level: info@' \
-		-e 's@ip:.*@ip: 0.0.0.0@' \
-		-i /etc/ssdb.conf && \
-	echo "# ! /bin/sh " > /usr/src/app/run.sh && \
-	echo "cd Run" >> /usr/src/app/run.sh && \
-	echo "/usr/bin/ssdb-server /etc/ssdb.conf &" >> /usr/src/app/run.sh && \
-	echo "python main.py" >> /usr/src/app/run.sh && \
-	chmod 777 run.sh
+
+RUN apt-get update
+RUN apt-get install vim -y
+
+RUN apt-get install -y redis-server
+RUN sed -i 's/^\(bind .*\)$/# \1/' /etc/redis/redis.conf \
+    && sed -i 's/^\(databases .*\)$/databases 1/' /etc/redis/redis.conf \
+    && sed -i 's/^\(daemonize .*\)$/daemonize yes/' /etc/redis/redis.conf
+#    && sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/redis/redis.conf  \
+#    && sed -i 's/^\(logfile .*\)$/# \1/' /etc/redis/redis.conf
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+
+RUN echo "# ! /bin/sh " > run.sh \
+    && echo "redis-server /etc/redis/redis.conf&" >> run.sh \
+	&& echo "cd Run" >> run.sh \
+	&& echo "python main.py" >> run.sh  \
+	&& chmod 777 run.sh
+
 EXPOSE 5010
 CMD [ "sh", "run.sh" ]

Manager/ProxyManager.py

@@ -2,13 +2,13 @@
 # !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     ProxyManager.py  
-   Description :  
+   File Name：     ProxyManager.py
+   Description :
    Author :       JHao
    date：          2016/12/3
 -------------------------------------------------
    Change Activity:
-                   2016/12/3: 
+                   2016/12/3:
 -------------------------------------------------
 """
 __author__ = 'JHao'
@@ -40,30 +40,22 @@ class ProxyManager(object):
         fetch proxy into Db by ProxyGetter
         :return:
         """
+        self.db.changeTable(self.raw_proxy_queue)
         for proxyGetter in self.config.proxy_getter_functions:
             # fetch
-            proxy_set = set()
             try:
                 self.log.info("{func}: fetch proxy start".format(func=proxyGetter))
-                proxy_iter = [_ for _ in getattr(GetFreeProxy, proxyGetter.strip())()]
+                for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
+                    # 挨个存储 proxy，优化raw 队列的 push 速度，进而加快 check proxy 的速度
+                    proxy = proxy.strip()
+                    if proxy and verifyProxyFormat(proxy):
+                        self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy))
+                        self.db.put(proxy)
+                    else:
+                        self.log.error('{func}: fetch proxy {proxy} error'.format(func=proxyGetter, proxy=proxy))
             except Exception as e:
                 self.log.error("{func}: fetch proxy fail".format(func=proxyGetter))
                 continue
-            for proxy in proxy_iter:
-                proxy = proxy.strip()
-                if proxy and verifyProxyFormat(proxy):
-                    self.log.info('{func}: fetch proxy {proxy}'.format(func=proxyGetter, proxy=proxy))
-                    proxy_set.add(proxy)
-                else:
-                    self.log.error('{func}: fetch proxy {proxy} error'.format(func=proxyGetter, proxy=proxy))
-
-            # store
-            for proxy in proxy_set:
-                self.db.changeTable(self.useful_proxy_queue)
-                if self.db.exists(proxy):
-                    continue
-                self.db.changeTable(self.raw_proxy_queue)
-                self.db.put(proxy)
 
     def get(self):
         """

Schedule/ProxyRefreshSchedule.py

@@ -18,7 +18,8 @@ import sys
 import time
 import logging
 from threading import Thread
-from apscheduler.schedulers.blocking import BlockingScheduler
+# 使用后台调度，不使用阻塞式~
+from apscheduler.schedulers.background import BackgroundScheduler as Sch
 
 sys.path.append('../')
 
@@ -73,12 +74,7 @@ def refreshPool():
     pp.validProxy()
 
 
-def main(process_num=30):
-    p = ProxyRefreshSchedule()
-
-    # 获取新代理
-    p.refresh()
-
+def batch_refresh(process_num=30):
     # 检验新代理
     pl = []
     for num in range(process_num):
@@ -93,11 +89,21 @@ def main(process_num=30):
         pl[num].join()
 
 
+def fetch_all():
+    p = ProxyRefreshSchedule()
+    # 获取新代理
+    p.refresh()
+
+
 def run():
-    main()
-    sch = BlockingScheduler()
-    sch.add_job(main, 'interval', minutes=10)  # 每10分钟抓取一次
+    sch = Sch()
+    sch.add_job(fetch_all, 'interval', minutes=5)  # 每5分钟抓取一次
+    sch.add_job(batch_refresh, "interval", minutes=1)  # 每分钟检查一次
     sch.start()
+    fetch_all()
+
+    while True:
+        time.sleep(1)
 
 
 if __name__ == '__main__':

Util/GetConfig.py

@@ -2,7 +2,7 @@
 # !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     GetConfig.py  
+   File Name：     GetConfig.py
    Description :  fetch config from config.ini
    Author :       JHao
    date：          2016/12/3
@@ -51,11 +51,15 @@ class GetConfig(object):
 
     @LazyProperty
     def host_ip(self):
-        return self.config_file.get('HOST','ip')
+        return self.config_file.get('API','ip')
 
     @LazyProperty
     def host_port(self):
-        return int(self.config_file.get('HOST', 'port'))
+        return int(self.config_file.get('API', 'port'))
+
+    @LazyProperty
+    def processes(self):
+        return int(self.config_file.get('API', 'processes'))
 
 if __name__ == '__main__':
     gg = GetConfig()
@@ -66,3 +70,4 @@ if __name__ == '__main__':
     print(gg.proxy_getter_functions)
     print(gg.host_ip)
     print(gg.host_port)
+    print(gg.processes)