[update] 为验证的代理不再入库，只储存useful :electric_plug:

DB/SsdbClient.py

@@ -2,9 +2,9 @@
 # !/usr/bin/env python
 """
 -------------------------------------------------
-   File Name：     SsdbClient.py
-   Description :  封装SSDB操作
-   Author :       JHao
+   File Name：     ssdbClient.py
+   Description :   封装SSDB操作
+   Author :        JHao
    date：          2016/12/2
 -------------------------------------------------
    Change Activity:

doc/release_notes.md

@@ -4,9 +4,10 @@
 
     1. 新增免费代理源 `西拉代理`  （2020-03-30）
     2. Fix Bug [#401](https://github.com/jhao104/proxy_pool/issues/401) [#356](https://github.com/jhao104/proxy_pool/issues/356)
-    3. 优化Docker镜像体积 (2020-06-19)
-    4. 优化配置
-    5. 优化代码结构
+    3. 优化Docker镜像体积; (2020-06-19)
+    4. 优化配置方式;
+    5. 优化代码结构;
+    6. 不再储存raw_proxy, 抓取后直接验证入库;
 
 * 2.0.1 (2020)
 

handler/configHandler.py

@@ -38,12 +38,8 @@ class ConfigHandler(object):
         return os.getenv("DB_CONN", setting.DB_CONN)
 
     @LazyProperty
-    def rawProxy(self):
-        return setting.RAW_PROXY
-
-    @LazyProperty
-    def useProxy(self):
-        return setting.USE_PROXY
+    def tableName(self):
+        return os.getenv("TABLE_NAME", setting.TABLE_NAME)
 
     @property
     def fetchers(self):

handler/proxyHandler.py

@@ -23,13 +23,13 @@ class ProxyHandler(object):
     def __init__(self):
         self.conf = ConfigHandler()
         self.db = DbClient(self.conf.dbConn)
+        self.db.changeTable(self.conf.tableName)
 
     def get(self):
         """
         return a useful proxy
         :return:
         """
-        self.db.changeTable(self.conf.useProxy)
         proxy = self.db.get()
         if proxy:
             return Proxy.createFromJson(proxy)
@@ -40,7 +40,6 @@ class ProxyHandler(object):
         return and delete a useful proxy
         :return:
         """
-        self.db.changeTable(self.conf.useProxy)
         proxy = self.db.pop()
         if proxy:
             return Proxy.createFromJson(proxy)
@@ -51,7 +50,6 @@ class ProxyHandler(object):
         put proxy into use proxy
         :return:
         """
-        self.db.changeTable(self.conf.useProxy)
         self.db.put(proxy)
 
     def delete(self, proxy):
@@ -60,7 +58,6 @@ class ProxyHandler(object):
         :param proxy:
         :return:
         """
-        self.db.changeTable(self.conf.useProxy)
         return self.db.delete(proxy.proxy)
 
     def getAll(self):
@@ -68,7 +65,6 @@ class ProxyHandler(object):
         get all proxy from pool as Proxy list
         :return:
         """
-        self.db.changeTable(self.conf.useProxy)
         proxies_dict = self.db.getAll()
         return [Proxy.createFromJson(value) for _, value in proxies_dict.items()]
 
@@ -78,7 +74,6 @@ class ProxyHandler(object):
         :param proxy:
         :return:
         """
-        self.db.changeTable(self.conf.useProxy)
         return self.db.exists(proxy.proxy)
 
     def getCount(self):
@@ -86,8 +81,5 @@ class ProxyHandler(object):
         return raw_proxy and use_proxy count
         :return:
         """
-        self.db.changeTable(self.conf.useProxy)
-        total_raw_proxy = self.db.getCount()
-        self.db.changeTable(self.conf.rawProxy)
         total_use_proxy = self.db.getCount()
-        return {'raw_proxy': total_raw_proxy, 'use_proxy': total_use_proxy}
+        return {'count': total_use_proxy}

setting.py

@@ -39,11 +39,8 @@ PORT = 5000
 #      Ssdb:  ssdb://:password@ip:port
 DB_CONN = 'redis://:_@Fintell@10.10.61.65:6379'
 
-# raw proxy db
-RAW_PROXY = 'raw_proxy'
-
-# useful proxy db
-USE_PROXY = 'use_proxy'
+# proxy table name
+TABLE_NAME = 'use_proxy'
 
 
 # ###### config the proxy fetch function ######