Merge branch 'master' of github.com:jhao104/proxy_pool into develop

Api/ProxyApi.py

@@ -19,7 +19,7 @@ from flask import Flask, jsonify, request
 
 sys.path.append('../')
 
-from Util.GetConfig import GetConfig
+from Util.GetConfig import config
 from Manager.ProxyManager import ProxyManager
 
 app = Flask(__name__)
@@ -84,7 +84,6 @@ def getStatus():
 
 
 def run():
-    config = GetConfig()
     if sys.platform.startswith("win"):
         app.run(host=config.host_ip, port=config.host_port)
     else:

Config.ini

@@ -5,7 +5,8 @@ type = SSDB
 host = 127.0.0.1
 port = 6379
 name = proxy
-#password = yourpassword
+;username = your_username (Only Mongodb)
+;password = your_password
 
 [ProxyGetter]
 ;register the proxy getter function

DB/DbClient.py

@@ -16,7 +16,7 @@ __author__ = 'JHao'
 import os
 import sys
 
-from Util.GetConfig import GetConfig
+from Util.GetConfig import config
 from Util.utilClass import Singleton
 
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
@@ -55,7 +55,6 @@ class DbClient(object):
         init
         :return:
         """
-        self.config = GetConfig()
         self.__initDbClient()
 
     def __initDbClient(self):
@@ -64,19 +63,19 @@ class DbClient(object):
         :return:
         """
         __type = None
-        if "SSDB" == self.config.db_type:
+        if "SSDB" == config.db_type:
             __type = "SsdbClient"
-        elif "REDIS" == self.config.db_type:
+        elif "REDIS" == config.db_type:
             __type = "RedisClient"
-        elif "MONGODB" == self.config.db_type:
+        elif "MONGODB" == config.db_type:
             __type = "MongodbClient"
         else:
             pass
-        assert __type, 'type error, Not support DB type: {}'.format(self.config.db_type)
-        self.client = getattr(__import__(__type), __type)(name=self.config.db_name,
-                                                          host=self.config.db_host,
-                                                          port=self.config.db_port,
-                                                          password=self.config.db_password)
+        assert __type, 'type error, Not support DB type: {}'.format(config.db_type)
+        self.client = getattr(__import__(__type), __type)(name=config.db_name,
+                                                          host=config.db_host,
+                                                          port=config.db_port,
+                                                          password=config.db_password)
 
     def get(self, key, **kwargs):
         return self.client.get(key, **kwargs)

DB/MongodbClient.py

@@ -17,9 +17,9 @@ from pymongo import MongoClient
 
 
 class MongodbClient(object):
-    def __init__(self, name, host, port):
+    def __init__(self, name, host, port, **kwargs):
         self.name = name
-        self.client = MongoClient(host, port)
+        self.client = MongoClient(host, port, **kwargs)
         self.db = self.client.proxy
 
     def changeTable(self, name):

DB/RedisClient.py

@@ -22,7 +22,11 @@ class RedisClient(object):
     Reids client
     """
 
-    def __init__(self, name, host, port):
+    # 为了保持DbClient的标准
+    # 在RedisClient里面接受username参数, 但不进行使用.
+    # 因为不能将username通过kwargs传进redis.Redis里面, 会报错:
+    # TypeError: __init__() got an unexpected keyword argument 'username'
+    def __init__(self, name, host, port, username, **kwargs):
         """
         init
         :param name:
@@ -31,7 +35,7 @@ class RedisClient(object):
         :return:
         """
         self.name = name
-        self.__conn = redis.Redis(host=host, port=port, db=0)
+        self.__conn = redis.Redis(host=host, port=port, db=0, **kwargs)
 
     def get(self):
         """

DB/SsdbClient.py

@@ -31,8 +31,11 @@ class SsdbClient(object):
         验证后的代理存放在name为useful_proxy的hash中，key为代理的ip:port，value为一个计数,初始为1，每校验失败一次减1；
 
     """
-
-    def __init__(self, name, **kwargs):
+    # 为了保持DbClient的标准
+    # 在SsdbClient里面接受username参数, 但不进行使用.
+    # 因为不能将username通过kwargs传进redis.Redis里面, 会报错:
+    # TypeError: __init__() got an unexpected keyword argument 'username'
+    def __init__(self, name, username, **kwargs):
         """
         init
         :param name: hash name

Manager/ProxyManager.py

@@ -17,7 +17,7 @@ import random
 
 from Util import EnvUtil
 from DB.DbClient import DbClient
-from Util.GetConfig import GetConfig
+from Util.GetConfig import config
 from Util.LogHandler import LogHandler
 from Util.utilFunction import verifyProxyFormat
 from ProxyGetter.getFreeProxy import GetFreeProxy
@@ -30,7 +30,6 @@ class ProxyManager(object):
 
     def __init__(self):
         self.db = DbClient()
-        self.config = GetConfig()
         self.raw_proxy_queue = 'raw_proxy'
         self.log = LogHandler('proxy_manager')
         self.useful_proxy_queue = 'useful_proxy'
@@ -41,7 +40,7 @@ class ProxyManager(object):
         :return:
         """
         self.db.changeTable(self.raw_proxy_queue)
-        for proxyGetter in self.config.proxy_getter_functions:
+        for proxyGetter in config.proxy_getter_functions:
             # fetch
             try:
                 self.log.info("{func}: fetch proxy start".format(func=proxyGetter))

Util/GetConfig.py

@@ -53,14 +53,13 @@ class GetConfig(object):
             password = None
         return password
 
-
     @LazyProperty
     def proxy_getter_functions(self):
         return self.config_file.options('ProxyGetter')
 
     @LazyProperty
     def host_ip(self):
-        return self.config_file.get('API','ip')
+        return self.config_file.get('API', 'ip')
 
     @LazyProperty
     def host_port(self):
@@ -70,6 +69,9 @@ class GetConfig(object):
     def processes(self):
         return int(self.config_file.get('API', 'processes'))
 
+
+config = GetConfig()
+
 if __name__ == '__main__':
     gg = GetConfig()
     print(gg.db_type)