[update] 使用setting.py替换Config.ini配置文件

Api/ProxyApi.py

@@ -19,7 +19,7 @@ from flask import Flask, jsonify, request
 
 sys.path.append('../')
 
-from Util.GetConfig import config
+from Config.ConfigGetter import config
 from Manager.ProxyManager import ProxyManager
 
 app = Flask(__name__)

Config.ini

@@ -1,6 +1,6 @@
 [DB]
 ;Configure the database information
-;type: SSDB/MONGODB if use redis, only modify the host port，the type should be SSDB
+;type: SSDB/MONGODB if use redis, only modify the host port, the type should be SSDB
 type = SSDB
 host = 127.0.0.1
 port = 8888

Config/ConfigGetter.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     ConfigGetter
+   Description :   读取配置
+   Author :        JHao
+   date：          2019/2/15
+-------------------------------------------------
+   Change Activity:
+                   2019/2/15:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+
+from Util.utilClass import LazyProperty
+from Config.setting import *
+
+
+class ConfigGetter(object):
+    """
+    get config
+    """
+
+    def __init__(self):
+        pass
+
+    @LazyProperty
+    def db_type(self):
+        return DATABASES.get("default", {}).get("TYPE", "SSDB")
+
+    @LazyProperty
+    def db_name(self):
+        return DATABASES.get("default", {}).get("NAME", "proxy")
+
+    @LazyProperty
+    def db_host(self):
+        return DATABASES.get("default", {}).get("HOST", "127.0.0.1")
+
+    @LazyProperty
+    def db_port(self):
+        return DATABASES.get("default", {}).get("PORT", 8080)
+
+    @LazyProperty
+    def db_password(self):
+        return DATABASES.get("default", {}).get("PASSWORD", "")
+
+    @LazyProperty
+    def proxy_getter_functions(self):
+        return PROXY_GETTER
+
+    @LazyProperty
+    def host_ip(self):
+        return SERVER_API.get("HOST", "127.0.0.1")
+
+    @LazyProperty
+    def host_port(self):
+        return SERVER_API.get("PORT", 5010)
+
+
+config = ConfigGetter()
+
+if __name__ == '__main__':
+    print(config.db_type)
+    print(config.db_name)
+    print(config.db_host)
+    print(config.db_port)
+    print(config.proxy_getter_functions)
+    print(config.host_ip)
+    print(config.host_port)
+    print(config.db_password)

Test/__init__.py → Config/__init__.py

@@ -1,13 +1,12 @@
 # -*- coding: utf-8 -*-
 """
 -------------------------------------------------
-   File Name：     __init__.py
-   Description :   
-   Author :        J_hao
-   date：          2017/7/31
+   File Name：     __init__
+   Description :
+   Author :        JHao
+   date：          2019/2/15
 -------------------------------------------------
    Change Activity:
-                   2017/7/31:
+                   2019/2/15:
 -------------------------------------------------
 """
-__author__ = 'J_hao'

Config/setting.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     setting.py
+   Description :   配置文件
+   Author :        JHao
+   date：          2019/2/15
+-------------------------------------------------
+   Change Activity:
+                   2019/2/15:
+-------------------------------------------------
+"""
+
+# database config
+
+DATABASES = {
+    "default": {
+        "TYPE": "SSDB",  # TYPE SSDB/MONGODB if use redis, only modify the host port, the type should be SSDB
+        "HOST": "127.0.0.1",
+        "PORT": 8888,
+        "NAME": "proxy",
+        "PASSWORD": ""
+
+    }
+}
+
+# register the proxy getter function
+
+PROXY_GETTER = [
+    "freeProxyFirst",
+    "freeProxySecond",
+    # "freeProxyThird",
+    "freeProxyFourth",
+    "freeProxyFifth",
+    # "freeProxySixth"
+    "freeProxySeventh",
+    # "freeProxyEight",
+    # "freeProxyNinth",
+    "freeProxyTen",
+    "freeProxyEleven",
+    "freeProxyTwelve",
+    # foreign website, outside the wall
+    "freeProxyWallFirst",
+    "freeProxyWallSecond",
+    "freeProxyWallThird"
+]
+
+
+# # API config http://127.0.0.1:5010
+
+SERVER_API = {
+    "HOST": "0.0.0.0",  # The ip specified which starting the web API
+    "PORT": 5010  # port number to which the server listens to
+}

DB/DbClient.py

@@ -16,7 +16,7 @@ __author__ = 'JHao'
 import os
 import sys
 
-from Util.GetConfig import config
+from Config.ConfigGetter import config
 from Util.utilClass import Singleton
 
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))

DB/SsdbClient.py

@@ -3,7 +3,7 @@
 """
 -------------------------------------------------
    File Name：     SsdbClient.py
-   Description :  封装SSDB操作
+   Description :  封装SSDB/Redis操作
    Author :       JHao
    date：          2016/12/2
 -------------------------------------------------
@@ -27,7 +27,7 @@ class SsdbClient(object):
     SSDB client
 
     SSDB中代理存放的容器为hash：
-        原始代理存放在name为raw_proxy的hash中，key为代理的ip:port，value为为None,以后扩展可能会加入代理属性；
+        原始代理存放在name为raw_proxy的hash中，key为代理的ip:port，value为None,以后扩展可能会加入代理属性；
         验证后的代理存放在name为useful_proxy的hash中，key为代理的ip:port，value为一个计数,初始为1，每校验失败一次减1；
 
     """

Manager/ProxyManager.py

@@ -17,7 +17,7 @@ import random
 
 from Util import EnvUtil
 from DB.DbClient import DbClient
-from Util.GetConfig import config
+from Config.ConfigGetter import config
 from Util.LogHandler import LogHandler
 from Util.utilFunction import verifyProxyFormat
 from ProxyGetter.getFreeProxy import GetFreeProxy

Schedule/ProxyValidSchedule.py

@@ -56,8 +56,8 @@ class ProxyValidSchedule(ProxyManager, object):
                 self.log.info("Start valid useful proxy")
                 self.__validProxy()
             else:
-                self.log.info('Valid Complete! sleep 5 minutes.')
-                time.sleep(60 * 5)
+                self.log.info('Valid Complete! sleep 5 sec.')
+                time.sleep(5)
                 self.putQueue()
 
     def putQueue(self):

Test/.pytest_cache/v/cache/lastfailed

@@ -1,3 +0,0 @@
-{
-  "testGetFreeProxy.py::testGetFreeProxy": true
-}

Test/.pytest_cache/v/cache/nodeids

@@ -1,3 +0,0 @@
-[
-  "testGetFreeProxy.py::testGetFreeProxy"
-]

Test/testGetConfig.py → Test/testConfig.py

@@ -12,22 +12,22 @@
 """
 __author__ = 'J_hao'
 
-from Util.GetConfig import GetConfig
+from Config.ConfigGetter import config
 
 
 # noinspection PyPep8Naming
-def testGetConfig():
+def testConfig():
     """
-    test class GetConfig in Util/GetConfig
     :return:
     """
-    gg = GetConfig()
-    print(gg.db_type)
-    print(gg.db_name)
-    print(gg.db_host)
-    print(gg.db_port)
-    assert isinstance(gg.proxy_getter_functions, list)
-    print(gg.proxy_getter_functions)
+    print(config.db_type)
+    print(config.db_name)
+    print(config.db_host)
+    print(config.db_port)
+    print(config.db_password)
+    assert isinstance(config.proxy_getter_functions, list)
+    print(config.proxy_getter_functions)
+
 
 if __name__ == '__main__':
-    testGetConfig()
+    testConfig()

Util/GetConfig.py

@@ -13,8 +13,6 @@
 """
 __author__ = 'JHao'
 
-import os
-from Util.utilClass import ConfigParse
 from Util.utilClass import LazyProperty
 
 
@@ -24,10 +22,7 @@ class GetConfig(object):
     """
 
     def __init__(self):
-        self.pwd = os.path.split(os.path.realpath(__file__))[0]
-        self.config_path = os.path.join(os.path.split(self.pwd)[0], 'Config.ini')
-        self.config_file = ConfigParse(defaults={"password": None})
-        self.config_file.read(self.config_path)
+        pass
 
     @LazyProperty
     def db_type(self):

Util/utilClass.py

@@ -9,7 +9,6 @@
 -------------------------------------------------
    Change Activity:
                    2016/12/3: Class LazyProperty
-                   2016/12/4: rewrite ConfigParser
 -------------------------------------------------
 """
 __author__ = 'JHao'
@@ -33,24 +32,6 @@ class LazyProperty(object):
             return value
 
 
-try:
-    from configparser import ConfigParser  # py3
-except:
-    from ConfigParser import ConfigParser  # py2
-
-
-class ConfigParse(ConfigParser):
-    """
-    rewrite ConfigParser, for support upper option
-    """
-
-    def __init__(self, *args, **kwargs):
-        ConfigParser.__init__(self, *args, **kwargs)
-
-    def optionxform(self, optionstr):
-        return optionstr
-
-
 class Singleton(type):
     """
     Singleton Metaclass

test.py

@@ -12,4 +12,7 @@
 """
 __author__ = 'JHao'
 
-from Schedule import ProxyRefreshSchedule
+from Test import testConfig
+
+if __name__ == '__main__':
+    testConfig.testConfig()