Merge pull request #245 from jhao104/develop

使用setting.py替换Config.ini配置文件

Api/ProxyApi.py

@@ -19,7 +19,7 @@ from flask import Flask, jsonify, request
 
 sys.path.append('../')
 
-from Util.GetConfig import config
+from Config.ConfigGetter import config
 from Manager.ProxyManager import ProxyManager
 
 app = Flask(__name__)

Config.ini

@@ -1,35 +0,0 @@
-[DB]
-;Configure the database information
-;type: SSDB/MONGODB if use redis, only modify the host port，the type should be SSDB
-type = SSDB
-host = 127.0.0.1
-port = 8888
-name = proxy
-;username = your_username (Only Mongodb)
-;password = your_password
-
-[ProxyGetter]
-;register the proxy getter function
-freeProxyFirst = 1
-freeProxySecond = 1
-;freeProxyThird  = 1
-freeProxyFourth = 1
-freeProxyFifth = 1
-;freeProxySixth = 1
-freeProxySeventh = 1
-;freeProxyEight = 1
-;freeProxyNinth = 1
-freeProxyTen = 1
-freeProxyEleven = 1
-freeProxyTwelve = 1
-;foreign website, outside the wall
-;freeProxyWallFirst = 1
-;freeProxyWallSecond = 1
-;freeProxyWallThird = 1
-
-[API]
-# API config http://127.0.0.1:5010
-# The ip specified when starting the web API
-ip = 0.0.0.0
-# he port on which to run the web API
-port = 8080

Config/ConfigGetter.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     ConfigGetter
+   Description :   读取配置
+   Author :        JHao
+   date：          2019/2/15
+-------------------------------------------------
+   Change Activity:
+                   2019/2/15:
+-------------------------------------------------
+"""
+__author__ = 'JHao'
+
+
+from Util.utilClass import LazyProperty
+from Config.setting import *
+
+
+class ConfigGetter(object):
+    """
+    get config
+    """
+
+    def __init__(self):
+        pass
+
+    @LazyProperty
+    def db_type(self):
+        return DATABASES.get("default", {}).get("TYPE", "SSDB")
+
+    @LazyProperty
+    def db_name(self):
+        return DATABASES.get("default", {}).get("NAME", "proxy")
+
+    @LazyProperty
+    def db_host(self):
+        return DATABASES.get("default", {}).get("HOST", "127.0.0.1")
+
+    @LazyProperty
+    def db_port(self):
+        return DATABASES.get("default", {}).get("PORT", 8080)
+
+    @LazyProperty
+    def db_password(self):
+        return DATABASES.get("default", {}).get("PASSWORD", "")
+
+    @LazyProperty
+    def proxy_getter_functions(self):
+        return PROXY_GETTER
+
+    @LazyProperty
+    def host_ip(self):
+        return SERVER_API.get("HOST", "127.0.0.1")
+
+    @LazyProperty
+    def host_port(self):
+        return SERVER_API.get("PORT", 5010)
+
+
+config = ConfigGetter()
+
+if __name__ == '__main__':
+    print(config.db_type)
+    print(config.db_name)
+    print(config.db_host)
+    print(config.db_port)
+    print(config.proxy_getter_functions)
+    print(config.host_ip)
+    print(config.host_port)
+    print(config.db_password)

Config/__init__.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     __init__
+   Description :
+   Author :        JHao
+   date：          2019/2/15
+-------------------------------------------------
+   Change Activity:
+                   2019/2/15:
+-------------------------------------------------
+"""

Config/setting.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""
+-------------------------------------------------
+   File Name：     setting.py
+   Description :   配置文件
+   Author :        JHao
+   date：          2019/2/15
+-------------------------------------------------
+   Change Activity:
+                   2019/2/15:
+-------------------------------------------------
+"""
+
+# database config
+
+DATABASES = {
+    "default": {
+        "TYPE": "SSDB",  # TYPE SSDB/MONGODB if use redis, only modify the host port, the type should be SSDB
+        "HOST": "127.0.0.1",
+        "PORT": 8888,
+        "NAME": "proxy",
+        "PASSWORD": ""
+
+    }
+}
+
+# register the proxy getter function
+
+PROXY_GETTER = [
+    "freeProxyFirst",
+    "freeProxySecond",
+    # "freeProxyThird",
+    "freeProxyFourth",
+    "freeProxyFifth",
+    # "freeProxySixth"
+    "freeProxySeventh",
+    # "freeProxyEight",
+    # "freeProxyNinth",
+    "freeProxyTen",
+    "freeProxyEleven",
+    "freeProxyTwelve",
+    # foreign website, outside the wall
+    "freeProxyWallFirst",
+    "freeProxyWallSecond",
+    "freeProxyWallThird"
+]
+
+
+# # API config http://127.0.0.1:5010
+
+SERVER_API = {
+    "HOST": "0.0.0.0",  # The ip specified which starting the web API
+    "PORT": 5010  # port number to which the server listens to
+}

DB/DbClient.py

@@ -16,7 +16,7 @@ __author__ = 'JHao'
 import os
 import sys
 
-from Util.GetConfig import config
+from Config.ConfigGetter import config
 from Util.utilClass import Singleton
 
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))

DB/SsdbClient.py

@@ -3,7 +3,7 @@
 """
 -------------------------------------------------
    File Name：     SsdbClient.py
-   Description :  封装SSDB操作
+   Description :  封装SSDB/Redis操作
    Author :       JHao
    date：          2016/12/2
 -------------------------------------------------
@@ -27,7 +27,7 @@ class SsdbClient(object):
     SSDB client
 
     SSDB中代理存放的容器为hash：
-        原始代理存放在name为raw_proxy的hash中，key为代理的ip:port，value为为None,以后扩展可能会加入代理属性；
+        原始代理存放在name为raw_proxy的hash中，key为代理的ip:port，value为None,以后扩展可能会加入代理属性；
         验证后的代理存放在name为useful_proxy的hash中，key为代理的ip:port，value为一个计数,初始为1，每校验失败一次减1；
 
     """

Manager/ProxyManager.py

@@ -17,7 +17,7 @@ import random
 
 from Util import EnvUtil
 from DB.DbClient import DbClient
-from Util.GetConfig import config
+from Config.ConfigGetter import config
 from Util.LogHandler import LogHandler
 from Util.utilFunction import verifyProxyFormat
 from ProxyGetter.getFreeProxy import GetFreeProxy

README.md

@@ -39,25 +39,41 @@ git clone git@github.com:jhao104/proxy_pool.git
 pip install -r requirements.txt
 ```
 
-* 配置Config.ini:
+* 配置Config/setting.py:
 
 ```shell
-# Config.ini 为项目配置文件
-# 配置DB
-type = SSDB       # 如果使用SSDB或redis数据库，均配置为SSDB
-host = localhost  # db host
-port = 8888       # db port
-name = proxy      # 默认配置
+# Config/setting.py 为项目配置文件
+
+# 配置DB     
+DATABASES = {
+    "default": {
+        "TYPE": "SSDB",        # 如果使用SSDB或redis数据库，均配置为SSDB
+        "HOST": "127.0.0.1",   # db host
+        "PORT": 8888,          # db port
+        "NAME": "proxy",       # 默认配置
+        "PASSWORD": ""         # db password
+
+    }
+}
+
 
 # 配置 ProxyGetter
-freeProxyFirst  = 1  # 这里是启动的抓取函数，可在ProxyGetter/getFreeProxy.py 扩展
-freeProxySecond = 1
-....
 
-# 配置 HOST (api服务)
-ip = 127.0.0.1       # 监听ip,0.0.0.0开启外网访问
-port = 5010          # 监听端口
-# 上面配置启动后，代理api地址为 http://127.0.0.1:5010
+PROXY_GETTER = [
+    "freeProxyFirst",      # 这里是启用的代理抓取函数名，可在ProxyGetter/getFreeProxy.py 扩展
+    "freeProxySecond",
+    ....
+]
+
+
+# 配置 API服务
+
+SERVER_API = {
+    "HOST": "0.0.0.0",  # 监听ip, 0.0.0.0 监听所有IP
+    "PORT": 5010        # 监听端口
+}
+       
+# 上面配置启动后，代理池访问地址为 http://127.0.0.1:5010
 
 ```
 
@@ -164,18 +180,17 @@ class GetFreeProxy(object):
         # 确保每个proxy都是 host:ip正确的格式就行
 ```
 
-* 2、添加好方法后，修改Config.ini文件中的`[ProxyGetter]`项：
+* 2、添加好方法后，修改Config/setting.py文件中的`PROXY_GETTER`项：
 
-　　在`Config.ini`的`[ProxyGetter]`下添加自定义的方法的名字:
+　　在`PROXY_GETTER`下添加自定义的方法的名字:
 
 ```shell
-
-[ProxyGetter]
-;register the proxy getter function
-freeProxyFirst  = 0  # 如果要取消某个方法，将其删除或赋为0即可
-....
-freeProxyCustom  = 1  # 确保名字和你添加方法名字一致
-
+PROXY_GETTER = [
+    "freeProxyFirst",    
+    "freeProxySecond",
+    ....
+    "freeProxyCustom"  #  # 确保名字和你添加方法名字一致
+]
 ```
 
 

Schedule/ProxyValidSchedule.py

@@ -56,8 +56,8 @@ class ProxyValidSchedule(ProxyManager, object):
                 self.log.info("Start valid useful proxy")
                 self.__validProxy()
             else:
-                self.log.info('Valid Complete! sleep 5 minutes.')
-                time.sleep(60 * 5)
+                self.log.info('Valid Complete! sleep 5 sec.')
+                time.sleep(5)
                 self.putQueue()
 
     def putQueue(self):

Test/.pytest_cache/v/cache/lastfailed

@@ -1,3 +0,0 @@
-{
-  "testGetFreeProxy.py::testGetFreeProxy": true
-}

Test/.pytest_cache/v/cache/nodeids

@@ -1,3 +0,0 @@
-[
-  "testGetFreeProxy.py::testGetFreeProxy"
-]

Test/__init__.py

@@ -1,13 +1,13 @@
 # -*- coding: utf-8 -*-
 """
 -------------------------------------------------
-   File Name：     __init__.py
-   Description :   
-   Author :        J_hao
-   date：          2017/7/31
+   File Name：     __init__
+   Description :
+   Author :        JHao
+   date：          2019/2/15
 -------------------------------------------------
    Change Activity:
-                   2017/7/31:
+                   2019/2/15:
 -------------------------------------------------
 """
-__author__ = 'J_hao'
+__author__ = 'JHao'

Test/testGetConfig.py → Test/testConfig.py

@@ -2,7 +2,7 @@
 """
 -------------------------------------------------
    File Name：     testGetConfig
-   Description :   test all function in GetConfig.py
+   Description :   testGetConfig
    Author :        J_hao
    date：          2017/7/31
 -------------------------------------------------
@@ -12,22 +12,22 @@
 """
 __author__ = 'J_hao'
 
-from Util.GetConfig import GetConfig
+from Config.ConfigGetter import config
 
 
 # noinspection PyPep8Naming
-def testGetConfig():
+def testConfig():
     """
-    test class GetConfig in Util/GetConfig
     :return:
     """
-    gg = GetConfig()
-    print(gg.db_type)
-    print(gg.db_name)
-    print(gg.db_host)
-    print(gg.db_port)
-    assert isinstance(gg.proxy_getter_functions, list)
-    print(gg.proxy_getter_functions)
+    print(config.db_type)
+    print(config.db_name)
+    print(config.db_host)
+    print(config.db_port)
+    print(config.db_password)
+    assert isinstance(config.proxy_getter_functions, list)
+    print(config.proxy_getter_functions)
+
 
 if __name__ == '__main__':
-    testGetConfig()
+    testConfig()

Test/testGetFreeProxy.py

@@ -16,7 +16,6 @@ import re
 import sys
 import requests
 
-
 try:
     from importlib import reload  # py3 实际不会实用，只是为了不显示语法错误
 except:
@@ -25,7 +24,7 @@ except:
 
 sys.path.append('..')
 from ProxyGetter.getFreeProxy import GetFreeProxy
-from Util.GetConfig import GetConfig
+from Config.ConfigGetter import config
 
 
 # noinspection PyPep8Naming
@@ -34,15 +33,15 @@ def testGetFreeProxy():
     test class GetFreeProxy in ProxyGetter/GetFreeProxy
     :return:
     """
-    gc = GetConfig()
-    proxy_getter_functions = gc.proxy_getter_functions
+    proxy_getter_functions = config.proxy_getter_functions
     for proxyGetter in proxy_getter_functions:
         proxy_count = 0
         for proxy in getattr(GetFreeProxy, proxyGetter.strip())():
             if proxy:
-                print('{func}: fetch proxy {proxy},proxy_count:{proxy_count}'.format(func=proxyGetter, proxy=proxy,proxy_count=proxy_count))
+                print('{func}: fetch proxy {proxy},proxy_count:{proxy_count}'.format(func=proxyGetter, proxy=proxy,
+                                                                                     proxy_count=proxy_count))
                 proxy_count += 1
-        #assert proxy_count >= 20, '{} fetch proxy fail'.format(proxyGetter)
+        # assert proxy_count >= 20, '{} fetch proxy fail'.format(proxyGetter)
 
 
 if __name__ == '__main__':

Util/GetConfig.py

@@ -1,76 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-"""
--------------------------------------------------
-   File Name：     GetConfig.py
-   Description :  fetch config from config.ini
-   Author :       JHao
-   date：          2016/12/3
--------------------------------------------------
-   Change Activity:
-                   2016/12/3: get db property func
--------------------------------------------------
-"""
-__author__ = 'JHao'
-
-import os
-from Util.utilClass import ConfigParse
-from Util.utilClass import LazyProperty
-
-
-class GetConfig(object):
-    """
-    to get config from config.ini
-    """
-
-    def __init__(self):
-        self.pwd = os.path.split(os.path.realpath(__file__))[0]
-        self.config_path = os.path.join(os.path.split(self.pwd)[0], 'Config.ini')
-        self.config_file = ConfigParse(defaults={"password": None})
-        self.config_file.read(self.config_path)
-
-    @LazyProperty
-    def db_type(self):
-        return self.config_file.get('DB', 'type')
-
-    @LazyProperty
-    def db_name(self):
-        return self.config_file.get('DB', 'name')
-
-    @LazyProperty
-    def db_host(self):
-        return self.config_file.get('DB', 'host')
-
-    @LazyProperty
-    def db_port(self):
-        return int(self.config_file.get('DB', 'port'))
-
-    @LazyProperty
-    def db_password(self):
-        return self.config_file.get('DB', 'password')
-
-    @LazyProperty
-    def proxy_getter_functions(self):
-        return self.config_file.options('ProxyGetter')
-
-    @LazyProperty
-    def host_ip(self):
-        return self.config_file.get('API', 'ip')
-
-    @LazyProperty
-    def host_port(self):
-        return int(self.config_file.get('API', 'port'))
-
-
-config = GetConfig()
-
-if __name__ == '__main__':
-    gg = GetConfig()
-    print(gg.db_type)
-    print(gg.db_name)
-    print(gg.db_host)
-    print(gg.db_port)
-    print(gg.proxy_getter_functions)
-    print(gg.host_ip)
-    print(gg.host_port)
-    print(gg.db_password)

Util/utilClass.py

@@ -9,7 +9,6 @@
 -------------------------------------------------
    Change Activity:
                    2016/12/3: Class LazyProperty
-                   2016/12/4: rewrite ConfigParser
 -------------------------------------------------
 """
 __author__ = 'JHao'
@@ -33,24 +32,6 @@ class LazyProperty(object):
             return value
 
 
-try:
-    from configparser import ConfigParser  # py3
-except:
-    from ConfigParser import ConfigParser  # py2
-
-
-class ConfigParse(ConfigParser):
-    """
-    rewrite ConfigParser, for support upper option
-    """
-
-    def __init__(self, *args, **kwargs):
-        ConfigParser.__init__(self, *args, **kwargs)
-
-    def optionxform(self, optionstr):
-        return optionstr
-
-
 class Singleton(type):
     """
     Singleton Metaclass

test.py

@@ -12,4 +12,7 @@
 """
 __author__ = 'JHao'
 
-from Schedule import ProxyRefreshSchedule
+from Test import testConfig
+
+if __name__ == '__main__':
+    testConfig.testConfig()