import time from multiprocessing import Process from proxypool.api import app from proxypool.getter import Getter from proxypool.tester import Tester from proxypool.db import RedisClient from proxypool.setting import * class Scheduler(object): def schedule_tester(cycle=TESTER_CYCLE): """ 定时测试代理 """ redis = RedisClient() tester = Tester() while True: print('测试器开始运行') count = redis.count() if count == 0: print('代理池已枯竭,等待添加代理') time.sleep(cycle) continue proxies = redis.all() tester.set_proxies(proxies) print('开始检测全部代理') tester.run() time.sleep(cycle) def schedule_getter(cycle=GETTER_CYCLE): """ 定时获取代理 """ while True: print('开始抓取代理') getter = Getter() getter.run() time.sleep(cycle) def schedule_api(self): """ 开启API """ app.run(API_HOST, API_PORT) def run(self): print('代理池开始运行') if TESTER_ENABLED: tester_process = Process(target=self.schedule_tester) tester_process.start() if GETTER_ENABLED: getter_process = Process(target=self.schedule_getter) getter_process.start() if API_ENABLED: api_process = Process(target=self.schedule_api) api_process.start()