Browse Source

[bugfix] fix freeProxy09 with request params, and add freeProxy05 page param

shuzhang 6 years ago
parent
commit
29c5fafe8f
2 changed files with 10 additions and 5 deletions
  1. 9 4
      fetcher/proxyFetcher.py
  2. 1 1
      setting.py

+ 9 - 4
fetcher/proxyFetcher.py

@@ -164,10 +164,15 @@ class ProxyFetcher(object):
         """
         快代理 https://www.kuaidaili.com
         """
-        url_list = [
-            'https://www.kuaidaili.com/free/inha/',
-            'https://www.kuaidaili.com/free/intr/'
+        url_pattern = [
+            'https://www.kuaidaili.com/free/inha/{}/',
+            'https://www.kuaidaili.com/free/intr/{}/'
         ]
+        url_list = []
+        for page_index in range(1, 3):
+            for pattern in url_pattern:
+                url_list.append(pattern.format(page_index))
+
         for url in url_list:
             tree = WebRequest().get(url).tree
             proxy_list = tree.xpath('.//table//tr')
@@ -229,7 +234,7 @@ class ProxyFetcher(object):
         :return:
         """
         for i in range(1, page_count + 1):
-            url = 'http://ip.jiangxianli.com/?country=中国&?page={}'.format(i)
+            url = 'http://ip.jiangxianli.com/?country=中国&page={}'.format(i)
             html_tree = WebRequest().get(url).tree
             for index, tr in enumerate(html_tree.xpath("//table//tr")):
                 if index == 0:

+ 1 - 1
setting.py

@@ -53,7 +53,7 @@ PROXY_FETCHER = [
     # "freeProxy06",
     "freeProxy07",
     # "freeProxy08",
-    # "freeProxy09",
+    "freeProxy09",
     "freeProxy13",
     "freeProxy14",
     "freeProxy15",