|
|
@@ -36,7 +36,7 @@ def verifyProxy(proxy):
|
|
|
return True if re.findall(verify_regex, proxy) else False
|
|
|
|
|
|
|
|
|
-def getHtmlTree(url, header=None, **kwargs):
|
|
|
+def getHtmlTree(url, **kwargs):
|
|
|
"""
|
|
|
获取html树
|
|
|
:param url:
|
|
|
@@ -45,5 +45,13 @@ def getHtmlTree(url, header=None, **kwargs):
|
|
|
"""
|
|
|
import requests
|
|
|
from lxml import etree
|
|
|
- html = requests.get(url=url, headers=header).content
|
|
|
+ header = {'Connection': 'keep-alive',
|
|
|
+ 'Cache-Control': 'max-age=0',
|
|
|
+ 'Upgrade-Insecure-Requests': '1',
|
|
|
+ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)',
|
|
|
+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
|
|
+ 'Accept-Encoding': 'gzip, deflate, sdch',
|
|
|
+ 'Accept-Language': 'zh-CN,zh;q=0.8',
|
|
|
+ }
|
|
|
+ html = requests.get(url=url, headers=header, timeout=30).content
|
|
|
return etree.HTML(html)
|
jhao104