소스 검색

[update] :heavy_check_mark: readthedocs

jhao 6 년 전
부모
커밋
dedf727671
7개의 변경된 파일227개의 추가작업 그리고 0개의 파일을 삭제
  1. 1 0
      .gitignore
  2. 1 0
      README.md
  3. 20 0
      docs/Makefile
  4. 62 0
      docs/conf.py
  5. 100 0
      docs/index.rst
  6. 35 0
      docs/make.bat
  7. 8 0
      docs/user/intro.rst

+ 1 - 0
.gitignore

@@ -1,3 +1,4 @@
 .idea/
+docs/_build
 *.pyc
 *.log

+ 1 - 0
README.md

@@ -228,4 +228,5 @@ PROXY_FETCHER = [
 ### AD
 
   最后, 开源不易, 有时间的小伙伴可以点下[推广](https://tracking.gitads.io/?repo=proxy_pool)广告。
+
   [![AD](https://images.gitads.io/proxy_pool)](https://tracking.gitads.io/?repo=proxy_pool)

+ 20 - 0
docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

+ 62 - 0
docs/conf.py

@@ -0,0 +1,62 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'ProxyPool'
+copyright = '2020, jhao104'
+author = 'jhao104'
+
+# The full version, including alpha/beta/rc tags
+release = '2.1.0'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = 'zh_CN'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']

+ 100 - 0
docs/index.rst

@@ -0,0 +1,100 @@
+.. ProxyPool documentation master file, created by
+   sphinx-quickstart on Wed Jul  8 16:13:42 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+ProxyPool
+=====================================
+
+Python爬虫代理IP池
+
+安装
+-----
+
+* 下载代码
+
+.. code-block:: console
+
+    $ git clone git@github.com:jhao104/proxy_pool.git
+
+* 安装依赖
+
+.. code-block:: console
+
+    $ pip install -r requirements.txt
+
+* 更新配置
+
+.. code-block:: python
+
+   HOST = "0.0.0.0"
+   PORT = 5000
+
+   DB_CONN = 'redis://@127.0.0.1:8888'
+
+   PROXY_FETCHER = [
+       "freeProxy01",
+       "freeProxy02",
+       # ....
+   ]
+
+* 启动项目
+
+.. code-block:: console
+
+    $ python proxyPool.py schedule
+    $ python proxyPool.py server
+
+使用
+______
+
+* API
+
+============     ========    ================       ==============
+Api               Method      Description            Arg
+============     ========    ================       ==============
+/                GET         API介绍                 无
+/get             GET         随机返回一个代理         无
+/get_all         GET         返回所有代理             无
+/get_status      GET         返回代理数量             无
+/delete          GET         删除指定代理             proxy=host:ip
+============     ========    ================       ==============
+
+
+* 爬虫
+
+.. code-block:: python
+
+   import requests
+
+   def get_proxy():
+       return requests.get("http://127.0.0.1:5010/get/").json()
+
+   def delete_proxy(proxy):
+       requests.get("http://127.0.0.1:5010/delete/?proxy={}".format(proxy))
+
+   # your spider code
+
+   def getHtml():
+       # ....
+       retry_count = 5
+       proxy = get_proxy().get("proxy")
+       while retry_count > 0:
+           try:
+               html = requests.get('http://www.example.com', proxies={"http": "http://{}".format(proxy)})
+               # 使用代理访问
+               return html
+           except Exception:
+               retry_count -= 1
+       # 删除代理池中代理
+       delete_proxy(proxy)
+       return None
+
+Contents
+--------
+
+.. toctree::
+   :maxdepth: 2
+
+   user/intro.rst
+

+ 35 - 0
docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

+ 8 - 0
docs/user/intro.rst

@@ -0,0 +1,8 @@
+.. intro
+
+用户手册
+========
+
+安装
+------
+