test_main_summary.py 50 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400
  1. import asyncio
  2. import json
  3. import os
  4. from pathlib import Path
  5. from core.settings import AppSettings
  6. from dashboard.api import _account_survival_payload
  7. from main import _apply_runtime_mode, _build_background_tasks, _recent_pool_files, _rotate_log_tail, _runtime_payload, _summary_payload, create_app
  8. from ops.scan import ScanResult
  9. def _request_via_asgi(
  10. app,
  11. method: str,
  12. path: str,
  13. headers: dict[str, str] | None = None,
  14. body: bytes = b"",
  15. ) -> tuple[int, dict[str, str], bytes]:
  16. request_headers = [
  17. (key.lower().encode("latin-1"), value.encode("latin-1"))
  18. for key, value in (headers or {}).items()
  19. ]
  20. scope = {
  21. "type": "http",
  22. "asgi": {"version": "3.0"},
  23. "http_version": "1.1",
  24. "method": method,
  25. "scheme": "http",
  26. "path": path,
  27. "raw_path": path.encode("ascii"),
  28. "query_string": b"",
  29. "headers": request_headers,
  30. "client": ("127.0.0.1", 12345),
  31. "server": ("testserver", 80),
  32. "root_path": "",
  33. "app": app,
  34. }
  35. response: dict[str, object] = {"status": 500, "headers": {}, "body": b""}
  36. request_sent = False
  37. async def receive() -> dict[str, object]:
  38. nonlocal request_sent
  39. if request_sent:
  40. return {"type": "http.disconnect"}
  41. request_sent = True
  42. return {"type": "http.request", "body": body, "more_body": False}
  43. async def send(message: dict[str, object]) -> None:
  44. if message["type"] == "http.response.start":
  45. response["status"] = int(message["status"])
  46. response["headers"] = {
  47. key.decode("latin-1"): value.decode("latin-1")
  48. for key, value in message.get("headers", [])
  49. }
  50. return
  51. if message["type"] == "http.response.body":
  52. response["body"] = bytes(response["body"]) + bytes(message.get("body", b""))
  53. asyncio.run(app(scope, receive, send))
  54. return int(response["status"]), dict(response["headers"]), bytes(response["body"])
  55. def test_summary_payload_exposes_register_and_ops_commands(tmp_path: Path) -> None:
  56. app = create_app(enable_background_tasks=False)
  57. app.state.settings = AppSettings(
  58. pool_dir=tmp_path,
  59. runtime_state_file=tmp_path / "runtime_state.json",
  60. cleanup_enabled=False,
  61. validate_enabled=False,
  62. )
  63. app.state.background_tasks = []
  64. payload = _summary_payload(app)
  65. commands = payload["commands"]
  66. routes = payload["routes"]
  67. assert commands["chatgpt_preflight"].startswith("uv run python scripts/chatgpt_preflight.py")
  68. assert commands["chatgpt_register_once"].startswith("uv run python scripts/chatgpt_register_once.py")
  69. assert commands["chatgpt_callback_exchange"].startswith("uv run python scripts/chatgpt_exchange_callback.py")
  70. assert commands["update_priority_dry_run"].startswith("uv run python -m ops.update_priority")
  71. assert commands["validate_used_dry_run"].startswith("uv run python -m ops.validate --scope used")
  72. assert routes["chatgpt_callback_exchange"] == "/api/register/chatgpt/callback-exchange"
  73. assert "dashboard" not in routes
  74. def test_dashboard_route_removed(tmp_path: Path) -> None:
  75. app = create_app(enable_background_tasks=False)
  76. app.state.settings = AppSettings(
  77. pool_dir=tmp_path,
  78. runtime_state_file=tmp_path / "runtime_state.json",
  79. cleanup_enabled=False,
  80. validate_enabled=False,
  81. )
  82. app.state.background_tasks = []
  83. status, _headers, _body = _request_via_asgi(app, "GET", "/dashboard")
  84. assert status == 404
  85. def test_summary_payload_exposes_register_log_tail(tmp_path: Path) -> None:
  86. log_path = tmp_path / "register.log"
  87. log_path.write_text("line-1\nline-2\nline-3\n", encoding="utf-8")
  88. app = create_app(enable_background_tasks=False)
  89. app.state.settings = AppSettings(
  90. pool_dir=tmp_path,
  91. register_log_file=str(log_path),
  92. cleanup_enabled=False,
  93. validate_enabled=False,
  94. )
  95. app.state.background_tasks = []
  96. payload = _summary_payload(app)
  97. log_tail = payload["register_log_tail"]
  98. assert log_tail["available"] is True
  99. assert log_tail["path"] == str(log_path)
  100. assert log_tail["error"] is None
  101. assert log_tail["lines"] == ["line-1", "line-2", "line-3"]
  102. def test_summary_payload_exposes_rotate_log_tail_and_runtime_state_meta(monkeypatch, tmp_path: Path) -> None:
  103. runtime_state_file = tmp_path / "runtime_state.json"
  104. runtime_state_file.write_text('{"updated_at":"2026-03-23T22:25:02"}', encoding="utf-8")
  105. app = create_app(enable_background_tasks=False)
  106. app.state.settings = AppSettings(
  107. pool_dir=tmp_path,
  108. runtime_state_file=runtime_state_file,
  109. cleanup_enabled=False,
  110. validate_enabled=False,
  111. )
  112. app.state.background_tasks = []
  113. monkeypatch.setattr(
  114. "main._rotate_log_tail",
  115. lambda **_kwargs: {
  116. "available": True,
  117. "path": "/home/sophomores/zhuce6/logs/dashboard.log",
  118. "updated_at": 1774276100.0,
  119. "updated_at_iso": "2026-03-23T22:28:20",
  120. "error": None,
  121. "lines": [
  122. "[22:18:20] [rotate] summary | 主池: 800 → 790 | 401删除: 10 | quota探测: 797 | probe401: 14 | probe429: 0 | probe跳过: 0 | 429删除: 0",
  123. ],
  124. "recent_events": [
  125. "[22:17:35] [rotate] 🔎 ocd0553a1fb1@mail.example.test.json quota probe → 401 invalidated",
  126. "[22:17:35] [rotate] ❌ ocd0553a1fb1@mail.example.test.json 401删除",
  127. ],
  128. "latest_summary": {
  129. "time": "22:18:20",
  130. "main_before": 800,
  131. "main_after": 790,
  132. "deleted_401": 10,
  133. "quota_probed": 797,
  134. "quota_probe_401": 14,
  135. "quota_probe_429": 0,
  136. "quota_probe_skipped": 0,
  137. "deleted_429": 0,
  138. },
  139. "current_summary": {
  140. "time": "22:19:10",
  141. "main_before": None,
  142. "main_after": None,
  143. "deleted_401": 1,
  144. "quota_probed": 1,
  145. "quota_probe_401": 1,
  146. "quota_probe_429": 0,
  147. "quota_probe_skipped": 0,
  148. "deleted_429": 0,
  149. "partial": True,
  150. "event_count": 2,
  151. },
  152. },
  153. )
  154. payload = _summary_payload(app)
  155. assert payload["rotate_latest_summary"]["deleted_401"] == 10
  156. assert payload["rotate_latest_summary"]["quota_probe_401"] == 14
  157. assert payload["rotate_current_summary"]["deleted_401"] == 1
  158. assert payload["rotate_log_tail"]["available"] is True
  159. assert len(payload["rotate_log_tail"]["recent_events"]) == 2
  160. assert payload["runtime_state_file"]["exists"] is True
  161. assert payload["runtime_state_file"]["path"] == str(runtime_state_file)
  162. def test_summary_payload_exposes_account_survival_payload(monkeypatch, tmp_path: Path) -> None:
  163. state_file = tmp_path / "account_survival.json"
  164. state_file.write_text(
  165. (
  166. "{\n"
  167. ' "updated_at": "2026-03-26T13:10:00+08:00",\n'
  168. ' "summary": {"tracked": 4, "alive": 3, "invalid": 1}\n'
  169. "}\n"
  170. ),
  171. encoding="utf-8",
  172. )
  173. app = create_app(enable_background_tasks=False)
  174. app.state.settings = AppSettings(
  175. pool_dir=tmp_path,
  176. account_survival_enabled=True,
  177. account_survival_state_file=state_file,
  178. responses_survival_state_file=tmp_path / "responses_survival_missing.json",
  179. cleanup_enabled=False,
  180. validate_enabled=False,
  181. )
  182. app.state.background_tasks = []
  183. payload = _summary_payload(app)
  184. assert payload["account_survival"]["available"] is True
  185. assert payload["account_survival"]["summary"]["tracked"] == 4
  186. assert payload["routes"]["account_survival"] == "/api/account-survival"
  187. def test_summary_payload_prefers_responses_survival_payload_when_available(tmp_path: Path) -> None:
  188. account_state_file = tmp_path / "account_survival.json"
  189. account_state_file.write_text(
  190. (
  191. "{\n"
  192. ' "probe_mode": "usage",\n'
  193. ' "summary": {"tracked": 4, "alive": 4, "invalid": 0}\n'
  194. "}\n"
  195. ),
  196. encoding="utf-8",
  197. )
  198. responses_state_file = tmp_path / "responses_survival.json"
  199. responses_state_file.write_text(
  200. (
  201. "{\n"
  202. ' "probe_mode": "responses",\n'
  203. ' "updated_at": "2026-03-30T20:30:00+08:00",\n'
  204. ' "summary": {"tracked": 8, "alive": 7, "invalid": 1, "first_invalid_count": 1}\n'
  205. "}\n"
  206. ),
  207. encoding="utf-8",
  208. )
  209. app = create_app(enable_background_tasks=False)
  210. app.state.settings = AppSettings(
  211. pool_dir=tmp_path,
  212. account_survival_enabled=True,
  213. account_survival_state_file=account_state_file,
  214. responses_survival_state_file=responses_state_file,
  215. cleanup_enabled=False,
  216. validate_enabled=False,
  217. )
  218. app.state.background_tasks = []
  219. payload = _summary_payload(app)
  220. assert payload["account_survival"]["available"] is True
  221. assert payload["account_survival"]["probe_mode"] == "responses"
  222. assert payload["account_survival"]["summary"]["tracked"] == 8
  223. def test_account_survival_payload_derives_promotion_stats_from_member_files(tmp_path: Path) -> None:
  224. warmup_file = tmp_path / "warmup@example.com.json"
  225. warmup_file.write_text(
  226. json.dumps(
  227. {
  228. "email": "warmup@example.com",
  229. "access_token": "tok",
  230. "account_id": "acct",
  231. "created_at": "2026-03-31T17:00:00+08:00",
  232. "warmup_required": True,
  233. "cpa_sync_status": "synced",
  234. },
  235. ensure_ascii=False,
  236. ),
  237. encoding="utf-8",
  238. )
  239. responses_state_file = tmp_path / "responses_survival.json"
  240. responses_state_file.write_text(
  241. json.dumps(
  242. {
  243. "probe_mode": "responses",
  244. "updated_at": "2026-03-31T17:30:00+08:00",
  245. "summary": {"tracked": 1, "alive": 1, "invalid": 0},
  246. "members": [
  247. {
  248. "email": "warmup@example.com",
  249. "path": str(warmup_file),
  250. "warmup_state": "passed",
  251. }
  252. ],
  253. "promotion_stats": {
  254. "promoted_success_total": 0,
  255. "promoted_failure_total": 0,
  256. },
  257. },
  258. ensure_ascii=False,
  259. ),
  260. encoding="utf-8",
  261. )
  262. settings = AppSettings(
  263. pool_dir=tmp_path,
  264. responses_survival_state_file=responses_state_file,
  265. account_survival_enabled=True,
  266. cleanup_enabled=False,
  267. validate_enabled=False,
  268. )
  269. payload = _account_survival_payload(settings)
  270. assert payload["promotion_stats"]["promoted_success_total"] == 1
  271. assert payload["promotion_stats"]["promoted_failure_total"] == 0
  272. def test_summary_payload_formats_survival_durations_as_hms(tmp_path: Path) -> None:
  273. responses_state_file = tmp_path / "responses_survival.json"
  274. responses_state_file.write_text(
  275. (
  276. "{\n"
  277. ' "probe_mode": "responses",\n'
  278. ' "updated_at": "2026-03-31T13:30:00+08:00",\n'
  279. ' "summary": {"tracked": 2, "alive": 1, "invalid": 1, "first_invalid_count": 1},\n'
  280. ' "changes": [{"email": "a@example.com", "survival_seconds": 3661}],\n'
  281. ' "members": [\n'
  282. ' {"email": "a@example.com", "survival_seconds": 3661},\n'
  283. ' {"email": "b@example.com", "survival_seconds": 59}\n'
  284. " ]\n"
  285. "}\n"
  286. ),
  287. encoding="utf-8",
  288. )
  289. app = create_app(enable_background_tasks=False)
  290. app.state.settings = AppSettings(
  291. pool_dir=tmp_path,
  292. account_survival_enabled=True,
  293. account_survival_state_file=tmp_path / "account_survival_missing.json",
  294. responses_survival_state_file=responses_state_file,
  295. cleanup_enabled=False,
  296. validate_enabled=False,
  297. )
  298. app.state.background_tasks = []
  299. payload = _summary_payload(app)
  300. assert payload["account_survival"]["changes"][0]["survival_text"] == "1h 1m 1s"
  301. assert payload["account_survival"]["members"][0]["survival_text"] == "1h 1m 1s"
  302. assert payload["account_survival"]["members"][1]["survival_text"] == "59s"
  303. def test_account_survival_payload_exposes_fresh_unauthorized_experiment(tmp_path: Path) -> None:
  304. responses_state_file = tmp_path / "responses_survival.json"
  305. responses_state_file.write_text(
  306. json.dumps(
  307. {
  308. "probe_mode": "responses",
  309. "updated_at": "2026-04-01T09:59:00+08:00",
  310. "summary": {"tracked": 1, "alive": 1, "invalid": 0},
  311. "members": [],
  312. },
  313. ensure_ascii=False,
  314. ),
  315. encoding="utf-8",
  316. )
  317. experiment_state = tmp_path / "track_new8_unauthorized_20260401.json"
  318. experiment_state.write_text(
  319. json.dumps(
  320. {
  321. "started_at": "2026-04-01T09:57:26+08:00",
  322. "cohort_size": 8,
  323. "members": [
  324. {
  325. "email": "fresh@example.com",
  326. "created_at": "2026-04-01T09:58:00+08:00",
  327. "registration_post_create_gate": "add_phone",
  328. "registration_proxy_key": "台湾-三网备用",
  329. "probe_count": 3,
  330. "last_status_code": 401,
  331. "last_category": "invalid",
  332. "last_detail": '{"detail":"Unauthorized"}',
  333. "first_401_at": "2026-04-01T10:01:05+08:00",
  334. "first_401_seconds": 185,
  335. "first_401_detail": '{"detail":"Unauthorized"}',
  336. }
  337. ],
  338. "history": [],
  339. },
  340. ensure_ascii=False,
  341. ),
  342. encoding="utf-8",
  343. )
  344. settings = AppSettings(
  345. pool_dir=tmp_path,
  346. state_dir=tmp_path,
  347. responses_survival_state_file=responses_state_file,
  348. account_survival_enabled=True,
  349. cleanup_enabled=False,
  350. validate_enabled=False,
  351. )
  352. payload = _account_survival_payload(settings)
  353. experiment = payload["fresh_unauthorized_experiment"]
  354. assert experiment["available"] is True
  355. assert experiment["summary"]["tracked"] == 1
  356. assert experiment["summary"]["first_401_count"] == 1
  357. assert experiment["members"][0]["first_401_text"] == "3m 5s"
  358. assert experiment["members"][0]["first_401_detail"] == '{"detail":"Unauthorized"}'
  359. def test_account_survival_reseed_api_rebuilds_latest_ten_cohort(monkeypatch, tmp_path: Path) -> None:
  360. for idx in range(12):
  361. (tmp_path / f"user{idx:02d}@example.com.json").write_text(
  362. (
  363. "{\n"
  364. f' "email": "user{idx:02d}@example.com",\n'
  365. ' "access_token": "tok",\n'
  366. ' "account_id": "acct",\n'
  367. f' "created_at": "2026-03-26T12:{idx:02d}:00+08:00"\n'
  368. "}\n"
  369. ),
  370. encoding="utf-8",
  371. )
  372. state_file = tmp_path / "responses_survival.json"
  373. state_file.write_text(
  374. (
  375. "{\n"
  376. ' "probe_mode": "responses",\n'
  377. ' "members": []\n'
  378. "}\n"
  379. ),
  380. encoding="utf-8",
  381. )
  382. app = create_app(enable_background_tasks=False)
  383. app.state.settings = AppSettings(
  384. pool_dir=tmp_path,
  385. account_survival_enabled=True,
  386. account_survival_cohort_size=10,
  387. account_survival_state_file=tmp_path / "account_survival_unused.json",
  388. responses_survival_state_file=state_file,
  389. responses_survival_require_provenance=False,
  390. responses_survival_recent_window_seconds=0,
  391. cleanup_enabled=False,
  392. validate_enabled=False,
  393. )
  394. app.state.background_tasks = []
  395. monkeypatch.setattr(
  396. "ops.responses_survival.probe_responses_token_file",
  397. lambda path, proxy, timeout: ScanResult(file=path.name, category="normal", status_code=200, detail="ok"),
  398. )
  399. status, _headers, body = _request_via_asgi(app, "POST", "/api/account-survival/reseed")
  400. payload = json.loads(body.decode("utf-8"))
  401. assert status == 200
  402. assert payload["available"] is True
  403. assert payload["summary"]["tracked"] == 10
  404. assert payload["seed_source"] == "latest_generated_pool_files"
  405. assert payload["members"][0]["email"] == "user11@example.com"
  406. def test_summary_payload_exposes_register_burst_plan(tmp_path: Path) -> None:
  407. app = create_app(enable_background_tasks=False)
  408. app.state.settings = AppSettings(
  409. pool_dir=tmp_path,
  410. register_batch_threads=1,
  411. register_batch_target_count=20,
  412. register_batch_interval_seconds=10800,
  413. cleanup_enabled=False,
  414. validate_enabled=False,
  415. )
  416. app.state.background_tasks = []
  417. payload = _summary_payload(app)
  418. burst_plan = payload["register_burst_plan"]
  419. assert burst_plan["mode"] == "burst"
  420. assert burst_plan["threads"] == 1
  421. assert burst_plan["target_count"] == 20
  422. assert burst_plan["interval_seconds"] == 10800
  423. assert burst_plan["accounts_per_day"] == 160
  424. assert burst_plan["accounts_needed_for_one_day_target"] == 20
  425. assert burst_plan["accounts_needed_for_sustained_daily_target"] == 140
  426. def test_rotate_log_tail_builds_current_summary_for_in_progress_rotate(monkeypatch, tmp_path: Path) -> None:
  427. log_path = tmp_path / "dashboard.log"
  428. log_path.write_text(
  429. "\n".join(
  430. [
  431. "[10:00:00] [rotate] summary | 主池: 800 → 789 | 401删除: 10 | quota探测: 20 | probe401: 10 | probe429: 1 | probe跳过: 2 | 429删除: 1",
  432. "[10:05:00] [rotate] 🔎 a@example.com.json quota probe → 401 invalidated",
  433. "[10:05:01] [rotate] ❌ a@example.com.json 401删除",
  434. "[10:05:02] [rotate] 🔎 b@example.com.json quota probe → 429",
  435. "[10:05:03] [rotate] ❌ b@example.com.json 429删除",
  436. "",
  437. ]
  438. )
  439. + "\n",
  440. encoding="utf-8",
  441. )
  442. monkeypatch.setattr("main.DEFAULT_DASHBOARD_LOG_FILE", log_path)
  443. payload = _rotate_log_tail()
  444. assert payload["latest_summary"]["deleted_401"] == 10
  445. assert payload["current_summary"]["quota_probed"] == 2
  446. assert payload["current_summary"]["quota_probe_401"] == 1
  447. assert payload["current_summary"]["quota_probe_429"] == 1
  448. assert payload["current_summary"]["deleted_401"] == 1
  449. assert payload["current_summary"]["deleted_429"] == 1
  450. def test_rotate_log_tail_prefers_live_stdout_log_over_stale_dashboard_log(monkeypatch, tmp_path: Path) -> None:
  451. stale_log = tmp_path / "dashboard.log"
  452. stale_log.write_text(
  453. "[10:00:00] [rotate] summary | 主池: 800 → 790 | 401删除: 10 | quota探测: 20 | probe401: 10 | probe429: 0 | probe跳过: 0\n",
  454. encoding="utf-8",
  455. )
  456. live_log = tmp_path / "main_full_8threads.log"
  457. live_log.write_text(
  458. "[10:05:00] [rotate] summary | 主池: 790 → 788 | 401删除: 2 | quota探测: 5 | probe401: 2 | probe429: 0 | probe跳过: 0\n",
  459. encoding="utf-8",
  460. )
  461. monkeypatch.setattr("main.DEFAULT_DASHBOARD_LOG_FILE", stale_log)
  462. original_readlink = os.readlink
  463. def fake_readlink(path: str) -> str:
  464. if path == "/proc/self/fd/1":
  465. return str(live_log)
  466. return original_readlink(path)
  467. monkeypatch.setattr(os, "readlink", fake_readlink)
  468. payload = _rotate_log_tail()
  469. assert payload["path"] == str(live_log)
  470. assert payload["latest_summary"]["deleted_401"] == 2
  471. def test_recent_pool_files_returns_latest_entries_without_glob_expansion_issue(tmp_path: Path) -> None:
  472. older = tmp_path / "older@example.com.json"
  473. newer = tmp_path / "newer@example.com.json"
  474. older.write_text('{"email":"older@example.com"}', encoding="utf-8")
  475. newer.write_text('{"email":"newer@example.com"}', encoding="utf-8")
  476. older.touch()
  477. newer.touch()
  478. items = _recent_pool_files(tmp_path, limit=2)
  479. assert len(items) == 2
  480. assert {item["name"] for item in items} == {"older@example.com.json", "newer@example.com.json"}
  481. assert all(item["size_bytes"] > 0 for item in items)
  482. def test_summary_payload_exposes_dashboard_overview_fields(monkeypatch, tmp_path: Path) -> None:
  483. app = create_app(enable_background_tasks=False)
  484. app.state.settings = AppSettings(
  485. pool_dir=tmp_path,
  486. cleanup_enabled=False,
  487. validate_enabled=False,
  488. )
  489. app.state.background_tasks = []
  490. class FakeRegistrationLoop:
  491. def snapshot(self) -> dict[str, object]:
  492. return {
  493. "name": "register",
  494. "status": "running",
  495. "threads_alive": 2,
  496. "threads_total": 4,
  497. "total_attempts": 10,
  498. "total_success": 8,
  499. "total_success_registered": 8,
  500. "total_cpa_sync_success": 5,
  501. "total_cpa_sync_failure": 2,
  502. "total_failure": 2,
  503. "success_rate": 80.0,
  504. "registered_success_rate": 80.0,
  505. "cpa_sync_success_rate": 50.0,
  506. "target_count": None,
  507. "target_reached": False,
  508. "last_error": None,
  509. "proxy": None,
  510. "proxy_pool_enabled": False,
  511. "mail_provider": "mailtm",
  512. "interval_seconds": 5,
  513. "run_count": 10,
  514. "success_count": 8,
  515. "failure_count": 2,
  516. "is_running": True,
  517. "last_started_at": None,
  518. "last_finished_at": None,
  519. "last_duration_seconds": None,
  520. "next_run_at": None,
  521. "failure_by_stage": {},
  522. "failure_signals": {},
  523. "recent_failure_hotspots": [],
  524. "recent_attempts": [],
  525. "cfmail_domain_pool": {
  526. "target_count": 3,
  527. "active_count": 2,
  528. "active_domains": [
  529. {
  530. "name": "cfmail-tw",
  531. "domain": "tw.example.test",
  532. "inflight": 1,
  533. "recent_attempts": 6,
  534. "recent_success": 5,
  535. "recent_failure": 1,
  536. }
  537. ],
  538. "replenishing": False,
  539. "replenish_reason": "",
  540. },
  541. "cfmail_add_phone_stoploss": {
  542. "active_domain": "demo.example.test",
  543. "in_cooldown": True,
  544. "cooldown_remaining_seconds": 120,
  545. "last_triggered_at": "2026-03-23T00:00:00",
  546. "last_reason": "add_phone threshold reached",
  547. "last_add_phone_failures": 8,
  548. "last_successes": 0,
  549. "last_window_size": 12,
  550. "window_size": 12,
  551. "threshold": 8,
  552. "max_successes_in_window": 2,
  553. },
  554. }
  555. app.state.registration_loop = FakeRegistrationLoop()
  556. monkeypatch.setattr(
  557. "main._fetch_management_auth_files",
  558. lambda settings: (
  559. True,
  560. [
  561. {"name": "a@example.com.json", "unavailable": False},
  562. {"name": "b@example.com.json", "unavailable": True, "status_message": "usage_limit_reached"},
  563. {"name": "c@example.com.json", "status_message": "token invalidated by upstream"},
  564. ],
  565. ),
  566. )
  567. monkeypatch.setattr("main._count_today_new", lambda pool_dir: 3)
  568. payload = _summary_payload(app)
  569. assert payload["cpa_count"] == 3
  570. assert payload["regular_accounts"]["source_available"] is True
  571. assert payload["regular_accounts"]["available"] == 1
  572. assert payload["regular_accounts"]["waiting_reset"] == 1
  573. assert payload["regular_accounts"]["invalid"] == 1
  574. assert payload["tokens"]["estimation_mode"] == "count_based"
  575. assert payload["tokens"]["baseline_source"] == "configured"
  576. assert payload["tokens"]["available_now"] == 5000000
  577. assert payload["tokens"]["available_with_reset"] == 10000000
  578. assert payload["today_new"] == 3
  579. assert payload["success_rate"] == 80.0
  580. assert payload["registered_success_total"] == 8
  581. assert payload["cpa_sync_success_total"] == 5
  582. assert payload["cpa_sync_failure_total"] == 2
  583. assert payload["registered_success_rate"] == 80.0
  584. assert payload["cpa_sync_success_rate"] == 50.0
  585. assert payload["observed_loss"] == 2
  586. assert payload["register_failure_by_stage"] == {}
  587. assert payload["register_failure_signals"] == {}
  588. assert payload["register_recent_failure_hotspots"] == []
  589. assert payload["register_cfmail_domain_pool"]["active_count"] == 2
  590. assert payload["register_cfmail_domain_pool"]["active_domains"][0]["domain"] == "tw.example.test"
  591. assert payload["register_cfmail_add_phone_stoploss"]["in_cooldown"] is True
  592. def test_summary_payload_falls_back_when_management_inventory_unavailable(monkeypatch, tmp_path: Path) -> None:
  593. app = create_app(enable_background_tasks=False)
  594. app.state.settings = AppSettings(
  595. pool_dir=tmp_path,
  596. runtime_state_file=tmp_path / "runtime_state.json",
  597. cleanup_enabled=False,
  598. validate_enabled=False,
  599. )
  600. app.state.background_tasks = []
  601. app.state.registration_loop = None
  602. monkeypatch.setattr("main._fetch_management_auth_files", lambda settings: (False, []))
  603. monkeypatch.setattr("main._count_cpa_files", lambda settings: 9)
  604. payload = _summary_payload(app)
  605. assert payload["cpa_count"] == 9
  606. assert payload["cpa_inventory"]["management_available"] is False
  607. assert payload["regular_accounts"]["source_available"] is False
  608. assert payload["regular_accounts"]["source_error"] == "management_data_unavailable"
  609. assert payload["tokens"]["estimation_mode"] == "count_based"
  610. assert payload["tokens"]["fallback_reason"] == "missing_management_inventory"
  611. assert payload["success_rate"] is None
  612. def test_runtime_payload_exposes_proxy_pool_snapshot(tmp_path: Path) -> None:
  613. app = create_app(enable_background_tasks=False)
  614. app.state.settings = AppSettings(
  615. pool_dir=tmp_path,
  616. cleanup_enabled=False,
  617. validate_enabled=False,
  618. )
  619. app.state.background_tasks = []
  620. class FakePool:
  621. def snapshot(self) -> list[dict[str, object]]:
  622. return [
  623. {
  624. "name": "sg-node-1",
  625. "region": "sg",
  626. "proxy_url": "socks5://127.0.0.1:17891",
  627. "local_port": 17891,
  628. "in_use": True,
  629. "disabled": False,
  630. "successes": 7,
  631. "failures": 1,
  632. "last_error": "",
  633. }
  634. ]
  635. class FakeRegistrationLoop:
  636. def __init__(self) -> None:
  637. self._proxy_pool = FakePool()
  638. def snapshot(self) -> dict[str, object]:
  639. return {
  640. "name": "register",
  641. "status": "running",
  642. "threads_alive": 1,
  643. "threads_total": 1,
  644. "total_attempts": 8,
  645. "total_success": 7,
  646. "total_failure": 1,
  647. "success_rate": 87.5,
  648. "target_count": None,
  649. "target_reached": False,
  650. "last_error": None,
  651. "proxy": None,
  652. "proxy_pool_enabled": True,
  653. "mail_provider": "mailtm",
  654. "interval_seconds": 5,
  655. "run_count": 8,
  656. "success_count": 7,
  657. "failure_count": 1,
  658. "is_running": True,
  659. "last_started_at": None,
  660. "last_finished_at": None,
  661. "last_duration_seconds": None,
  662. "next_run_at": None,
  663. }
  664. app.state.registration_loop = FakeRegistrationLoop()
  665. payload = _runtime_payload(app)
  666. proxy_pool = payload["proxy_pool"]
  667. assert proxy_pool["enabled"] is True
  668. assert proxy_pool["node_count"] == 1
  669. assert proxy_pool["in_use_count"] == 1
  670. assert proxy_pool["disabled_count"] == 0
  671. assert proxy_pool["nodes"][0]["name"] == "sg-node-1"
  672. def test_runtime_payload_uses_external_runtime_state_when_loop_runs_out_of_process(tmp_path: Path) -> None:
  673. runtime_state_file = tmp_path / "runtime_state.json"
  674. runtime_state_file.write_text(
  675. """
  676. {
  677. "updated_at": "2026-03-22T22:40:00",
  678. "register_snapshot": {
  679. "name": "register",
  680. "status": "running",
  681. "threads_alive": 3,
  682. "threads_total": 3,
  683. "total_attempts": 12,
  684. "total_success": 9,
  685. "total_failure": 3,
  686. "success_rate": 75.0,
  687. "target_count": null,
  688. "target_reached": false,
  689. "last_error": "token acquisition failed",
  690. "proxy": "http://127.0.0.1:7899",
  691. "proxy_pool_enabled": true,
  692. "mail_provider": "cfmail",
  693. "interval_seconds": 5,
  694. "run_count": 12,
  695. "success_count": 9,
  696. "failure_count": 3,
  697. "is_running": true,
  698. "last_started_at": "2026-03-22T22:39:00",
  699. "last_finished_at": null,
  700. "last_duration_seconds": null,
  701. "next_run_at": null,
  702. "failure_by_stage": {"token_acquisition": 3},
  703. "failure_signals": {"add_phone_gate": 2},
  704. "recent_failure_hotspots": [{"key": "add_phone_gate", "stage": "add_phone_gate", "count": 2}],
  705. "recent_attempts": [{"timestamp": "2026-03-22T22:39:30", "success": false, "stage": "add_phone_gate", "signal": "add_phone_gate"}],
  706. "cfmail_add_phone_stoploss": {"active_domain": "demo.example.test", "in_cooldown": true}
  707. },
  708. "proxy_pool": {
  709. "configured": true,
  710. "enabled": true,
  711. "snapshot_error": null,
  712. "node_count": 2,
  713. "in_use_count": 1,
  714. "disabled_count": 0,
  715. "nodes": [
  716. {"name": "sg-1", "in_use": true, "disabled": false},
  717. {"name": "tw-1", "in_use": false, "disabled": false}
  718. ]
  719. }
  720. }
  721. """.strip(),
  722. encoding="utf-8",
  723. )
  724. app = create_app(enable_background_tasks=False)
  725. app.state.settings = AppSettings(
  726. pool_dir=tmp_path,
  727. runtime_state_file=runtime_state_file,
  728. cleanup_enabled=False,
  729. validate_enabled=False,
  730. )
  731. app.state.background_tasks = []
  732. app.state.registration_loop = None
  733. payload = _runtime_payload(app)
  734. assert payload["architecture"] == "split-runtime-fastapi+loop"
  735. register_state = next(task for task in payload["task_states"] if task["name"] == "register")
  736. assert register_state["threads_alive"] == 3
  737. assert register_state["failure_by_stage"]["token_acquisition"] == 3
  738. assert register_state["cfmail_add_phone_stoploss"]["in_cooldown"] is True
  739. assert payload["proxy_pool"]["enabled"] is True
  740. assert payload["proxy_pool"]["node_count"] == 2
  741. def test_runtime_payload_marks_proxy_pool_configured_for_direct_urls(tmp_path: Path) -> None:
  742. app = create_app(enable_background_tasks=False)
  743. app.state.settings = AppSettings(
  744. pool_dir=tmp_path,
  745. runtime_state_file=tmp_path / "runtime_state.json",
  746. proxy_pool_direct_urls="http://5.6.7.8:8080",
  747. cleanup_enabled=False,
  748. validate_enabled=False,
  749. )
  750. app.state.background_tasks = []
  751. app.state.registration_loop = None
  752. payload = _runtime_payload(app)
  753. assert payload["proxy_pool"]["configured"] is True
  754. assert payload["proxy_pool"]["enabled"] is False
  755. def test_dashboard_cors_preflight_allows_configured_origin(monkeypatch) -> None:
  756. monkeypatch.setenv("ZHUCE6_DASHBOARD_ALLOWED_ORIGINS", "http://127.0.0.1:8317")
  757. app = create_app(enable_background_tasks=False)
  758. app.state.settings = AppSettings(cleanup_enabled=False, validate_enabled=False)
  759. app.state.background_tasks = []
  760. app.state.registration_loop = None
  761. status_code, headers, _ = _request_via_asgi(
  762. app,
  763. "OPTIONS",
  764. "/api/summary",
  765. {
  766. "Origin": "http://127.0.0.1:8317",
  767. "Access-Control-Request-Method": "GET",
  768. },
  769. )
  770. assert status_code == 204
  771. assert headers["access-control-allow-origin"] == "http://127.0.0.1:8317"
  772. def test_settings_api_returns_current_runtime_mode(tmp_path: Path) -> None:
  773. app = create_app(enable_background_tasks=False, mode="lite")
  774. app.state.settings = AppSettings(
  775. runtime_mode="lite",
  776. register_enabled=True,
  777. register_threads=2,
  778. register_batch_target_count=30,
  779. register_batch_interval_seconds=3600,
  780. register_mail_provider="cfmail",
  781. register_proxy="http://127.0.0.1:7899",
  782. enable_proxy_pool=True,
  783. proxy_pool_size=10,
  784. proxy_pool_direct_urls="http://1.2.3.4:8080",
  785. proxy_pool_regions=("jp", "tw"),
  786. rotate_enabled=False,
  787. rotate_interval=120,
  788. pool_dir=tmp_path,
  789. )
  790. app.state.background_tasks = []
  791. app.state.registration_loop = None
  792. status, _headers, body = _request_via_asgi(app, "GET", "/api/settings")
  793. payload = json.loads(body.decode("utf-8"))
  794. assert status == 200
  795. assert payload["mode"] == "lite"
  796. assert payload["register"]["threads"] == 2
  797. assert payload["proxy_pool"]["size"] == 10
  798. assert payload["cpa"]["rotate_interval"] == 120
  799. def test_settings_api_persists_whitelisted_updates(monkeypatch, tmp_path: Path) -> None:
  800. env_file = tmp_path / ".env"
  801. env_file.write_text("", encoding="utf-8")
  802. monkeypatch.setenv("ZHUCE6_ENV_FILE", str(env_file))
  803. app = create_app(enable_background_tasks=False, mode="full")
  804. app.state.settings = AppSettings(
  805. runtime_mode="full",
  806. register_threads=1,
  807. register_batch_target_count=20,
  808. register_batch_interval_seconds=10800,
  809. register_mail_provider="cfmail",
  810. register_proxy="http://127.0.0.1:7899",
  811. enable_proxy_pool=True,
  812. proxy_pool_size=20,
  813. proxy_pool_direct_urls="",
  814. proxy_pool_regions=("jp", "tw", "hk", "sg"),
  815. rotate_interval=120,
  816. pool_dir=tmp_path,
  817. )
  818. app.state.background_tasks = []
  819. app.state.registration_loop = None
  820. status, _headers, body = _request_via_asgi(
  821. app,
  822. "PUT",
  823. "/api/settings",
  824. headers={"content-type": "application/json"},
  825. body=json.dumps(
  826. {
  827. "register.threads": 3,
  828. "register.batch_target_count": 25,
  829. "proxy_pool.size": 12,
  830. "cpa.rotate_interval": 300,
  831. }
  832. ).encode("utf-8"),
  833. )
  834. payload = json.loads(body.decode("utf-8"))
  835. persisted = env_file.read_text(encoding="utf-8")
  836. assert status == 200
  837. assert payload["register"]["threads"] == 3
  838. assert payload["proxy_pool"]["size"] == 12
  839. assert payload["cpa"]["rotate_interval"] == 300
  840. assert payload["restart_required"] is True
  841. assert "ZHUCE6_REGISTER_THREADS=3" in persisted
  842. assert "ZHUCE6_REGISTER_BATCH_TARGET_COUNT=25" in persisted
  843. assert "ZHUCE6_PROXY_POOL_SIZE=12" in persisted
  844. assert "ZHUCE6_ROTATE_INTERVAL=300" in persisted
  845. def test_register_control_api_starts_and_stops_loop(monkeypatch, tmp_path: Path) -> None:
  846. events: list[str] = []
  847. class FakeLoop:
  848. def __init__(self, settings): # type: ignore[no-untyped-def]
  849. self.settings = settings
  850. def start(self) -> None:
  851. events.append("start")
  852. def stop(self) -> None:
  853. events.append("stop")
  854. monkeypatch.setattr("main.RegistrationLoop", FakeLoop)
  855. app = create_app(enable_background_tasks=False, mode="dashboard")
  856. app.state.settings = AppSettings(runtime_mode="dashboard", pool_dir=tmp_path)
  857. app.state.background_tasks = []
  858. app.state.registration_loop = None
  859. status, _headers, body = _request_via_asgi(
  860. app,
  861. "POST",
  862. "/api/control/register",
  863. headers={"content-type": "application/json"},
  864. body=b'{"action":"start"}',
  865. )
  866. payload = json.loads(body.decode("utf-8"))
  867. assert status == 200
  868. assert payload["status"] == "started"
  869. assert events == ["start"]
  870. status, _headers, body = _request_via_asgi(
  871. app,
  872. "POST",
  873. "/api/control/register",
  874. headers={"content-type": "application/json"},
  875. body=b'{"action":"stop"}',
  876. )
  877. payload = json.loads(body.decode("utf-8"))
  878. assert status == 200
  879. assert payload["status"] == "stopped"
  880. assert events == ["start", "stop"]
  881. def test_health_dependencies_api_skips_cpa_checks_in_lite_mode(monkeypatch, tmp_path: Path) -> None:
  882. def fail_fetch(*_args, **_kwargs): # type: ignore[no-untyped-def]
  883. raise AssertionError("lite mode should not query CPA")
  884. monkeypatch.setattr("main._fetch_management_auth_files", fail_fetch)
  885. app = create_app(enable_background_tasks=False, mode="lite")
  886. app.state.settings = AppSettings(
  887. runtime_mode="lite",
  888. register_mail_provider="cfmail",
  889. enable_proxy_pool=False,
  890. pool_dir=tmp_path,
  891. )
  892. app.state.background_tasks = []
  893. app.state.registration_loop = None
  894. status, _headers, body = _request_via_asgi(app, "GET", "/api/health/dependencies")
  895. payload = json.loads(body.decode("utf-8"))
  896. assert status == 200
  897. assert payload["cpa"]["status"] == "unconfigured"
  898. assert "docker" not in payload
  899. def test_lite_mode_summary_skips_management_inventory(monkeypatch, tmp_path: Path) -> None:
  900. def fail_fetch(*_args, **_kwargs): # type: ignore[no-untyped-def]
  901. raise AssertionError("lite mode should not fetch CPA inventory")
  902. monkeypatch.setattr("main._fetch_management_auth_files", fail_fetch)
  903. app = create_app(enable_background_tasks=False, mode="lite")
  904. app.state.settings = AppSettings(runtime_mode="lite", pool_dir=tmp_path)
  905. app.state.background_tasks = []
  906. app.state.registration_loop = None
  907. payload = _summary_payload(app)
  908. assert payload["runtime"]["runtime_mode"] == "lite"
  909. assert payload["cpa_count"] is None
  910. assert payload["regular_accounts"] is None
  911. assert payload["tokens"] is None
  912. def test_dashboard_html_contains_settings_tab_and_control_api_hooks() -> None:
  913. html = Path("/home/sophomores/zhuce6/dashboard/zhuce6.html").read_text(encoding="utf-8")
  914. assert "Settings" in html
  915. assert "/api/settings" in html
  916. assert "/api/control/register" in html
  917. assert "/api/health/dependencies" in html
  918. assert "http://localhost:8317/management.html" not in html
  919. assert "settings.cpa.management_url" in html
  920. def test_create_app_lite_mode_registers_only_register_task(monkeypatch) -> None:
  921. class FakeLoop:
  922. def __init__(self, settings): # type: ignore[no-untyped-def]
  923. self.settings = settings
  924. def start(self) -> None:
  925. return None
  926. def stop(self) -> None:
  927. return None
  928. def snapshot(self) -> dict[str, object]:
  929. return {
  930. "name": "register",
  931. "status": "running",
  932. "run_count": 0,
  933. "success_count": 0,
  934. "failure_count": 0,
  935. "threads_alive": 1,
  936. "threads_total": 1,
  937. }
  938. monkeypatch.setattr("main.RegistrationLoop", FakeLoop)
  939. async def run_lifespan() -> None:
  940. app = create_app(enable_background_tasks=True, mode="lite")
  941. async with app.router.lifespan_context(app):
  942. payload = _runtime_payload(app)
  943. assert payload["runtime_mode"] == "lite"
  944. assert payload["registered_tasks"] == ["register"]
  945. asyncio.run(run_lifespan())
  946. def test_dashboard_cors_get_adds_origin_header_for_allowed_origin(monkeypatch, tmp_path: Path) -> None:
  947. monkeypatch.setenv("ZHUCE6_DASHBOARD_ALLOWED_ORIGINS", "http://localhost:8317")
  948. app = create_app(enable_background_tasks=False)
  949. app.state.settings = AppSettings(pool_dir=tmp_path, cleanup_enabled=False, validate_enabled=False)
  950. app.state.background_tasks = []
  951. app.state.registration_loop = None
  952. status_code, headers, _ = _request_via_asgi(
  953. app,
  954. "GET",
  955. "/api/summary",
  956. {"Origin": "http://localhost:8317"},
  957. )
  958. assert status_code == 200
  959. assert headers["access-control-allow-origin"] == "http://localhost:8317"
  960. def test_dashboard_cors_headers_are_not_added_for_other_origins(monkeypatch) -> None:
  961. monkeypatch.setenv("ZHUCE6_DASHBOARD_ALLOWED_ORIGINS", "http://localhost:8317")
  962. app = create_app(enable_background_tasks=False)
  963. app.state.settings = AppSettings(cleanup_enabled=False, validate_enabled=False)
  964. app.state.background_tasks = []
  965. app.state.registration_loop = None
  966. status_code, headers, _ = _request_via_asgi(
  967. app,
  968. "GET",
  969. "/api/runtime",
  970. {"Origin": "http://127.0.0.1:9999"},
  971. )
  972. assert status_code == 200
  973. assert "access-control-allow-origin" not in headers
  974. def test_build_background_tasks_registers_validate_when_enabled() -> None:
  975. tasks = _build_background_tasks(
  976. AppSettings(
  977. cleanup_enabled=False,
  978. d1_cleanup_enabled=False,
  979. validate_enabled=True,
  980. validate_interval=90,
  981. validate_scope="used",
  982. rotate_enabled=False,
  983. account_survival_enabled=False,
  984. )
  985. )
  986. assert [task.name for task in tasks] == ["validate"]
  987. assert tasks[0].interval_seconds == 90
  988. def test_build_background_tasks_registers_d1_cleanup_when_enabled() -> None:
  989. tasks = _build_background_tasks(
  990. AppSettings(
  991. cleanup_enabled=False,
  992. validate_enabled=False,
  993. rotate_enabled=False,
  994. d1_cleanup_enabled=True,
  995. d1_cleanup_interval=1800,
  996. account_survival_enabled=False,
  997. )
  998. )
  999. assert [task.name for task in tasks] == ["d1_cleanup"]
  1000. assert tasks[0].interval_seconds == 1800
  1001. def test_build_background_tasks_uses_responses_survival_when_enabled(monkeypatch, tmp_path: Path) -> None:
  1002. tasks = _build_background_tasks(
  1003. AppSettings(
  1004. cleanup_enabled=False,
  1005. validate_enabled=False,
  1006. rotate_enabled=False,
  1007. d1_cleanup_enabled=False,
  1008. account_survival_enabled=True,
  1009. account_survival_interval=123,
  1010. pool_dir=tmp_path / "pool",
  1011. responses_survival_state_file=tmp_path / "responses_survival.json",
  1012. responses_survival_recent_window_seconds=900,
  1013. responses_survival_require_provenance=True,
  1014. warmup_min_age_seconds=600,
  1015. warmup_min_successful_probes=2,
  1016. )
  1017. )
  1018. assert [task.name for task in tasks] == ["account_survival"]
  1019. assert tasks[0].interval_seconds == 123
  1020. assert "responses_survival_once" in tasks[0].fn.__code__.co_names
  1021. assert "print_responses_survival_summary" in tasks[0].fn.__code__.co_names
  1022. def test_apply_runtime_mode_forces_account_survival_for_full_cpa_register() -> None:
  1023. settings = AppSettings(
  1024. runtime_mode="full",
  1025. backend="cpa",
  1026. register_enabled=True,
  1027. account_survival_enabled=False,
  1028. )
  1029. updated = _apply_runtime_mode(settings, "full")
  1030. assert updated.account_survival_enabled is True
  1031. def test_runtime_payload_counts_warmup_promotions_as_success(tmp_path: Path) -> None:
  1032. responses_state_file = tmp_path / "responses_survival.json"
  1033. responses_state_file.write_text("{}", encoding="utf-8")
  1034. for idx in range(3):
  1035. (tmp_path / f"warmup-{idx}.json").write_text(
  1036. json.dumps(
  1037. {
  1038. "email": f"warmup-{idx}@example.com",
  1039. "access_token": "tok",
  1040. "account_id": f"acct-{idx}",
  1041. "created_at": "2026-03-31T17:30:10+08:00",
  1042. "warmup_required": True,
  1043. "cpa_sync_status": "synced",
  1044. },
  1045. ensure_ascii=False,
  1046. ),
  1047. encoding="utf-8",
  1048. )
  1049. class _Task:
  1050. def __init__(self, snapshot: dict[str, object]) -> None:
  1051. self._snapshot = snapshot
  1052. def snapshot(self) -> dict[str, object]:
  1053. return dict(self._snapshot)
  1054. register_snapshot = {
  1055. "name": "register",
  1056. "status": "running",
  1057. "threads_alive": 4,
  1058. "threads_total": 4,
  1059. "total_attempts": 10,
  1060. "total_success": 1,
  1061. "total_success_registered": 1,
  1062. "total_success_direct": 1,
  1063. "total_warmup_pending": 2,
  1064. "total_cpa_sync_success": 1,
  1065. "total_cpa_sync_failure": 0,
  1066. "total_failure": 4,
  1067. "success_rate": 10.0,
  1068. "registered_success_rate": 10.0,
  1069. "cpa_sync_success_rate": 10.0,
  1070. "last_started_at": "2026-03-31T17:30:00+08:00",
  1071. }
  1072. app = create_app(enable_background_tasks=False)
  1073. app.state.settings = AppSettings(
  1074. pool_dir=tmp_path,
  1075. responses_survival_state_file=responses_state_file,
  1076. cleanup_enabled=False,
  1077. validate_enabled=False,
  1078. rotate_enabled=False,
  1079. )
  1080. app.state.background_tasks = [_Task(register_snapshot)]
  1081. app.state.registration_loop = None
  1082. payload = _runtime_payload(app)
  1083. register_task = next(task for task in payload["task_states"] if task.get("name") == "register")
  1084. assert register_task["total_success"] == 4
  1085. assert register_task["total_success_registered"] == 4
  1086. assert register_task["total_success_promoted"] == 3
  1087. assert register_task["total_cpa_sync_success"] == 4
  1088. assert register_task["success_rate"] == 40.0
  1089. assert register_task["registered_success_rate"] == 40.0
  1090. assert register_task["cpa_sync_success_rate"] == 40.0
  1091. def test_runtime_payload_exposes_worker_retry_split_and_current_warmup_backlog(tmp_path: Path) -> None:
  1092. responses_state_file = tmp_path / "responses_survival.json"
  1093. responses_state_file.write_text("{}", encoding="utf-8")
  1094. runtime_start = "2026-04-01T10:00:00+08:00"
  1095. (tmp_path / "pending.json").write_text(
  1096. json.dumps(
  1097. {
  1098. "email": "pending@example.com",
  1099. "access_token": "tok",
  1100. "account_id": "acct-pending",
  1101. "created_at": "2026-04-01T10:00:30+08:00",
  1102. "warmup_required": True,
  1103. "cpa_sync_status": "warmup_pending",
  1104. },
  1105. ensure_ascii=False,
  1106. ),
  1107. encoding="utf-8",
  1108. )
  1109. (tmp_path / "synced.json").write_text(
  1110. json.dumps(
  1111. {
  1112. "email": "synced@example.com",
  1113. "access_token": "tok",
  1114. "account_id": "acct-synced",
  1115. "created_at": "2026-04-01T10:00:40+08:00",
  1116. "warmup_required": True,
  1117. "cpa_sync_status": "synced",
  1118. },
  1119. ensure_ascii=False,
  1120. ),
  1121. encoding="utf-8",
  1122. )
  1123. class _Task:
  1124. def __init__(self, snapshot: dict[str, object]) -> None:
  1125. self._snapshot = snapshot
  1126. def snapshot(self) -> dict[str, object]:
  1127. return dict(self._snapshot)
  1128. register_snapshot = {
  1129. "name": "register",
  1130. "status": "running",
  1131. "threads_alive": 9,
  1132. "threads_total": 9,
  1133. "last_started_at": runtime_start,
  1134. "pending_token_queue": {"queue_size": 0},
  1135. "total_attempts": 2,
  1136. "total_success": 0,
  1137. "total_success_direct": 0,
  1138. "total_failure": 0,
  1139. "total_warmup_pending": 2,
  1140. }
  1141. app = create_app(enable_background_tasks=False)
  1142. app.state.settings = AppSettings(
  1143. pool_dir=tmp_path,
  1144. responses_survival_state_file=responses_state_file,
  1145. cleanup_enabled=False,
  1146. validate_enabled=False,
  1147. rotate_enabled=False,
  1148. )
  1149. app.state.background_tasks = [_Task(register_snapshot)]
  1150. app.state.registration_loop = None
  1151. payload = _runtime_payload(app)
  1152. register_task = next(task for task in payload["task_states"] if task.get("name") == "register")
  1153. assert register_task["register_worker_threads"] == 8
  1154. assert register_task["retry_sidecar_threads"] == 1
  1155. assert register_task["current_warmup_backlog"] == 1
  1156. def test_summary_payload_uses_effective_success_totals_from_warmup_promotions(tmp_path: Path) -> None:
  1157. responses_state_file = tmp_path / "responses_survival.json"
  1158. responses_state_file.write_text("{}", encoding="utf-8")
  1159. for idx in range(3):
  1160. (tmp_path / f"warmup-{idx}.json").write_text(
  1161. json.dumps(
  1162. {
  1163. "email": f"warmup-{idx}@example.com",
  1164. "access_token": "tok",
  1165. "account_id": f"acct-{idx}",
  1166. "created_at": "2026-03-31T17:30:10+08:00",
  1167. "warmup_required": True,
  1168. "cpa_sync_status": "synced",
  1169. },
  1170. ensure_ascii=False,
  1171. ),
  1172. encoding="utf-8",
  1173. )
  1174. class _Task:
  1175. def __init__(self, snapshot: dict[str, object]) -> None:
  1176. self._snapshot = snapshot
  1177. def snapshot(self) -> dict[str, object]:
  1178. return dict(self._snapshot)
  1179. app = create_app(enable_background_tasks=False)
  1180. app.state.settings = AppSettings(
  1181. pool_dir=tmp_path,
  1182. responses_survival_state_file=responses_state_file,
  1183. cleanup_enabled=False,
  1184. validate_enabled=False,
  1185. rotate_enabled=False,
  1186. )
  1187. app.state.background_tasks = [
  1188. _Task(
  1189. {
  1190. "name": "register",
  1191. "status": "running",
  1192. "threads_alive": 4,
  1193. "threads_total": 4,
  1194. "total_attempts": 10,
  1195. "total_success": 1,
  1196. "total_success_registered": 1,
  1197. "total_success_direct": 1,
  1198. "total_cpa_sync_success": 1,
  1199. "total_cpa_sync_failure": 0,
  1200. "total_failure": 4,
  1201. "success_rate": 10.0,
  1202. "registered_success_rate": 10.0,
  1203. "cpa_sync_success_rate": 10.0,
  1204. "last_started_at": "2026-03-31T17:30:00+08:00",
  1205. }
  1206. )
  1207. ]
  1208. app.state.registration_loop = None
  1209. payload = _summary_payload(app)
  1210. assert payload["registered_success_total"] == 4
  1211. assert payload["cpa_sync_success_total"] == 4
  1212. assert payload["registered_success_rate"] == 40.0
  1213. assert payload["cpa_sync_success_rate"] == 40.0