重置项目
This commit is contained in:
@@ -8,7 +8,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from datetime import date, datetime, timezone
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -200,15 +200,82 @@ class FinancialScheduler:
|
||||
# 手动同步(run_now)是否正在进行。前端据此显示"同步中"并防重复点击。
|
||||
self._is_syncing = False
|
||||
|
||||
def start(self, data_dir: Path, capset: CapabilitySet) -> None:
|
||||
def start(self, data_dir: Path, capset: CapabilitySet, *, auto_schedule: bool = False) -> None:
|
||||
"""初始化调度器，并按需启动周期同步后台任务。
|
||||
|
||||
auto_schedule=False (默认): 仅初始化 (设置数据目录/能力 + 恢复 last_sync),
|
||||
供 /api/financials/sync/* 手动同步使用, 不启动自动调度。
|
||||
auto_schedule=True: 额外启动每周一次的 metrics 自动同步 (启动后 60s 首跑)。
|
||||
"""
|
||||
# 先记录 data_dir/capset, 即使当前无 FINANCIAL 也保留引用:
|
||||
# 用户稍后在「设置」页升级到 Expert Key 时, update_capabilities() 会把新 capset
|
||||
# 推进来,trigger()/run_now() 才能用上 FINANCIAL。否则 _capset 永远是 None,
|
||||
# 即便 app.state.capabilities 已更新, 调度器仍报 "no FINANCIAL capability"。
|
||||
self._data_dir = data_dir
|
||||
self._capset = capset
|
||||
if not capset.has(Cap.FINANCIAL):
|
||||
logger.info("FinancialScheduler skipped: no FINANCIAL capability")
|
||||
return
|
||||
self._data_dir = data_dir
|
||||
self._capset = capset
|
||||
# 从持久化恢复上次同步时间: 重启后前端仍能显示真实最后同步时间,而非"尚未同步"
|
||||
try:
|
||||
from app.services import preferences
|
||||
restored = dict(preferences.get_financial_sync_times())
|
||||
# 老用户迁移兜底: 若某表在 preferences 无记录但 parquet 已存在(升级前同步过),
|
||||
# 用 parquet 文件的修改时间作为同步时间并补写持久化。
|
||||
for table in FINANCIAL_TABLES:
|
||||
if table in restored:
|
||||
continue
|
||||
parquet = data_dir / "financials" / table / "part.parquet"
|
||||
if parquet.exists():
|
||||
mtime = datetime.fromtimestamp(parquet.stat().st_mtime, tz=timezone.utc).isoformat()
|
||||
restored[table] = mtime
|
||||
preferences.set_financial_sync_time(table, mtime)
|
||||
logger.info("FinancialScheduler backfilled last_sync for %s from parquet mtime", table)
|
||||
self._last_sync = restored
|
||||
if self._last_sync:
|
||||
logger.info("FinancialScheduler restored last_sync: %s", list(self._last_sync.keys()))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("restore financial_sync_times failed: %s", e)
|
||||
|
||||
if not auto_schedule:
|
||||
# 仅初始化 (手动同步用), 不启动周期任务。
|
||||
logger.info("FinancialScheduler initialized (auto-schedule disabled; manual sync only)")
|
||||
return
|
||||
|
||||
self._running = True
|
||||
self._task = asyncio.create_task(self._run_loop())
|
||||
logger.info("FinancialScheduler started")
|
||||
logger.info("FinancialScheduler started (auto-schedule enabled)")
|
||||
|
||||
def _record_sync(self, table: str) -> None:
|
||||
"""记录一张表的同步完成时间: 更新内存 + 持久化到 preferences.json。
|
||||
|
||||
持久化确保即使重启,前端 /status 仍返回真实的最后同步时间,
|
||||
不会错误地显示"尚未同步"。
|
||||
"""
|
||||
ts = datetime.now(timezone.utc).isoformat()
|
||||
self._last_sync[table] = ts
|
||||
try:
|
||||
from app.services import preferences
|
||||
preferences.set_financial_sync_time(table, ts)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("persist financial_sync_time(%s) failed: %s", e)
|
||||
|
||||
def update_capabilities(self, capset: CapabilitySet) -> None:
|
||||
"""刷新调度器持有的能力集。
|
||||
|
||||
用户在「设置」页新增/清除 API Key 后, settings API 会重新探测能力并更新
|
||||
app.state.capabilities; 必须同步推给本调度器, 否则 trigger()/run_now() 仍读
|
||||
启动时的旧 capset, 即便 app.state 已含 FINANCIAL, 调度器仍报
|
||||
"no FINANCIAL capability" 而拒绝同步 (表现为前端「全部同步」按钮闪一下无动作)。
|
||||
"""
|
||||
prev = self._capset
|
||||
self._capset = capset
|
||||
had = bool(prev) and prev.has(Cap.FINANCIAL)
|
||||
now = capset.has(Cap.FINANCIAL)
|
||||
if had != now:
|
||||
logger.info(
|
||||
"FinancialScheduler capabilities updated: FINANCIAL %s -> %s", had, now
|
||||
)
|
||||
|
||||
def stop(self) -> None:
|
||||
self._running = False
|
||||
@@ -217,22 +284,6 @@ class FinancialScheduler:
|
||||
self._task = None
|
||||
logger.info("FinancialScheduler stopped")
|
||||
|
||||
def update(self, data_dir: Path, capset: CapabilitySet) -> None:
|
||||
"""运行时更新数据目录和能力集。
|
||||
|
||||
用户在设置页更换/清除 Key 后,能力集可能变化,无需重启服务即可让
|
||||
财务调度器生效或失效。
|
||||
"""
|
||||
had_financial = self._capset is not None and self._capset.has(Cap.FINANCIAL)
|
||||
has_financial = capset.has(Cap.FINANCIAL)
|
||||
self._data_dir = data_dir
|
||||
self._capset = capset
|
||||
|
||||
if has_financial and not self._running:
|
||||
self.start(data_dir, capset)
|
||||
elif had_financial and not has_financial and self._running:
|
||||
self.stop()
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
"""每周执行一次 metrics 同步。"""
|
||||
try:
|
||||
@@ -245,7 +296,7 @@ class FinancialScheduler:
|
||||
# 每周: 只同步 metrics
|
||||
try:
|
||||
rows = sync_metrics(self._data_dir, self._capset)
|
||||
self._last_sync["metrics"] = datetime.now(timezone.utc).isoformat()
|
||||
self._record_sync("metrics")
|
||||
logger.info("FinancialScheduler: metrics synced, %d rows", rows)
|
||||
except Exception as e:
|
||||
logger.warning("FinancialScheduler: metrics sync failed: %s", e)
|
||||
@@ -259,8 +310,39 @@ class FinancialScheduler:
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
def _run_body(self, table: str | None) -> dict[str, int]:
|
||||
"""同步逻辑本体(不加锁,假设调用方已持有 _is_syncing)。
|
||||
|
||||
table=None 同步全部 4 张表;否则只同步指定表。
|
||||
每张表完成立即更新 last_sync,让前端轮询 /status 能看到进度递增。
|
||||
"""
|
||||
if table:
|
||||
fn = {
|
||||
"metrics": sync_metrics,
|
||||
"income": sync_income,
|
||||
"balance_sheet": sync_balance_sheet,
|
||||
"cash_flow": sync_cash_flow,
|
||||
}.get(table)
|
||||
if not fn:
|
||||
return {}
|
||||
rows = fn(self._data_dir, self._capset)
|
||||
self._record_sync(table)
|
||||
return {table: rows}
|
||||
# 全部同步
|
||||
symbols = _get_symbols(self._data_dir)
|
||||
result: dict[str, int] = {}
|
||||
for t in FINANCIAL_TABLES:
|
||||
result[t] = _sync_table(t, symbols, self._data_dir, self._capset, latest_only=True)
|
||||
self._record_sync(t)
|
||||
_refresh_financials_views(self._data_dir)
|
||||
return result
|
||||
|
||||
def run_now(self, table: str | None = None) -> dict[str, int]:
|
||||
"""手动触发同步。table=None 同步全部。
|
||||
"""同步执行一次同步(阻塞调用线程)。
|
||||
|
||||
⚠ 全量同步需数分钟,务必在后台线程调用,不要直接在 HTTP 请求线程里阻塞,
|
||||
否则请求会长时间 pending 直至被浏览器/代理超时掐断(表现为"点击无反应")。
|
||||
HTTP 接口应调用 trigger() 立即返回,再让前端轮询 /status.syncing 看进度。
|
||||
|
||||
用 _is_syncing 标志防并发:若已有同步在进行,本次直接跳过,
|
||||
避免重复请求拖慢服务端 / 触发上游限流。
|
||||
@@ -273,32 +355,46 @@ class FinancialScheduler:
|
||||
return {"_skipped": 1}
|
||||
self._is_syncing = True
|
||||
try:
|
||||
if table:
|
||||
fn = {
|
||||
"metrics": sync_metrics,
|
||||
"income": sync_income,
|
||||
"balance_sheet": sync_balance_sheet,
|
||||
"cash_flow": sync_cash_flow,
|
||||
}.get(table)
|
||||
if not fn:
|
||||
return {}
|
||||
rows = fn(self._data_dir, self._capset)
|
||||
self._last_sync[table] = datetime.now(timezone.utc).isoformat()
|
||||
return {table: rows}
|
||||
else:
|
||||
# 全部同步: 逐表执行, 每张完成立即更新 last_sync,
|
||||
# 让前端轮询 /status 能看到进度递增 (而非等全部完成才一次性更新)。
|
||||
symbols = _get_symbols(self._data_dir)
|
||||
result: dict[str, int] = {}
|
||||
for t in FINANCIAL_TABLES:
|
||||
result[t] = _sync_table(t, symbols, self._data_dir, self._capset, latest_only=True)
|
||||
self._last_sync[t] = datetime.now(timezone.utc).isoformat()
|
||||
_refresh_financials_views(self._data_dir)
|
||||
return result
|
||||
return self._run_body(table)
|
||||
finally:
|
||||
with self._lock:
|
||||
self._is_syncing = False
|
||||
|
||||
def trigger(self, table: str | None = None) -> dict[str, int]:
|
||||
"""触发一次同步(非阻塞,立即返回)。
|
||||
|
||||
在后台线程执行同步体,HTTP 请求无需等待。
|
||||
返回 {"started": True/False}:
|
||||
- False = 能力不足或已有同步在进行(被防并发跳过)
|
||||
- True = 已在后台开始,前端应轮询 /status.syncing 观察进度
|
||||
|
||||
⚠ _is_syncing 在此处置 True(持锁),确保 trigger 返回时前端轮询
|
||||
/status 已能看到 syncing=True,无竞态窗口;同时防止快速重复点击
|
||||
启动多个后台线程。后台线程复用 _run_body 执行真正的同步逻辑。
|
||||
"""
|
||||
if not self._capset or not self._capset.has(Cap.FINANCIAL):
|
||||
return {"started": False, "reason": "no FINANCIAL capability"}
|
||||
with self._lock:
|
||||
if self._is_syncing:
|
||||
logger.info("financial sync trigger skipped: already running")
|
||||
return {"started": False, "reason": "already running"}
|
||||
# 持锁置位:保证 trigger 返回前 syncing 已为 True
|
||||
self._is_syncing = True
|
||||
|
||||
def _bg() -> None:
|
||||
try:
|
||||
self._run_body(table)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.exception("background financial sync failed: %s", e)
|
||||
finally:
|
||||
with self._lock:
|
||||
self._is_syncing = False
|
||||
|
||||
t = threading.Thread(target=_bg, name="financial-sync", daemon=True)
|
||||
t.start()
|
||||
logger.info("financial sync triggered in background: table=%s", table or "all")
|
||||
return {"started": True}
|
||||
|
||||
@property
|
||||
def is_syncing(self) -> bool:
|
||||
"""手动同步是否正在进行(供 /status 返回,前端据此显示"同步中")。"""
|
||||
|
||||
Reference in New Issue
Block a user