serve 设为只读模式,禁用外部数据拉取

serve 端不再启动 pull_scheduler 和 financial_scheduler,
阻塞 ext_data 和 financials 的外部获取/同步 API 端点,
数据仅通过 local→serve 同步推送。

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2026-07-06 12:04:46 +08:00
parent 817d130fd3
commit 580743853a
3 changed files with 9 additions and 155 deletions
+6 -114
View File
@@ -26,7 +26,6 @@ from app.services.ext_data import (
write_ext_parquet, write_ext_parquet,
rows_to_parquet, rows_to_parquet,
) )
from app.services.ext_pull import fetch_and_ingest, pull_scheduler
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/ext-data", tags=["ext-data"]) router = APIRouter(prefix="/api/ext-data", tags=["ext-data"])
@@ -330,20 +329,8 @@ async def fetch_preset_data(request: Request, config_id: str):
"""手动触发内置预设 (概念/行业) 的数据拉取。 """手动触发内置预设 (概念/行业) 的数据拉取。
注意: 必须在 /{config_id}/... 动态路由之前声明, 否则 'presets' 会被当成 config_id。 注意: 必须在 /{config_id}/... 动态路由之前声明, 否则 'presets' 会被当成 config_id。
与通用 pull/run 不同: 走 ext_presets 的结构转换 (接口的 concepts/industries
数组 → 拼接成字符串), 保证 schema 与现有数据一致。
""" """
from app.services.ext_presets import fetch_preset raise HTTPException(403, "serve 端为只读模式,数据仅通过 local→serve 同步")
try:
n = await fetch_preset(config_id, _data_dir(request))
except ValueError as e:
raise HTTPException(404, str(e)) from e
except Exception as e:
raise HTTPException(400, f"拉取失败: {e}") from e
_refresh_views(request)
return {"status": "ok", "rows": n}
@router.post("") @router.post("")
@@ -548,115 +535,20 @@ def ingest_data(request: Request, config_id: str, body: IngestReq):
@router.put("/{config_id}/pull") @router.put("/{config_id}/pull")
def configure_pull(request: Request, config_id: str, body: PullConfigReq): def configure_pull(request: Request, config_id: str, body: PullConfigReq):
"""配置(或更新)定时拉取。""" """配置(或更新)定时拉取 — serve 端为只读模式,已禁用"""
store = _store(request) raise HTTPException(403, "serve 端为只读模式,数据仅通过 local→serve 同步")
config = store.get(config_id)
if not config:
raise HTTPException(404, f"配置 '{config_id}' 不存在")
# 保留历史状态字段
old_pull = config.pull
config.pull = PullConfig(
url=body.url,
method=body.method,
headers=body.headers,
body=body.body,
response_path=body.response_path,
field_map=body.field_map,
schedule_minutes=body.schedule_minutes,
enabled=body.enabled,
last_run=old_pull.last_run if old_pull else None,
last_status=old_pull.last_status if old_pull else None,
last_message=old_pull.last_message if old_pull else None,
last_rows=old_pull.last_rows if old_pull else None,
)
store.upsert(config)
# 刷新调度器
pull_scheduler.refresh(_data_dir(request))
# 关闭定时拉取时清理残留的 next_run, 避免前端展示一个永不执行的"下次"
if not config.pull.enabled:
cleared = store.get(config_id)
if cleared and cleared.pull and cleared.pull.next_run:
cleared.pull.next_run = None
store.upsert(cleared)
return {"status": "ok", "pull": config.pull.to_dict()}
@router.post("/{config_id}/pull/test") @router.post("/{config_id}/pull/test")
async def test_pull(request: Request, config_id: str): async def test_pull(request: Request, config_id: str):
"""测试拉取:请求外部 API 并返回预览数据,不写入""" """测试拉取 — serve 端为只读模式,已禁用"""
store = _store(request) raise HTTPException(403, "serve 端为只读模式,数据仅通过 local→serve 同步")
config = store.get(config_id)
if not config:
raise HTTPException(404, f"配置 '{config_id}' 不存在")
if not config.pull or not config.pull.url:
raise HTTPException(400, "拉取未配置或 URL 为空")
# 临时构建一个带新配置的 config 用于测试
from app.services.ext_pull import _extract_rows, _apply_field_map
import httpx
pull = config.pull
try:
async with httpx.AsyncClient(timeout=30) as client:
headers = pull.headers or {}
kwargs: dict = {"headers": headers}
if pull.method.upper() == "POST" and pull.body:
kwargs["content"] = pull.body
if "content-type" not in {k.lower() for k in headers}:
kwargs["headers"]["Content-Type"] = "application/json"
resp = await client.request(pull.method.upper(), pull.url, **kwargs)
resp.raise_for_status()
data = resp.json()
rows = _extract_rows(data, pull.response_path)
preview = _apply_field_map(rows[:5], pull.field_map)
return {
"status": "ok",
"total_rows": len(rows),
"preview": preview,
"has_symbol": bool(rows and "symbol" in rows[0]),
}
except Exception as e:
raise HTTPException(400, f"测试失败: {e}") from e
@router.post("/{config_id}/pull/run") @router.post("/{config_id}/pull/run")
async def run_pull(request: Request, config_id: str): async def run_pull(request: Request, config_id: str):
"""手动触发一次拉取并写入。""" """手动触发一次拉取并写入。"""
store = _store(request) raise HTTPException(403, "serve 端为只读模式,数据仅通过 local→serve 同步")
config = store.get(config_id)
if not config:
raise HTTPException(404, f"配置 '{config_id}' 不存在")
if not config.pull or not config.pull.url:
raise HTTPException(400, "拉取未配置或 URL 为空")
try:
n, d = await fetch_and_ingest(config, _data_dir(request))
_refresh_views(request)
# 写回执行状态, 让前端"上次执行"面板立即反映
updated = store.get(config_id)
if updated and updated.pull:
from datetime import datetime, timezone
updated.pull.last_run = datetime.now(timezone.utc).isoformat()
updated.pull.last_status = "success"
updated.pull.last_message = f"{n} rows @ {d}"
updated.pull.last_rows = n
store.upsert(updated)
return {"status": "ok", "rows": n, "date": d}
except Exception as e:
# 失败也写回状态, 记录错误信息
failed = store.get(config_id)
if failed and failed.pull:
from datetime import datetime, timezone
failed.pull.last_run = datetime.now(timezone.utc).isoformat()
failed.pull.last_status = "error"
failed.pull.last_message = str(e)[:200]
store.upsert(failed)
raise HTTPException(400, f"拉取失败: {e}") from e
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
+2 -21
View File
@@ -98,27 +98,8 @@ def get_cash_flow(request: Request, symbol: str | None = None):
@router.post("/sync/{table}") @router.post("/sync/{table}")
def sync_table(request: Request, table: str): def sync_table(request: Request, table: str):
"""手动触发同步(立即返回,后台异步执行)。 """手动触发同步 — serve 端为只读模式,已禁用。"""
raise HTTPException(403, "serve 端为只读模式,数据仅通过 local→serve 同步")
table: metrics / income / balance_sheet / cash_flow / all
同步在后台线程执行,全量同步需数分钟。本接口立即返回 started 状态,
前端通过轮询 GET /status 的 syncing 字段观察进度。
"""
capset = request.app.state.capabilities
capset.require(Cap.FINANCIAL)
valid_tables = {"metrics", "income", "balance_sheet", "cash_flow", "all"}
if table not in valid_tables:
raise HTTPException(400, f"invalid table: {table}, expected one of {valid_tables}")
fs = getattr(request.app.state, "financial_scheduler", None)
if not fs:
return {"status": "error", "message": "FinancialScheduler not available"}
target = None if table == "all" else table
result = fs.trigger(target)
return {"status": "ok", "synced": result}
class AnalyzeRequest(BaseModel): class AnalyzeRequest(BaseModel):
+1 -20
View File
@@ -63,26 +63,13 @@ async def lifespan(app: FastAPI):
logger.warning("scheduler not started: %s", e) logger.warning("scheduler not started: %s", e)
app.state.scheduler = None app.state.scheduler = None
# 扩展数据定时拉取 # 内置扩展表 (概念/行业): 只创建 config 供 sync 解压时用, 不拉数据。
from app.services.ext_pull import pull_scheduler
pull_scheduler.start(store.data_dir)
pull_scheduler.refresh(store.data_dir)
app.state.pull_scheduler = pull_scheduler
# 内置扩展表 (概念/行业): 只创建 config (含拉取配置), 不自动拉数据
# 数据获取由用户在概念/行业页点「获取数据」手动触发 (POST /api/ext-data/presets/{id}/fetch)
try: try:
from app.services.ext_presets import ensure_builtin_presets from app.services.ext_presets import ensure_builtin_presets
await ensure_builtin_presets(store.data_dir) await ensure_builtin_presets(store.data_dir)
except Exception as e: # noqa: BLE001 except Exception as e: # noqa: BLE001
logger.warning("内置扩展表初始化失败 (不影响启动): %s", e) logger.warning("内置扩展表初始化失败 (不影响启动): %s", e)
# 财务数据 (需 Expert 套餐): 仅初始化调度器供 /api/financials/sync/* 手动同步,
# 不启动自动调度——用户在「财务分析」页点「同步」手动拉取。
from app.services.financial_sync import financial_scheduler
financial_scheduler.start(store.data_dir, capset)
app.state.financial_scheduler = financial_scheduler
# 策略引擎 # 策略引擎
from app.strategy.engine import StrategyEngine from app.strategy.engine import StrategyEngine
from app.services.screener import ScreenerService from app.services.screener import ScreenerService
@@ -105,12 +92,6 @@ async def lifespan(app: FastAPI):
if app.state.scheduler: if app.state.scheduler:
app.state.scheduler.shutdown(wait=False) app.state.scheduler.shutdown(wait=False)
ps = getattr(app.state, "pull_scheduler", None)
if ps:
ps.stop()
fsc = getattr(app.state, "financial_scheduler", None)
if fsc:
fsc.stop()
logger.info("shutdown") logger.info("shutdown")