初始化工程
This commit is contained in:
@@ -0,0 +1,15 @@
|
||||
"""Stock Panel backend."""
|
||||
|
||||
import sys
|
||||
|
||||
__version__ = "0.1.44"
|
||||
|
||||
# Windows 默认 stdout/stderr 编码为 GBK(cp936),数据源 SDK 内部输出含 emoji 的
|
||||
# 指数/标的名称(如 \U0001f193)时会抛 UnicodeEncodeError,导致请求失败。
|
||||
# 进程加载最早阶段强制 UTF-8,根治此类编码崩溃。
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(_stream, "reconfigure"):
|
||||
try:
|
||||
_stream.reconfigure(encoding="utf-8", errors="replace")
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
@@ -0,0 +1 @@
|
||||
"""FastAPI 路由汇总。"""
|
||||
@@ -0,0 +1,144 @@
|
||||
"""告警触发记录 API — 查询/清空/生成演示数据 alerts.jsonl。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
|
||||
from app.services import alert_store
|
||||
|
||||
router = APIRouter(prefix="/api/alerts", tags=["alerts"])
|
||||
|
||||
|
||||
def _data_dir(request: Request) -> Path:
|
||||
return request.app.state.repo.store.data_dir
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_alerts(
|
||||
request: Request,
|
||||
days: int = 7,
|
||||
limit: int = 5000,
|
||||
source: str | None = None,
|
||||
type: str | None = None,
|
||||
):
|
||||
"""查询触发记录 (时间倒序)。"""
|
||||
events = alert_store.list_recent(
|
||||
_data_dir(request), days=days, limit=limit, source=source, type=type,
|
||||
)
|
||||
total = alert_store.count(_data_dir(request))
|
||||
return {"alerts": events, "total": total}
|
||||
|
||||
|
||||
@router.delete("")
|
||||
def clear_alerts(request: Request):
|
||||
"""清空全部触发记录。"""
|
||||
n = alert_store.clear(_data_dir(request))
|
||||
return {"ok": True, "cleared": n}
|
||||
|
||||
|
||||
@router.delete("/{ts}")
|
||||
def delete_alert(ts: int, request: Request):
|
||||
"""删除单条触发记录 (按 ts 毫秒时间戳)。"""
|
||||
deleted = alert_store.delete_one(_data_dir(request), ts)
|
||||
if not deleted:
|
||||
raise HTTPException(status_code=404, detail="记录不存在")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# ── 演示数据生成 (仅 Dev 页用) ─────────────────────────
|
||||
|
||||
_DEMO_STOCKS = [
|
||||
("600519.SH", "贵州茅台"), ("000001.SZ", "平安银行"), ("300750.SZ", "宁德时代"),
|
||||
("002594.SZ", "比亚迪"), ("000858.SZ", "五粮液"), ("601318.SH", "中国平安"),
|
||||
("002475.SZ", "立讯精密"), ("600036.SH", "招商银行"), ("000725.SZ", "京东方A"),
|
||||
("300059.SZ", "东方财富"),
|
||||
]
|
||||
_DEMO_TEMPLATES = [
|
||||
("signal", "MA金叉触发", ["signal_ma_golden_5_20"], "info"),
|
||||
("signal", "放量突破新高", ["signal_volume_surge", "signal_n_day_high"], "warn"),
|
||||
("signal", "MACD金叉", ["signal_macd_golden"], "info"),
|
||||
("signal", "跌破MA20", ["signal_ma20_breakdown"], "info"),
|
||||
("price", "涨幅超 5%", [], "warn"),
|
||||
("price", "RSI 极度超卖", [], "warn"),
|
||||
("price", "跌幅超 3%", [], "info"),
|
||||
("market", "涨停封板", ["signal_limit_up"], "critical"),
|
||||
("market", "连板异动", ["signal_limit_up"], "warn"),
|
||||
("market", "炸板", ["signal_broken_limit_up"], "warn"),
|
||||
# 新策略变更格式
|
||||
("strategy", "策略「趋势突破」进入 贵州茅台 +2.3%", ["signal_n_day_high", "signal_volume_surge"], "info"),
|
||||
("strategy", "策略「趋势突破」移出 五粮液 -1.5%", ["signal_ma20_breakdown"], "info"),
|
||||
("strategy", "策略「新低反转」进入 平安银行 +1.1%", ["signal_n_day_low"], "warn"),
|
||||
("strategy", "策略「MACD金叉」移出 比亚迪 -0.8%", ["signal_macd_golden"], "info"),
|
||||
# 批量变更
|
||||
("strategy", "策略「趋势突破」进入 6 只：平安银行、宁德时代、比亚迪、东方财富、招商银行、立讯精密", [], "info"),
|
||||
("strategy", "策略「MACD金叉」移出 7 只：京东方A、平安银行、五粮液、立讯精密、招商银行、东方财富、比亚迪", [], "warn"),
|
||||
]
|
||||
|
||||
|
||||
@router.post("/seed")
|
||||
def seed_demo_alerts(request: Request, count: int = 12, recent: bool = True):
|
||||
"""生成演示触发记录 (Dev 页用)。
|
||||
|
||||
Args:
|
||||
count: 生成条数 (1-50)
|
||||
recent: True=时间戳设为"刚刚"(用于测试闪烁效果); False=分散在近3天
|
||||
"""
|
||||
count = max(1, min(50, count))
|
||||
now_ms = int(time.time() * 1000)
|
||||
events = []
|
||||
for i in range(count):
|
||||
source, message, signals, severity = _DEMO_TEMPLATES[i % len(_DEMO_TEMPLATES)]
|
||||
sym, name = _DEMO_STOCKS[i % len(_DEMO_STOCKS)]
|
||||
# 策略类型按消息推导 type: new_entry / dropped, 否则沿用 source
|
||||
if source == "strategy":
|
||||
if "进入" in message:
|
||||
ev_type = "new_entry"
|
||||
elif "移出" in message:
|
||||
ev_type = "dropped"
|
||||
else:
|
||||
ev_type = "strategy"
|
||||
else:
|
||||
ev_type = source
|
||||
# recent 模式: 时间戳从现在往前每条错开 30 秒 (最新在前)
|
||||
ts = now_ms - (i * 30000) if recent else now_ms - random.randint(60, 4320) * 60 * 1000
|
||||
events.append({
|
||||
"ts": ts,
|
||||
"rule_id": f"demo_rule_{i}",
|
||||
"rule_name": message,
|
||||
"source": source,
|
||||
"type": ev_type,
|
||||
"symbol": "" if source == "strategy" and ("只：" in message) else sym,
|
||||
"name": name,
|
||||
"message": message,
|
||||
"price": round(random.uniform(8, 1800), 2) if not (source == "strategy" and "只：" in message) else None,
|
||||
"change_pct": round(random.uniform(-0.06, 0.098), 4) if not (source == "strategy" and "只：" in message) else None,
|
||||
"signals": signals,
|
||||
"severity": severity,
|
||||
})
|
||||
alert_store.append_many(_data_dir(request), events)
|
||||
|
||||
# 同步推入 SSE 队列, 让所有连着 SSE 的客户端实时收到 (不依赖轮询)
|
||||
qs = getattr(request.app.state, "quote_service", None)
|
||||
if qs:
|
||||
# 转成 SSE 推送格式 (和 _evaluate_monitors 一致)
|
||||
sse_alerts = [{
|
||||
"source": ev["source"],
|
||||
"type": ev["type"],
|
||||
"rule_id": ev.get("rule_id"),
|
||||
"symbol": ev["symbol"],
|
||||
"name": ev["name"],
|
||||
"message": ev["message"],
|
||||
"price": ev["price"],
|
||||
"change_pct": ev["change_pct"],
|
||||
"signals": ev["signals"],
|
||||
"severity": ev.get("severity", "info"),
|
||||
} for ev in events]
|
||||
with qs._lock:
|
||||
qs._pending_alerts.extend(sse_alerts)
|
||||
qs._alert_event.set()
|
||||
|
||||
return {"ok": True, "generated": len(events)}
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
"""自定义分析菜单 API。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.services.ext_data import ExtConfigStore
|
||||
|
||||
router = APIRouter(prefix="/api/analysis-menus", tags=["analysis-menus"])
|
||||
|
||||
|
||||
class AnalysisColumn(BaseModel):
|
||||
field: str
|
||||
label: str = ""
|
||||
type: Literal["string", "number", "percent", "amount", "date"] = "string"
|
||||
width: int | None = None
|
||||
sortable: bool = False
|
||||
precision: int | None = None
|
||||
format: str | None = None
|
||||
aggregate: Literal["count", "avg", "sum", "min", "max"] | None = None
|
||||
visible: bool = True
|
||||
|
||||
|
||||
class DefaultSort(BaseModel):
|
||||
field: str
|
||||
order: Literal["asc", "desc"] = "desc"
|
||||
|
||||
|
||||
class AnalysisMenu(BaseModel):
|
||||
id: str = Field(..., min_length=1, max_length=64, pattern=r"^[a-zA-Z0-9_]+$")
|
||||
label: str = Field(..., min_length=1, max_length=64)
|
||||
icon: str = "chart"
|
||||
data_source: str = Field(..., min_length=1)
|
||||
template: Literal["dimension_rank", "ranking", "table"] = "dimension_rank"
|
||||
dimension_field: str | None = None
|
||||
rank_field: str | None = None
|
||||
group_columns: list[AnalysisColumn] = Field(default_factory=list)
|
||||
detail_columns: list[AnalysisColumn] = Field(default_factory=list)
|
||||
default_sort: DefaultSort | None = None
|
||||
visible: bool = True
|
||||
order: int = 0
|
||||
created_at: str | None = None
|
||||
updated_at: str | None = None
|
||||
builtin: bool = False
|
||||
|
||||
|
||||
class UpsertAnalysisMenu(BaseModel):
|
||||
label: str = Field(..., min_length=1, max_length=64)
|
||||
icon: str = "chart"
|
||||
data_source: str = Field(..., min_length=1)
|
||||
template: Literal["dimension_rank", "ranking", "table"] = "dimension_rank"
|
||||
dimension_field: str | None = None
|
||||
rank_field: str | None = None
|
||||
group_columns: list[AnalysisColumn] = Field(default_factory=list)
|
||||
detail_columns: list[AnalysisColumn] = Field(default_factory=list)
|
||||
default_sort: DefaultSort | None = None
|
||||
visible: bool = True
|
||||
order: int = 0
|
||||
|
||||
|
||||
class ReorderMenusReq(BaseModel):
|
||||
ids: list[str] = Field(..., min_length=1)
|
||||
|
||||
|
||||
def _data_dir(request: Request) -> Path:
|
||||
return request.app.state.repo.store.data_dir
|
||||
|
||||
|
||||
def _base_dir(request: Request) -> Path:
|
||||
return _data_dir(request) / "analysis_menus"
|
||||
|
||||
|
||||
def _path(request: Request, menu_id: str) -> Path:
|
||||
return _base_dir(request) / f"{menu_id}.json"
|
||||
|
||||
|
||||
def _load_saved(request: Request) -> list[AnalysisMenu]:
|
||||
base = _base_dir(request)
|
||||
if not base.exists():
|
||||
return []
|
||||
items: list[AnalysisMenu] = []
|
||||
for p in sorted(base.glob("*.json")):
|
||||
try:
|
||||
raw = json.loads(p.read_text(encoding="utf-8"))
|
||||
items.append(AnalysisMenu(**raw))
|
||||
except Exception:
|
||||
continue
|
||||
return items
|
||||
|
||||
|
||||
def _ordered(items: list[AnalysisMenu]) -> list[AnalysisMenu]:
|
||||
return sorted(items, key=lambda m: (m.order, m.label, m.id))
|
||||
|
||||
|
||||
def _save(request: Request, menu: AnalysisMenu) -> AnalysisMenu:
|
||||
now = datetime.now().isoformat()
|
||||
if not menu.created_at:
|
||||
menu.created_at = now
|
||||
menu.updated_at = now
|
||||
menu.builtin = False
|
||||
base = _base_dir(request)
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
_path(request, menu.id).write_text(
|
||||
json.dumps(menu.model_dump(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return menu
|
||||
|
||||
|
||||
def _default_menus(request: Request) -> list[AnalysisMenu]:
|
||||
ext_store = ExtConfigStore(_data_dir(request))
|
||||
menus: list[AnalysisMenu] = []
|
||||
for cfg in ext_store.load_all():
|
||||
fields = cfg.fields
|
||||
concept = next((f for f in fields if "概念" in f.name or "概念" in f.label or "concept" in f.name.lower()), None)
|
||||
if concept:
|
||||
detail_names = ["股票简称", "股票代码", concept.name, "人气排名", "资金流向", "PE", "PB"]
|
||||
detail_columns = []
|
||||
for name in detail_names:
|
||||
f = next((x for x in fields if x.name == name), None)
|
||||
if not f:
|
||||
continue
|
||||
is_num = f.dtype in ("int", "float")
|
||||
detail_columns.append(AnalysisColumn(
|
||||
field=f.name,
|
||||
label=f.label or f.name,
|
||||
type="number" if is_num else "string",
|
||||
sortable=is_num,
|
||||
precision=2 if f.dtype == "float" else None,
|
||||
))
|
||||
menus.append(AnalysisMenu(
|
||||
id="concept_analysis",
|
||||
label="概念分析",
|
||||
icon="tags",
|
||||
data_source=cfg.id,
|
||||
template="dimension_rank",
|
||||
dimension_field=concept.name,
|
||||
group_columns=[
|
||||
AnalysisColumn(field="__dimension", label="概念"),
|
||||
AnalysisColumn(field="__count", label="股票数", type="number", sortable=True),
|
||||
],
|
||||
detail_columns=detail_columns,
|
||||
default_sort=DefaultSort(field="人气排名", order="asc") if any(c.field == "人气排名" for c in detail_columns) else None,
|
||||
order=100,
|
||||
builtin=True,
|
||||
))
|
||||
break
|
||||
return menus
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_menus(request: Request):
|
||||
saved = _load_saved(request)
|
||||
saved_ids = {m.id for m in saved}
|
||||
defaults = [m for m in _default_menus(request) if m.id not in saved_ids]
|
||||
return {"items": _ordered(saved + defaults)}
|
||||
|
||||
|
||||
@router.get("/{menu_id}")
|
||||
def get_menu(request: Request, menu_id: str):
|
||||
for menu in _ordered(_load_saved(request) + _default_menus(request)):
|
||||
if menu.id == menu_id:
|
||||
return menu
|
||||
raise HTTPException(404, f"分析菜单 '{menu_id}' 不存在")
|
||||
|
||||
|
||||
@router.post("/reorder")
|
||||
def reorder_menus(request: Request, body: ReorderMenusReq):
|
||||
saved = {m.id: m for m in _load_saved(request)}
|
||||
defaults = {m.id: m for m in _default_menus(request)}
|
||||
for idx, menu_id in enumerate(body.ids):
|
||||
menu = saved.get(menu_id) or defaults.get(menu_id)
|
||||
if not menu:
|
||||
continue
|
||||
menu.order = idx
|
||||
_save(request, menu)
|
||||
return {"items": _ordered(_load_saved(request))}
|
||||
|
||||
|
||||
@router.post("/{menu_id}")
|
||||
def upsert_menu(request: Request, menu_id: str, body: UpsertAnalysisMenu):
|
||||
if not menu_id.replace("_", "").isalnum():
|
||||
raise HTTPException(400, "菜单标识只能包含字母、数字和下划线")
|
||||
existing = next((m for m in _load_saved(request) if m.id == menu_id), None)
|
||||
menu = AnalysisMenu(
|
||||
id=menu_id,
|
||||
created_at=existing.created_at if existing else None,
|
||||
**body.model_dump(),
|
||||
)
|
||||
return _save(request, menu)
|
||||
|
||||
|
||||
@router.delete("/{menu_id}")
|
||||
def delete_menu(request: Request, menu_id: str):
|
||||
p = _path(request, menu_id)
|
||||
if not p.exists():
|
||||
raise HTTPException(404, f"分析菜单 '{menu_id}' 不存在或为默认菜单")
|
||||
p.unlink()
|
||||
return {"status": "deleted"}
|
||||
@@ -0,0 +1,80 @@
|
||||
"""访问门控 API 与管理接口。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
|
||||
from app import auth
|
||||
from app import uuid_store
|
||||
|
||||
router = APIRouter(prefix="/api/auth", tags=["auth"])
|
||||
admin_router = APIRouter(prefix="/api/admin", tags=["admin"])
|
||||
|
||||
|
||||
@router.post("/verify")
|
||||
def verify_credential(req: auth.VerifyIn) -> auth.VerifyOut:
|
||||
"""校验管理员令牌或普通 UUID,成功后返回访问令牌及角色。"""
|
||||
role = auth.verify_credential(req.credential)
|
||||
if role:
|
||||
token = auth.create_access_token(role)
|
||||
return auth.VerifyOut(valid=True, role=role.value, token=token)
|
||||
return auth.VerifyOut(valid=False, role=None, token=None)
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
def auth_status(request: Request) -> auth.AuthStatusOut:
|
||||
"""返回当前门控状态、当前请求是否通过校验及角色。"""
|
||||
enabled = auth.access_control_enabled()
|
||||
token = auth.get_access_token_from_request(request)
|
||||
role = auth.validate_access_token(token)
|
||||
return auth.AuthStatusOut(
|
||||
enabled=enabled,
|
||||
verified=role is not None,
|
||||
role=role.value if role else None,
|
||||
)
|
||||
|
||||
|
||||
# ===== 管理员 UUID 管理 =====
|
||||
|
||||
@admin_router.get("/uuids")
|
||||
def list_uuids(request: Request) -> list[auth.UuidRecordOut]:
|
||||
"""列出所有动态 UUID(仅管理员)。"""
|
||||
auth.require_admin(request)
|
||||
records = uuid_store.list_uuids()
|
||||
return [
|
||||
auth.UuidRecordOut(
|
||||
uuid=r["uuid"],
|
||||
label=r.get("label", ""),
|
||||
enabled=r.get("enabled", True),
|
||||
created_at=r.get("created_at", 0),
|
||||
)
|
||||
for r in records
|
||||
]
|
||||
|
||||
|
||||
@admin_router.post("/uuids")
|
||||
def create_uuid(req: auth.UuidCreateIn, request: Request) -> auth.UuidRecordOut:
|
||||
"""创建新的访问 UUID(仅管理员)。"""
|
||||
auth.require_admin(request)
|
||||
record = uuid_store.create(req.label)
|
||||
return auth.UuidRecordOut(
|
||||
uuid=record["uuid"],
|
||||
label=record["label"],
|
||||
enabled=record["enabled"],
|
||||
created_at=record["created_at"],
|
||||
)
|
||||
|
||||
|
||||
@admin_router.delete("/uuids/{uuid}")
|
||||
def delete_uuid(uuid: str, request: Request) -> dict:
|
||||
"""删除访问 UUID(仅管理员)。"""
|
||||
auth.require_admin(request)
|
||||
ok = uuid_store.delete(uuid)
|
||||
return {"ok": ok}
|
||||
|
||||
|
||||
@admin_router.put("/uuids/{uuid}/toggle")
|
||||
def toggle_uuid(uuid: str, request: Request, enabled: bool) -> dict:
|
||||
"""启用/禁用访问 UUID(仅管理员)。"""
|
||||
auth.require_admin(request)
|
||||
ok = uuid_store.toggle(uuid, enabled)
|
||||
return {"ok": ok}
|
||||
@@ -0,0 +1,474 @@
|
||||
"""回测 API — 信号回测 + 因子回测 + 策略回测。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import queue
|
||||
import threading
|
||||
from dataclasses import asdict
|
||||
from datetime import date, timedelta
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.config import settings
|
||||
from app.services.backtest import (
|
||||
BacktestConfig,
|
||||
BacktestService,
|
||||
VectorbtUnavailable,
|
||||
is_available,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/backtest", tags=["backtest"])
|
||||
|
||||
FACTOR_DEFAULT_DAYS = 180
|
||||
STRATEGY_DEFAULT_DAYS = 365 * 3
|
||||
BACKTEST_MAX_SERVER_DAYS = 186
|
||||
FACTOR_MAX_SYMBOLS = 1000
|
||||
BACKTEST_SERVER_GUARD_MESSAGE = (
|
||||
"当前服务器内存约 1.8GB，回测区间最多支持 6 个月；"
|
||||
"更长周期容易触发 OOM，建议在 8GB 以上内存环境或本机运行。"
|
||||
)
|
||||
|
||||
|
||||
def _get_engine(request: Request):
|
||||
"""获取或创建 BacktestEngine (单例，PanelCache 跨请求生效)。"""
|
||||
from app.backtest.engine import BacktestEngine
|
||||
engine = getattr(request.app.state, "backtest_engine", None)
|
||||
if engine is None:
|
||||
engine = BacktestEngine(request.app.state.repo)
|
||||
request.app.state.backtest_engine = engine
|
||||
return engine
|
||||
|
||||
|
||||
def _resolve_start(req: BaseModel, end: date, default_days: int) -> date:
|
||||
"""未传 start 使用默认区间；显式传 null/空值表示全部历史。"""
|
||||
start = getattr(req, "start")
|
||||
if start is not None:
|
||||
return start
|
||||
if "start" in req.model_fields_set:
|
||||
return date(1900, 1, 1)
|
||||
return end - timedelta(days=default_days)
|
||||
|
||||
|
||||
def _guard_server_backtest_range(start: date, end: date):
|
||||
if not settings.backtest_range_guard:
|
||||
return
|
||||
days = (end - start).days + 1
|
||||
if days > BACKTEST_MAX_SERVER_DAYS:
|
||||
raise HTTPException(status_code=400, detail=BACKTEST_SERVER_GUARD_MESSAGE)
|
||||
|
||||
|
||||
# ================================================================
|
||||
# 状态
|
||||
# ================================================================
|
||||
|
||||
@router.get("/status")
|
||||
def status():
|
||||
"""前端可用此接口判断回测页是否要灰显。"""
|
||||
return {"available": True}
|
||||
|
||||
|
||||
# ================================================================
|
||||
# 信号回测 (现有接口，保持不变)
|
||||
# ================================================================
|
||||
|
||||
class BacktestRequest(BaseModel):
|
||||
symbols: list[str] = Field(..., min_length=1)
|
||||
start: date | None = None
|
||||
end: date | None = None
|
||||
entries: list[str] = []
|
||||
exits: list[str] = []
|
||||
stop_loss_pct: float | None = None
|
||||
max_hold_days: int | None = None
|
||||
fees_pct: float = 0.0002
|
||||
slippage_bps: float = 5
|
||||
matching: Literal["close_t", "open_t+1"] = "close_t"
|
||||
|
||||
|
||||
@router.post("/run")
|
||||
def run(req: BacktestRequest, request: Request):
|
||||
"""信号回测 — 现有接口，向后兼容。"""
|
||||
repo = request.app.state.repo
|
||||
svc = BacktestService(repo)
|
||||
end = req.end or date.today()
|
||||
start = req.start or (end - timedelta(days=365 * 3))
|
||||
|
||||
cfg = BacktestConfig(
|
||||
symbols=req.symbols,
|
||||
start=start,
|
||||
end=end,
|
||||
entries=req.entries,
|
||||
exits=req.exits,
|
||||
stop_loss_pct=req.stop_loss_pct,
|
||||
max_hold_days=req.max_hold_days,
|
||||
fees_pct=req.fees_pct,
|
||||
slippage_bps=req.slippage_bps,
|
||||
matching=req.matching,
|
||||
)
|
||||
try:
|
||||
result = svc.run(cfg)
|
||||
except VectorbtUnavailable as e:
|
||||
raise HTTPException(status_code=503, detail=str(e)) from e
|
||||
return asdict(result)
|
||||
|
||||
|
||||
# ================================================================
|
||||
# 因子回测
|
||||
# ================================================================
|
||||
|
||||
class FactorColumnsResponse(BaseModel):
|
||||
columns: list[dict]
|
||||
|
||||
|
||||
@router.get("/factor/columns")
|
||||
def factor_columns():
|
||||
"""返回可用的因子列列表。"""
|
||||
from app.backtest.factor import FACTOR_COLUMNS
|
||||
return {"columns": FACTOR_COLUMNS}
|
||||
|
||||
|
||||
class FactorBacktestRequest(BaseModel):
|
||||
factor_name: str
|
||||
symbols: list[str] | None = None
|
||||
start: date | None = None
|
||||
end: date | None = None
|
||||
n_groups: int = 5
|
||||
rebalance: Literal["daily", "weekly", "monthly"] = "monthly"
|
||||
weight: Literal["equal", "factor_weight"] = "equal"
|
||||
fees_pct: float = 0.0002
|
||||
slippage_bps: float = 5.0
|
||||
|
||||
|
||||
@router.post("/factor/run")
|
||||
def factor_run(req: FactorBacktestRequest, request: Request):
|
||||
"""因子回测 — IC/IR 分析 + 分层回测。"""
|
||||
from app.backtest.factor import FactorBacktestService, FactorConfig
|
||||
|
||||
engine = _get_engine(request)
|
||||
svc = FactorBacktestService(engine)
|
||||
|
||||
end = req.end or date.today()
|
||||
start = _resolve_start(req, end, STRATEGY_DEFAULT_DAYS)
|
||||
_guard_server_backtest_range(start, end)
|
||||
symbols = req.symbols if req.symbols else None
|
||||
if symbols is not None and len(symbols) > FACTOR_MAX_SYMBOLS:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"指定标的最多支持 {FACTOR_MAX_SYMBOLS} 只，请缩小标的范围。",
|
||||
)
|
||||
|
||||
cfg = FactorConfig(
|
||||
factor_name=req.factor_name,
|
||||
symbols=symbols,
|
||||
start=start,
|
||||
end=end,
|
||||
n_groups=req.n_groups,
|
||||
rebalance=req.rebalance,
|
||||
weight=req.weight,
|
||||
fees_pct=req.fees_pct,
|
||||
slippage_bps=req.slippage_bps,
|
||||
)
|
||||
result = svc.run(cfg)
|
||||
return asdict(result)
|
||||
|
||||
|
||||
# ================================================================
|
||||
# 策略回测
|
||||
# ================================================================
|
||||
|
||||
class StrategyBacktestRequest(BaseModel):
|
||||
strategy_id: str
|
||||
symbols: list[str] | None = None
|
||||
start: date | None = None
|
||||
end: date | None = None
|
||||
params: dict | None = None
|
||||
overrides: dict | None = None
|
||||
# matching 向后兼容; 显式传 entry_fill/exit_fill 时以二者为准。
|
||||
matching: Literal["close_t", "open_t+1"] = "open_t+1"
|
||||
entry_fill: Literal["close_t", "open_t+1"] | None = None
|
||||
exit_fill: Literal["close_t", "open_t+1"] | None = None
|
||||
fees_pct: float = 0.0002
|
||||
slippage_bps: float = 5.0
|
||||
max_positions: int = 10
|
||||
max_exposure_pct: float = 1.0
|
||||
initial_capital: float = 1_000_000.0
|
||||
position_sizing: Literal["equal", "score_weight"] = "equal"
|
||||
mode: Literal["position", "full"] = "position"
|
||||
holding_days: int = 5
|
||||
|
||||
|
||||
@router.post("/strategy/run")
|
||||
def strategy_run(req: StrategyBacktestRequest, request: Request):
|
||||
"""策略回测 — 复用 StrategyDef 体系做全周期回测。"""
|
||||
from app.backtest.strategy import StrategyBacktestService, StrategyBacktestConfig
|
||||
|
||||
engine = _get_engine(request)
|
||||
strategy_engine = request.app.state.strategy_engine
|
||||
svc = StrategyBacktestService(engine, strategy_engine)
|
||||
|
||||
end = req.end or date.today()
|
||||
start = _resolve_start(req, end, FACTOR_DEFAULT_DAYS)
|
||||
_guard_server_backtest_range(start, end)
|
||||
|
||||
cfg = StrategyBacktestConfig(
|
||||
strategy_id=req.strategy_id,
|
||||
symbols=req.symbols if req.symbols else None,
|
||||
start=start,
|
||||
end=end,
|
||||
params=req.params,
|
||||
overrides=req.overrides,
|
||||
matching=req.matching,
|
||||
entry_fill=req.entry_fill,
|
||||
exit_fill=req.exit_fill,
|
||||
fees_pct=req.fees_pct,
|
||||
slippage_bps=req.slippage_bps,
|
||||
max_positions=req.max_positions,
|
||||
max_exposure_pct=req.max_exposure_pct,
|
||||
initial_capital=req.initial_capital,
|
||||
position_sizing=req.position_sizing,
|
||||
mode=req.mode,
|
||||
holding_days=req.holding_days,
|
||||
)
|
||||
result = svc.run(cfg)
|
||||
return asdict(result)
|
||||
|
||||
|
||||
# ── SSE 流式回测 (实时进度 + 可取消 + 支持重连) ───────────────────
|
||||
|
||||
import time
|
||||
import hashlib
|
||||
|
||||
|
||||
class _BacktestJob:
|
||||
"""单个回测任务的状态, 存模块级供重连使用。"""
|
||||
__slots__ = ("key", "cancel_event", "progress", "result", "error", "done", "finish_ts")
|
||||
|
||||
def __init__(self, key: str):
|
||||
self.key = key
|
||||
self.cancel_event = threading.Event()
|
||||
self.progress: list[dict] = [] # 进度历史 (新连接可回放)
|
||||
self.result = None # 完成后的结果
|
||||
self.error: str | None = None
|
||||
self.done = False
|
||||
self.finish_ts: float = 0.0
|
||||
|
||||
|
||||
# 模块级任务表: key -> _BacktestJob
|
||||
_running_jobs: dict[str, _BacktestJob] = {}
|
||||
_jobs_lock = threading.Lock()
|
||||
_JOB_TTL = 300 # 完成后保留 5 分钟
|
||||
|
||||
|
||||
def _cleanup_stale_jobs():
|
||||
"""清理过期任务 (完成超过 TTL 的)。"""
|
||||
now = time.time()
|
||||
stale = [k for k, j in _running_jobs.items() if j.done and now - j.finish_ts > _JOB_TTL]
|
||||
for k in stale:
|
||||
_running_jobs.pop(k, None)
|
||||
|
||||
|
||||
def _make_job_key(
|
||||
strategy_id: str, symbols: str | None, start: str | None, end: str | None,
|
||||
matching: str, entry_fill: str | None, exit_fill: str | None,
|
||||
fees_pct: float, slippage_bps: float,
|
||||
max_positions: int, max_exposure_pct: float, initial_capital: float, position_sizing: str,
|
||||
params: str | None, overrides: str | None,
|
||||
mode: str = "position", holding_days: int = 5,
|
||||
) -> str:
|
||||
raw = f"{strategy_id}|{symbols}|{start}|{end}|{matching}|{entry_fill}|{exit_fill}|{fees_pct}|{slippage_bps}|{max_positions}|{max_exposure_pct}|{initial_capital}|{position_sizing}|{params}|{overrides}|{mode}|{holding_days}"
|
||||
return hashlib.md5(raw.encode()).hexdigest()[:12]
|
||||
|
||||
|
||||
@router.get("/strategy/stream")
|
||||
async def strategy_stream(
|
||||
request: Request,
|
||||
strategy_id: str,
|
||||
symbols: str | None = None,
|
||||
start: str | None = None,
|
||||
end: str | None = None,
|
||||
matching: str = "open_t+1",
|
||||
entry_fill: str | None = None,
|
||||
exit_fill: str | None = None,
|
||||
fees_pct: float = 0.0002,
|
||||
slippage_bps: float = 5.0,
|
||||
max_positions: int = 10,
|
||||
max_exposure_pct: float = 1.0,
|
||||
initial_capital: float = 1_000_000.0,
|
||||
position_sizing: str = "equal",
|
||||
params: str | None = None,
|
||||
overrides: str | None = None,
|
||||
mode: str = "position",
|
||||
holding_days: int = 5,
|
||||
):
|
||||
"""SSE 流式策略回测: 实时推送进度, 完成后推送结果, 支持重连 (刷新/切页后恢复)。
|
||||
|
||||
- 相同参数的任务只启动一次, 多次连接订阅同一个任务
|
||||
- 断开连接不会取消任务 (除非显式调用 cancel)
|
||||
- 结果保留 5 分钟供重连
|
||||
|
||||
事件类型:
|
||||
- progress: {day, total, date, equity}
|
||||
- done: {result} (完整回测结果)
|
||||
- error: {message}
|
||||
"""
|
||||
from app.backtest.strategy import StrategyBacktestService, StrategyBacktestConfig
|
||||
|
||||
engine = _get_engine(request)
|
||||
strategy_engine = request.app.state.strategy_engine
|
||||
svc = StrategyBacktestService(engine, strategy_engine)
|
||||
|
||||
end_date = date.fromisoformat(end) if end else date.today()
|
||||
if start:
|
||||
start_date = date.fromisoformat(start)
|
||||
else:
|
||||
# 空 start = 全部历史: 用本地最早日K日期, 查不到再回退到默认窗口
|
||||
earliest = request.app.state.repo.earliest_daily_date()
|
||||
start_date = earliest or (end_date - timedelta(days=FACTOR_DEFAULT_DAYS))
|
||||
|
||||
# 服务端范围保护
|
||||
guard_violated = False
|
||||
if settings.backtest_range_guard:
|
||||
days = (end_date - start_date).days + 1
|
||||
if days > BACKTEST_MAX_SERVER_DAYS:
|
||||
guard_violated = True
|
||||
|
||||
job_key = _make_job_key(
|
||||
strategy_id, symbols, start, end,
|
||||
matching, entry_fill, exit_fill,
|
||||
fees_pct, slippage_bps, max_positions, max_exposure_pct, initial_capital, position_sizing,
|
||||
params, overrides,
|
||||
mode, holding_days,
|
||||
)
|
||||
|
||||
_cleanup_stale_jobs()
|
||||
|
||||
# 获取或创建任务
|
||||
with _jobs_lock:
|
||||
job = _running_jobs.get(job_key)
|
||||
if job is None:
|
||||
job = _BacktestJob(job_key)
|
||||
_running_jobs[job_key] = job
|
||||
is_new = True
|
||||
else:
|
||||
is_new = False
|
||||
|
||||
async def event_generator():
|
||||
# 范围保护: 直接报错
|
||||
if guard_violated:
|
||||
yield f"event: error\ndata: {json.dumps({'message': BACKTEST_SERVER_GUARD_MESSAGE}, ensure_ascii=False)}\n\n"
|
||||
return
|
||||
|
||||
# 如果是新任务, 启动回测线程
|
||||
if is_new and not job.done:
|
||||
cfg = StrategyBacktestConfig(
|
||||
strategy_id=strategy_id,
|
||||
symbols=[s.strip() for s in symbols.split(",") if s.strip()] if symbols else None,
|
||||
start=start_date,
|
||||
end=end_date,
|
||||
params=json.loads(params) if params else None,
|
||||
overrides=json.loads(overrides) if overrides else None,
|
||||
matching=matching,
|
||||
entry_fill=entry_fill,
|
||||
exit_fill=exit_fill,
|
||||
fees_pct=fees_pct,
|
||||
slippage_bps=slippage_bps,
|
||||
max_positions=int(max_positions),
|
||||
max_exposure_pct=float(max_exposure_pct),
|
||||
initial_capital=float(initial_capital),
|
||||
position_sizing=position_sizing,
|
||||
mode=mode,
|
||||
holding_days=int(holding_days),
|
||||
)
|
||||
|
||||
def _run_backtest():
|
||||
try:
|
||||
result = svc.run(cfg, lambda d: job.progress.append(d), job.cancel_event)
|
||||
job.result = result
|
||||
job.done = True
|
||||
job.finish_ts = time.time()
|
||||
except Exception as e:
|
||||
job.error = str(e)
|
||||
job.done = True
|
||||
job.finish_ts = time.time()
|
||||
|
||||
# 启动后台线程 (不阻塞事件循环)
|
||||
threading.Thread(target=_run_backtest, daemon=True).start()
|
||||
|
||||
# 订阅进度: 用读指针读 job.progress 列表 (多连接互不干扰)
|
||||
cursor = 0
|
||||
tick = 0
|
||||
|
||||
try:
|
||||
while True:
|
||||
# 已完成: 推送最终结果/错误并退出
|
||||
if job.done:
|
||||
if job.error:
|
||||
yield f"event: error\ndata: {json.dumps({'message': job.error}, ensure_ascii=False)}\n\n"
|
||||
elif job.result is not None:
|
||||
r = job.result
|
||||
if hasattr(r, "error") and r.error == "cancelled":
|
||||
yield f"event: error\ndata: {json.dumps({'message': '回测已取消'}, ensure_ascii=False)}\n\n"
|
||||
elif hasattr(r, "error") and r.error:
|
||||
yield f"event: error\ndata: {json.dumps({'message': r.error}, ensure_ascii=False)}\n\n"
|
||||
else:
|
||||
yield f"event: done\ndata: {json.dumps(asdict(r), ensure_ascii=False, default=str)}\n\n"
|
||||
return
|
||||
|
||||
# 断开检测: 每 4 轮检查一次 (降低 GIL 抢占频率)
|
||||
tick += 1
|
||||
if tick % 4 == 0 and await request.is_disconnected():
|
||||
break
|
||||
|
||||
# 推送新进度 (从 cursor 开始读)
|
||||
prog_list = job.progress
|
||||
while cursor < len(prog_list):
|
||||
msg = prog_list[cursor]
|
||||
cursor += 1
|
||||
yield f"event: progress\ndata: {json.dumps(msg, ensure_ascii=False, default=str)}\n\n"
|
||||
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
|
||||
return StreamingResponse(event_generator(), media_type="text/event-stream")
|
||||
|
||||
|
||||
@router.post("/strategy/cancel")
|
||||
async def strategy_cancel(request: Request):
|
||||
"""取消正在运行的回测任务 (前端传 query string, 后端算 job_key)。"""
|
||||
body = await request.json()
|
||||
qs = body.get("qs", "")
|
||||
# 解析 qs 得到参数
|
||||
from urllib.parse import parse_qs
|
||||
p = parse_qs(qs)
|
||||
def _get(key: str, default: str = "") -> str:
|
||||
return p.get(key, [default])[0]
|
||||
job_key = _make_job_key(
|
||||
_get("strategy_id"),
|
||||
_get("symbols") or None,
|
||||
_get("start") or None,
|
||||
_get("end") or None,
|
||||
_get("matching", "open_t+1"),
|
||||
_get("entry_fill") or None,
|
||||
_get("exit_fill") or None,
|
||||
float(_get("fees_pct", "0.0002")),
|
||||
float(_get("slippage_bps", "5")),
|
||||
int(_get("max_positions", "10")),
|
||||
float(_get("max_exposure_pct", "1")),
|
||||
float(_get("initial_capital", "1000000")),
|
||||
_get("position_sizing", "equal"),
|
||||
_get("params") or None,
|
||||
_get("overrides") or None,
|
||||
_get("mode", "position"),
|
||||
int(_get("holding_days", "5")),
|
||||
)
|
||||
job = _running_jobs.get(job_key)
|
||||
if job and not job.done:
|
||||
job.cancel_event.set()
|
||||
return {"ok": True}
|
||||
return {"ok": False, "message": "任务不存在或已完成"}
|
||||
|
||||
@@ -0,0 +1,747 @@
|
||||
"""数据画像 API —— 让前端知道"我们本地有什么数据"。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
|
||||
from app.indicators.pipeline import ENRICHED_COLUMNS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/data", tags=["data"])
|
||||
|
||||
# ===== 缓存:storage(文件扫描) + 每张表 aggregate 各自缓存 =====
|
||||
# 同步期间前端 2s 轮一次 status,每张表 aggregate 全表 count + min/max + distinct
|
||||
# 太重,加 TTL + 事件失效。stage 写完只清对应那张表的缓存。
|
||||
|
||||
_TABLE_TTL = 30.0 # 兜底 TTL,即使没人调 invalidate 也会过期
|
||||
_TABLE_TTL_LARGE = 120.0 # 大表(分钟K等)单独 TTL，避免多分区聚合反复重算
|
||||
_STORAGE_TTL = 60.0 # storage 文件扫描独立 TTL,stage 写完不触发重算
|
||||
|
||||
# 聚合慢的大表（分区数多、行数多），使用更长的 TTL
|
||||
_LARGE_TABLES = {"minute"}
|
||||
|
||||
_storage_cache: dict[str, Any] | None = None
|
||||
_storage_cache_ts: float = 0.0
|
||||
_storage_lock = threading.Lock()
|
||||
|
||||
_table_cache: dict[str, dict | None] = {
|
||||
"daily": None,
|
||||
"enriched": None,
|
||||
"index_daily": None,
|
||||
"index_enriched": None,
|
||||
"index_instruments": None,
|
||||
"minute": None,
|
||||
"adj_factor": None,
|
||||
"instruments": None,
|
||||
"financials": None,
|
||||
}
|
||||
_table_cache_ts: dict[str, float] = {k: 0.0 for k in _table_cache}
|
||||
_table_cache_lock = threading.Lock()
|
||||
|
||||
_last_finished_cache: dict[str, str | None] | None = None
|
||||
_last_finished_lock = threading.Lock()
|
||||
|
||||
|
||||
def invalidate_data_cache(table: str | None = None) -> None:
|
||||
"""数据写入/清除后调用。
|
||||
|
||||
table=None 时清所有表 cache + storage(粗粒度,用于 pipeline 完成/clear);
|
||||
指定 table 时只清那张表,不影响 storage(细粒度,用于单 stage 写完)。
|
||||
"""
|
||||
with _table_cache_lock:
|
||||
if table is None:
|
||||
global _storage_cache, _storage_cache_ts, _last_finished_cache
|
||||
_storage_cache = None
|
||||
_storage_cache_ts = 0.0
|
||||
_last_finished_cache = None
|
||||
for k in _table_cache:
|
||||
_table_cache[k] = None
|
||||
_table_cache_ts[k] = 0.0
|
||||
elif table in _table_cache:
|
||||
_table_cache[table] = None
|
||||
_table_cache_ts[table] = 0.0
|
||||
|
||||
|
||||
def invalidate_storage_cache() -> None:
|
||||
"""向后兼容入口 — 清全部缓存。新代码请用 invalidate_data_cache(table)。"""
|
||||
invalidate_data_cache(None)
|
||||
|
||||
|
||||
def _get_table_stats(name: str, fetch: Callable[[], dict | None]) -> dict | None:
|
||||
"""走 TTL+事件 双重缓存。fetch 在锁外执行避免阻塞别的请求。"""
|
||||
ttl = _TABLE_TTL_LARGE if name in _LARGE_TABLES else _TABLE_TTL
|
||||
now = time.time()
|
||||
with _table_cache_lock:
|
||||
cached = _table_cache.get(name)
|
||||
cached_ts = _table_cache_ts.get(name, 0.0)
|
||||
if cached is not None and (now - cached_ts) < ttl:
|
||||
return cached
|
||||
|
||||
fresh = fetch()
|
||||
|
||||
with _table_cache_lock:
|
||||
_table_cache[name] = fresh
|
||||
_table_cache_ts[name] = now
|
||||
return fresh
|
||||
|
||||
|
||||
def _safe_aggregate(repo, view: str) -> dict | None:
|
||||
"""聚合视图基础统计;视图不存在或为空时返 None。"""
|
||||
try:
|
||||
row = repo.execute_one(
|
||||
f"""SELECT count(*) AS rows,
|
||||
min(date) AS earliest,
|
||||
max(date) AS latest,
|
||||
count(DISTINCT symbol) AS symbols,
|
||||
count(DISTINCT date) AS trading_days
|
||||
FROM {view}"""
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("aggregate %s failed: %s", view, e)
|
||||
return None
|
||||
if not row or not row[0]:
|
||||
return None
|
||||
return {
|
||||
"rows": int(row[0]),
|
||||
"earliest_date": str(row[1]) if row[1] else None,
|
||||
"latest_date": str(row[2]) if row[2] else None,
|
||||
"symbols_covered": int(row[3] or 0),
|
||||
"trading_days": int(row[4] or 0),
|
||||
}
|
||||
|
||||
|
||||
def _safe_aggregate_daily(repo, view: str = "kline_daily") -> dict | None:
|
||||
"""日K轻量统计 — 零数据扫描。
|
||||
|
||||
从分区目录名获取日期范围和交易日数，不读任何 parquet。
|
||||
标的数从 instruments 小表获取（~5000行，毫秒级）。
|
||||
"""
|
||||
daily_dir = repo.store.data_dir / "kline_daily"
|
||||
if not daily_dir.exists():
|
||||
return None
|
||||
dates: list[str] = []
|
||||
for d in daily_dir.iterdir():
|
||||
if d.is_dir() and d.name.startswith("date="):
|
||||
dates.append(d.name[5:])
|
||||
if not dates:
|
||||
return None
|
||||
dates.sort()
|
||||
|
||||
symbols = _count_instruments_symbols(repo)
|
||||
|
||||
return {
|
||||
"rows": 0,
|
||||
"earliest_date": dates[0],
|
||||
"latest_date": dates[-1],
|
||||
"symbols_covered": symbols,
|
||||
"trading_days": len(dates),
|
||||
}
|
||||
|
||||
|
||||
def _safe_aggregate_enriched(repo) -> dict | None:
|
||||
"""Enriched 轻量统计 — 零数据扫描。
|
||||
|
||||
字段数从 DESCRIBE 读 schema（不碰数据），毫秒级。
|
||||
日期范围从分区目录名获取（同 minute 策略），不读任何 parquet。
|
||||
标的数从 instruments 小表取。
|
||||
"""
|
||||
# 字段数：读 schema，不碰数据
|
||||
fields = 0
|
||||
try:
|
||||
cols = repo.execute_all("DESCRIBE kline_enriched")
|
||||
fields = len(cols)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
# 日期范围：从分区目录名获取，不扫数据
|
||||
enriched_dir = repo.store.data_dir / "kline_daily_enriched"
|
||||
if not enriched_dir.exists():
|
||||
return None
|
||||
dates: list[str] = []
|
||||
for d in enriched_dir.iterdir():
|
||||
if d.is_dir() and d.name.startswith("date="):
|
||||
dates.append(d.name[5:])
|
||||
if not dates:
|
||||
return None
|
||||
dates.sort()
|
||||
|
||||
symbols = _count_instruments_symbols(repo)
|
||||
|
||||
return {
|
||||
"rows": 0,
|
||||
"fields": fields,
|
||||
"earliest_date": dates[0],
|
||||
"latest_date": dates[-1],
|
||||
"symbols_covered": symbols,
|
||||
"trading_days": len(dates),
|
||||
}
|
||||
|
||||
|
||||
def _count_instruments_symbols(repo) -> int:
|
||||
"""从 instruments 小表取标的数（~5000行，毫秒级）。"""
|
||||
try:
|
||||
sym_row = repo.execute_one(
|
||||
"SELECT count(DISTINCT symbol) FROM instruments"
|
||||
)
|
||||
if sym_row and sym_row[0]:
|
||||
return int(sym_row[0])
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def _safe_aggregate_instruments(repo) -> dict | None:
|
||||
"""instruments 视图统计(无 date 列,用 as_of)。"""
|
||||
try:
|
||||
row = repo.execute_one(
|
||||
"""SELECT count(*) AS rows,
|
||||
count(DISTINCT symbol) AS symbols,
|
||||
max(as_of) AS latest_as_of,
|
||||
count_if(name IS NOT NULL AND name != '') AS named
|
||||
FROM instruments"""
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("aggregate instruments failed: %s", e)
|
||||
return None
|
||||
if not row or not row[0]:
|
||||
return None
|
||||
return {
|
||||
"rows": int(row[0]),
|
||||
"symbols_covered": int(row[1] or 0),
|
||||
"latest_as_of": str(row[2]) if row[2] else None,
|
||||
"named": int(row[3] or 0),
|
||||
}
|
||||
|
||||
|
||||
def _safe_aggregate_index_daily(repo) -> dict | None:
|
||||
"""指数日K统计。指数数据量较小，直接读取 parquet 元数据统计真实行数。"""
|
||||
return _safe_aggregate(repo, "kline_index_daily")
|
||||
|
||||
|
||||
def _safe_aggregate_index_enriched(repo) -> dict | None:
|
||||
"""指数 enriched 统计。指数数据量较小，直接读取 parquet 元数据统计真实行数。"""
|
||||
fields = 0
|
||||
try:
|
||||
cols = repo.execute_all("DESCRIBE kline_index_enriched")
|
||||
fields = len(cols)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
stats = _safe_aggregate(repo, "kline_index_enriched")
|
||||
if not stats:
|
||||
return None
|
||||
return {**stats, "fields": fields}
|
||||
|
||||
|
||||
def _safe_aggregate_index_instruments(repo) -> dict | None:
|
||||
"""指数 instruments 视图统计。"""
|
||||
try:
|
||||
row = repo.execute_one(
|
||||
"""SELECT count(*) AS rows,
|
||||
count(DISTINCT symbol) AS symbols,
|
||||
count_if(name IS NOT NULL AND name != '') AS named
|
||||
FROM instruments_index"""
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("aggregate instruments_index failed: %s", e)
|
||||
return None
|
||||
if not row or not row[0]:
|
||||
return None
|
||||
return {
|
||||
"rows": int(row[0]),
|
||||
"symbols_covered": int(row[1] or 0),
|
||||
"latest_as_of": None,
|
||||
"named": int(row[2] or 0),
|
||||
}
|
||||
|
||||
|
||||
def _safe_aggregate_adj_factor(repo) -> dict | None:
|
||||
"""adj_factor 视图统计,日期范围对齐日 K 覆盖区间。"""
|
||||
try:
|
||||
# 取日 K 的日期范围作为过滤条件
|
||||
dr = repo.execute_one(
|
||||
"SELECT min(date), max(date) FROM kline_daily"
|
||||
)
|
||||
if not dr or not dr[0]:
|
||||
return None
|
||||
d_min, d_max = dr[0], dr[1]
|
||||
row = repo.execute_one(
|
||||
"""SELECT count(*) AS rows,
|
||||
count(DISTINCT symbol) AS symbols,
|
||||
count(DISTINCT trade_date) AS trading_days
|
||||
FROM adj_factor
|
||||
WHERE trade_date BETWEEN ? AND ?""",
|
||||
[str(d_min), str(d_max)],
|
||||
)
|
||||
if not row or not row[0]:
|
||||
return None
|
||||
return {
|
||||
"rows": int(row[0]),
|
||||
"symbols_covered": int(row[1]) if isinstance(row[1], (int, float)) else 0,
|
||||
"earliest_date": str(d_min),
|
||||
"latest_date": str(d_max),
|
||||
"trading_days": int(row[2] or 0),
|
||||
}
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("aggregate adj_factor failed: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def _safe_aggregate_minute(repo) -> dict | None:
|
||||
"""kline_minute 统计 — 从分区目录名获取交易日数，跳过全表扫描。
|
||||
|
||||
分钟 K 按 date=YYYY-MM-DD 分区存储，直接数目录即可，
|
||||
无需 count(*) / count(DISTINCT ...) 等昂贵查询。
|
||||
"""
|
||||
minute_dir = repo.store.data_dir / "kline_minute"
|
||||
if not minute_dir.exists():
|
||||
return None
|
||||
|
||||
# 从 date=YYYY-MM-DD 目录名提取交易日
|
||||
dates: list[str] = []
|
||||
for d in minute_dir.iterdir():
|
||||
if d.is_dir() and d.name.startswith("date="):
|
||||
dates.append(d.name[5:])
|
||||
|
||||
if not dates:
|
||||
return None
|
||||
|
||||
dates.sort()
|
||||
return {
|
||||
"rows": 0, # 不再查询行数
|
||||
"earliest_date": dates[0],
|
||||
"latest_date": dates[-1],
|
||||
"symbols_covered": 0, # 不再查询标的数
|
||||
"trading_days": len(dates),
|
||||
}
|
||||
|
||||
|
||||
def _safe_aggregate_financials(repo) -> dict | None:
|
||||
"""财务数据统计 — 检查各表文件是否存在及行数。"""
|
||||
data_dir = repo.store.data_dir
|
||||
tables_info: dict[str, dict] = {}
|
||||
total_rows = 0
|
||||
|
||||
for table in ("metrics", "income", "balance_sheet", "cash_flow"):
|
||||
path = data_dir / "financials" / table / "part.parquet"
|
||||
if path.exists():
|
||||
try:
|
||||
import polars as pl
|
||||
df = pl.read_parquet(path, columns=["symbol"])
|
||||
rows = len(df)
|
||||
symbols = df["symbol"].n_unique() if not df.is_empty() else 0
|
||||
tables_info[table] = {"rows": rows, "symbols": symbols}
|
||||
total_rows += rows
|
||||
except Exception:
|
||||
tables_info[table] = {"rows": 0, "symbols": 0}
|
||||
else:
|
||||
tables_info[table] = {"rows": 0, "symbols": 0}
|
||||
|
||||
if total_rows == 0:
|
||||
return None
|
||||
|
||||
return {
|
||||
"rows": total_rows,
|
||||
"tables": tables_info,
|
||||
}
|
||||
|
||||
|
||||
def _scan_dir_stats(dirpath: Path) -> tuple[int, float]:
|
||||
"""单次遍历统计目录下文件数和总大小(MB)。比 rglob+stat 快很多。"""
|
||||
if not dirpath.exists():
|
||||
return 0, 0.0
|
||||
count = 0
|
||||
total = 0
|
||||
for entry in os.scandir(dirpath):
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
c, s = _scan_dir_recursive(entry)
|
||||
count += c
|
||||
total += s
|
||||
elif entry.is_file(follow_symlinks=False):
|
||||
try:
|
||||
total += entry.stat().st_size
|
||||
except OSError:
|
||||
pass
|
||||
count += 1
|
||||
return count, round(total / 1048576, 2)
|
||||
|
||||
|
||||
def _scan_dir_recursive(entry: os.DirEntry) -> tuple[int, int]:
|
||||
"""递归统计一个 DirEntry 下的文件数和总字节数。"""
|
||||
count = 0
|
||||
total = 0
|
||||
try:
|
||||
for sub in os.scandir(entry.path):
|
||||
if sub.is_dir(follow_symlinks=False):
|
||||
c, s = _scan_dir_recursive(sub)
|
||||
count += c
|
||||
total += s
|
||||
elif sub.is_file(follow_symlinks=False):
|
||||
try:
|
||||
total += sub.stat().st_size
|
||||
except OSError:
|
||||
pass
|
||||
count += 1
|
||||
except PermissionError:
|
||||
pass
|
||||
return count, total
|
||||
|
||||
|
||||
def _compute_storage(data_dir: Path) -> dict:
|
||||
"""单次遍历计算 storage 统计，避免多次 rglob。"""
|
||||
import os
|
||||
|
||||
# 只统计关心的子目录
|
||||
subdirs = {
|
||||
"daily": data_dir / "kline_daily",
|
||||
"enriched": data_dir / "kline_daily_enriched",
|
||||
"index_daily": data_dir / "kline_index_daily",
|
||||
"index_enriched": data_dir / "kline_index_enriched",
|
||||
"index_instruments": data_dir / "instruments_index",
|
||||
"minute": data_dir / "kline_minute",
|
||||
"adj_factor": data_dir / "adj_factor",
|
||||
"instruments": data_dir / "instruments",
|
||||
"ext_data": data_dir / "ext_data",
|
||||
}
|
||||
stats = {}
|
||||
total_size = 0
|
||||
for key, d in subdirs.items():
|
||||
fc, sz = _scan_dir_stats(d)
|
||||
total_size += sz
|
||||
stats[f"{key}_files"] = fc
|
||||
stats[f"{key}_size_mb"] = sz
|
||||
|
||||
# total: 再加上其他零散文件(pools, financials, capabilities.json 等)
|
||||
other_dirs = ["pools", "financials", "backtest_results", "screener_results", "ai_cache"]
|
||||
for name in other_dirs:
|
||||
d = data_dir / name
|
||||
if d.exists():
|
||||
_, s = _scan_dir_stats(d)
|
||||
total_size += s
|
||||
|
||||
# financials 单独统计
|
||||
fin_dir = data_dir / "financials"
|
||||
if fin_dir.exists():
|
||||
fc, sz = _scan_dir_stats(fin_dir)
|
||||
stats["financials_files"] = fc
|
||||
stats["financials_size_mb"] = sz
|
||||
total_size += sz
|
||||
for name in other_dirs:
|
||||
d = data_dir / name
|
||||
if d.exists():
|
||||
_, s = _scan_dir_stats(d)
|
||||
total_size += s
|
||||
# 根目录散文件
|
||||
for entry in os.scandir(data_dir):
|
||||
if entry.is_file(follow_symlinks=False):
|
||||
try:
|
||||
total_size += entry.stat().st_size / 1048576
|
||||
except OSError:
|
||||
pass
|
||||
stats["total_size_mb"] = round(total_size, 2)
|
||||
return stats
|
||||
|
||||
|
||||
def _next_cron_run(scheduler, job_id: str) -> str | None:
|
||||
"""读 APScheduler 下次执行时间。"""
|
||||
if not scheduler:
|
||||
return None
|
||||
try:
|
||||
job = scheduler.get_job(job_id)
|
||||
if job and job.next_run_time:
|
||||
return job.next_run_time.isoformat(timespec="seconds")
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _get_storage(data_dir: Path) -> dict:
|
||||
"""返回缓存的 storage 统计；走独立 TTL，stage 写完不触发重算。"""
|
||||
global _storage_cache, _storage_cache_ts
|
||||
now = time.time()
|
||||
with _storage_lock:
|
||||
if _storage_cache is not None and (now - _storage_cache_ts) < _STORAGE_TTL:
|
||||
return _storage_cache
|
||||
fresh = _compute_storage(data_dir)
|
||||
with _storage_lock:
|
||||
_storage_cache = fresh
|
||||
_storage_cache_ts = now
|
||||
return fresh
|
||||
|
||||
|
||||
def _last_finished(job_label: str) -> str | None:
|
||||
"""从 JobStore 读最近一次该类型任务的完成时间（缓存到 pipeline 终态失效）。"""
|
||||
global _last_finished_cache
|
||||
with _last_finished_lock:
|
||||
if _last_finished_cache is not None:
|
||||
return _last_finished_cache.get(job_label)
|
||||
|
||||
from app.services.pipeline_jobs import job_store
|
||||
jobs = job_store.list_recent(limit=50)
|
||||
cache: dict[str, str | None] = {}
|
||||
for j in jobs:
|
||||
if j["status"] not in ("succeeded", "failed"):
|
||||
continue
|
||||
if "instruments_rows" in (j.get("result") or {}) and "instruments" not in cache:
|
||||
cache["instruments"] = j["finished_at"]
|
||||
if "daily_days" in (j.get("result") or {}) and "pipeline" not in cache:
|
||||
cache["pipeline"] = j["finished_at"]
|
||||
with _last_finished_lock:
|
||||
_last_finished_cache = cache
|
||||
return cache.get(job_label)
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
def status(request: Request) -> dict:
|
||||
repo = request.app.state.repo
|
||||
scheduler = getattr(request.app.state, "scheduler", None)
|
||||
data_dir = repo.store.data_dir
|
||||
|
||||
return {
|
||||
"daily": _get_table_stats("daily", lambda: _safe_aggregate_daily(repo)),
|
||||
"enriched": _get_table_stats("enriched", lambda: _safe_aggregate_enriched(repo)),
|
||||
"index_daily": _get_table_stats("index_daily", lambda: _safe_aggregate_index_daily(repo)),
|
||||
"index_enriched": _get_table_stats("index_enriched", lambda: _safe_aggregate_index_enriched(repo)),
|
||||
"index_instruments": _get_table_stats("index_instruments", lambda: _safe_aggregate_index_instruments(repo)),
|
||||
"minute": _get_table_stats("minute", lambda: _safe_aggregate_minute(repo)),
|
||||
"adj_factor": _get_table_stats("adj_factor", lambda: _safe_aggregate_adj_factor(repo)),
|
||||
"instruments": _get_table_stats("instruments", lambda: _safe_aggregate_instruments(repo)),
|
||||
"financials": _get_table_stats("financials", lambda: _safe_aggregate_financials(repo)),
|
||||
|
||||
# 文件层面信息(缓存)
|
||||
"storage": _get_storage(data_dir),
|
||||
|
||||
# 调度
|
||||
"next_instruments_run": _next_cron_run(scheduler, "pre_market_instruments"),
|
||||
"next_pipeline_run": _next_cron_run(scheduler, "daily_pipeline"),
|
||||
"last_instruments_run": _last_finished("instruments"),
|
||||
"last_pipeline_run": _last_finished("pipeline"),
|
||||
"checked_at": datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/clear")
|
||||
def clear_data(request: Request):
|
||||
"""清除所有本地 Parquet 数据（保留 capabilities.json 和目录结构）。"""
|
||||
import shutil
|
||||
|
||||
repo = request.app.state.repo
|
||||
data_dir = repo.store.data_dir
|
||||
deleted = 0
|
||||
|
||||
for sub in (
|
||||
"kline_daily", "kline_daily_enriched", "kline_index_daily", "kline_index_enriched", "kline_minute",
|
||||
"adj_factor", "instruments", "instruments_index", "pools", "financials",
|
||||
"backtest_results", "screener_results", "ai_cache",
|
||||
):
|
||||
d = data_dir / sub
|
||||
if d.exists():
|
||||
# 先删所有 parquet 文件
|
||||
for f in d.rglob("*.parquet"):
|
||||
f.unlink()
|
||||
deleted += 1
|
||||
# 再删除空的日期分区子目录（date=YYYY-MM-DD 等）
|
||||
for child in list(d.iterdir()):
|
||||
if child.is_dir():
|
||||
shutil.rmtree(child, ignore_errors=True)
|
||||
|
||||
# 清除同步历史（内存 + 磁盘 job_store/ 文件夹）
|
||||
from app.services.pipeline_jobs import job_store
|
||||
job_store.clear()
|
||||
|
||||
# 清除财务数据
|
||||
fin_dir = data_dir / "financials"
|
||||
for sub in ("metrics", "income", "balance_sheet", "cash_flow"):
|
||||
fp = fin_dir / sub / "part.parquet"
|
||||
if fp.exists():
|
||||
fp.unlink()
|
||||
deleted += 1
|
||||
|
||||
# 清除监控运行数据 (user_data 下仅清运行产物, 不动 monitor_rules/preferences/secrets 等用户配置)
|
||||
# - 触发记录 alerts.jsonl
|
||||
from app.services import alert_store
|
||||
alert_store.clear(data_dir)
|
||||
# - 待推送的实时通知队列 (进程内存)
|
||||
qs = getattr(request.app.state, "quote_service", None)
|
||||
if qs is not None:
|
||||
with qs._lock:
|
||||
qs._pending_alerts.clear()
|
||||
|
||||
# 清除 Polars 缓存
|
||||
# 先 clear_cache 无条件清空内存 (refresh_cache 在磁盘无数据时会提前 return,
|
||||
# 导致 _enriched_cache 等旧数据残留 —— 清数据后看板仍显示旧数据的根因),
|
||||
# 再 refresh_cache 尝试重载 (磁盘有数据则重建缓存)。
|
||||
repo.clear_cache()
|
||||
repo.refresh_cache()
|
||||
|
||||
# 清除 Screener 进程级 _history_cache (TTL 缓存)
|
||||
from app.services.screener import ScreenerService
|
||||
ScreenerService.clear_history_cache()
|
||||
|
||||
# 清除 Overview 总览聚合结果缓存 (5s TTL)
|
||||
from app.api.overview import invalidate_overview_cache
|
||||
invalidate_overview_cache()
|
||||
|
||||
# 刷新 DuckDB 视图（空 parquet 目录也需要重新挂载）
|
||||
d = data_dir.as_posix()
|
||||
for name, path in {
|
||||
"kline_daily": f"{d}/kline_daily/**/*.parquet",
|
||||
"kline_enriched": f"{d}/kline_daily_enriched/**/*.parquet",
|
||||
"kline_index_daily": f"{d}/kline_index_daily/**/*.parquet",
|
||||
"kline_index_enriched": f"{d}/kline_index_enriched/**/*.parquet",
|
||||
"kline_minute": f"{d}/kline_minute/**/*.parquet",
|
||||
"adj_factor": f"{d}/adj_factor/**/*.parquet",
|
||||
"instruments": f"{d}/instruments/**/*.parquet",
|
||||
"instruments_index": f"{d}/instruments_index/**/*.parquet",
|
||||
}.items():
|
||||
try:
|
||||
repo.db.execute(
|
||||
f"CREATE OR REPLACE VIEW {name} AS "
|
||||
f"SELECT * FROM read_parquet('{path}', union_by_name=true)"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.info("数据已清除: 删除 %d 个 parquet 文件", deleted)
|
||||
invalidate_data_cache(None)
|
||||
return {"deleted_files": deleted}
|
||||
|
||||
|
||||
# 各表字段说明
|
||||
_TABLE_FIELD_DESC: dict[str, dict[str, str]] = {
|
||||
"kline_daily": {
|
||||
"symbol": "股票代码",
|
||||
"date": "交易日期",
|
||||
"open": "开盘价",
|
||||
"high": "最高价",
|
||||
"low": "最低价",
|
||||
"close": "收盘价",
|
||||
"volume": "成交量",
|
||||
"amount": "成交额",
|
||||
},
|
||||
"kline_enriched": ENRICHED_COLUMNS,
|
||||
"kline_index_daily": {
|
||||
"symbol": "指数代码",
|
||||
"date": "交易日期",
|
||||
"open": "开盘点位",
|
||||
"high": "最高点位",
|
||||
"low": "最低点位",
|
||||
"close": "收盘点位",
|
||||
"volume": "成交量",
|
||||
"amount": "成交额",
|
||||
},
|
||||
"kline_index_enriched": ENRICHED_COLUMNS,
|
||||
"kline_minute": {
|
||||
"symbol": "股票代码",
|
||||
"datetime": "分钟时间戳",
|
||||
"open": "开盘价",
|
||||
"high": "最高价",
|
||||
"low": "最低价",
|
||||
"close": "收盘价",
|
||||
"volume": "成交量",
|
||||
"amount": "成交额",
|
||||
},
|
||||
"adj_factor": {
|
||||
"symbol": "股票代码",
|
||||
"timestamp": "除权除息时间戳(ms)",
|
||||
"trade_date": "除权除息日",
|
||||
"ex_factor": "复权因子",
|
||||
},
|
||||
"instruments": {
|
||||
"symbol": "股票代码",
|
||||
"name": "股票名称",
|
||||
"code": "股票编码(纯数字)",
|
||||
"exchange": "交易所(SH/SZ/BJ)",
|
||||
"region": "地区",
|
||||
"type": "证券类型",
|
||||
"listing_date": "上市日期",
|
||||
"total_shares": "总股本",
|
||||
"float_shares": "流通股本",
|
||||
"tick_size": "最小价格变动单位",
|
||||
"limit_up": "涨停限制(%)",
|
||||
"limit_down": "跌停限制(%)",
|
||||
"as_of": "快照日期",
|
||||
},
|
||||
"instruments_index": {
|
||||
"symbol": "指数代码",
|
||||
"name": "指数名称",
|
||||
"code": "指数编码(纯数字)",
|
||||
"asset_type": "资产类型(index)",
|
||||
},
|
||||
}
|
||||
|
||||
# view 名 → DuckDB 视图名
|
||||
_SCHEMA_VIEWS: dict[str, str] = {
|
||||
"daily": "kline_daily",
|
||||
"enriched": "kline_enriched",
|
||||
"index_daily": "kline_index_daily",
|
||||
"index_enriched": "kline_index_enriched",
|
||||
"index_instruments": "instruments_index",
|
||||
"minute": "kline_minute",
|
||||
"adj_factor": "adj_factor",
|
||||
"instruments": "instruments",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/schema/{table}")
|
||||
def table_schema(request: Request, table: str) -> list[dict]:
|
||||
"""返回指定表的字段名、类型和中文说明。
|
||||
|
||||
优先从 DuckDB DESCRIBE 读取(有数据时含精确类型)；
|
||||
视图不存在(无数据)时回退到 _TABLE_FIELD_DESC 静态定义。
|
||||
"""
|
||||
view = _SCHEMA_VIEWS.get(table)
|
||||
if not view:
|
||||
return []
|
||||
desc_map = _TABLE_FIELD_DESC.get(view, {})
|
||||
repo = request.app.state.repo
|
||||
fields: list[dict] = []
|
||||
try:
|
||||
cols = repo.execute_all(f"DESCRIBE {view}")
|
||||
for col in cols:
|
||||
name = col[0]
|
||||
dtype = col[1]
|
||||
fields.append({
|
||||
"name": name,
|
||||
"type": dtype,
|
||||
"desc": desc_map.get(name, ""),
|
||||
})
|
||||
except Exception: # noqa: BLE001
|
||||
# 视图不存在(本地无数据)，用静态字段定义兜底
|
||||
if desc_map:
|
||||
for name, desc in desc_map.items():
|
||||
fields.append({"name": name, "type": "—", "desc": desc})
|
||||
return fields
|
||||
|
||||
|
||||
@router.get("/version")
|
||||
def get_version(request: Request) -> dict:
|
||||
"""返回当前项目版本号。
|
||||
|
||||
优先读 app.__version__ (与 /health 接口同源, 唯一权威版本),
|
||||
回退到项目根 VERSION 文件, 最后兜底 v0.0.0。
|
||||
"""
|
||||
from app import __version__
|
||||
|
||||
# 1. 优先用 app.__version__ (开发期 bump_version.py 写入)
|
||||
if __version__:
|
||||
v = __version__.strip()
|
||||
return {"version": v if v.startswith("v") else f"v{v}"}
|
||||
|
||||
# 2. 回退到项目根 VERSION 文件
|
||||
from app.config import settings
|
||||
project_root = Path(settings.data_dir).parent
|
||||
version_file = project_root / "VERSION"
|
||||
if version_file.exists():
|
||||
v = version_file.read_text(encoding="utf-8").strip()
|
||||
if v:
|
||||
return {"version": v}
|
||||
|
||||
return {"version": "v0.0.0"}
|
||||
@@ -0,0 +1,826 @@
|
||||
"""扩展数据 API — CRUD + 文件上传 + JSON 写入 + 定时拉取 + schema 发现。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import shutil
|
||||
import tempfile
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import polars as pl
|
||||
from fastapi import APIRouter, File, HTTPException, Query, Request, UploadFile
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.services.ext_data import (
|
||||
ExtConfig,
|
||||
ExtConfigStore,
|
||||
ExtField,
|
||||
PullConfig,
|
||||
ensure_utf8_csv,
|
||||
fix_symbol_format,
|
||||
normalize_symbol,
|
||||
parse_upload_file,
|
||||
write_ext_parquet,
|
||||
rows_to_parquet,
|
||||
)
|
||||
from app.services.ext_pull import fetch_and_ingest, pull_scheduler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/ext-data", tags=["ext-data"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic 模型
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class FieldDef(BaseModel):
|
||||
name: str
|
||||
dtype: str = "string" # string | int | float | bool
|
||||
label: str = ""
|
||||
|
||||
|
||||
class CreateExtReq(BaseModel):
|
||||
id: str = Field(..., min_length=1, max_length=64, pattern=r"^[a-zA-Z0-9_]+$")
|
||||
label: str = Field(..., min_length=1, max_length=64)
|
||||
mode: Literal["snapshot", "timeseries"]
|
||||
fields: list[FieldDef] = Field(..., min_length=1)
|
||||
description: str = ""
|
||||
symbol_map: dict = {} # {"type": "mapped", "col": "..."} 或 {"type": "computed", "from": "code", "method": "append_exchange"}
|
||||
code_map: dict = {} # {"type": "mapped", "col": "..."} 或 {"type": "computed", "from": "symbol", "method": "strip_exchange"}
|
||||
|
||||
|
||||
class UpdateExtReq(BaseModel):
|
||||
label: str | None = None
|
||||
fields: list[FieldDef] | None = None
|
||||
description: str | None = None
|
||||
symbol_map: dict | None = None
|
||||
code_map: dict | None = None
|
||||
|
||||
|
||||
class IngestReq(BaseModel):
|
||||
"""JSON 批量写入请求。"""
|
||||
date: str | None = None # YYYY-MM-DD，不传默认今天
|
||||
rows: list[dict] = Field(..., min_length=1)
|
||||
|
||||
|
||||
class PullConfigReq(BaseModel):
|
||||
"""定时拉取配置请求。"""
|
||||
url: str = Field(..., min_length=1)
|
||||
method: str = "GET"
|
||||
headers: dict[str, str] | None = None
|
||||
body: str | None = None
|
||||
response_path: str = "" # dot-path to rows array
|
||||
field_map: dict[str, str] | None = None # external → internal field name
|
||||
schedule_minutes: int = Field(1440, ge=1)
|
||||
enabled: bool = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 辅助
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _store(request: Request) -> ExtConfigStore:
|
||||
return ExtConfigStore(request.app.state.repo.store.data_dir)
|
||||
|
||||
|
||||
def _data_dir(request: Request) -> Path:
|
||||
return request.app.state.repo.store.data_dir
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _apply_mapping(df: pl.DataFrame, config: ExtConfig, data_dir: Path) -> pl.DataFrame:
|
||||
"""根据 config 的 symbol_map / code_map 自动生成 symbol 和 code 列。
|
||||
|
||||
执行顺序：先 mapped（从文件列复制），再 computed（从已生成的列计算）。
|
||||
"""
|
||||
sm = config.symbol_map or {}
|
||||
cm = config.code_map or {}
|
||||
|
||||
# --- 第一步：mapped 类型，直接从文件列映射 ---
|
||||
if sm.get("type") == "mapped" and sm["col"] in df.columns:
|
||||
df = df.with_columns(df[sm["col"]].cast(pl.Utf8).alias("symbol"))
|
||||
|
||||
if cm.get("type") == "mapped" and cm["col"] in df.columns:
|
||||
df = df.with_columns(df[cm["col"]].cast(pl.Utf8).alias("code"))
|
||||
|
||||
# --- 第二步：computed 类型，从已生成的列计算 ---
|
||||
if "symbol" not in df.columns and sm.get("type") == "computed":
|
||||
if sm.get("from") == "code" and "code" in df.columns:
|
||||
# code → symbol: 000001 → 000001.SZ
|
||||
from app.services.ext_data import build_code_lookup
|
||||
lookup = build_code_lookup(data_dir)
|
||||
df = df.with_columns(normalize_symbol(df["code"].cast(pl.Utf8), lookup).alias("symbol"))
|
||||
|
||||
if "code" not in df.columns and cm.get("type") == "computed":
|
||||
if cm.get("from") == "symbol" and "symbol" in df.columns:
|
||||
# symbol → code: 000001.SZ → 000001
|
||||
df = df.with_columns(
|
||||
df["symbol"].cast(pl.Utf8).str.split(".").list.first().alias("code")
|
||||
)
|
||||
|
||||
# --- 兜底：如果只有一个，自动生成另一个 ---
|
||||
if "symbol" in df.columns and "code" not in df.columns:
|
||||
df = df.with_columns(
|
||||
df["symbol"].cast(pl.Utf8).str.split(".").list.first().alias("code")
|
||||
)
|
||||
elif "code" in df.columns and "symbol" not in df.columns:
|
||||
from app.services.ext_data import build_code_lookup
|
||||
lookup = build_code_lookup(data_dir)
|
||||
df = df.with_columns(normalize_symbol(df["code"].cast(pl.Utf8), lookup).alias("symbol"))
|
||||
|
||||
# 标准化 symbol 列
|
||||
if "symbol" in df.columns:
|
||||
from app.services.ext_data import build_code_lookup
|
||||
lookup = build_code_lookup(data_dir)
|
||||
df = df.with_columns(normalize_symbol(df["symbol"].cast(pl.Utf8), lookup))
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def _clean_col_names(df: pl.DataFrame) -> pl.DataFrame:
|
||||
"""清洗列名：去掉所有 (...) 及其内容，避免时间戳导致列名不稳定。"""
|
||||
import re
|
||||
renames = {col: re.sub(r"\([^)]*\)", "", col).strip() for col in df.columns}
|
||||
# 去重：如果清洗后重名，加序号后缀
|
||||
seen: dict[str, int] = {}
|
||||
final = {}
|
||||
for old, new in renames.items():
|
||||
if new in seen:
|
||||
seen[new] += 1
|
||||
final[old] = f"{new}_{seen[new]}"
|
||||
else:
|
||||
seen[new] = 0
|
||||
final[old] = new
|
||||
return df.rename(final)
|
||||
|
||||
|
||||
def _ext_data_dir(config: ExtConfig, data_dir: Path) -> Path:
|
||||
"""返回扩展数据的数据目录。
|
||||
|
||||
- snapshot: data/ext_data/{id}/（part.parquet 与 config.json 同级）
|
||||
- timeseries: data/ext_data/{id}/timeseries/
|
||||
"""
|
||||
cfg_dir = data_dir / "ext_data" / config.id
|
||||
if config.mode == "timeseries":
|
||||
return cfg_dir / "timeseries"
|
||||
return cfg_dir
|
||||
|
||||
|
||||
def _parquet_glob(config: ExtConfig, data_dir: Path) -> str:
|
||||
"""返回该扩展配置下所有 parquet 文件的 glob 模式。
|
||||
|
||||
snapshot: 'data/ext_data/{id}/*.parquet'（只有 part.parquet）
|
||||
timeseries: 'data/ext_data/{id}/timeseries/**/*.parquet'
|
||||
"""
|
||||
cfg_dir = data_dir / "ext_data" / config.id
|
||||
if config.mode == "snapshot":
|
||||
return str(cfg_dir / "*.parquet")
|
||||
return str(cfg_dir / "timeseries" / "**" / "*.parquet")
|
||||
|
||||
|
||||
def _safe_json_value(value):
|
||||
if isinstance(value, float) and not math.isfinite(value):
|
||||
return None
|
||||
if isinstance(value, (date, datetime)):
|
||||
return value.isoformat()
|
||||
return value
|
||||
|
||||
|
||||
def _read_ext_dataframe(
|
||||
config: ExtConfig,
|
||||
data_dir: Path,
|
||||
snapshot_date: str | None = None,
|
||||
) -> tuple[pl.DataFrame, str | None]:
|
||||
cfg_dir = data_dir / "ext_data" / config.id
|
||||
|
||||
if config.mode == "snapshot":
|
||||
path = cfg_dir / "part.parquet"
|
||||
if not path.exists():
|
||||
return pl.DataFrame(), None
|
||||
return pl.read_parquet(path), _latest_sync_date(config, data_dir)
|
||||
|
||||
base = cfg_dir / "timeseries"
|
||||
if not base.exists():
|
||||
return pl.DataFrame(), None
|
||||
|
||||
if snapshot_date:
|
||||
path = base / f"date={snapshot_date}" / "part.parquet"
|
||||
if not path.exists():
|
||||
return pl.DataFrame(), snapshot_date
|
||||
return pl.read_parquet(path), snapshot_date
|
||||
|
||||
partitions = sorted(
|
||||
d for d in base.iterdir()
|
||||
if d.is_dir() and d.name.startswith("date=") and (d / "part.parquet").exists()
|
||||
)
|
||||
if not partitions:
|
||||
return pl.DataFrame(), None
|
||||
|
||||
latest = partitions[-1]
|
||||
latest_date = latest.name[5:]
|
||||
return pl.read_parquet(latest / "part.parquet"), latest_date
|
||||
|
||||
|
||||
def _with_instrument_name(df: pl.DataFrame, data_dir: Path) -> pl.DataFrame:
|
||||
if df.is_empty() or "symbol" not in df.columns or "name" in df.columns:
|
||||
return df
|
||||
path = data_dir / "instruments" / "instruments.parquet"
|
||||
if not path.exists():
|
||||
return df
|
||||
try:
|
||||
inst = pl.read_parquet(path)
|
||||
if "symbol" in inst.columns and "name" in inst.columns:
|
||||
inst = inst.select(["symbol", "name"]).unique(subset=["symbol"], keep="last")
|
||||
return df.join(inst, on="symbol", how="left")
|
||||
except Exception:
|
||||
return df
|
||||
return df
|
||||
|
||||
|
||||
def _latest_sync_date(config: ExtConfig, data_dir: Path) -> str | None:
|
||||
"""扫描数据文件，返回该扩展配置的最新同步时间（含时分秒）。
|
||||
|
||||
- snapshot: 直接取 ext_data/{id}/part.parquet 的 mtime
|
||||
- timeseries: 扫描 ext_data/{id}/timeseries/date=xxx 分区目录
|
||||
"""
|
||||
from datetime import datetime
|
||||
|
||||
if config.mode == "snapshot":
|
||||
# 快照: part.parquet 与 config.json 同级
|
||||
p = data_dir / "ext_data" / config.id / "part.parquet"
|
||||
if p.exists():
|
||||
ts = datetime.fromtimestamp(p.stat().st_mtime).strftime("%Y-%m-%d %H:%M:%S")
|
||||
return ts
|
||||
# 兼容旧路径
|
||||
old = data_dir / "instruments_ext"
|
||||
if old.exists():
|
||||
return _latest_sync_from_partitions(old)
|
||||
return None
|
||||
|
||||
# 时序: 扫描 timeseries/date=xxx
|
||||
base = _ext_data_dir(config, data_dir)
|
||||
if not base.exists():
|
||||
# 兼容旧路径
|
||||
base = data_dir / "kline_ext"
|
||||
if not base.exists():
|
||||
return None
|
||||
return _latest_sync_from_partitions(base)
|
||||
|
||||
|
||||
def _latest_sync_from_partitions(base: Path) -> str | None:
|
||||
"""从 date=xxx 分区目录中找到最新分区的修改时间。"""
|
||||
from datetime import datetime
|
||||
latest_ts: float = 0
|
||||
latest_date: str | None = None
|
||||
for d in base.iterdir():
|
||||
if d.is_dir() and d.name.startswith("date="):
|
||||
for f in d.glob("*.parquet"):
|
||||
mtime = f.stat().st_mtime
|
||||
if mtime > latest_ts:
|
||||
latest_ts = mtime
|
||||
latest_date = d.name[5:]
|
||||
if latest_date and latest_ts > 0:
|
||||
ts = datetime.fromtimestamp(latest_ts).strftime("%H:%M:%S")
|
||||
return f"{latest_date} {ts}"
|
||||
return latest_date
|
||||
|
||||
|
||||
def _date_range(config: ExtConfig, data_dir: Path) -> list[str] | None:
|
||||
"""返回时序型扩展数据的日期范围 [最早, 最新]。"""
|
||||
if config.mode != "timeseries":
|
||||
return None
|
||||
base = _ext_data_dir(config, data_dir)
|
||||
if not base.exists():
|
||||
# 兼容旧路径
|
||||
base = data_dir / "kline_ext"
|
||||
if not base.exists():
|
||||
return None
|
||||
dates: list[str] = []
|
||||
for d in base.iterdir():
|
||||
if d.is_dir() and d.name.startswith("date="):
|
||||
dates.append(d.name[5:])
|
||||
if len(dates) < 1:
|
||||
return None
|
||||
dates.sort()
|
||||
return [dates[0], dates[-1]]
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_configs(request: Request):
|
||||
"""列出所有扩展数据配置。"""
|
||||
configs = _store(request).load_all()
|
||||
data_dir = _data_dir(request)
|
||||
items = []
|
||||
for c in configs:
|
||||
d = c.to_dict()
|
||||
d["latest_sync_date"] = _latest_sync_date(c, data_dir)
|
||||
d["date_range"] = _date_range(c, data_dir)
|
||||
items.append(d)
|
||||
return {"items": items}
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_config(request: Request, body: CreateExtReq):
|
||||
"""创建扩展数据配置。"""
|
||||
store = _store(request)
|
||||
if store.get(body.id):
|
||||
raise HTTPException(400, f"配置 '{body.id}' 已存在")
|
||||
config = ExtConfig(
|
||||
id=body.id,
|
||||
label=body.label,
|
||||
mode=body.mode,
|
||||
fields=[ExtField(f.name, f.dtype, f.label) for f in body.fields],
|
||||
description=body.description,
|
||||
symbol_map=body.symbol_map,
|
||||
code_map=body.code_map,
|
||||
)
|
||||
store.upsert(config)
|
||||
return config.to_dict()
|
||||
|
||||
|
||||
@router.put("/{config_id}")
|
||||
def update_config(request: Request, config_id: str, body: UpdateExtReq):
|
||||
"""更新扩展数据配置。"""
|
||||
store = _store(request)
|
||||
config = store.get(config_id)
|
||||
if not config:
|
||||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||||
if body.label is not None:
|
||||
config.label = body.label
|
||||
if body.fields is not None:
|
||||
config.fields = [ExtField(f.name, f.dtype, f.label) for f in body.fields]
|
||||
if body.description is not None:
|
||||
config.description = body.description
|
||||
if body.symbol_map is not None:
|
||||
config.symbol_map = body.symbol_map
|
||||
if body.code_map is not None:
|
||||
config.code_map = body.code_map
|
||||
store.upsert(config)
|
||||
return config.to_dict()
|
||||
|
||||
|
||||
@router.delete("/{config_id}")
|
||||
def delete_config(request: Request, config_id: str):
|
||||
"""删除扩展数据配置。"""
|
||||
store = _store(request)
|
||||
if not store.delete(config_id):
|
||||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||||
return {"status": "deleted"}
|
||||
|
||||
|
||||
@router.get("/{config_id}/rows")
|
||||
def list_rows(
|
||||
request: Request,
|
||||
config_id: str,
|
||||
snapshot_date: str | None = Query(None, alias="date"),
|
||||
columns: str | None = Query(None, description="逗号分隔的字段列表"),
|
||||
limit: int = Query(1000, ge=1, le=20000),
|
||||
):
|
||||
"""读取扩展数据明细。
|
||||
|
||||
- snapshot: 返回当前快照。
|
||||
- timeseries: 默认返回最新日期分区，也可通过 date=YYYY-MM-DD 指定。
|
||||
"""
|
||||
config = _store(request).get(config_id)
|
||||
if not config:
|
||||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||||
|
||||
data_dir = _data_dir(request)
|
||||
df, active_date = _read_ext_dataframe(config, data_dir, snapshot_date)
|
||||
df = _with_instrument_name(df, data_dir)
|
||||
requested = [c.strip() for c in (columns or "").split(",") if c.strip()]
|
||||
if requested:
|
||||
keep = [c for c in ["symbol", "code", "name", *requested] if c in df.columns]
|
||||
if keep:
|
||||
df = df.select(list(dict.fromkeys(keep)))
|
||||
total = len(df)
|
||||
if total > limit:
|
||||
df = df.head(limit)
|
||||
|
||||
rows = []
|
||||
for row in df.to_dicts():
|
||||
rows.append({k: _safe_json_value(v) for k, v in row.items()})
|
||||
|
||||
return {
|
||||
"id": config.id,
|
||||
"label": config.label,
|
||||
"mode": config.mode,
|
||||
"date": active_date,
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"fields": [f.to_dict() for f in config.fields],
|
||||
"rows": rows,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 文件上传
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.post("/{config_id}/upload")
|
||||
async def upload_data(
|
||||
request: Request,
|
||||
config_id: str,
|
||||
file: UploadFile = File(...),
|
||||
snapshot_date: str | None = None,
|
||||
):
|
||||
"""上传 CSV/Excel 文件写入扩展数据。"""
|
||||
store = _store(request)
|
||||
config = store.get(config_id)
|
||||
if not config:
|
||||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||||
|
||||
# 校验文件后缀
|
||||
suffix = Path(file.filename or "").suffix.lower()
|
||||
if suffix not in (".csv", ".xlsx", ".xls"):
|
||||
raise HTTPException(400, "仅支持 CSV / Excel 文件")
|
||||
|
||||
# 写到临时文件再解析
|
||||
tmp_dir = Path(tempfile.mkdtemp())
|
||||
tmp_path = tmp_dir / f"upload{suffix}"
|
||||
try:
|
||||
with tmp_path.open("wb") as f:
|
||||
content = await file.read()
|
||||
f.write(content)
|
||||
|
||||
# 直接读取文件，不做列重命名
|
||||
if suffix == ".csv":
|
||||
df = pl.read_csv(ensure_utf8_csv(tmp_path), infer_schema_length=10000)
|
||||
elif suffix in (".xlsx", ".xls"):
|
||||
df = pl.read_excel(tmp_path)
|
||||
else:
|
||||
raise HTTPException(400, f"不支持的文件格式: {suffix}")
|
||||
|
||||
# 清洗列名：去掉括号内的时间戳等信息
|
||||
df = _clean_col_names(df)
|
||||
|
||||
# 按映射关系自动生成 symbol 和 code 列
|
||||
df = _apply_mapping(df, config, _data_dir(request))
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
finally:
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
# 确保配置的字段列存在于上传数据中（symbol/code 由映射自动生成，不校验）
|
||||
auto_fields = {"symbol", "code"}
|
||||
config_cols = {f.name for f in config.fields} - auto_fields
|
||||
missing = config_cols - set(df.columns)
|
||||
if missing:
|
||||
raise HTTPException(400, f"上传数据缺少字段: {', '.join(sorted(missing))}")
|
||||
|
||||
# 只保留配置中定义的列（包括自动生成的 symbol/code），忽略文件中多余的字段
|
||||
all_config_cols = {f.name for f in config.fields}
|
||||
keep = [c for c in df.columns if c in all_config_cols]
|
||||
df = df.select(keep)
|
||||
|
||||
# 解析快照日期
|
||||
snap = date.fromisoformat(snapshot_date) if snapshot_date else date.today()
|
||||
|
||||
rows = write_ext_parquet(df, config, _data_dir(request), snapshot_date=snap)
|
||||
|
||||
# 刷新 DuckDB 视图
|
||||
_refresh_views(request)
|
||||
|
||||
return {"status": "ok", "rows": rows, "date": snap.isoformat()}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSON 接口写入
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.post("/{config_id}/ingest")
|
||||
def ingest_data(request: Request, config_id: str, body: IngestReq):
|
||||
"""通过 JSON 接口批量写入扩展数据。"""
|
||||
store = _store(request)
|
||||
config = store.get(config_id)
|
||||
if not config:
|
||||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||||
|
||||
# 校验必填字段
|
||||
configured = {f.name for f in config.fields}
|
||||
required = configured - {"symbol"}
|
||||
for i, row in enumerate(body.rows):
|
||||
if "symbol" not in row:
|
||||
raise HTTPException(400, f"第 {i + 1} 行缺少 symbol 字段")
|
||||
missing = required - set(row.keys())
|
||||
if missing:
|
||||
raise HTTPException(400, f"第 {i + 1} 行缺少字段: {', '.join(sorted(missing))}")
|
||||
|
||||
snap = date.fromisoformat(body.date) if body.date else date.today()
|
||||
|
||||
rows_written = rows_to_parquet(body.rows, config, _data_dir(request), snapshot_date=snap)
|
||||
|
||||
_refresh_views(request)
|
||||
|
||||
return {"status": "ok", "rows": rows_written, "date": snap.isoformat()}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 定时拉取
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.put("/{config_id}/pull")
|
||||
def configure_pull(request: Request, config_id: str, body: PullConfigReq):
|
||||
"""配置（或更新）定时拉取。"""
|
||||
store = _store(request)
|
||||
config = store.get(config_id)
|
||||
if not config:
|
||||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||||
|
||||
# 保留历史状态字段
|
||||
old_pull = config.pull
|
||||
config.pull = PullConfig(
|
||||
url=body.url,
|
||||
method=body.method,
|
||||
headers=body.headers,
|
||||
body=body.body,
|
||||
response_path=body.response_path,
|
||||
field_map=body.field_map,
|
||||
schedule_minutes=body.schedule_minutes,
|
||||
enabled=body.enabled,
|
||||
last_run=old_pull.last_run if old_pull else None,
|
||||
last_status=old_pull.last_status if old_pull else None,
|
||||
last_message=old_pull.last_message if old_pull else None,
|
||||
last_rows=old_pull.last_rows if old_pull else None,
|
||||
)
|
||||
store.upsert(config)
|
||||
|
||||
# 刷新调度器
|
||||
pull_scheduler.refresh(_data_dir(request))
|
||||
|
||||
return {"status": "ok", "pull": config.pull.to_dict()}
|
||||
|
||||
|
||||
@router.post("/{config_id}/pull/test")
|
||||
async def test_pull(request: Request, config_id: str):
|
||||
"""测试拉取：请求外部 API 并返回预览数据，不写入。"""
|
||||
store = _store(request)
|
||||
config = store.get(config_id)
|
||||
if not config:
|
||||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||||
if not config.pull or not config.pull.url:
|
||||
raise HTTPException(400, "拉取未配置或 URL 为空")
|
||||
|
||||
# 临时构建一个带新配置的 config 用于测试
|
||||
from app.services.ext_pull import _extract_rows, _apply_field_map
|
||||
import httpx
|
||||
|
||||
pull = config.pull
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
headers = pull.headers or {}
|
||||
kwargs: dict = {"headers": headers}
|
||||
if pull.method.upper() == "POST" and pull.body:
|
||||
kwargs["content"] = pull.body
|
||||
if "content-type" not in {k.lower() for k in headers}:
|
||||
kwargs["headers"]["Content-Type"] = "application/json"
|
||||
resp = await client.request(pull.method.upper(), pull.url, **kwargs)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
rows = _extract_rows(data, pull.response_path)
|
||||
preview = _apply_field_map(rows[:5], pull.field_map)
|
||||
return {
|
||||
"status": "ok",
|
||||
"total_rows": len(rows),
|
||||
"preview": preview,
|
||||
"has_symbol": bool(rows and "symbol" in rows[0]),
|
||||
}
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"测试失败: {e}") from e
|
||||
|
||||
|
||||
@router.post("/{config_id}/pull/run")
|
||||
async def run_pull(request: Request, config_id: str):
|
||||
"""手动触发一次拉取并写入。"""
|
||||
store = _store(request)
|
||||
config = store.get(config_id)
|
||||
if not config:
|
||||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||||
if not config.pull or not config.pull.url:
|
||||
raise HTTPException(400, "拉取未配置或 URL 为空")
|
||||
|
||||
try:
|
||||
n, d = await fetch_and_ingest(config, _data_dir(request))
|
||||
_refresh_views(request)
|
||||
return {"status": "ok", "rows": n, "date": d}
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"拉取失败: {e}") from e
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ---------------------------------------------------------------------------
|
||||
# Symbol 格式修复
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.post("/{config_id}/fix-symbol")
|
||||
def fix_symbol(request: Request, config_id: str):
|
||||
"""扫描已有 Parquet 数据，将 symbol 列标准化为 代码.交易所 格式。"""
|
||||
store = _store(request)
|
||||
config = store.get(config_id)
|
||||
if not config:
|
||||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||||
|
||||
fixed = fix_symbol_format(config, _data_dir(request))
|
||||
_refresh_views(request)
|
||||
return {"status": "ok", "fixed_files": fixed}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schema 发现
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_POLARS_TYPE_MAP = {
|
||||
"Int64": "int", "Int32": "int", "Int16": "int", "Int8": "int",
|
||||
"UInt64": "int", "UInt32": "int", "UInt16": "int", "UInt8": "int",
|
||||
"Float64": "float", "Float32": "float",
|
||||
"Boolean": "bool",
|
||||
"Utf8": "string", "String": "string",
|
||||
"Date": "string", "Datetime": "string", "Duration": "string",
|
||||
"Categorical": "string",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/detect-fields")
|
||||
async def detect_fields(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
):
|
||||
"""上传 CSV/Excel 文件，自动检测列名和类型。
|
||||
|
||||
返回 symbol_candidates（数据匹配 000001.SZ 格式的列）和
|
||||
code_candidates（数据匹配 6位纯数字的列）。
|
||||
"""
|
||||
suffix = Path(file.filename or "").suffix.lower()
|
||||
if suffix not in (".csv", ".xlsx", ".xls"):
|
||||
raise HTTPException(400, "仅支持 CSV / Excel 文件")
|
||||
|
||||
tmp_dir = Path(tempfile.mkdtemp())
|
||||
tmp_path = tmp_dir / f"upload{suffix}"
|
||||
try:
|
||||
with tmp_path.open("wb") as f:
|
||||
content = await file.read()
|
||||
f.write(content)
|
||||
|
||||
# 直接读取，不要求 symbol 列
|
||||
if suffix == ".csv":
|
||||
df = pl.read_csv(ensure_utf8_csv(tmp_path), infer_schema_length=10000)
|
||||
elif suffix in (".xlsx", ".xls"):
|
||||
df = pl.read_excel(tmp_path)
|
||||
else:
|
||||
raise HTTPException(400, f"不支持的文件格式: {suffix}")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(400, str(e)) from e
|
||||
finally:
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
# 清洗列名：去掉括号内的时间戳等信息
|
||||
df = _clean_col_names(df)
|
||||
|
||||
# 构建字段列表
|
||||
fields = []
|
||||
for col_name in df.columns:
|
||||
pl_type = df[col_name].dtype
|
||||
dtype = _POLARS_TYPE_MAP.get(str(pl_type.base_type()), "string")
|
||||
fields.append({"name": col_name, "dtype": dtype, "label": col_name})
|
||||
|
||||
# --- 分别检测 symbol 候选 (000001.SZ) 和 code 候选 (6位纯数字) ---
|
||||
import re
|
||||
_CODE_PAT = re.compile(r"^\d{6}$")
|
||||
_SYMBOL_PAT = re.compile(r"^\d{6}\.[A-Z]{2}$")
|
||||
|
||||
symbol_candidates: list[str] = [] # 数据匹配 000001.SZ 格式
|
||||
code_candidates: list[str] = [] # 数据匹配 000001 格式
|
||||
|
||||
for col in df.columns:
|
||||
col_data = df[col].cast(pl.Utf8).drop_nulls()
|
||||
if len(col_data) == 0:
|
||||
continue
|
||||
sample = col_data.head(200).to_list()
|
||||
sym_hits = sum(1 for v in sample if _SYMBOL_PAT.match(str(v).strip()))
|
||||
code_hits = sum(1 for v in sample if _CODE_PAT.match(str(v).strip()))
|
||||
total = len(sample)
|
||||
if total > 0:
|
||||
if sym_hits / total > 0.5:
|
||||
symbol_candidates.append(col)
|
||||
elif code_hits / total > 0.5:
|
||||
code_candidates.append(col)
|
||||
|
||||
return {
|
||||
"fields": fields,
|
||||
"rows": len(df),
|
||||
"symbol_candidates": symbol_candidates,
|
||||
"code_candidates": code_candidates,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/schema/{config_id}")
|
||||
def discover_schema(request: Request, config_id: str):
|
||||
"""发现扩展数据的实际 Parquet schema（基于已有数据）。"""
|
||||
config = _store(request).get(config_id)
|
||||
if not config:
|
||||
raise HTTPException(404, f"配置 '{config_id}' 不存在")
|
||||
|
||||
data_dir = _data_dir(request)
|
||||
glob = _parquet_glob(config, data_dir)
|
||||
|
||||
try:
|
||||
import duckdb
|
||||
rows = duckdb.query(
|
||||
f"SELECT column_name, data_type FROM (DESCRIBE SELECT * FROM read_parquet('{glob}', union_by_name=true))"
|
||||
).fetchall()
|
||||
return {"columns": [{"name": r[0], "type": r[1]} for r in rows]}
|
||||
except Exception:
|
||||
# 无数据时返回配置中定义的字段
|
||||
return {"columns": [f.to_dict() for f in config.fields]}
|
||||
|
||||
|
||||
@router.get("/schema-all")
|
||||
def discover_all_schemas(request: Request):
|
||||
"""发现所有扩展表的 schema（用于前端动态列选择）。"""
|
||||
configs = _store(request).load_all()
|
||||
result = []
|
||||
for config in configs:
|
||||
data_dir = _data_dir(request)
|
||||
glob = _parquet_glob(config, data_dir)
|
||||
|
||||
try:
|
||||
import duckdb
|
||||
cols = duckdb.query(
|
||||
f"SELECT column_name, data_type FROM (DESCRIBE SELECT * FROM read_parquet('{glob}', union_by_name=true))"
|
||||
).fetchall()
|
||||
field_labels = {f.name: f.label for f in config.fields}
|
||||
columns = [{"name": r[0], "type": r[1], "label": field_labels.get(r[0], r[0])} for r in cols]
|
||||
except Exception:
|
||||
columns = [f.to_dict() for f in config.fields]
|
||||
|
||||
result.append({
|
||||
"id": config.id,
|
||||
"label": config.label,
|
||||
"mode": config.mode,
|
||||
"columns": columns,
|
||||
})
|
||||
return {"items": result}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 视图刷新
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _refresh_views(request: Request) -> None:
|
||||
"""重新注册 DuckDB 视图以包含新的扩展数据。"""
|
||||
repo = request.app.state.repo
|
||||
db = repo.store.db
|
||||
d = repo.store.data_dir.as_posix()
|
||||
|
||||
# 注册旧路径视图（兼容）
|
||||
for name, subdir in [("instruments_ext", "instruments_ext"), ("kline_ext", "kline_ext")]:
|
||||
old_glob = f"{d}/{subdir}/**/*.parquet"
|
||||
old_dir = Path(d) / subdir
|
||||
if old_dir.exists():
|
||||
sql = (
|
||||
f"CREATE OR REPLACE VIEW {name} AS "
|
||||
f"SELECT * FROM read_parquet('{old_glob}', union_by_name=true)"
|
||||
)
|
||||
try:
|
||||
db.execute(sql)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 注册新路径视图：每个扩展表一个视图 ext_{config_id}
|
||||
ext_base = Path(d) / "ext_data"
|
||||
if ext_base.exists():
|
||||
for cfg_dir in ext_base.iterdir():
|
||||
if not cfg_dir.is_dir():
|
||||
continue
|
||||
cp = cfg_dir / "config.json"
|
||||
if not cp.exists():
|
||||
continue
|
||||
try:
|
||||
raw = json.loads(cp.read_text(encoding="utf-8"))
|
||||
cfg_id = raw["id"]
|
||||
# 检查是否有数据文件（snapshot: part.parquet, timeseries: timeseries/ 目录）
|
||||
has_data = (cfg_dir / "part.parquet").exists() or (cfg_dir / "timeseries").exists()
|
||||
if has_data:
|
||||
view_name = f"ext_{cfg_id}"
|
||||
# snapshot: part.parquet 在 cfg_dir/ 根下; timeseries: 在 timeseries/ 子目录
|
||||
mode = raw.get("mode", "snapshot")
|
||||
if mode == "snapshot":
|
||||
glob_pattern = f"{cfg_dir.as_posix()}/*.parquet"
|
||||
else:
|
||||
glob_pattern = f"{cfg_dir.as_posix()}/timeseries/**/*.parquet"
|
||||
sql = (
|
||||
f"CREATE OR REPLACE VIEW {view_name} AS "
|
||||
f"SELECT * FROM read_parquet('{glob_pattern}', union_by_name=true)"
|
||||
)
|
||||
db.execute(sql)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,127 @@
|
||||
"""财务数据 API — 独立路由, Cap.FINANCIAL 门控。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import polars as pl
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
|
||||
from app.services.financial_sync import get_financial_df
|
||||
from app.tickflow.capabilities import Cap
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/financials", tags=["financials"])
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
def financial_status(request: Request):
|
||||
"""返回各财务表的同步状态。无需 FINANCIAL 权限（前端根据 available 决定是否展示）。"""
|
||||
capset = request.app.state.capabilities
|
||||
if not capset.has(Cap.FINANCIAL):
|
||||
return {"available": False, "tables": {}}
|
||||
|
||||
data_dir = request.app.state.repo.store.data_dir
|
||||
tables = {}
|
||||
|
||||
for table in ("metrics", "income", "balance_sheet", "cash_flow"):
|
||||
path = data_dir / "financials" / table / "part.parquet"
|
||||
if path.exists():
|
||||
try:
|
||||
df = pl.read_parquet(path, columns=["symbol"])
|
||||
tables[table] = {
|
||||
"rows": len(df),
|
||||
"symbols": df["symbol"].n_unique() if not df.is_empty() else 0,
|
||||
}
|
||||
except Exception:
|
||||
tables[table] = {"rows": 0, "symbols": 0}
|
||||
else:
|
||||
tables[table] = {"rows": 0, "symbols": 0}
|
||||
|
||||
fs = getattr(request.app.state, "financial_scheduler", None)
|
||||
last_sync = fs.last_sync if fs else {}
|
||||
|
||||
return {
|
||||
"available": True,
|
||||
"tables": tables,
|
||||
"last_sync": last_sync,
|
||||
# 服务端是否正在同步(手动触发)——前端据此显示"同步中"并防重复点击,
|
||||
# 且刷新页面后仍能正确反映服务端状态。
|
||||
"syncing": bool(fs and fs.is_syncing),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/metrics")
|
||||
def get_metrics(request: Request, symbol: str | None = None):
|
||||
"""查询核心财务指标。"""
|
||||
capset = request.app.state.capabilities
|
||||
capset.require(Cap.FINANCIAL)
|
||||
|
||||
df = get_financial_df(request.app.state.repo.store.data_dir, "metrics")
|
||||
if df.is_empty():
|
||||
return {"data": []}
|
||||
if symbol:
|
||||
df = df.filter(pl.col("symbol") == symbol)
|
||||
return {"data": df.to_dicts()}
|
||||
|
||||
|
||||
@router.get("/income")
|
||||
def get_income(request: Request, symbol: str | None = None):
|
||||
"""查询利润表。"""
|
||||
capset = request.app.state.capabilities
|
||||
capset.require(Cap.FINANCIAL)
|
||||
|
||||
df = get_financial_df(request.app.state.repo.store.data_dir, "income")
|
||||
if df.is_empty():
|
||||
return {"data": []}
|
||||
if symbol:
|
||||
df = df.filter(pl.col("symbol") == symbol)
|
||||
return {"data": df.to_dicts()}
|
||||
|
||||
|
||||
@router.get("/balance-sheet")
|
||||
def get_balance_sheet(request: Request, symbol: str | None = None):
|
||||
"""查询资产负债表。"""
|
||||
capset = request.app.state.capabilities
|
||||
capset.require(Cap.FINANCIAL)
|
||||
|
||||
df = get_financial_df(request.app.state.repo.store.data_dir, "balance_sheet")
|
||||
if df.is_empty():
|
||||
return {"data": []}
|
||||
if symbol:
|
||||
df = df.filter(pl.col("symbol") == symbol)
|
||||
return {"data": df.to_dicts()}
|
||||
|
||||
|
||||
@router.get("/cash-flow")
|
||||
def get_cash_flow(request: Request, symbol: str | None = None):
|
||||
"""查询现金流量表。"""
|
||||
capset = request.app.state.capabilities
|
||||
capset.require(Cap.FINANCIAL)
|
||||
|
||||
df = get_financial_df(request.app.state.repo.store.data_dir, "cash_flow")
|
||||
if df.is_empty():
|
||||
return {"data": []}
|
||||
if symbol:
|
||||
df = df.filter(pl.col("symbol") == symbol)
|
||||
return {"data": df.to_dicts()}
|
||||
|
||||
|
||||
@router.post("/sync/{table}")
|
||||
def sync_table(request: Request, table: str):
|
||||
"""手动触发同步。table: metrics / income / balance_sheet / cash_flow / all"""
|
||||
capset = request.app.state.capabilities
|
||||
capset.require(Cap.FINANCIAL)
|
||||
|
||||
valid_tables = {"metrics", "income", "balance_sheet", "cash_flow", "all"}
|
||||
if table not in valid_tables:
|
||||
raise HTTPException(400, f"invalid table: {table}, expected one of {valid_tables}")
|
||||
|
||||
fs = getattr(request.app.state, "financial_scheduler", None)
|
||||
if not fs:
|
||||
return {"status": "error", "message": "FinancialScheduler not available"}
|
||||
|
||||
target = None if table == "all" else table
|
||||
result = fs.run_now(target)
|
||||
|
||||
return {"status": "ok", "synced": result}
|
||||
@@ -0,0 +1,149 @@
|
||||
"""指数 API。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date, datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
import polars as pl
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
|
||||
from app.indicators.pipeline import compute_enriched
|
||||
from app.services import index_sync, kline_sync
|
||||
from app.tickflow.capabilities import Cap
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/index", tags=["index"])
|
||||
|
||||
|
||||
def _index_info(repo, symbol: str) -> dict:
|
||||
df = repo.get_index_instruments()
|
||||
if df.is_empty() or "symbol" not in df.columns:
|
||||
return {}
|
||||
hit = df.filter(pl.col("symbol") == symbol).head(1)
|
||||
if hit.is_empty():
|
||||
return {}
|
||||
return hit.to_dicts()[0]
|
||||
|
||||
|
||||
@router.get("/list")
|
||||
def list_indices(request: Request):
|
||||
"""返回已缓存的 CN_Index 指数列表。"""
|
||||
repo = request.app.state.repo
|
||||
df = repo.get_index_instruments()
|
||||
if df.is_empty():
|
||||
return {"results": [], "count": 0}
|
||||
cols = [c for c in ["symbol", "name", "code", "asset_type"] if c in df.columns]
|
||||
rows = df.select(cols).sort("symbol").to_dicts()
|
||||
return {"results": rows, "count": len(rows)}
|
||||
|
||||
|
||||
@router.get("/search")
|
||||
def search_indices(
|
||||
request: Request,
|
||||
q: str = Query("", min_length=0, max_length=50, description="搜索关键词"),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
):
|
||||
"""模糊搜索指数。"""
|
||||
repo = request.app.state.repo
|
||||
df = repo.get_index_instruments()
|
||||
if df.is_empty():
|
||||
return {"results": []}
|
||||
if not q.strip():
|
||||
rows = df.head(limit).to_dicts()
|
||||
return {"results": rows}
|
||||
|
||||
keyword = q.strip().upper()
|
||||
masks = []
|
||||
if "code" in df.columns:
|
||||
masks.append(pl.col("code").cast(pl.Utf8).str.contains(keyword, literal=True))
|
||||
masks.append(pl.col("symbol").cast(pl.Utf8).str.to_uppercase().str.contains(keyword, literal=True))
|
||||
if "name" in df.columns:
|
||||
masks.append(pl.col("name").cast(pl.Utf8).str.contains(q.strip(), literal=True))
|
||||
|
||||
mask = masks[0]
|
||||
for m in masks[1:]:
|
||||
mask = mask | m
|
||||
rows = df.filter(mask).head(limit).to_dicts()
|
||||
return {"results": rows}
|
||||
|
||||
|
||||
@router.get("/daily")
|
||||
def get_index_daily(
|
||||
request: Request,
|
||||
symbol: str = Query(..., description="指数代码, 如 000001.SH"),
|
||||
days: int = Query(120, ge=10, le=2000),
|
||||
start_date: Optional[str] = Query(None, description="起始日期 YYYY-MM-DD, 优先于 days"),
|
||||
end_date: Optional[str] = Query(None, description="截止日期 YYYY-MM-DD, 默认今天"),
|
||||
):
|
||||
"""读取指数日 K。指数数据使用独立 kline_index_* parquet。"""
|
||||
repo = request.app.state.repo
|
||||
end = date.fromisoformat(end_date) if end_date else date.today()
|
||||
start = date.fromisoformat(start_date) if start_date else end - timedelta(days=days)
|
||||
info = _index_info(repo, symbol)
|
||||
|
||||
df = repo.get_index_daily(symbol, start, end)
|
||||
if not df.is_empty():
|
||||
return {"symbol": symbol, "name": info.get("name"), "index_info": info, "rows": df.to_dicts(), "source": "index_enriched"}
|
||||
|
||||
capset = request.app.state.capabilities
|
||||
if not capset.has(Cap.KLINE_DAILY_BATCH):
|
||||
return {"symbol": symbol, "name": info.get("name"), "index_info": info, "rows": [], "source": "none"}
|
||||
|
||||
try:
|
||||
raw = kline_sync.sync_daily_batch([symbol], count=days + 150)
|
||||
except Exception as e: # noqa: BLE001
|
||||
raise HTTPException(status_code=502, detail=f"数据源 fetch failed: {e}") from e
|
||||
if raw.is_empty():
|
||||
return {"symbol": symbol, "name": info.get("name"), "index_info": info, "rows": [], "source": "none"}
|
||||
|
||||
enriched = compute_enriched(raw, factors=None, instruments=None)
|
||||
rows = enriched.filter((pl.col("date") >= start) & (pl.col("date") <= end)).to_dicts()
|
||||
return {"symbol": symbol, "name": info.get("name"), "index_info": info, "rows": rows, "source": "live"}
|
||||
|
||||
|
||||
@router.get("/minute")
|
||||
def get_index_minute(
|
||||
request: Request,
|
||||
symbol: str = Query(..., description="指数代码, 如 000001.SH"),
|
||||
trade_date: date | None = Query(None, alias="date", description="交易日期, 默认今天"),
|
||||
):
|
||||
"""实时读取指数分钟 K。不写入股票分钟 parquet。"""
|
||||
repo = request.app.state.repo
|
||||
info = _index_info(repo, symbol)
|
||||
day = trade_date or date.today()
|
||||
df = kline_sync.fetch_minute_single(symbol, day)
|
||||
return {
|
||||
"symbol": symbol,
|
||||
"name": info.get("name"),
|
||||
"index_info": info,
|
||||
"date": str(day),
|
||||
"rows": df.to_dicts(),
|
||||
"source": "live" if not df.is_empty() else "none",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/sync_instruments")
|
||||
def sync_index_instruments(request: Request):
|
||||
"""同步 CN_Index 指数标的列表。"""
|
||||
repo = request.app.state.repo
|
||||
count = index_sync.sync_index_instruments(repo)
|
||||
return {"status": "ok", "count": count}
|
||||
|
||||
|
||||
@router.post("/sync_daily")
|
||||
def sync_index_daily(
|
||||
request: Request,
|
||||
days: int = Query(365, ge=30, le=5000),
|
||||
):
|
||||
"""同步指数日K到独立 parquet。"""
|
||||
repo = request.app.state.repo
|
||||
capset = request.app.state.capabilities
|
||||
if not capset.has(Cap.KLINE_DAILY_BATCH):
|
||||
raise HTTPException(status_code=403, detail="需要 Pro+ 权限 (batch K-line)")
|
||||
end = datetime.now()
|
||||
start = end - timedelta(days=days)
|
||||
count = index_sync.sync_index_instruments(repo)
|
||||
rows = index_sync.sync_and_persist_index_daily(repo, capset, start_date=start, end_date=end)
|
||||
return {"status": "ok", "index_count": count, "rows_written": rows}
|
||||
@@ -0,0 +1,201 @@
|
||||
"""行情状态 / SSE 推送 API。
|
||||
|
||||
盘中选股相关端点已迁移至策略页面，此处仅保留全局行情基础设施。
|
||||
SSE 推送三种事件 (使用标准 SSE event 字段):
|
||||
- quotes_updated: 行情数据刷新，前端 invalidate 对应 query
|
||||
- strategy_alert: 策略监控/告警触发，前端弹通知
|
||||
- depth_updated: 五档盘口修正完成，前端刷新连板梯队/看板封单数据
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
|
||||
from fastapi import APIRouter, Query, Request
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
|
||||
router = APIRouter(prefix="/api/intraday", tags=["quotes"])
|
||||
|
||||
|
||||
def _get_quote_service(request: Request):
|
||||
"""获取全局 QuoteService。"""
|
||||
return getattr(request.app.state, "quote_service", None)
|
||||
|
||||
|
||||
def _fallback_index_quotes_from_daily(request: Request, symbols: list[str] | None = None) -> list[dict]:
|
||||
"""实时指数缓存为空时，从本地指数日 K 取最近收盘价作为兜底。"""
|
||||
repo = getattr(request.app.state, "repo", None)
|
||||
if not repo:
|
||||
return []
|
||||
|
||||
params: list[str] = []
|
||||
symbol_filter = ""
|
||||
if symbols:
|
||||
placeholders = ", ".join("?" for _ in symbols)
|
||||
symbol_filter = f"WHERE symbol IN ({placeholders})"
|
||||
params.extend(symbols)
|
||||
|
||||
try:
|
||||
rows = repo.execute_all(
|
||||
f"""
|
||||
WITH ranked AS (
|
||||
SELECT symbol, date, close,
|
||||
row_number() OVER (PARTITION BY symbol ORDER BY date DESC) AS rn
|
||||
FROM kline_index_daily
|
||||
{symbol_filter}
|
||||
), latest AS (
|
||||
SELECT symbol,
|
||||
max(CASE WHEN rn = 1 THEN date END) AS date,
|
||||
max(CASE WHEN rn = 1 THEN close END) AS last_price,
|
||||
max(CASE WHEN rn = 2 THEN close END) AS prev_close
|
||||
FROM ranked
|
||||
WHERE rn <= 2
|
||||
GROUP BY symbol
|
||||
)
|
||||
SELECT latest.symbol, latest.date, latest.last_price, latest.prev_close
|
||||
FROM latest
|
||||
ORDER BY latest.symbol
|
||||
""",
|
||||
params,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
return []
|
||||
|
||||
out: list[dict] = []
|
||||
for symbol, dt, last_price, prev_close in rows:
|
||||
change_amount = None
|
||||
change_pct = None
|
||||
if last_price is not None and prev_close not in (None, 0):
|
||||
change_amount = float(last_price) - float(prev_close)
|
||||
change_pct = change_amount / float(prev_close) * 100
|
||||
out.append({
|
||||
"symbol": symbol,
|
||||
"name": None,
|
||||
"date": str(dt) if dt else None,
|
||||
"last_price": float(last_price) if last_price is not None else None,
|
||||
"close": float(last_price) if last_price is not None else None,
|
||||
"prev_close": float(prev_close) if prev_close is not None else None,
|
||||
"change_amount": change_amount,
|
||||
"change_pct": change_pct,
|
||||
"source": "index_daily",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
def status(request: Request):
|
||||
"""行情状态 (来自全局 QuoteService)。"""
|
||||
qs = _get_quote_service(request)
|
||||
if qs:
|
||||
return qs.status()
|
||||
return {"enabled": False, "running": False, "symbol_count": 0, "index_symbol_count": 0,
|
||||
"quote_age_ms": None, "is_trading_hours": False, "last_fetch_ms": None}
|
||||
|
||||
|
||||
@router.get("/indices")
|
||||
def index_quotes(
|
||||
request: Request,
|
||||
symbols: str | None = Query(None, description="逗号分隔的指数 symbol 列表"),
|
||||
):
|
||||
"""返回实时指数行情缓存，不触发数据源请求。"""
|
||||
symbol_list = [s.strip() for s in symbols.split(",") if s.strip()] if symbols else None
|
||||
qs = _get_quote_service(request)
|
||||
if not qs:
|
||||
rows = _fallback_index_quotes_from_daily(request, symbol_list)
|
||||
return {"rows": rows, "count": len(rows), "source": "index_daily"}
|
||||
df = qs.get_index_quotes(symbol_list)
|
||||
rows = df.to_dicts() if not df.is_empty() else []
|
||||
if not rows:
|
||||
rows = _fallback_index_quotes_from_daily(request, symbol_list)
|
||||
return {"rows": rows, "count": len(rows), "source": "index_daily"}
|
||||
return {"rows": rows, "count": len(rows), "source": "realtime"}
|
||||
|
||||
|
||||
@router.get("/stream")
|
||||
async def quote_stream(request: Request):
|
||||
"""SSE 端点: 行情更新 + 告警推送 + 五档修正。
|
||||
|
||||
使用 sse-starlette EventSourceResponse:
|
||||
- 标准 SSE event 字段，前端按 event name 监听
|
||||
- 内置断线检测，客户端断开立即终止 generator
|
||||
- 内置 ping 心跳，保持连接活跃
|
||||
"""
|
||||
qs = _get_quote_service(request)
|
||||
|
||||
async def event_generator():
|
||||
while True:
|
||||
# 同时等待三类信号: 行情更新 / 告警 / 五档修正
|
||||
tasks: dict[str, asyncio.Future] = {
|
||||
"quote": asyncio.ensure_future(
|
||||
asyncio.to_thread(qs.wait_for_update, timeout=5.0) if qs else asyncio.sleep(5)
|
||||
),
|
||||
"alert": asyncio.ensure_future(
|
||||
asyncio.to_thread(qs.wait_for_alert, timeout=5.0) if qs else asyncio.sleep(5)
|
||||
),
|
||||
"depth": asyncio.ensure_future(
|
||||
asyncio.to_thread(qs.wait_for_depth_update, timeout=5.0) if qs else asyncio.sleep(5)
|
||||
),
|
||||
}
|
||||
|
||||
done, pending = await asyncio.wait(
|
||||
list(tasks.values()),
|
||||
timeout=30.0,
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
for t in pending:
|
||||
t.cancel()
|
||||
|
||||
# 先推送告警 (如果有)
|
||||
if qs:
|
||||
alerts = qs.pop_alerts()
|
||||
if alerts:
|
||||
for chunk_start in range(0, len(alerts), 20):
|
||||
chunk = alerts[chunk_start:chunk_start + 20]
|
||||
yield {
|
||||
"event": "strategy_alert",
|
||||
"data": json.dumps({
|
||||
"ts": int(time.time() * 1000),
|
||||
"alerts": chunk,
|
||||
}, ensure_ascii=False),
|
||||
}
|
||||
|
||||
# 推送行情更新 (行情信号触发)
|
||||
if tasks["quote"] in done:
|
||||
try:
|
||||
update_result = tasks["quote"].result()
|
||||
except Exception: # noqa: BLE001
|
||||
update_result = False
|
||||
if update_result:
|
||||
yield {
|
||||
"event": "quotes_updated",
|
||||
"data": json.dumps({
|
||||
"ts": int(time.time() * 1000),
|
||||
"symbol_count": qs._symbol_count if qs else 0,
|
||||
}),
|
||||
}
|
||||
|
||||
# 推送五档修正完成 (depth 信号触发) — 前端刷新连板梯队封单数据
|
||||
if tasks["depth"] in done:
|
||||
try:
|
||||
depth_result = tasks["depth"].result()
|
||||
except Exception: # noqa: BLE001
|
||||
depth_result = False
|
||||
if depth_result:
|
||||
yield {
|
||||
"event": "depth_updated",
|
||||
"data": json.dumps({
|
||||
"ts": int(time.time() * 1000),
|
||||
}),
|
||||
}
|
||||
|
||||
return EventSourceResponse(event_generator())
|
||||
|
||||
|
||||
@router.post("/refresh")
|
||||
def refresh_quotes(request: Request):
|
||||
"""手动刷新一次行情数据。"""
|
||||
qs = _get_quote_service(request)
|
||||
if qs:
|
||||
return qs.refresh()
|
||||
return {"error": "QuoteService not available"}
|
||||
@@ -0,0 +1,813 @@
|
||||
"""K 线 / 同步 API。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
|
||||
from app.indicators.pipeline import compute_enriched_single
|
||||
from app.services import kline_sync
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/kline", tags=["kline"])
|
||||
|
||||
|
||||
@router.get("/instruments/search")
|
||||
def search_instruments(
|
||||
request: Request,
|
||||
q: str = Query("", min_length=0, max_length=50, description="搜索关键词"),
|
||||
limit: int = Query(20, ge=1, le=50),
|
||||
):
|
||||
"""模糊搜索标的 (代码 / 名称)。从内存 instruments 缓存中查。"""
|
||||
repo = request.app.state.repo
|
||||
df = repo.get_instruments()
|
||||
if df.is_empty() or not q.strip():
|
||||
return {"results": []}
|
||||
|
||||
keyword = q.strip().upper()
|
||||
import polars as pl
|
||||
|
||||
# code/symbol 前缀优先，再 name 包含匹配
|
||||
prefix_mask = (
|
||||
pl.col("code").str.starts_with(keyword)
|
||||
| pl.col("symbol").str.to_uppercase().str.starts_with(keyword)
|
||||
)
|
||||
contains_mask = (
|
||||
pl.col("code").str.contains(keyword, literal=True)
|
||||
| pl.col("symbol").str.to_uppercase().str.contains(keyword, literal=True)
|
||||
| pl.col("name").str.contains(keyword, literal=True)
|
||||
)
|
||||
|
||||
# 前缀匹配优先，剩余名额用包含匹配补充
|
||||
prefix_hits = df.filter(prefix_mask).head(limit)
|
||||
if prefix_hits.height >= limit:
|
||||
matched = prefix_hits
|
||||
else:
|
||||
remaining = limit - prefix_hits.height
|
||||
# 排除已匹配的 symbol
|
||||
prefix_symbols = set(prefix_hits["symbol"].to_list()) if not prefix_hits.is_empty() else set()
|
||||
contain_hits = df.filter(contains_mask & ~pl.col("symbol").is_in(prefix_symbols)).head(remaining)
|
||||
matched = pl.concat([prefix_hits, contain_hits]) if not prefix_hits.is_empty() else contain_hits
|
||||
rows = matched.select(["symbol", "name", "code"]).to_dicts()
|
||||
return {"results": rows}
|
||||
|
||||
|
||||
@router.post("/instruments/names")
|
||||
def instruments_names(request: Request, symbols: list[str]):
|
||||
"""批量查股票名称。传入 symbol 列表, 返回 {symbol: name}。"""
|
||||
if not symbols:
|
||||
return {"names": {}}
|
||||
repo = request.app.state.repo
|
||||
df = repo.get_instruments()
|
||||
if df.is_empty():
|
||||
return {"names": {}}
|
||||
import polars as pl
|
||||
matched = df.filter(pl.col("symbol").is_in(symbols)).select(["symbol", "name"])
|
||||
names = {row["symbol"]: row["name"] for row in matched.iter_rows(named=True)}
|
||||
return {"names": names}
|
||||
|
||||
|
||||
def _get_stock_info(repo, symbol: str) -> dict:
|
||||
"""从 instruments 视图查标的名称 + 股本。"""
|
||||
try:
|
||||
row = repo.execute_one(
|
||||
"SELECT name, total_shares, float_shares FROM instruments WHERE symbol = ? LIMIT 1",
|
||||
[symbol],
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
return {}
|
||||
if not row:
|
||||
return {}
|
||||
return {
|
||||
"name": row[0],
|
||||
"total_shares": row[1],
|
||||
"float_shares": row[2],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/daily")
|
||||
def get_daily(
|
||||
request: Request,
|
||||
symbol: str = Query(..., description="标的代码,如 000001.SZ"),
|
||||
days: int = Query(120, ge=10, le=2000),
|
||||
start_date: Optional[str] = Query(None, description="起始日期 YYYY-MM-DD, 优先于 days"),
|
||||
end_date: Optional[str] = Query(None, description="截止日期 YYYY-MM-DD, 默认今天"),
|
||||
ext_columns: Optional[str] = Query(None, description="逗号分隔的 ext 列: config_id.field_name"),
|
||||
):
|
||||
"""读取本地 enriched 表中某只股票的日 K。
|
||||
|
||||
- 若 QuoteService 有实时行情, 追加/覆盖今日实时蜡烛
|
||||
- Free 用户: 若 enriched 表里没有该股票, 实时拉取 + 本地算 enriched 返回
|
||||
- ext_columns: 可选，动态 LEFT JOIN 扩展数据表，结果平铺到 stock_info.ext 下
|
||||
(key 为 "{config_id}__{field_name}")，供日K信息条等场景展示自定义字段
|
||||
"""
|
||||
import polars as pl
|
||||
|
||||
repo = request.app.state.repo
|
||||
end = date.fromisoformat(end_date) if end_date else date.today()
|
||||
if start_date:
|
||||
start = date.fromisoformat(start_date)
|
||||
else:
|
||||
start = end - timedelta(days=days)
|
||||
|
||||
stock_info = _get_stock_info(repo, symbol)
|
||||
stock_name = stock_info.get("name")
|
||||
|
||||
# 从 enriched 表读取 (已含前复权 OHLCV + 技术指标 + 信号)
|
||||
df = repo.get_daily(symbol, start, end)
|
||||
|
||||
if df.is_empty():
|
||||
try:
|
||||
raw = kline_sync.sync_daily_batch([symbol], count=days + 30)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=502, detail=f"数据源 fetch failed: {e}") from e
|
||||
if raw.is_empty():
|
||||
return {"symbol": symbol, "name": stock_name, "stock_info": stock_info, "rows": []}
|
||||
enriched = compute_enriched_single(raw)
|
||||
rows = enriched.tail(days).to_dicts()
|
||||
# 即使 live 模式也尝试追加实时蜡烛
|
||||
rows = _maybe_inject_live_candle(request, symbol, rows)
|
||||
resp = {"symbol": symbol, "name": stock_name, "stock_info": stock_info, "rows": rows, "source": "live"}
|
||||
return _attach_ext(resp, repo, symbol, ext_columns)
|
||||
|
||||
rows = df.to_dicts()
|
||||
|
||||
# 追加/覆盖今日实时蜡烛
|
||||
rows = _maybe_inject_live_candle(request, symbol, rows)
|
||||
|
||||
resp = {"symbol": symbol, "name": stock_name, "stock_info": stock_info, "rows": rows, "source": "enriched"}
|
||||
return _attach_ext(resp, repo, symbol, ext_columns)
|
||||
|
||||
|
||||
def _attach_ext(resp: dict, repo, symbol: str, ext_columns: Optional[str]) -> dict:
|
||||
"""按 ext_columns 规格为单只股票 LEFT JOIN 扩展数据，平铺到 stock_info['ext']。
|
||||
|
||||
key 形如 "{config_id}__{field_name}"，与自选列表 enriched 接口保持一致。
|
||||
JOIN 逻辑参考 watchlist.watchlist_enriched；任何 ext 表/字段缺失都静默跳过。
|
||||
"""
|
||||
if not ext_columns or not ext_columns.strip():
|
||||
return resp
|
||||
|
||||
specs: list[tuple[str, str]] = []
|
||||
for part in ext_columns.split(","):
|
||||
part = part.strip()
|
||||
if "." not in part:
|
||||
continue
|
||||
config_id, field_name = part.split(".", 1)
|
||||
config_id, field_name = config_id.strip(), field_name.strip()
|
||||
if config_id and field_name:
|
||||
specs.append((config_id, field_name))
|
||||
if not specs:
|
||||
return resp
|
||||
|
||||
import polars as pl
|
||||
data_dir = repo.store.data_dir
|
||||
try:
|
||||
from app.services.ext_data import ExtConfigStore
|
||||
from app.api.ext_data import _read_ext_dataframe
|
||||
ext_store = ExtConfigStore(data_dir)
|
||||
configs = {c.id: c for c in ext_store.load_all()}
|
||||
except Exception: # noqa: BLE001
|
||||
configs = {}
|
||||
|
||||
ext_values: dict = {}
|
||||
for config_id, field_name in specs:
|
||||
ext_col_name = f"{config_id}__{field_name}"
|
||||
value = None
|
||||
try:
|
||||
cfg = configs.get(config_id)
|
||||
if cfg:
|
||||
ext_df, _ = _read_ext_dataframe(cfg, data_dir)
|
||||
else:
|
||||
ext_df = pl.from_arrow(
|
||||
repo.store.db.query(
|
||||
f'SELECT symbol, "{field_name}" FROM ext_{config_id}'
|
||||
).arrow()
|
||||
)
|
||||
if not ext_df.is_empty() and "symbol" in ext_df.columns and field_name in ext_df.columns:
|
||||
# 时序表取最新分区，避免一个 symbol 多行
|
||||
row = (
|
||||
ext_df
|
||||
.select(["symbol", field_name])
|
||||
.unique(subset=["symbol"], keep="last")
|
||||
.filter(pl.col("symbol") == symbol)
|
||||
)
|
||||
if not row.is_empty():
|
||||
value = row[field_name][0]
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("kline ext join failed for %s.%s: %s", config_id, field_name, e)
|
||||
ext_values[ext_col_name] = value
|
||||
|
||||
stock_info = dict(resp.get("stock_info") or {})
|
||||
stock_info["ext"] = ext_values
|
||||
resp["stock_info"] = stock_info
|
||||
return resp
|
||||
|
||||
|
||||
def _maybe_inject_live_candle(request: Request, symbol: str, rows: list[dict]) -> list[dict]:
|
||||
"""如果 QuoteService 有实时 enriched 数据, 用实时数据生成今日蜡烛并追加/覆盖。"""
|
||||
qs = getattr(request.app.state, "quote_service", None)
|
||||
if not qs:
|
||||
return rows
|
||||
|
||||
df_today, enriched_date = qs.get_enriched_today()
|
||||
if df_today.is_empty():
|
||||
return rows
|
||||
|
||||
# 非交易日（周末/假日）缓存的行情日期 != 今天，跳过注入避免产生重复蜡烛
|
||||
if not enriched_date or enriched_date != date.today():
|
||||
return rows
|
||||
|
||||
# 查找该 symbol 的实时 enriched 行
|
||||
import polars as pl
|
||||
try:
|
||||
q = df_today.filter(pl.col("symbol") == symbol).to_dicts()
|
||||
if not q:
|
||||
return rows
|
||||
q = q[0]
|
||||
except Exception: # noqa: BLE001
|
||||
return rows
|
||||
|
||||
close_price = q.get("close")
|
||||
if not close_price or close_price <= 0:
|
||||
return rows
|
||||
|
||||
today_str = str(enriched_date)
|
||||
|
||||
# enriched 行已包含 OHLCV + 全套指标, 直接用它
|
||||
# 修复: API 在非交易时段可能返回 open/high/low=0, 用 close 填充避免异常蜡烛
|
||||
raw_open = q.get("open")
|
||||
raw_high = q.get("high")
|
||||
raw_low = q.get("low")
|
||||
live_row: dict = {
|
||||
"date": today_str,
|
||||
"symbol": symbol,
|
||||
"open": raw_open if raw_open and raw_open > 0 else close_price,
|
||||
"high": raw_high if raw_high and raw_high > 0 else close_price,
|
||||
"low": raw_low if raw_low and raw_low > 0 else close_price,
|
||||
"close": close_price,
|
||||
"volume": q.get("volume"),
|
||||
"amount": q.get("amount"),
|
||||
"change_pct": q.get("change_pct"),
|
||||
"is_live": True,
|
||||
}
|
||||
# 补上 enriched 的技术指标字段
|
||||
for key in ("ma5", "ma10", "ma20", "ma30", "ma60",
|
||||
"macd_dif", "macd_dea", "macd_hist",
|
||||
"kdj_k", "kdj_d", "kdj_j",
|
||||
"boll_upper", "boll_lower",
|
||||
"rsi_6", "rsi_14", "rsi_24",
|
||||
"atr_14", "vol_ratio_5d"):
|
||||
if key in q and q[key] is not None:
|
||||
live_row[key] = q[key]
|
||||
|
||||
# 如果已有今天的 enriched 行, 覆盖; 否则追加
|
||||
found = False
|
||||
for i, r in enumerate(rows):
|
||||
if str(r.get("date")) == today_str:
|
||||
r.update(live_row)
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
rows.append(live_row)
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
class DailyBatchRequest:
|
||||
"""批量日K请求。"""
|
||||
symbols: list[str]
|
||||
days: int = 12
|
||||
|
||||
|
||||
@router.post("/daily-batch")
|
||||
def get_daily_batch(request: Request, body: dict):
|
||||
"""批量获取多只股票最近 N 天日K (OHLCV)。
|
||||
|
||||
用于自选列表迷你蜡烛图等场景，只返回基础列，不返回全部 enriched 指标。
|
||||
"""
|
||||
symbols = body.get("symbols", [])
|
||||
days = body.get("days", 12)
|
||||
if not symbols:
|
||||
return {"data": {}}
|
||||
days = max(5, min(60, days))
|
||||
|
||||
repo = request.app.state.repo
|
||||
import polars as pl
|
||||
from datetime import date, timedelta
|
||||
|
||||
end = date.today()
|
||||
start = end - timedelta(days=days * 2) # 多取一些确保交易日够
|
||||
|
||||
cols = ["symbol", "date", "open", "high", "low", "close", "volume"]
|
||||
df = repo.get_daily_batch(symbols, start, end, columns=cols)
|
||||
|
||||
if df.is_empty():
|
||||
return {"data": {}}
|
||||
|
||||
# 按 symbol 分组, 每只取最近 N 条
|
||||
result: dict[str, list[dict]] = {}
|
||||
for sym in symbols:
|
||||
sub = df.filter(pl.col("symbol") == sym).sort("date").tail(days)
|
||||
if not sub.is_empty():
|
||||
result[sym] = sub.to_dicts()
|
||||
|
||||
return {"data": result}
|
||||
|
||||
|
||||
@router.get("/minute")
|
||||
def get_minute(
|
||||
request: Request,
|
||||
symbol: str = Query(..., description="标的代码"),
|
||||
trade_date: date | None = Query(None, alias="date", description="交易日期, 默认最新"),
|
||||
):
|
||||
"""读取某只股票某天的分钟 K 线。
|
||||
|
||||
- 本地有完整数据(240条) → 直接返回
|
||||
- 本地无数据或不完整 → 从数据源实时拉取返回（不写入）
|
||||
"""
|
||||
repo = request.app.state.repo
|
||||
stock_info = _get_stock_info(repo, symbol)
|
||||
stock_name = stock_info.get("name")
|
||||
|
||||
if trade_date is None:
|
||||
trade_date = repo.latest_minute_date(symbol)
|
||||
if trade_date is None:
|
||||
# 本地无任何分钟K，尝试从数据源拉取当天
|
||||
trade_date = date.today()
|
||||
df = kline_sync.fetch_minute_single(symbol, trade_date)
|
||||
return {
|
||||
"symbol": symbol, "name": stock_name, "stock_info": stock_info,
|
||||
"date": str(trade_date), "rows": df.to_dicts(), "source": "live",
|
||||
}
|
||||
|
||||
df = repo.get_minute(symbol, trade_date)
|
||||
|
||||
# 完整交易日应有 240 条分钟K；如果是今天(盘中)，期望条数按已交易分钟估算
|
||||
expected = 240
|
||||
today = date.today()
|
||||
if trade_date == today:
|
||||
from datetime import datetime as _dt
|
||||
now = _dt.now()
|
||||
h, m = now.hour, now.minute
|
||||
if h < 9 or (h == 9 and m < 30):
|
||||
expected = 0 # 还没开盘
|
||||
elif h < 12 or (h == 12 and m == 0):
|
||||
expected = (h - 9) * 60 + m - 30 # 9:30 起
|
||||
elif h < 13:
|
||||
expected = 120 # 午休
|
||||
elif h < 15:
|
||||
expected = 120 + (h - 13) * 60 + m
|
||||
else:
|
||||
expected = 240
|
||||
|
||||
is_complete = not df.is_empty() and len(df) >= expected * 0.9 # 允许 10% 容差
|
||||
|
||||
if is_complete:
|
||||
return {
|
||||
"symbol": symbol, "name": stock_name, "stock_info": stock_info,
|
||||
"date": str(trade_date), "rows": df.to_dicts(), "source": "local",
|
||||
}
|
||||
|
||||
# 本地不完整或无数据 → 从数据源实时拉取
|
||||
live_df = kline_sync.fetch_minute_single(symbol, trade_date)
|
||||
return {
|
||||
"symbol": symbol, "name": stock_name, "stock_info": stock_info,
|
||||
"date": str(trade_date), "rows": live_df.to_dicts(),
|
||||
"source": "live" if not live_df.is_empty() else "none",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/sync")
|
||||
def sync_symbol(
|
||||
request: Request,
|
||||
symbol: str = Query(...),
|
||||
days: int = Query(250, ge=10, le=2000),
|
||||
):
|
||||
"""手动触发单股同步(Free 用户在 K 线页用)。"""
|
||||
repo = request.app.state.repo
|
||||
capset = request.app.state.capabilities
|
||||
n = kline_sync.sync_and_persist_daily_batch([symbol], repo, capset, count=days)
|
||||
return {"symbol": symbol, "rows_written": n}
|
||||
|
||||
|
||||
@router.post("/sync_batch")
|
||||
def sync_batch(
|
||||
request: Request,
|
||||
symbols: list[str],
|
||||
days: int = Query(250, ge=10, le=2000),
|
||||
):
|
||||
repo = request.app.state.repo
|
||||
capset = request.app.state.capabilities
|
||||
n = kline_sync.sync_and_persist_daily_batch(symbols, repo, capset, count=days)
|
||||
return {"symbols": symbols, "rows_written": n}
|
||||
|
||||
|
||||
@router.post("/refresh_views")
|
||||
def refresh_views(request: Request):
|
||||
"""刷新所有 DuckDB 视图(解决视图状态不一致问题)。"""
|
||||
from app.jobs.daily_pipeline import _refresh_views
|
||||
repo = request.app.state.repo
|
||||
_refresh_views(repo)
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.post("/sync_minute")
|
||||
async def sync_minute(request: Request):
|
||||
"""手动触发分钟 K 同步(全市场)。返回 pipeline job_id 可轮询进度。"""
|
||||
import asyncio
|
||||
|
||||
from app.services.pipeline_jobs import job_store
|
||||
from app.api.data import invalidate_storage_cache
|
||||
from app.services.preferences import get_minute_sync_days
|
||||
from app.tickflow.capabilities import Cap
|
||||
from app.tickflow.pools import get_pool
|
||||
|
||||
repo = request.app.state.repo
|
||||
capset = request.app.state.capabilities
|
||||
|
||||
if not capset.has(Cap.KLINE_MINUTE_BATCH):
|
||||
raise HTTPException(status_code=403, detail="需要 Pro+ 权限")
|
||||
|
||||
job_id = job_store.create()
|
||||
existing = job_store.get(job_id)
|
||||
if existing and existing["status"] == "running":
|
||||
return {"status": "reused", "job_id": job_id}
|
||||
|
||||
async def task() -> None:
|
||||
job_store.start(job_id)
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def progress(stage: str, pct: int, msg: str) -> None:
|
||||
job_store.progress(job_id, stage, pct, msg)
|
||||
|
||||
try:
|
||||
progress("sync_minute", 5, "解析标的池…")
|
||||
universe = sorted(set(get_pool("watchlist")) | set(get_pool("CN_Equity_A")))
|
||||
# 补充 instruments 全量标的，覆盖北交所、新股等
|
||||
inst_path = repo.store.data_dir / "instruments" / "instruments.parquet"
|
||||
if inst_path.exists():
|
||||
try:
|
||||
import polars as pl
|
||||
inst = pl.read_parquet(inst_path, columns=["symbol"])
|
||||
universe = sorted(set(universe) | set(inst["symbol"].to_list()))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
progress("sync_minute", 10, f"标的池 {len(universe)} 只")
|
||||
|
||||
days = get_minute_sync_days()
|
||||
|
||||
def _run():
|
||||
return kline_sync.sync_and_persist_minute(universe, repo, capset, days=days)
|
||||
|
||||
written = await loop.run_in_executor(_long_task_executor, _run)
|
||||
|
||||
# 刷新视图
|
||||
from app.jobs.daily_pipeline import _refresh_single_view
|
||||
_refresh_single_view(repo, "kline_minute")
|
||||
|
||||
progress("done", 100, f"分钟 K 同步完成,{written} 行")
|
||||
job_store.succeed(job_id, {"minute_rows": written, "universe_size": len(universe)})
|
||||
invalidate_storage_cache()
|
||||
except Exception as e: # noqa: BLE001
|
||||
job_store.fail(job_id, str(e))
|
||||
invalidate_storage_cache()
|
||||
|
||||
asyncio.create_task(task())
|
||||
return {"status": "started", "job_id": job_id}
|
||||
|
||||
|
||||
@router.post("/extend_history")
|
||||
async def extend_history(request: Request):
|
||||
"""向前扩展历史日K数据 — 独立于盘后管道。
|
||||
|
||||
body: { "value": int, "unit": "day"|"month"|"year" }
|
||||
返回 job_id,可轮询 /api/pipeline/jobs 查看进度。
|
||||
"""
|
||||
import asyncio
|
||||
import traceback as _tb
|
||||
try:
|
||||
body = await request.json()
|
||||
value = body.get("value")
|
||||
unit = body.get("unit", "month")
|
||||
if not value or value <= 0:
|
||||
raise HTTPException(status_code=400, detail="value 必须为正整数")
|
||||
if unit not in ("day", "month", "year"):
|
||||
raise HTTPException(status_code=400, detail="unit 只支持 day/month/year")
|
||||
|
||||
repo = request.app.state.repo
|
||||
capset = request.app.state.capabilities
|
||||
|
||||
from app.tickflow.capabilities import Cap
|
||||
if not capset.has(Cap.KLINE_DAILY_BATCH):
|
||||
raise HTTPException(status_code=403, detail="需要 Pro+ 权限 (batch K-line)")
|
||||
|
||||
from app.services.extend_history import run_extend_history
|
||||
from app.services.pipeline_jobs import job_store
|
||||
from app.api.data import invalidate_storage_cache
|
||||
|
||||
job_id = job_store.create()
|
||||
existing = job_store.get(job_id)
|
||||
if existing and existing["status"] == "running":
|
||||
return {"status": "reused", "job_id": job_id}
|
||||
|
||||
async def task() -> None:
|
||||
job_store.start(job_id)
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def progress(stage: str, pct: int, msg: str,
|
||||
stage_pct: int | None = None, skip_log: bool = False) -> None:
|
||||
job_store.progress(job_id, stage, pct, msg,
|
||||
stage_pct=stage_pct, skip_log=skip_log)
|
||||
|
||||
try:
|
||||
result = await loop.run_in_executor(
|
||||
_long_task_executor,
|
||||
lambda: run_extend_history(repo, capset, value, unit, on_progress=progress),
|
||||
)
|
||||
if "error" in result:
|
||||
job_store.fail(job_id, result["error"])
|
||||
else:
|
||||
job_store.succeed(job_id, result)
|
||||
invalidate_storage_cache()
|
||||
except Exception as e:
|
||||
logger.exception("extend_history failed: job_id=%s", job_id)
|
||||
job_store.fail(job_id, str(e))
|
||||
invalidate_storage_cache()
|
||||
|
||||
asyncio.create_task(task())
|
||||
return {"status": "started", "job_id": job_id}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("extend_history error: %s\n%s", e, _tb.format_exc())
|
||||
raise HTTPException(status_code=500, detail=str(e)) from e
|
||||
|
||||
|
||||
@router.post("/rebuild_enriched")
|
||||
async def rebuild_enriched(request: Request):
|
||||
"""全量重算 enriched 表 — 不获取任何数据,仅基于已有 kline_daily + adj_factor 重算复权+指标。
|
||||
|
||||
返回 job_id,可轮询 /api/pipeline/jobs 查看进度。
|
||||
"""
|
||||
import asyncio
|
||||
try:
|
||||
repo = request.app.state.repo
|
||||
|
||||
from app.services.pipeline_jobs import job_store
|
||||
from app.api.data import invalidate_storage_cache
|
||||
|
||||
job_id = job_store.create()
|
||||
existing = job_store.get(job_id)
|
||||
if existing and existing["status"] == "running":
|
||||
return {"status": "reused", "job_id": job_id}
|
||||
|
||||
async def task() -> None:
|
||||
job_store.start(job_id)
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def progress(stage: str, pct: int, msg: str,
|
||||
stage_pct: int | None = None, skip_log: bool = False) -> None:
|
||||
job_store.progress(job_id, stage, pct, msg,
|
||||
stage_pct=stage_pct, skip_log=skip_log)
|
||||
|
||||
try:
|
||||
progress("rebuild_enriched", 10, "全量计算 enriched…")
|
||||
from app.indicators.pipeline import run_pipeline
|
||||
|
||||
def _batch_progress(cur: int, tot: int) -> None:
|
||||
pct = 10 + int(85 * cur / tot)
|
||||
progress("rebuild_enriched", pct,
|
||||
f"计算指标 批次 {cur}/{tot}",
|
||||
stage_pct=int(100 * cur / tot), skip_log=True)
|
||||
|
||||
written = await loop.run_in_executor(
|
||||
_long_task_executor,
|
||||
lambda: run_pipeline(on_batch_done=_batch_progress),
|
||||
)
|
||||
|
||||
enriched_dir = repo.store.data_dir / "kline_daily_enriched"
|
||||
enriched_days = len(list(enriched_dir.glob("date=*"))) if enriched_dir.exists() else 0
|
||||
|
||||
# 刷新视图
|
||||
d = repo.store.data_dir.as_posix()
|
||||
for view_name, glob in [
|
||||
("kline_enriched", f"{d}/kline_daily_enriched/**/*.parquet"),
|
||||
]:
|
||||
try:
|
||||
repo.db.execute(
|
||||
f"CREATE OR REPLACE VIEW {view_name} AS "
|
||||
f"SELECT * FROM read_parquet('{glob}', union_by_name=true)"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
progress("rebuild_enriched", 100, f"完成,覆盖 {enriched_days} 天")
|
||||
job_store.succeed(job_id, {
|
||||
"enriched_days": enriched_days,
|
||||
"enriched_rows": written,
|
||||
})
|
||||
invalidate_storage_cache()
|
||||
except Exception as e:
|
||||
logger.exception("rebuild_enriched failed: job_id=%s", job_id)
|
||||
job_store.fail(job_id, str(e))
|
||||
invalidate_storage_cache()
|
||||
|
||||
asyncio.create_task(task())
|
||||
return {"status": "started", "job_id": job_id}
|
||||
except Exception as e:
|
||||
import traceback as _tb
|
||||
logger.error("rebuild_enriched error: %s\n%s", e, _tb.format_exc())
|
||||
raise HTTPException(status_code=500, detail=str(e)) from e
|
||||
|
||||
|
||||
# 长时间任务专用线程池（隔离于 FastAPI 默认线程池，防止阻塞请求处理）
|
||||
import concurrent.futures as _cf
|
||||
_long_task_executor = _cf.ThreadPoolExecutor(max_workers=2, thread_name_prefix="long-task")
|
||||
|
||||
|
||||
@router.post("/extend_minute_history")
|
||||
async def extend_minute_history(request: Request):
|
||||
"""向前扩展分钟K历史数据 — 仅拉数据,不做任何后续处理。
|
||||
|
||||
body: { "value": int, "unit": "day"|"month" }
|
||||
- day 单位:1~15 天(所有有分钟K权限的套餐可用)
|
||||
- month 单位:1~6 月(每月按 30 天计,即最多 180 天)—— 仅 Expert+ 可用
|
||||
返回 job_id,可轮询 /api/pipeline/jobs 查看进度。
|
||||
"""
|
||||
import asyncio
|
||||
import traceback as _tb
|
||||
try:
|
||||
body = await request.json()
|
||||
value = body.get("value")
|
||||
unit = body.get("unit", "day")
|
||||
if not value or value <= 0:
|
||||
raise HTTPException(status_code=400, detail="value 必须为正整数")
|
||||
if unit not in ("day", "month"):
|
||||
raise HTTPException(status_code=400, detail="unit 只支持 day/month")
|
||||
|
||||
repo = request.app.state.repo
|
||||
capset = request.app.state.capabilities
|
||||
|
||||
from app.tickflow.capabilities import Cap
|
||||
if not capset.has(Cap.KLINE_MINUTE_BATCH):
|
||||
raise HTTPException(status_code=403, detail="需要 Pro+ 权限 (batch minute K-line)")
|
||||
|
||||
# month 单位(按月扩展更长的分钟K历史)仅 Expert+ 开放;Pro 仅可用 day
|
||||
if unit == "month":
|
||||
from app.tickflow.policy import tier_label
|
||||
base_tier = tier_label().split()[0].split("+")[0].strip().lower()
|
||||
if base_tier != "expert":
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="按月扩展分钟K历史需要 Expert 及以上套餐",
|
||||
)
|
||||
|
||||
# 计算天数上限:day 最多 15 天;month 最多 6 月(180 天)
|
||||
from datetime import timedelta
|
||||
if unit == "month":
|
||||
total_days = min(value * 30, 180)
|
||||
else:
|
||||
total_days = min(value, 15)
|
||||
|
||||
if total_days <= 0:
|
||||
raise HTTPException(status_code=400, detail="扩展范围无效")
|
||||
|
||||
from app.services.pipeline_jobs import job_store
|
||||
from app.api.data import invalidate_storage_cache
|
||||
|
||||
job_id = job_store.create()
|
||||
existing = job_store.get(job_id)
|
||||
if existing and existing["status"] == "running":
|
||||
return {"status": "reused", "job_id": job_id}
|
||||
|
||||
async def task() -> None:
|
||||
job_store.start(job_id)
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def progress(stage: str, pct: int, msg: str,
|
||||
stage_pct: int | None = None, skip_log: bool = False) -> None:
|
||||
job_store.progress(job_id, stage, pct, msg,
|
||||
stage_pct=stage_pct, skip_log=skip_log)
|
||||
|
||||
try:
|
||||
# 获取当前最早日期
|
||||
earliest = repo.earliest_minute_date()
|
||||
if not earliest:
|
||||
# 本地无分钟K数据 → 以今天为基准往前获取
|
||||
from datetime import date as _date
|
||||
latest = _date.today()
|
||||
else:
|
||||
latest = earliest
|
||||
|
||||
new_start = latest - timedelta(days=total_days)
|
||||
if new_start >= latest:
|
||||
job_store.fail(job_id, "扩展范围无效")
|
||||
invalidate_storage_cache()
|
||||
return
|
||||
|
||||
start_str = new_start.strftime("%Y-%m-%d")
|
||||
end_str = latest.strftime("%Y-%m-%d")
|
||||
|
||||
progress("extend_minute", 5, "解析标的池…")
|
||||
universe = _resolve_minute_universe(capset, repo)
|
||||
progress("extend_minute", 8, f"标的池: {len(universe)} 只")
|
||||
|
||||
from app.tickflow.capabilities import Cap
|
||||
|
||||
lim = capset.limits(Cap.KLINE_MINUTE_BATCH)
|
||||
batch_size = lim.batch if lim and lim.batch else 100
|
||||
rpm = lim.rpm if lim else 30
|
||||
|
||||
def _run():
|
||||
"""全部在 executor 线程里完成,避免阻塞事件循环。"""
|
||||
from app.services.kline_sync import sync_minute_batch
|
||||
from datetime import datetime as _dt
|
||||
|
||||
def _chunk(cur: int, tot: int) -> None:
|
||||
progress("extend_minute", 8 + int(85 * cur / tot),
|
||||
f"分钟K 批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
|
||||
|
||||
df = sync_minute_batch(
|
||||
universe,
|
||||
start_time=_dt.combine(new_start, _dt.min.time()),
|
||||
end_time=_dt.combine(latest, _dt.min.time()),
|
||||
batch_size=batch_size, rpm=rpm,
|
||||
on_chunk_done=_chunk,
|
||||
)
|
||||
|
||||
written = 0
|
||||
day_count = 0
|
||||
if not df.is_empty():
|
||||
import polars as pl
|
||||
df = df.with_columns(pl.col("datetime").dt.date().alias("_trade_date"))
|
||||
for day_df in df.partition_by("_trade_date"):
|
||||
trade_date = day_df["_trade_date"][0]
|
||||
out = repo.store.data_dir / "kline_minute" / f"date={trade_date}" / "part.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
if out.exists():
|
||||
existing_df = pl.read_parquet(out)
|
||||
if "datetime" in existing_df.columns:
|
||||
existing_df = existing_df.filter(pl.col("datetime").is_not_null())
|
||||
day_df = pl.concat([existing_df, day_df.drop("_trade_date")]).unique(
|
||||
subset=["symbol", "datetime"], keep="last",
|
||||
)
|
||||
else:
|
||||
day_df = day_df.drop("_trade_date")
|
||||
day_df = day_df.sort("symbol", "datetime")
|
||||
day_df.write_parquet(out)
|
||||
written += day_df.height
|
||||
day_count += 1
|
||||
|
||||
# 刷新视图
|
||||
d = repo.store.data_dir.as_posix()
|
||||
try:
|
||||
repo.db.execute(
|
||||
f"CREATE OR REPLACE VIEW kline_minute AS "
|
||||
f"SELECT * FROM read_parquet('{d}/kline_minute/**/*.parquet', union_by_name=true)"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return written, day_count
|
||||
|
||||
progress("extend_minute", 10, f"获取分钟K [{start_str} ~ {end_str}]…")
|
||||
written, day_count = await loop.run_in_executor(_long_task_executor, _run)
|
||||
|
||||
progress("extend_minute", 95, f"分钟K 完成,{day_count} 天")
|
||||
job_store.succeed(job_id, {
|
||||
"minute_days": day_count,
|
||||
"universe_size": len(universe),
|
||||
"earliest_before": (earliest or latest).isoformat(),
|
||||
"earliest_after": new_start.isoformat(),
|
||||
})
|
||||
invalidate_storage_cache()
|
||||
except Exception as e:
|
||||
logger.exception("extend_minute_history failed: job_id=%s", job_id)
|
||||
job_store.fail(job_id, str(e))
|
||||
invalidate_storage_cache()
|
||||
|
||||
asyncio.create_task(task())
|
||||
return {"status": "started", "job_id": job_id}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("extend_minute_history error: %s\n%s", e, _tb.format_exc())
|
||||
raise HTTPException(status_code=500, detail=str(e)) from e
|
||||
|
||||
|
||||
def _resolve_minute_universe(capset, repo) -> list[str]:
|
||||
"""分钟K标的池解析。"""
|
||||
from app.tickflow.capabilities import Cap
|
||||
if capset.has(Cap.KLINE_MINUTE_BATCH):
|
||||
try:
|
||||
from app.tickflow.pools import get_pool
|
||||
all_a = get_pool("CN_Equity_A", refresh=True)
|
||||
if all_a:
|
||||
return sorted(all_a)
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
@@ -0,0 +1,235 @@
|
||||
"""监控规则 API 路由 — HTTP 请求 → 调用 monitor_rules 模块 → 同步引擎内存态。
|
||||
|
||||
只做胶水: 校验 → 持久化 → 失效引擎内存态。不含评估逻辑。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.strategy import monitor_rules
|
||||
|
||||
router = APIRouter(prefix="/api/monitor-rules", tags=["monitor-rules"])
|
||||
|
||||
|
||||
def _data_dir(request: Request) -> Path:
|
||||
return request.app.state.repo.store.data_dir
|
||||
|
||||
|
||||
def _sync_engine(request: Request) -> None:
|
||||
"""保存/删除后,把最新规则集 reload 到引擎内存态。"""
|
||||
engine = getattr(request.app.state, "monitor_engine", None)
|
||||
if engine is not None:
|
||||
rules = monitor_rules.load_all(_data_dir(request))
|
||||
engine.set_rules(rules)
|
||||
|
||||
|
||||
# ── Pydantic 模型 ───────────────────────────────────────
|
||||
class ConditionModel(BaseModel):
|
||||
field: str
|
||||
op: str # truth | > >= < <= == !=
|
||||
value: float | None = None # op 非 truth 时必填
|
||||
|
||||
|
||||
class RuleModel(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
enabled: bool = True
|
||||
type: str # strategy | signal | price | market
|
||||
scope: str = "symbols" # symbols | all | sector
|
||||
symbols: list[str] = []
|
||||
sector: str | None = None
|
||||
strategy_id: str | None = None
|
||||
direction: str = "entry" # entry | exit | both
|
||||
conditions: list[ConditionModel] = []
|
||||
logic: str = "and" # and | or
|
||||
cooldown_seconds: int = 3600
|
||||
severity: str = "info" # info | warn | critical
|
||||
webhook_url: str = "" # Webhook 推送地址 (推送到 QMT 等外部软件, 开发中)
|
||||
webhook_enabled: bool = False
|
||||
message: str = ""
|
||||
|
||||
|
||||
# ── 字段选项 ─────────────────────────────────────────────
|
||||
@router.get("/options")
|
||||
def get_options(request: Request):
|
||||
"""返回可选字段、信号列、运算符、枚举,供前端表单使用。"""
|
||||
from app.indicators.pipeline import ENRICHED_COLUMNS
|
||||
from app.strategy.custom_signals import ALLOWED_FIELDS, load_all as load_csg
|
||||
|
||||
# 阈值字段 (带中文标签)
|
||||
threshold_fields = [
|
||||
{"key": f, "label": ENRICHED_COLUMNS.get(f, f)}
|
||||
for f in sorted(ALLOWED_FIELDS)
|
||||
]
|
||||
# 内置信号列 (布尔, 用于 op=truth)
|
||||
builtin_signals = [
|
||||
{"key": k, "label": v}
|
||||
for k, v in ENRICHED_COLUMNS.items()
|
||||
if k.startswith("signal_")
|
||||
]
|
||||
# 自定义信号列 (csg_)
|
||||
custom_sigs = []
|
||||
try:
|
||||
for cs in load_csg(_data_dir(request)):
|
||||
if cs.get("enabled") is not False:
|
||||
custom_sigs.append({
|
||||
"key": f"csg_{cs['id']}",
|
||||
"label": cs.get("name", cs["id"]),
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"threshold_fields": threshold_fields,
|
||||
"builtin_signals": builtin_signals,
|
||||
"custom_signals": custom_sigs,
|
||||
"operators": [">", ">=", "<", "<=", "==", "!="],
|
||||
"types": [
|
||||
{"key": "signal", "label": "个股信号"},
|
||||
{"key": "price", "label": "价格/涨跌"},
|
||||
{"key": "market", "label": "市场异动"},
|
||||
{"key": "strategy", "label": "策略监控"},
|
||||
],
|
||||
"scopes": [
|
||||
{"key": "symbols", "label": "指定股票"},
|
||||
{"key": "all", "label": "全市场"},
|
||||
{"key": "sector", "label": "板块"},
|
||||
],
|
||||
"logics": [
|
||||
{"key": "and", "label": "全部满足 (AND)"},
|
||||
{"key": "or", "label": "任一满足 (OR)"},
|
||||
],
|
||||
"severities": [
|
||||
{"key": "info", "label": "普通"},
|
||||
{"key": "warn", "label": "警告"},
|
||||
{"key": "critical", "label": "重要"},
|
||||
],
|
||||
"directions": [
|
||||
{"key": "entry", "label": "买入"},
|
||||
{"key": "exit", "label": "卖出"},
|
||||
{"key": "both", "label": "买卖都报"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# ── 列表 ───────────────────────────────────────────────
|
||||
@router.get("")
|
||||
def list_rules(request: Request):
|
||||
rules = monitor_rules.load_all(_data_dir(request))
|
||||
# 按 created_at 倒序
|
||||
rules.sort(key=lambda r: r.get("created_at", ""), reverse=True)
|
||||
return {"rules": rules}
|
||||
|
||||
|
||||
# ── 新建 / 更新 ────────────────────────────────────────
|
||||
@router.post("")
|
||||
def save_rule(req: RuleModel, request: Request):
|
||||
rule = monitor_rules.normalize(req.model_dump())
|
||||
# 编辑现有规则时, 保留原 created_at (避免按时间排序时位置跳动)
|
||||
existing = monitor_rules.load_one(_data_dir(request), rule["id"])
|
||||
if existing and existing.get("created_at"):
|
||||
rule["created_at"] = existing["created_at"]
|
||||
try:
|
||||
monitor_rules.validate(rule)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
monitor_rules.save_one(_data_dir(request), rule)
|
||||
_sync_engine(request)
|
||||
return {"ok": True, "rule": rule}
|
||||
|
||||
|
||||
# ── 删除 ───────────────────────────────────────────────
|
||||
@router.delete("/{rule_id}")
|
||||
def delete_rule(rule_id: str, request: Request):
|
||||
if not monitor_rules.ID_RE.match(rule_id):
|
||||
raise HTTPException(status_code=400, detail="规则 id 非法")
|
||||
deleted = monitor_rules.delete_one(_data_dir(request), rule_id)
|
||||
if not deleted:
|
||||
raise HTTPException(status_code=404, detail="规则不存在")
|
||||
_sync_engine(request)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# ── 演示数据生成 (仅 Dev 页用) ─────────────────────────
|
||||
|
||||
import time as _time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def _demo_rule(rule_id: str, name: str, rtype: str, scope: str, symbols: list[str],
|
||||
conditions: list[dict], logic: str = "or", cooldown: int = 3600,
|
||||
severity: str = "info", message: str = "",
|
||||
strategy_id: str | None = None, direction: str = "entry") -> dict:
|
||||
rule = monitor_rules.normalize({
|
||||
"id": rule_id,
|
||||
"name": name,
|
||||
"type": rtype,
|
||||
"scope": scope,
|
||||
"symbols": symbols,
|
||||
"conditions": conditions,
|
||||
"logic": logic,
|
||||
"cooldown_seconds": cooldown,
|
||||
"severity": severity,
|
||||
"message": message,
|
||||
"enabled": True,
|
||||
})
|
||||
if rtype == "strategy":
|
||||
rule["strategy_id"] = strategy_id
|
||||
rule["direction"] = direction
|
||||
return rule
|
||||
|
||||
|
||||
_DEMO_RULES_TEMPLATE = [
|
||||
("个股信号 · 茅台放量突破", "signal", "symbols", ["600519.SH"],
|
||||
[{"field": "signal_volume_surge", "op": "truth"},
|
||||
{"field": "signal_n_day_high", "op": "truth"}], "or", "info"),
|
||||
("个股信号 · 宁德金叉", "signal", "symbols", ["300750.SZ"],
|
||||
[{"field": "signal_ma_golden_5_20", "op": "truth"}], "or", "info"),
|
||||
("价格 · 平安跌幅监控", "price", "symbols", ["000001.SZ"],
|
||||
[{"field": "change_pct", "op": "<", "value": -0.03}], "or", "warn", "warn"),
|
||||
("价格 · 比亚迪RSI超卖", "price", "symbols", ["002594.SZ"],
|
||||
[{"field": "rsi_14", "op": "<", "value": 30}], "and", "warn", "warn"),
|
||||
("市场异动 · 全市场涨停", "market", "all", [],
|
||||
[{"field": "signal_limit_up", "op": "truth"}], "or", "critical", "critical"),
|
||||
("市场异动 · 全市场炸板", "market", "all", [],
|
||||
[{"field": "signal_broken_limit_up", "op": "truth"}], "or", "warn", "warn"),
|
||||
("市场异动 · 跌幅超5%", "market", "all", [],
|
||||
[{"field": "change_pct", "op": "<", "value": -0.05}], "or", "warn", "warn"),
|
||||
("个股信号 · 茅台跌破MA20", "signal", "symbols", ["600519.SH"],
|
||||
[{"field": "signal_ma20_breakdown", "op": "truth"}], "or", "info"),
|
||||
]
|
||||
|
||||
# 策略类型单独声明 (格式不同: 含 strategy_id + direction)
|
||||
_DEMO_STRATEGY_RULES: list[dict] = [
|
||||
{"name": "策略监控 · 趋势突破", "strategy_id": "trend_breakout", "direction": "entry"},
|
||||
{"name": "策略监控 · MACD金叉", "strategy_id": "macd_golden", "direction": "both"},
|
||||
]
|
||||
|
||||
|
||||
@router.post("/seed")
|
||||
def seed_demo_rules(request: Request):
|
||||
"""生成演示监控规则 (Dev 页用)。覆盖 signal/price/market/strategy 四类。"""
|
||||
ts = int(_time.time() * 1000)
|
||||
created = []
|
||||
i = 0
|
||||
for (name, rtype, scope, symbols, conditions, logic, severity, sev) in _DEMO_RULES_TEMPLATE:
|
||||
rule_id = f"demo_{ts}_{i}"
|
||||
rule = _demo_rule(rule_id, name, rtype, scope, symbols, conditions, logic, 3600, sev)
|
||||
monitor_rules.save_one(_data_dir(request), rule)
|
||||
created.append(rule_id)
|
||||
i += 1
|
||||
# 策略类型规则
|
||||
for sr in _DEMO_STRATEGY_RULES:
|
||||
rule_id = f"demo_{ts}_{i}"
|
||||
rule = _demo_rule(
|
||||
rule_id, sr["name"], "strategy", "all", [], [], "and", 3600, "info",
|
||||
strategy_id=sr["strategy_id"], direction=sr.get("direction", "entry"),
|
||||
)
|
||||
monitor_rules.save_one(_data_dir(request), rule)
|
||||
created.append(rule_id)
|
||||
i += 1
|
||||
_sync_engine(request)
|
||||
return {"ok": True, "generated": len(created), "ids": created}
|
||||
@@ -0,0 +1,570 @@
|
||||
"""市场总览聚合 API。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
import time
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
import polars as pl
|
||||
from fastapi import APIRouter, Request
|
||||
|
||||
from app.services.ext_data import ExtConfig, ExtConfigStore
|
||||
from app.services.screener import ScreenerService
|
||||
|
||||
router = APIRouter(prefix="/api/overview", tags=["overview"])
|
||||
|
||||
_CACHE_TTL = 5.0
|
||||
_cache: dict[str, Any] | None = None
|
||||
_cache_key: str | None = None
|
||||
_cache_ts: float = 0.0
|
||||
|
||||
|
||||
def invalidate_overview_cache() -> None:
|
||||
"""清空总览聚合结果缓存。
|
||||
|
||||
清除数据后调用, 避免看板在 TTL 窗口内继续返回旧的聚合结果。
|
||||
"""
|
||||
global _cache, _cache_key, _cache_ts
|
||||
_cache = None
|
||||
_cache_key = None
|
||||
_cache_ts = 0.0
|
||||
|
||||
|
||||
CORE_INDEX_NAMES = {
|
||||
"000001.SH": "上证指数",
|
||||
"399001.SZ": "深证成指",
|
||||
"399006.SZ": "创业板指",
|
||||
"000680.SH": "科创综指",
|
||||
}
|
||||
CORE_INDEX_SYMBOLS = tuple(CORE_INDEX_NAMES.keys())
|
||||
|
||||
_DIMENSION_SEP = re.compile(r"[、,，;；|/\s]+")
|
||||
|
||||
|
||||
def _dimension_field(config: ExtConfig, kind: str) -> str | None:
|
||||
candidates = ["概念", "concept", "theme"] if kind == "concept" else ["行业", "industry", "sector"]
|
||||
for candidate in candidates:
|
||||
needle = candidate.lower()
|
||||
for field in config.fields:
|
||||
haystack = f"{field.name} {field.label}".lower()
|
||||
if needle in haystack:
|
||||
return field.name
|
||||
return None
|
||||
|
||||
|
||||
def _ext_files(data_dir, config: ExtConfig) -> list[str]:
|
||||
base = data_dir / "ext_data" / config.id
|
||||
if config.mode == "timeseries":
|
||||
root = base / "timeseries"
|
||||
return [str(p) for p in sorted(root.rglob("*.parquet")) if p.is_file()]
|
||||
return [str(p) for p in sorted(base.glob("*.parquet")) if p.is_file()]
|
||||
|
||||
|
||||
def _read_ext_rows(data_dir, config: ExtConfig, dimension_field: str) -> list[dict]:
|
||||
files = _ext_files(data_dir, config)
|
||||
if not files:
|
||||
return []
|
||||
try:
|
||||
df = pl.read_parquet(files, hive_partitioning=True)
|
||||
except TypeError:
|
||||
try:
|
||||
df = pl.read_parquet(files)
|
||||
except Exception: # noqa: BLE001
|
||||
return []
|
||||
except Exception: # noqa: BLE001
|
||||
return []
|
||||
if df.is_empty() or dimension_field not in df.columns:
|
||||
return []
|
||||
|
||||
if config.mode == "timeseries" and "date" in df.columns:
|
||||
latest = df.get_column("date").max()
|
||||
if latest is not None:
|
||||
df = df.filter(pl.col("date") == latest)
|
||||
|
||||
symbol_cols = ["symbol", "code", "股票代码", "代码"]
|
||||
for mapping in (config.symbol_map, config.code_map):
|
||||
if isinstance(mapping, dict) and mapping.get("type") == "mapped" and mapping.get("col"):
|
||||
symbol_cols.append(str(mapping["col"]))
|
||||
cols = []
|
||||
for col in [dimension_field, *symbol_cols]:
|
||||
if col in df.columns and col not in cols:
|
||||
cols.append(col)
|
||||
return df.select(cols).to_dicts()
|
||||
|
||||
|
||||
def _dimension_values(raw: Any) -> list[str]:
|
||||
if raw is None:
|
||||
return []
|
||||
values = [v.strip() for v in _DIMENSION_SEP.split(str(raw).strip()) if v.strip()]
|
||||
return values
|
||||
|
||||
|
||||
def _symbol_keys(row: dict, config: ExtConfig) -> list[str]:
|
||||
fields = ["symbol", "code", "股票代码", "代码"]
|
||||
for mapping in (config.symbol_map, config.code_map):
|
||||
if isinstance(mapping, dict) and mapping.get("type") == "mapped" and mapping.get("col"):
|
||||
fields.append(str(mapping["col"]))
|
||||
|
||||
keys: list[str] = []
|
||||
for field in fields:
|
||||
raw = row.get(field)
|
||||
if raw is None:
|
||||
continue
|
||||
text = str(raw).strip().upper()
|
||||
if not text:
|
||||
continue
|
||||
keys.append(text)
|
||||
if "." in text:
|
||||
keys.append(text.split(".", 1)[0])
|
||||
return keys
|
||||
|
||||
|
||||
def _dimension_rank(rows: list[dict], request: Request, kind: str, limit: int = 5, level: int | None = None) -> dict:
|
||||
if not rows:
|
||||
return {"leading": [], "lagging": []}
|
||||
|
||||
quote_map: dict[str, dict] = {}
|
||||
for row in rows:
|
||||
symbol = str(row.get("symbol") or "").strip().upper()
|
||||
if not symbol:
|
||||
continue
|
||||
quote_map[symbol] = row
|
||||
quote_map[symbol.split(".", 1)[0]] = row
|
||||
|
||||
store = ExtConfigStore(request.app.state.repo.store.data_dir)
|
||||
groups: dict[str, dict[str, dict]] = {}
|
||||
for config in store.load_all():
|
||||
field = _dimension_field(config, kind)
|
||||
if not field:
|
||||
continue
|
||||
for ext_row in _read_ext_rows(request.app.state.repo.store.data_dir, config, field):
|
||||
quote = None
|
||||
for key in _symbol_keys(ext_row, config):
|
||||
quote = quote_map.get(key)
|
||||
if quote:
|
||||
break
|
||||
if not quote:
|
||||
continue
|
||||
symbol = str(quote.get("symbol") or "")
|
||||
for value in _dimension_values(ext_row.get(field)):
|
||||
# 行业按 "-" 拆分级: "银行-银行-股份制银行" → level=2 取"银行"(二级)
|
||||
if level is not None and "-" in value:
|
||||
parts = value.split("-")
|
||||
value = parts[level - 1] if level <= len(parts) else parts[-1]
|
||||
groups.setdefault(value, {})[symbol] = quote
|
||||
|
||||
items = []
|
||||
for name, by_symbol in groups.items():
|
||||
stocks = list(by_symbol.values())
|
||||
changes = [_finite(s.get("change_pct")) for s in stocks]
|
||||
changes = [v for v in changes if v is not None]
|
||||
if not changes:
|
||||
continue
|
||||
leader = max(stocks, key=lambda s: _finite(s.get("change_pct")) or -999)
|
||||
items.append({
|
||||
"name": name,
|
||||
"count": len(stocks),
|
||||
"avg_pct": sum(changes) / len(changes),
|
||||
"up_count": sum(1 for v in changes if v > 0),
|
||||
"down_count": sum(1 for v in changes if v < 0),
|
||||
"amount": sum(_finite(s.get("amount")) or 0 for s in stocks),
|
||||
"leader": {
|
||||
"symbol": leader.get("symbol"),
|
||||
"name": leader.get("name"),
|
||||
"change_pct": _finite(leader.get("change_pct")),
|
||||
},
|
||||
})
|
||||
|
||||
leading = sorted(items, key=lambda x: x["avg_pct"], reverse=True)[:limit]
|
||||
lagging = sorted(items, key=lambda x: x["avg_pct"])[:limit]
|
||||
return {"leading": leading, "lagging": lagging}
|
||||
|
||||
|
||||
def _finite(v: Any) -> float | None:
|
||||
if v is None:
|
||||
return None
|
||||
try:
|
||||
f = float(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return f if math.isfinite(f) else None
|
||||
|
||||
|
||||
def _json_safe(value: Any) -> Any:
|
||||
if isinstance(value, dict):
|
||||
return {k: _json_safe(v) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_json_safe(v) for v in value]
|
||||
if isinstance(value, float) and not math.isfinite(value):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _board(symbol: str) -> str:
|
||||
if symbol.endswith(".BJ"):
|
||||
return "北交所"
|
||||
if symbol.startswith(("300", "301")):
|
||||
return "创业板"
|
||||
if symbol.startswith(("688", "689")):
|
||||
return "科创板"
|
||||
if symbol.endswith(".SH"):
|
||||
return "沪主板"
|
||||
if symbol.endswith(".SZ"):
|
||||
return "深主板"
|
||||
return "其他"
|
||||
|
||||
|
||||
def _score(value: float, low: float, high: float) -> int:
|
||||
if high <= low:
|
||||
return 50
|
||||
return max(0, min(100, round((value - low) / (high - low) * 100)))
|
||||
|
||||
|
||||
def _quote_status(request: Request) -> dict:
|
||||
qs = getattr(request.app.state, "quote_service", None)
|
||||
if not qs:
|
||||
return {"enabled": False, "running": False, "quote_age_ms": None, "is_trading_hours": False}
|
||||
return qs.status()
|
||||
|
||||
|
||||
def _index_quotes(request: Request, as_of: date | None = None) -> list[dict]:
|
||||
qs = getattr(request.app.state, "quote_service", None)
|
||||
rows: list[dict] = []
|
||||
if qs and as_of is None:
|
||||
df = qs.get_index_quotes(list(CORE_INDEX_SYMBOLS))
|
||||
if not df.is_empty():
|
||||
rows = df.to_dicts()
|
||||
|
||||
if not rows:
|
||||
repo = getattr(request.app.state, "repo", None)
|
||||
if repo:
|
||||
placeholders = ", ".join("?" for _ in CORE_INDEX_SYMBOLS)
|
||||
try:
|
||||
db_rows = repo.execute_all(
|
||||
f"""
|
||||
WITH ranked AS (
|
||||
SELECT symbol, date, close,
|
||||
row_number() OVER (PARTITION BY symbol ORDER BY date DESC) AS rn
|
||||
FROM kline_index_daily
|
||||
WHERE symbol IN ({placeholders})
|
||||
AND (? IS NULL OR date <= ?)
|
||||
), latest AS (
|
||||
SELECT symbol,
|
||||
max(CASE WHEN rn = 1 THEN date END) AS date,
|
||||
max(CASE WHEN rn = 1 THEN close END) AS last_price,
|
||||
max(CASE WHEN rn = 2 THEN close END) AS prev_close
|
||||
FROM ranked
|
||||
WHERE rn <= 2
|
||||
GROUP BY symbol
|
||||
)
|
||||
SELECT symbol, date, last_price, prev_close
|
||||
FROM latest
|
||||
""",
|
||||
[*CORE_INDEX_SYMBOLS, as_of, as_of],
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
db_rows = []
|
||||
for symbol, dt, last_price, prev_close in db_rows:
|
||||
change_amount = None
|
||||
change_pct = None
|
||||
lp = _finite(last_price)
|
||||
pc = _finite(prev_close)
|
||||
if lp is not None and pc not in (None, 0):
|
||||
change_amount = lp - pc
|
||||
change_pct = change_amount / pc * 100
|
||||
rows.append({
|
||||
"symbol": symbol,
|
||||
"name": CORE_INDEX_NAMES.get(symbol),
|
||||
"date": str(dt) if dt else None,
|
||||
"last_price": lp,
|
||||
"close": lp,
|
||||
"prev_close": pc,
|
||||
"change_amount": change_amount,
|
||||
"change_pct": change_pct,
|
||||
})
|
||||
|
||||
by_symbol = {r.get("symbol"): r for r in rows}
|
||||
out = []
|
||||
for symbol in CORE_INDEX_SYMBOLS:
|
||||
r = by_symbol.get(symbol, {"symbol": symbol})
|
||||
out.append({
|
||||
"symbol": symbol,
|
||||
"name": r.get("name") or CORE_INDEX_NAMES[symbol],
|
||||
"last_price": _finite(r.get("last_price") if r.get("last_price") is not None else r.get("close")),
|
||||
"change_pct": _finite(r.get("change_pct")),
|
||||
"change_amount": _finite(r.get("change_amount")),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _top_rows(rows: list[dict], key: str, descending: bool, limit: int = 8) -> list[dict]:
|
||||
filtered = [r for r in rows if _finite(r.get(key)) is not None]
|
||||
filtered.sort(key=lambda r: _finite(r.get(key)) or 0, reverse=descending)
|
||||
return [
|
||||
{
|
||||
"symbol": r.get("symbol"),
|
||||
"name": r.get("name"),
|
||||
"close": _finite(r.get("close")),
|
||||
"change_pct": _finite(r.get("change_pct")),
|
||||
"amount": _finite(r.get("amount")),
|
||||
"turnover_rate": _finite(r.get("turnover_rate")),
|
||||
"board": _board(str(r.get("symbol") or "")),
|
||||
}
|
||||
for r in filtered[:limit]
|
||||
]
|
||||
|
||||
|
||||
def _pct_band_rows(values: list[float]) -> list[dict]:
|
||||
bands = [
|
||||
("<-5%", None, -0.05),
|
||||
("-5~-3%", -0.05, -0.03),
|
||||
("-3~-1%", -0.03, -0.01),
|
||||
("-1~0%", -0.01, 0),
|
||||
("0~1%", 0, 0.01),
|
||||
("1~3%", 0.01, 0.03),
|
||||
("3~5%", 0.03, 0.05),
|
||||
(">5%", 0.05, None),
|
||||
]
|
||||
total = len(values) or 1
|
||||
out = []
|
||||
for label, low, high in bands:
|
||||
count = 0
|
||||
for v in values:
|
||||
if low is None and v < high:
|
||||
count += 1
|
||||
elif high is None and v >= low:
|
||||
count += 1
|
||||
elif low is not None and high is not None and low <= v < high:
|
||||
count += 1
|
||||
out.append({"label": label, "count": count, "pct": count / total * 100})
|
||||
return out
|
||||
|
||||
|
||||
def _build_overview(request: Request, as_of: date | None = None) -> dict:
|
||||
repo = request.app.state.repo
|
||||
svc = ScreenerService(repo)
|
||||
as_of = as_of or svc.latest_date()
|
||||
status = _quote_status(request)
|
||||
indices = _index_quotes(request, as_of)
|
||||
|
||||
if not as_of:
|
||||
return {
|
||||
"as_of": None,
|
||||
"quote_status": status,
|
||||
"indices": indices,
|
||||
"breadth": {"total": 0, "up": 0, "down": 0, "flat": 0, "up_pct": 0, "down_pct": 0},
|
||||
"amount": {"total": 0, "avg": 0},
|
||||
"boards": [],
|
||||
"limit": {"limit_up": 0, "broken": 0, "failed": 0, "limit_down": 0, "max_boards": 0, "tiers": []},
|
||||
"distribution": [],
|
||||
"trend": {"above_ma5": 0, "above_ma20": 0, "above_ma60": 0, "above_ma5_pct": 0, "above_ma20_pct": 0, "above_ma60_pct": 0, "new_high": 0, "new_low": 0},
|
||||
"activity": {"avg_turnover": 0, "high_turnover": 0, "high_vol_ratio": 0, "vol_ratio": 1},
|
||||
"radar": [],
|
||||
"emotion": {"score": 50, "label": "暂无"},
|
||||
"top_gainers": [],
|
||||
"top_losers": [],
|
||||
"turnover_leaders": [],
|
||||
"active_leaders": [],
|
||||
"concept_rank": {"leading": [], "lagging": []},
|
||||
"industry_rank": {"leading": [], "lagging": []},
|
||||
}
|
||||
|
||||
df = svc._load_enriched_for_date(as_of)
|
||||
if df.is_empty():
|
||||
rows: list[dict] = []
|
||||
else:
|
||||
cols = [
|
||||
"symbol", "name", "close", "change_pct", "amount", "turnover_rate", "volume",
|
||||
"vol_ratio_5d", "consecutive_limit_ups", "signal_limit_up", "signal_broken_limit_up", "signal_limit_down",
|
||||
"ma5", "ma20", "ma60", "high_60d", "low_60d", "signal_n_day_high", "signal_n_day_low",
|
||||
]
|
||||
df = df.select([c for c in cols if c in df.columns])
|
||||
rows = df.to_dicts()
|
||||
|
||||
# 过滤真停牌（volume=0 且 change_pct=0），保留有涨跌幅的浮点误差股以对齐同花顺口径
|
||||
if rows and "volume" in rows[0]:
|
||||
rows = [r for r in rows
|
||||
if (_finite(r.get("volume")) or 0) > 0
|
||||
or (_finite(r.get("change_pct")) or 0) != 0]
|
||||
|
||||
total = len(rows)
|
||||
up = sum(1 for r in rows if (_finite(r.get("change_pct")) or 0) > 0)
|
||||
down = sum(1 for r in rows if (_finite(r.get("change_pct")) or 0) < 0)
|
||||
flat = max(0, total - up - down)
|
||||
up_pct = up / total * 100 if total else 0
|
||||
down_pct = down / total * 100 if total else 0
|
||||
|
||||
amounts = [_finite(r.get("amount")) or 0 for r in rows]
|
||||
total_amount = sum(amounts)
|
||||
avg_amount = total_amount / total if total else 0
|
||||
|
||||
pct_values = [_finite(r.get("change_pct")) for r in rows]
|
||||
pct_values = [v for v in pct_values if v is not None]
|
||||
avg_pct = sum(pct_values) / len(pct_values) if pct_values else 0
|
||||
median_pct = sorted(pct_values)[len(pct_values) // 2] if pct_values else 0
|
||||
strong_up = sum(1 for v in pct_values if v >= 0.03)
|
||||
strong_down = sum(1 for v in pct_values if v <= -0.03)
|
||||
|
||||
limit_up = sum(1 for r in rows if bool(r.get("signal_limit_up")) or (_finite(r.get("consecutive_limit_ups")) or 0) > 0)
|
||||
broken = sum(1 for r in rows if bool(r.get("signal_broken_limit_up")))
|
||||
limit_down = sum(1 for r in rows if bool(r.get("signal_limit_down")))
|
||||
max_boards = max([int(_finite(r.get("consecutive_limit_ups")) or 0) for r in rows], default=0)
|
||||
|
||||
# 五档 sealed 修正: 假涨停/假跌停不计入(需 Pro+ depth5.batch 能力)
|
||||
depth_svc = getattr(request.app.state, "depth_service", None)
|
||||
sealed_ready = False
|
||||
fake_up = 0
|
||||
fake_down = 0
|
||||
if depth_svc:
|
||||
up_map = depth_svc.get_sealed_map(as_of, is_down=False)
|
||||
down_map = depth_svc.get_sealed_map(as_of, is_down=True)
|
||||
sealed_ready = bool(up_map or down_map) and depth_svc.is_sealed_ready(as_of)
|
||||
if up_map:
|
||||
fake_up = sum(1 for v in up_map.values() if v.get("sealed") is False)
|
||||
if down_map:
|
||||
fake_down = sum(1 for v in down_map.values() if v.get("sealed") is False)
|
||||
if sealed_ready:
|
||||
limit_up = max(0, limit_up - fake_up)
|
||||
limit_down = max(0, limit_down - fake_down)
|
||||
|
||||
seal_rate = limit_up / (limit_up + broken) * 100 if (limit_up + broken) > 0 else 0
|
||||
|
||||
def above_ma_count(ma_key: str) -> int:
|
||||
return sum(1 for r in rows if (_finite(r.get("close")) is not None and _finite(r.get(ma_key)) is not None and (_finite(r.get("close")) or 0) >= (_finite(r.get(ma_key)) or 0)))
|
||||
|
||||
above_ma5 = above_ma_count("ma5")
|
||||
above_ma20 = above_ma_count("ma20")
|
||||
above_ma60 = above_ma_count("ma60")
|
||||
new_high = sum(1 for r in rows if bool(r.get("signal_n_day_high")) or (_finite(r.get("close")) is not None and _finite(r.get("high_60d")) is not None and (_finite(r.get("close")) or 0) >= (_finite(r.get("high_60d")) or 0)))
|
||||
new_low = sum(1 for r in rows if bool(r.get("signal_n_day_low")) or (_finite(r.get("close")) is not None and _finite(r.get("low_60d")) is not None and (_finite(r.get("close")) or 0) <= (_finite(r.get("low_60d")) or 0)))
|
||||
|
||||
turnovers = [_finite(r.get("turnover_rate")) for r in rows]
|
||||
turnovers = [v for v in turnovers if v is not None]
|
||||
avg_turnover = sum(turnovers) / len(turnovers) if turnovers else 0
|
||||
high_turnover = sum(1 for v in turnovers if v >= 5)
|
||||
|
||||
boards_map: dict[str, dict] = {}
|
||||
for r in rows:
|
||||
b = _board(str(r.get("symbol") or ""))
|
||||
item = boards_map.setdefault(b, {"board": b, "count": 0, "up": 0, "down": 0, "amount": 0.0})
|
||||
item["count"] += 1
|
||||
change = _finite(r.get("change_pct")) or 0
|
||||
if change > 0:
|
||||
item["up"] += 1
|
||||
elif change < 0:
|
||||
item["down"] += 1
|
||||
item["amount"] += _finite(r.get("amount")) or 0
|
||||
boards = sorted(boards_map.values(), key=lambda x: x["amount"], reverse=True)
|
||||
for b in boards:
|
||||
count = b["count"] or 1
|
||||
b["up_pct"] = b["up"] / count * 100
|
||||
|
||||
tiers_map: dict[int, int] = {}
|
||||
for r in rows:
|
||||
n = int(_finite(r.get("consecutive_limit_ups")) or 0)
|
||||
if n > 0:
|
||||
tiers_map[n] = tiers_map.get(n, 0) + 1
|
||||
tiers = [{"boards": k, "count": v} for k, v in sorted(tiers_map.items(), key=lambda item: -item[0])]
|
||||
|
||||
index_changes = [_finite(r.get("change_pct")) for r in indices]
|
||||
index_changes = [v for v in index_changes if v is not None]
|
||||
avg_index_pct = sum(index_changes) / len(index_changes) if index_changes else 0
|
||||
vol_ratios = [_finite(r.get("vol_ratio_5d")) for r in rows]
|
||||
vol_ratios = [v for v in vol_ratios if v is not None]
|
||||
avg_vol_ratio = sum(vol_ratios) / len(vol_ratios) if vol_ratios else 1
|
||||
high_vol_ratio = sum(1 for v in vol_ratios if v >= 1.5)
|
||||
|
||||
concept_rank = _dimension_rank(rows, request, "concept")
|
||||
industry_rank = _dimension_rank(rows, request, "industry", level=2)
|
||||
|
||||
strong_diff_pct = (strong_up - strong_down) / total * 100 if total else 0
|
||||
high_vol_pct = high_vol_ratio / total * 100 if total else 0
|
||||
strong_down_pct = strong_down / total * 100 if total else 0
|
||||
tier2_count = sum(t["count"] for t in tiers if t["boards"] >= 2)
|
||||
mainline_items = [*concept_rank["leading"][:3], *industry_rank["leading"][:3]]
|
||||
mainline_avg = max([_finite(item.get("avg_pct")) or 0 for item in mainline_items], default=0)
|
||||
mainline_cover_pct = max([(_finite(item.get("count")) or 0) / total * 100 for item in mainline_items], default=0) if total else 0
|
||||
mainline_score = round(_score(mainline_avg, -0.005, 0.03) * 0.65 + _score(mainline_cover_pct, 1, 12) * 0.35) if mainline_items else 50
|
||||
|
||||
radar = [
|
||||
{"key": "index", "label": "指数", "value": _score(avg_index_pct, -2.5, 2.5)},
|
||||
{"key": "profit", "label": "赚钱", "value": round(_score(up_pct, 20, 80) * 0.45 + _score(avg_pct, -0.02, 0.02) * 0.25 + _score(median_pct, -0.02, 0.02) * 0.20 + _score(strong_diff_pct, -8, 8) * 0.10)},
|
||||
{"key": "money", "label": "量能", "value": round(_score(avg_vol_ratio, 0.6, 1.8) * 0.70 + _score(high_vol_pct, 2, 12) * 0.30)},
|
||||
{"key": "speculation", "label": "投机", "value": round(_score(limit_up, 5, 90) * 0.25 + _score(seal_rate, 30, 85) * 0.35 + _score(max_boards, 1, 8) * 0.25 + _score(tier2_count, 0, 30) * 0.15)},
|
||||
{"key": "resilience", "label": "抗跌", "value": 100 - round(_score(down_pct, 20, 80) * 0.55 + _score(strong_down_pct, 1, 12) * 0.45)},
|
||||
{"key": "mainline", "label": "主线", "value": mainline_score},
|
||||
]
|
||||
emotion_score = round(sum(r["value"] for r in radar) / len(radar)) if radar else 50
|
||||
if emotion_score >= 70:
|
||||
emotion_label = "强势"
|
||||
elif emotion_score >= 55:
|
||||
emotion_label = "偏暖"
|
||||
elif emotion_score >= 45:
|
||||
emotion_label = "震荡"
|
||||
elif emotion_score >= 30:
|
||||
emotion_label = "偏冷"
|
||||
else:
|
||||
emotion_label = "冰点"
|
||||
|
||||
return _json_safe({
|
||||
"as_of": str(as_of),
|
||||
"quote_status": status,
|
||||
"indices": indices,
|
||||
"breadth": {
|
||||
"total": total,
|
||||
"up": up,
|
||||
"down": down,
|
||||
"flat": flat,
|
||||
"up_pct": up_pct,
|
||||
"down_pct": down_pct,
|
||||
"avg_pct": avg_pct,
|
||||
"median_pct": median_pct,
|
||||
"strong_up": strong_up,
|
||||
"strong_down": strong_down,
|
||||
},
|
||||
"amount": {"total": total_amount, "avg": avg_amount},
|
||||
"boards": boards,
|
||||
"limit": {"limit_up": limit_up, "broken": broken, "failed": 0, "limit_down": limit_down, "max_boards": max_boards, "seal_rate": seal_rate, "tiers": tiers, "sealed_ready": sealed_ready, "fake_up": fake_up, "fake_down": fake_down},
|
||||
"distribution": _pct_band_rows(pct_values),
|
||||
"trend": {
|
||||
"above_ma5": above_ma5,
|
||||
"above_ma20": above_ma20,
|
||||
"above_ma60": above_ma60,
|
||||
"above_ma5_pct": above_ma5 / total * 100 if total else 0,
|
||||
"above_ma20_pct": above_ma20 / total * 100 if total else 0,
|
||||
"above_ma60_pct": above_ma60 / total * 100 if total else 0,
|
||||
"new_high": new_high,
|
||||
"new_low": new_low,
|
||||
},
|
||||
"activity": {
|
||||
"avg_turnover": avg_turnover,
|
||||
"high_turnover": high_turnover,
|
||||
"high_vol_ratio": high_vol_ratio,
|
||||
"vol_ratio": avg_vol_ratio,
|
||||
},
|
||||
"radar": radar,
|
||||
"emotion": {"score": emotion_score, "label": emotion_label},
|
||||
"top_gainers": _top_rows(rows, "change_pct", True),
|
||||
"top_losers": _top_rows(rows, "change_pct", False),
|
||||
"turnover_leaders": _top_rows(rows, "amount", True),
|
||||
"active_leaders": _top_rows(rows, "turnover_rate", True),
|
||||
"concept_rank": concept_rank,
|
||||
"industry_rank": industry_rank,
|
||||
})
|
||||
|
||||
|
||||
@router.get("/market")
|
||||
def market_overview(request: Request, as_of: date | None = None):
|
||||
"""总览页单次请求聚合数据，避免前端拉全市场明细后再计算。"""
|
||||
global _cache, _cache_key, _cache_ts
|
||||
now = time.time()
|
||||
cache_key = as_of.isoformat() if as_of else "latest"
|
||||
if _cache is not None and _cache_key == cache_key and (now - _cache_ts) < _CACHE_TTL:
|
||||
return _cache
|
||||
data = _build_overview(request, as_of)
|
||||
_cache = data
|
||||
_cache_key = cache_key
|
||||
_cache_ts = now
|
||||
return data
|
||||
@@ -0,0 +1,107 @@
|
||||
"""盘后管道 API — 异步触发 + 进度跟踪。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import concurrent.futures as _cf
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
|
||||
from app.jobs import daily_pipeline
|
||||
from app.services.pipeline_jobs import job_store
|
||||
from app.api.data import invalidate_storage_cache
|
||||
|
||||
# 长时间任务专用线程池（隔离于 FastAPI 默认线程池，防止阻塞请求处理）
|
||||
_long_task_executor = _cf.ThreadPoolExecutor(max_workers=2, thread_name_prefix="long-task")
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/pipeline", tags=["pipeline"])
|
||||
|
||||
|
||||
@router.post("/run")
|
||||
async def run_now(request: Request) -> dict:
|
||||
"""异步触发盘后管道,立即返回 job_id。客户端轮询 /jobs/{id} 拿进度。
|
||||
|
||||
若已有任务在跑,**返回该任务 id 而不是开新任务**(防止并发拉数据撞限流)。
|
||||
但如果该任务已运行超过 10 分钟 (可能因 reload 卡死), 强制标记为失败后重新创建。
|
||||
"""
|
||||
repo = request.app.state.repo
|
||||
capset = request.app.state.capabilities
|
||||
|
||||
# 检测卡死的 running job (如 reload 后孤儿 task)
|
||||
existing_id = job_store.active_id()
|
||||
if existing_id:
|
||||
existing = job_store.get(existing_id)
|
||||
if existing and existing["status"] == "running":
|
||||
from datetime import datetime, timezone
|
||||
started = existing.get("started_at")
|
||||
if started:
|
||||
try:
|
||||
start_dt = datetime.fromisoformat(started.replace("Z", "+00:00"))
|
||||
elapsed = (datetime.now(timezone.utc) - start_dt).total_seconds()
|
||||
if elapsed > 600: # 超过 10 分钟视为卡死
|
||||
logger.warning("强制取消卡死 job %s (已运行 %.0fs)", existing_id, elapsed)
|
||||
job_store.fail(existing_id, "超时自动取消 (疑似 reload 后孤儿 task)")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
job_id = job_store.create()
|
||||
|
||||
# 如果是复用的 active job,直接返回(不重启)
|
||||
existing = job_store.get(job_id)
|
||||
if existing and existing["status"] == "running":
|
||||
return {"job_id": job_id, "reused": True}
|
||||
|
||||
# 在 executor 里跑同步任务(pipeline 内部都是阻塞 IO + CPU)
|
||||
async def task() -> None:
|
||||
job_store.start(job_id)
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def progress(stage: str, pct: int, msg: str, stage_pct: int | None = None,
|
||||
skip_log: bool = False) -> None:
|
||||
job_store.progress(job_id, stage, pct, msg, stage_pct=stage_pct, skip_log=skip_log)
|
||||
|
||||
try:
|
||||
result = await loop.run_in_executor(
|
||||
_long_task_executor,
|
||||
lambda: daily_pipeline.run_now(repo, capset, on_progress=progress),
|
||||
)
|
||||
job_store.succeed(job_id, result)
|
||||
invalidate_storage_cache()
|
||||
repo.refresh_cache() # 刷新 Polars 缓存
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.exception("pipeline failed")
|
||||
job_store.fail(job_id, str(e))
|
||||
invalidate_storage_cache()
|
||||
|
||||
asyncio.create_task(task())
|
||||
return {"job_id": job_id, "reused": False}
|
||||
|
||||
|
||||
@router.get("/jobs/{job_id}")
|
||||
def get_job(job_id: str) -> dict:
|
||||
j = job_store.get(job_id)
|
||||
if not j:
|
||||
raise HTTPException(status_code=404, detail="job not found")
|
||||
return j
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/cancel")
|
||||
def cancel_job(job_id: str) -> dict:
|
||||
"""手动取消一个 running 的 job。"""
|
||||
j = job_store.get(job_id)
|
||||
if not j:
|
||||
raise HTTPException(status_code=404, detail="job not found")
|
||||
if j["status"] not in ("running", "pending"):
|
||||
raise HTTPException(status_code=400, detail=f"job status is {j['status']}, cannot cancel")
|
||||
job_store.fail(job_id, "用户手动取消")
|
||||
return {"cancelled": job_id}
|
||||
|
||||
|
||||
@router.get("/jobs")
|
||||
def list_jobs(limit: int = 20) -> dict:
|
||||
return {
|
||||
"active_id": job_store.active_id(),
|
||||
"jobs": job_store.list_recent(limit=limit),
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
"""API 路由 — Phase 0 仅 /health 与 /api/capabilities。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app import __version__
|
||||
from app.tickflow import client as tf_client
|
||||
from app.tickflow.policy import detect_capabilities, tier_label
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
def health() -> dict:
|
||||
return {
|
||||
"status": "ok",
|
||||
"version": __version__,
|
||||
# 三态: none(无key/无效) / free(免费key) / api_key(付费档)
|
||||
"mode": tf_client.current_mode(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/capabilities")
|
||||
def capabilities() -> dict:
|
||||
"""前端用来决定哪些功能可用、哪些灰显。"""
|
||||
capset = detect_capabilities()
|
||||
return {
|
||||
"label": tier_label(),
|
||||
"capabilities": capset.to_dict(),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/capabilities/redetect")
|
||||
def redetect() -> dict:
|
||||
"""用户在设置页"重新检测"按钮。"""
|
||||
capset = detect_capabilities(force=True)
|
||||
return {
|
||||
"label": tier_label(),
|
||||
"capabilities": capset.to_dict(),
|
||||
}
|
||||
@@ -0,0 +1,706 @@
|
||||
"""Screener API。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import time
|
||||
from dataclasses import asdict
|
||||
from datetime import date, datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.services.screener import PRESET_STRATEGIES, ScreenerService
|
||||
from app.services import strategy_cache
|
||||
from app.strategy import config as strategy_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/screener", tags=["screener"])
|
||||
|
||||
|
||||
class CustomRequest(BaseModel):
|
||||
conditions: list[str]
|
||||
order_by: Optional[str] = None
|
||||
limit: int = 30
|
||||
pool: Optional[list[str]] = None
|
||||
as_of: Optional[date] = None
|
||||
ext_columns: Optional[str] = None
|
||||
|
||||
|
||||
class PresetRequest(BaseModel):
|
||||
strategy_id: str
|
||||
pool: Optional[list[str]] = None
|
||||
as_of: Optional[date] = None
|
||||
ext_columns: Optional[str] = None
|
||||
|
||||
|
||||
def _safe(result_dict: dict) -> dict:
|
||||
"""sanitize for JSON(NaN / Inf → None)."""
|
||||
rows = result_dict.get("rows", [])
|
||||
for r in rows:
|
||||
for k, v in list(r.items()):
|
||||
if isinstance(v, float) and not math.isfinite(v):
|
||||
r[k] = None
|
||||
return result_dict
|
||||
|
||||
|
||||
_EXT_IDENT_RE = re.compile(r"^[A-Za-z0-9_]+$")
|
||||
|
||||
|
||||
def _safe_ext_value(value: Any) -> Any:
|
||||
if isinstance(value, float) and not math.isfinite(value):
|
||||
return None
|
||||
if isinstance(value, (date, datetime)):
|
||||
return value.isoformat()
|
||||
return value
|
||||
|
||||
|
||||
def _quote_ident(name: str) -> str:
|
||||
return '"' + name.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def _load_ext_value_maps(repo, ext_columns: Optional[str]) -> dict[str, dict[str, Any]]:
|
||||
"""按请求加载扩展列，返回 {输出列名: {symbol: value}}。
|
||||
|
||||
策略结果缓存是共享文件，不能被不同 ext_columns 组合污染；因此扩展列只在
|
||||
返回前通过该投影映射追加到结果副本中。
|
||||
"""
|
||||
ext_specs = _parse_ext_columns(ext_columns) if ext_columns else []
|
||||
if not ext_specs:
|
||||
return {}
|
||||
|
||||
import polars as pl
|
||||
from app.api.ext_data import _read_ext_dataframe
|
||||
from app.services.ext_data import ExtConfigStore
|
||||
|
||||
db = repo.store.db
|
||||
data_dir = repo.store.data_dir
|
||||
ext_store = ExtConfigStore(data_dir)
|
||||
configs = {c.id: c for c in ext_store.load_all()}
|
||||
value_maps: dict[str, dict[str, Any]] = {}
|
||||
|
||||
for config_id, field_name in ext_specs:
|
||||
out_col = f"{config_id}__{field_name}"
|
||||
cfg = configs.get(config_id)
|
||||
try:
|
||||
if cfg:
|
||||
# 时序扩展表只取最新分区，避免历史分区把同一 symbol JOIN 放大。
|
||||
ext_df, _ = _read_ext_dataframe(cfg, data_dir)
|
||||
else:
|
||||
view_name = f"ext_{config_id}"
|
||||
ext_df = pl.from_arrow(db.query(
|
||||
f"SELECT symbol, {_quote_ident(field_name)} FROM {view_name}"
|
||||
).arrow())
|
||||
|
||||
if ext_df.is_empty() or "symbol" not in ext_df.columns or field_name not in ext_df.columns:
|
||||
continue
|
||||
|
||||
ext_df = ext_df.select(["symbol", field_name]).unique(subset=["symbol"], keep="last")
|
||||
value_maps[out_col] = {
|
||||
str(row["symbol"]): _safe_ext_value(row.get(field_name))
|
||||
for row in ext_df.to_dicts()
|
||||
if row.get("symbol")
|
||||
}
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("screener ext column join skipped for %s.%s: %s", config_id, field_name, e)
|
||||
|
||||
return value_maps
|
||||
|
||||
|
||||
def _row_with_ext(row: dict, ext_values: dict[str, dict[str, Any]], symbol: Optional[str] = None) -> dict:
|
||||
next_row = dict(row)
|
||||
sym = symbol or next_row.get("symbol")
|
||||
for out_col, value_map in ext_values.items():
|
||||
next_row[out_col] = value_map.get(str(sym)) if sym else None
|
||||
return next_row
|
||||
|
||||
|
||||
def _rows_with_ext(rows: list[dict], ext_values: dict[str, dict[str, Any]]) -> list[dict]:
|
||||
if not ext_values:
|
||||
return rows
|
||||
return [_row_with_ext(r, ext_values) for r in rows]
|
||||
|
||||
|
||||
def _result_with_ext(result_dict: dict, ext_values: dict[str, dict[str, Any]]) -> dict:
|
||||
if not ext_values:
|
||||
return result_dict
|
||||
return {**result_dict, "rows": _rows_with_ext(result_dict.get("rows", []), ext_values)}
|
||||
|
||||
|
||||
def _results_with_ext(results: dict[str, dict], ext_values: dict[str, dict[str, Any]]) -> dict[str, dict]:
|
||||
if not ext_values:
|
||||
return results
|
||||
return {sid: _result_with_ext(r, ext_values) for sid, r in results.items()}
|
||||
|
||||
|
||||
def _cache_payload_with_ext(cached: dict, ext_values: dict[str, dict[str, Any]]) -> dict:
|
||||
if not ext_values:
|
||||
return cached
|
||||
|
||||
payload = dict(cached)
|
||||
payload["results"] = _results_with_ext(cached.get("results", {}), ext_values)
|
||||
|
||||
ever_rows = cached.get("today_ever_rows")
|
||||
if isinstance(ever_rows, dict):
|
||||
enriched_ever: dict[str, dict[str, dict]] = {}
|
||||
for sid, sym_map in ever_rows.items():
|
||||
if not isinstance(sym_map, dict):
|
||||
continue
|
||||
enriched_ever[sid] = {
|
||||
sym: _row_with_ext(row, ext_values, symbol=sym)
|
||||
for sym, row in sym_map.items()
|
||||
if isinstance(row, dict)
|
||||
}
|
||||
payload["today_ever_rows"] = enriched_ever
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def _update_cache_strategy(data_dir, as_of: str, strategy_id: str, safe_data: dict) -> None:
|
||||
"""单跑后更新缓存中该策略的结果，保持缓存与最新计算一致。"""
|
||||
from app.services import strategy_cache
|
||||
cached = strategy_cache.read_cache(data_dir)
|
||||
if cached and cached.get("as_of") == as_of:
|
||||
results = cached.get("results", {})
|
||||
results[strategy_id] = {
|
||||
"total": safe_data.get("total", 0),
|
||||
"as_of": as_of,
|
||||
"rows": safe_data.get("rows", []),
|
||||
}
|
||||
strategy_cache.write_cache(data_dir, as_of, results)
|
||||
|
||||
|
||||
@router.get("/strategies")
|
||||
def strategies(request: Request):
|
||||
"""策略清单（内置 + 自定义 + AI）。"""
|
||||
data_dir = request.app.state.repo.store.data_dir
|
||||
presets = []
|
||||
seen_ids: set[str] = set()
|
||||
|
||||
# 内置策略
|
||||
for k, v in PRESET_STRATEGIES.items():
|
||||
overrides = strategy_config.load_override(data_dir, k)
|
||||
name = (overrides.get("name") or v["name"]) if overrides else v["name"]
|
||||
desc = (overrides.get("description") or v["description"]) if overrides else v["description"]
|
||||
presets.append({"id": k, "name": name, "description": desc, "source": "builtin"})
|
||||
seen_ids.add(k)
|
||||
|
||||
# 自定义/AI 策略（不在 PRESET_STRATEGIES 中的）
|
||||
engine = getattr(request.app.state, "strategy_engine", None)
|
||||
if engine:
|
||||
for meta in engine.list_strategies():
|
||||
sid = meta["id"]
|
||||
if sid not in seen_ids:
|
||||
overrides = strategy_config.load_override(data_dir, sid)
|
||||
name = (overrides.get("name") or meta["name"]) if overrides else meta["name"]
|
||||
desc = (overrides.get("description") or meta.get("description", "")) if overrides else meta.get("description", "")
|
||||
presets.append({"id": sid, "name": name, "description": desc, "source": meta.get("source", "custom")})
|
||||
seen_ids.add(sid)
|
||||
|
||||
return {"presets": presets}
|
||||
|
||||
|
||||
@router.post("/run")
|
||||
def run_custom(req: CustomRequest, request: Request):
|
||||
repo = request.app.state.repo
|
||||
svc = ScreenerService(repo)
|
||||
as_of = req.as_of or svc.latest_date()
|
||||
if not as_of:
|
||||
raise HTTPException(status_code=400,
|
||||
detail="无可用数据日期 — enriched 表为空,请先运行盘后管道")
|
||||
result = svc.run(
|
||||
as_of=as_of,
|
||||
conditions=req.conditions,
|
||||
order_by=req.order_by,
|
||||
limit=req.limit,
|
||||
pool=req.pool,
|
||||
)
|
||||
safe_data = _safe(asdict(result))
|
||||
ext_values = _load_ext_value_maps(repo, req.ext_columns)
|
||||
return _result_with_ext(safe_data, ext_values)
|
||||
|
||||
|
||||
@router.post("/run_preset")
|
||||
def run_preset(req: PresetRequest, request: Request):
|
||||
repo = request.app.state.repo
|
||||
svc = ScreenerService(repo)
|
||||
as_of = req.as_of or svc.latest_date()
|
||||
if not as_of:
|
||||
raise HTTPException(status_code=400, detail="无可用数据日期")
|
||||
|
||||
# 加载用户保存的策略配置
|
||||
data_dir = request.app.state.repo.store.data_dir
|
||||
ext_values = _load_ext_value_maps(repo, req.ext_columns)
|
||||
overrides = strategy_config.load_override(data_dir, req.strategy_id)
|
||||
bf = overrides.get("basic_filter") if overrides else None
|
||||
dl = overrides.get("display_limit") if overrides else None
|
||||
if dl is None and overrides and "display_limit" in overrides:
|
||||
dl = 0
|
||||
|
||||
# 内置策略
|
||||
if req.strategy_id in PRESET_STRATEGIES:
|
||||
try:
|
||||
result = svc.run_preset(req.strategy_id, as_of=as_of, pool=req.pool, basic_filter=bf, display_limit=dl)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e)) from e
|
||||
safe_data = _safe(asdict(result))
|
||||
_update_cache_strategy(data_dir, str(as_of), req.strategy_id, safe_data)
|
||||
return _result_with_ext(safe_data, ext_values)
|
||||
|
||||
# 自定义/AI 策略 — 通过 StrategyEngine 执行
|
||||
engine = getattr(request.app.state, "strategy_engine", None)
|
||||
if not engine:
|
||||
raise HTTPException(status_code=404, detail=f"策略引擎未初始化或策略 {req.strategy_id} 不存在")
|
||||
|
||||
try:
|
||||
result = engine.run(req.strategy_id, as_of, pool=req.pool, overrides=overrides or None)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e)) from e
|
||||
|
||||
data = asdict(result)
|
||||
|
||||
if dl is not None and dl > 0:
|
||||
data["rows"] = data["rows"][:dl]
|
||||
data["total"] = min(data["total"], dl)
|
||||
|
||||
# 单跑后更新缓存中该策略的结果（保持缓存最新）
|
||||
safe_data = _safe(data)
|
||||
_update_cache_strategy(data_dir, str(as_of), req.strategy_id, safe_data)
|
||||
|
||||
return _result_with_ext(safe_data, ext_values)
|
||||
|
||||
|
||||
@router.get("/cached")
|
||||
def get_cached(
|
||||
request: Request,
|
||||
ext_columns: Optional[str] = Query(None, description="逗号分隔: config_id.field_name"),
|
||||
):
|
||||
"""读取策略结果缓存。返回 None 表示无缓存。"""
|
||||
data_dir = request.app.state.repo.store.data_dir
|
||||
cached = strategy_cache.read_cache(data_dir)
|
||||
if cached is None:
|
||||
return {"as_of": None, "results": {}, "updated_at": None}
|
||||
ext_values = _load_ext_value_maps(request.app.state.repo, ext_columns)
|
||||
return _cache_payload_with_ext(cached, ext_values)
|
||||
|
||||
|
||||
@router.get("/market-snapshot")
|
||||
def market_snapshot(request: Request):
|
||||
"""最新全市场轻量行情快照，供板块/概念聚合分析使用。"""
|
||||
import polars as pl
|
||||
|
||||
repo = request.app.state.repo
|
||||
svc = ScreenerService(repo)
|
||||
as_of = svc.latest_date()
|
||||
if not as_of:
|
||||
return {"as_of": None, "rows": []}
|
||||
|
||||
df = svc._load_enriched_for_date(as_of)
|
||||
if df.is_empty():
|
||||
return {"as_of": str(as_of), "rows": []}
|
||||
|
||||
if "close" in df.columns and "total_shares" in df.columns and "market_cap" not in df.columns:
|
||||
df = df.with_columns((pl.col("close") * pl.col("total_shares")).alias("market_cap"))
|
||||
if "close" in df.columns and "float_shares" in df.columns and "float_market_cap" not in df.columns:
|
||||
df = df.with_columns((pl.col("close") * pl.col("float_shares")).alias("float_market_cap"))
|
||||
|
||||
cols = [
|
||||
"symbol", "name", "close", "change_pct", "amount", "volume",
|
||||
"turnover_rate", "vol_ratio_5d", "total_shares", "float_shares",
|
||||
"market_cap", "float_market_cap", "consecutive_limit_ups",
|
||||
]
|
||||
df = df.select([c for c in cols if c in df.columns])
|
||||
rows = df.to_dicts()
|
||||
for r in rows:
|
||||
for k, v in list(r.items()):
|
||||
if isinstance(v, float) and not math.isfinite(v):
|
||||
r[k] = None
|
||||
|
||||
return {"as_of": str(as_of), "rows": rows}
|
||||
|
||||
|
||||
@router.post("/run_all")
|
||||
def run_all(request: Request, body: Optional[dict] = None):
|
||||
"""批量运行指定策略,只返回每个策略的命中数。
|
||||
|
||||
优化: 从 enriched 读取一次目标日期数据, 所有策略共享。
|
||||
body.strategy_ids: 只跑指定的策略 ID 列表, 为空则跑全部。
|
||||
"""
|
||||
from datetime import date as date_type
|
||||
|
||||
t_total = time.perf_counter()
|
||||
|
||||
body = body or {}
|
||||
repo = request.app.state.repo
|
||||
svc = ScreenerService(repo)
|
||||
|
||||
# 解析日期
|
||||
raw_date = body.get("as_of")
|
||||
if raw_date:
|
||||
as_of = date_type.fromisoformat(str(raw_date)) if isinstance(raw_date, str) else raw_date
|
||||
else:
|
||||
as_of = svc.latest_date()
|
||||
if not as_of:
|
||||
return {"as_of": None, "results": {}}
|
||||
|
||||
# 一次读取目标日期的全部数据
|
||||
t0 = time.perf_counter()
|
||||
precomputed = svc._load_enriched_for_date(as_of)
|
||||
logger.info("run_all: _load_enriched_for_date took %.1fms", (time.perf_counter() - t0) * 1000)
|
||||
|
||||
results: dict[str, dict] = {}
|
||||
data_dir = request.app.state.repo.store.data_dir
|
||||
|
||||
# 收集需要运行的策略 ID (如果指定了 strategy_ids 则只跑这些)
|
||||
requested_ids = body.get("strategy_ids")
|
||||
all_ids = list(PRESET_STRATEGIES.keys())
|
||||
engine = getattr(request.app.state, "strategy_engine", None)
|
||||
if engine:
|
||||
for meta in engine.list_strategies():
|
||||
sid = meta["id"]
|
||||
if sid not in PRESET_STRATEGIES:
|
||||
all_ids.append(sid)
|
||||
|
||||
if requested_ids and isinstance(requested_ids, list):
|
||||
id_set = set(requested_ids)
|
||||
all_ids = [sid for sid in all_ids if sid in id_set]
|
||||
|
||||
if not all_ids:
|
||||
return {"as_of": str(as_of), "results": {}}
|
||||
|
||||
# 批量预加载所有 override 配置
|
||||
t0 = time.perf_counter()
|
||||
all_overrides = strategy_config.list_overrides(data_dir)
|
||||
logger.info("run_all: list_overrides took %.1fms (%d overrides)", (time.perf_counter() - t0) * 1000, len(all_overrides))
|
||||
|
||||
# 历史策略: 只在需要时加载 (只加载 all_ids 中包含的 filter_history 策略)
|
||||
t0 = time.perf_counter()
|
||||
shared_history = None
|
||||
id_set = set(all_ids)
|
||||
if engine:
|
||||
history_strats = [
|
||||
(sid, s) for sid, s in engine._strategies.items()
|
||||
if s.filter_history_fn and sid in id_set
|
||||
]
|
||||
if history_strats:
|
||||
max_lb = min(max(s.lookback_days for _, s in history_strats), 30)
|
||||
shared_history = svc._load_enriched_history(as_of, max(1, max_lb))
|
||||
else:
|
||||
history_strats = []
|
||||
logger.info("run_all: _load_enriched_history took %.1fms (history_strats=%d)", (time.perf_counter() - t0) * 1000, len(history_strats))
|
||||
|
||||
for sid in all_ids:
|
||||
try:
|
||||
overrides = all_overrides.get(sid, {})
|
||||
bf = overrides.get("basic_filter") if overrides else None
|
||||
dl = overrides.get("display_limit") if overrides else None
|
||||
if dl is None and overrides and "display_limit" in overrides:
|
||||
dl = 0
|
||||
|
||||
if sid in PRESET_STRATEGIES:
|
||||
r = svc.run_preset(sid, as_of=as_of, precomputed=precomputed, basic_filter=bf, display_limit=dl)
|
||||
else:
|
||||
r = engine.run(
|
||||
sid, as_of, overrides=overrides or None,
|
||||
precomputed=precomputed, precomputed_history=shared_history,
|
||||
)
|
||||
if dl is not None and dl > 0:
|
||||
r.rows = r.rows[:dl]
|
||||
r.total = min(r.total, dl)
|
||||
|
||||
safe_rows = _safe(asdict(r)).get("rows", [])
|
||||
results[sid] = {"total": r.total, "as_of": str(as_of), "rows": safe_rows}
|
||||
except (ValueError, Exception):
|
||||
continue
|
||||
|
||||
elapsed = (time.perf_counter() - t_total) * 1000
|
||||
logger.info("run_all: total took %.1fms (%d strategies)", elapsed, len(all_ids))
|
||||
|
||||
# 写入策略缓存 (供页面秒加载)
|
||||
if results:
|
||||
try:
|
||||
strategy_cache.write_cache(data_dir, str(as_of), results)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
ext_values = _load_ext_value_maps(repo, body.get("ext_columns"))
|
||||
return {"as_of": str(as_of), "results": _results_with_ext(results, ext_values)}
|
||||
|
||||
|
||||
@router.get("/limit-ladder")
|
||||
def limit_ladder(
|
||||
request: Request,
|
||||
as_of: Optional[date] = None,
|
||||
direction: str = Query("up", description="up=涨停梯队 | down=跌停梯队"),
|
||||
ext_columns: Optional[str] = Query(None, description="逗号分隔: config_id.field_name"),
|
||||
):
|
||||
"""连板/连跌梯队 — 按连板数分组, 含三状态。
|
||||
返回: tiers = [{ boards, count, stocks: [{symbol,name,change_pct,status,...}] }]
|
||||
|
||||
direction=up (默认):
|
||||
status: limit_up=涨停 | broken=炸板(摸板未封) | failed=断板(晋级失败)
|
||||
direction=down:
|
||||
status: limit_down=跌停 | recovery=翘板(跌停后回升,含收阳条件) | failed=止跌(昨日跌停今日未跌停也未翘板)
|
||||
|
||||
ext_columns: 动态 JOIN 扩展数据, 如 "concept.concept,industry.industry"
|
||||
"""
|
||||
from datetime import timedelta
|
||||
|
||||
import polars as pl
|
||||
|
||||
is_down = direction == "down"
|
||||
|
||||
# 按 direction 参数化字段映射
|
||||
if is_down:
|
||||
sig_col = "signal_limit_down"
|
||||
consec_col = "consecutive_limit_downs"
|
||||
broken_col = "signal_limit_down_recovery"
|
||||
status_main, status_broken, status_failed = "limit_down", "recovery", "failed"
|
||||
else:
|
||||
sig_col = "signal_limit_up"
|
||||
consec_col = "consecutive_limit_ups"
|
||||
broken_col = "signal_broken_limit_up"
|
||||
status_main, status_broken, status_failed = "limit_up", "broken", "failed"
|
||||
|
||||
repo = request.app.state.repo
|
||||
svc = ScreenerService(repo)
|
||||
as_of = as_of or svc.latest_date()
|
||||
if not as_of:
|
||||
raise HTTPException(status_code=400, detail="无可用数据日期")
|
||||
|
||||
df = svc._load_enriched_for_date(as_of)
|
||||
if df.is_empty():
|
||||
return {"as_of": str(as_of), "tiers": [], "counts": {"up": 0, "down": 0}}
|
||||
|
||||
# 双方向涨跌停计数(不论当前 direction, 前端始终同时显示)
|
||||
count_up_raw = int(df.filter(pl.col("signal_limit_up").fill_null(False)).height) if "signal_limit_up" in df.columns else 0
|
||||
count_down_raw = int(df.filter(pl.col("signal_limit_down").fill_null(False)).height) if "signal_limit_down" in df.columns else 0
|
||||
|
||||
# 双方向 sealed 修正: 减去各自的假涨停(假涨停已归炸板, 不计入涨停数)
|
||||
depth_svc_global = getattr(request.app.state, "depth_service", None)
|
||||
fake_up = 0
|
||||
fake_down = 0
|
||||
sealed_up_ready = False
|
||||
sealed_down_ready = False
|
||||
if depth_svc_global:
|
||||
up_map = depth_svc_global.get_sealed_map(as_of, is_down=False)
|
||||
down_map = depth_svc_global.get_sealed_map(as_of, is_down=True)
|
||||
sealed_up_ready = bool(up_map) and depth_svc_global.is_sealed_ready(as_of)
|
||||
sealed_down_ready = bool(down_map) and depth_svc_global.is_sealed_ready(as_of)
|
||||
if up_map:
|
||||
fake_up = sum(1 for v in up_map.values() if v.get("sealed") is False)
|
||||
if down_map:
|
||||
fake_down = sum(1 for v in down_map.values() if v.get("sealed") is False)
|
||||
count_up = count_up_raw - fake_up if sealed_up_ready else count_up_raw
|
||||
count_down = count_down_raw - fake_down if sealed_down_ready else count_down_raw
|
||||
|
||||
# 双方向 sealed 明细(供前端弹窗同时显示涨跌停)
|
||||
def _count_sealed(m: dict, ready: bool):
|
||||
if not m or not ready:
|
||||
return {"real": 0, "fake": 0, "pending": 0}
|
||||
real = sum(1 for v in m.values() if v.get("sealed") is True)
|
||||
fake = sum(1 for v in m.values() if v.get("sealed") is False)
|
||||
pending = sum(1 for v in m.values() if v.get("sealed") is None)
|
||||
return {"real": real, "fake": fake, "pending": pending}
|
||||
sealed_counts_up = _count_sealed(up_map, sealed_up_ready)
|
||||
sealed_counts_down = _count_sealed(down_map, sealed_down_ready)
|
||||
|
||||
# 加载前一日数据获取 prev consecutive_limit_ups/downs
|
||||
prev_consec: pl.DataFrame = pl.DataFrame()
|
||||
for delta in range(1, 10):
|
||||
candidate = as_of - timedelta(days=delta)
|
||||
df_prev = svc._load_enriched_for_date(candidate)
|
||||
if not df_prev.is_empty() and consec_col in df_prev.columns:
|
||||
prev_consec = df_prev.select(
|
||||
"symbol",
|
||||
pl.col(consec_col).alias("prev_consec"),
|
||||
)
|
||||
break
|
||||
|
||||
if not prev_consec.is_empty():
|
||||
df = df.join(prev_consec, on="symbol", how="left")
|
||||
else:
|
||||
df = df.with_columns(pl.lit(0).cast(pl.UInt32).alias("prev_consec"))
|
||||
|
||||
# 表达式
|
||||
is_limit = pl.col(sig_col).fill_null(False) if sig_col in df.columns else pl.lit(False)
|
||||
is_broken = pl.col(broken_col).fill_null(False) if broken_col in df.columns else pl.lit(False)
|
||||
consec = pl.col(consec_col).fill_null(0) if consec_col in df.columns else pl.lit(0)
|
||||
prev_c = pl.col("prev_consec").fill_null(0)
|
||||
|
||||
# 计算 status + boards (结构涨跌停对称, 仅字段与字面量不同)
|
||||
is_failed = ~is_limit & ~is_broken & (prev_c > 0)
|
||||
df = df.with_columns([
|
||||
pl.when(is_limit).then(pl.lit(status_main))
|
||||
.when(is_broken).then(pl.lit(status_broken))
|
||||
.when(is_failed).then(pl.lit(status_failed))
|
||||
.otherwise(None).alias("status"),
|
||||
pl.when(is_limit).then(consec)
|
||||
.when(is_broken | is_failed).then(prev_c + 1)
|
||||
.otherwise(0).cast(pl.UInt32).alias("boards"),
|
||||
])
|
||||
|
||||
df = df.filter(pl.col("status").is_not_null() & (pl.col("boards") > 0))
|
||||
|
||||
# ── 五档 sealed 叠加(独立旁路, 不改 signal_limit_up) ──
|
||||
# 假涨停(收盘价=涨停价但卖一有量)从 limit 降级为 broken(归炸板视图)
|
||||
# 真涨停保留 + 附封单量; sealed=null(待确认/降级)保持原状
|
||||
depth_svc = getattr(request.app.state, "depth_service", None)
|
||||
sealed_ready = False
|
||||
sealed_age: float | None = None
|
||||
if depth_svc:
|
||||
sealed_map = depth_svc.get_sealed_map(as_of, is_down=is_down)
|
||||
sealed_ready = bool(sealed_map) and depth_svc.is_sealed_ready(as_of)
|
||||
sealed_age = depth_svc.get_sealed_age(as_of) if sealed_ready else None
|
||||
|
||||
if sealed_map:
|
||||
# 构建 sealed 列(symbol → sealed bool, vol)
|
||||
sym_sealed = {s: v.get("sealed") for s, v in sealed_map.items()}
|
||||
sym_vol = {s: v.get("vol") for s, v in sealed_map.items()}
|
||||
|
||||
# JOIN sealed: 对每只 status=main 的票, 看 sealed 值
|
||||
sealed_rows = pl.DataFrame({
|
||||
"symbol": list(sym_sealed.keys()),
|
||||
"_sealed": list(sym_sealed.values()),
|
||||
"_sealed_vol": list(sym_vol.values()),
|
||||
}) if sym_sealed else pl.DataFrame()
|
||||
|
||||
if not sealed_rows.is_empty():
|
||||
df = df.join(sealed_rows, on="symbol", how="left")
|
||||
# 假涨停(main 状态但 sealed=False)→ 降级为 broken
|
||||
df = df.with_columns(
|
||||
pl.when(
|
||||
(pl.col("status") == status_main)
|
||||
& pl.col("_sealed").is_not_null()
|
||||
& (pl.col("_sealed") == False) # noqa: E712
|
||||
).then(pl.lit(status_broken))
|
||||
.otherwise(pl.col("status")).alias("status"),
|
||||
# sealed_status: real/fake/pending/null
|
||||
pl.when(
|
||||
(pl.col("status") == status_main)
|
||||
& (pl.col("_sealed") == True) # noqa: E712
|
||||
).then(pl.lit("real"))
|
||||
.when(
|
||||
(pl.col("_sealed") == False) # noqa: E712
|
||||
).then(pl.lit("fake"))
|
||||
.when(
|
||||
(pl.col("status") == status_main)
|
||||
& pl.col("_sealed").is_null()
|
||||
).then(pl.lit("pending"))
|
||||
.otherwise(None).alias("sealed_status"),
|
||||
pl.col("_sealed_vol").alias("sealed_vol"),
|
||||
).drop(["_sealed", "_sealed_vol"])
|
||||
else:
|
||||
df = df.with_columns(
|
||||
pl.lit(None).alias("sealed_status"),
|
||||
pl.lit(None).alias("sealed_vol"),
|
||||
)
|
||||
else:
|
||||
df = df.with_columns(
|
||||
pl.lit(None).alias("sealed_status"),
|
||||
pl.lit(None).alias("sealed_vol"),
|
||||
)
|
||||
else:
|
||||
df = df.with_columns(
|
||||
pl.lit(None).alias("sealed_status"),
|
||||
pl.lit(None).alias("sealed_vol"),
|
||||
)
|
||||
|
||||
# 动态 JOIN 扩展数据
|
||||
ext_specs = _parse_ext_columns(ext_columns) if ext_columns else []
|
||||
ext_col_names: list[str] = []
|
||||
if ext_specs:
|
||||
db = repo.store.db
|
||||
data_dir = repo.store.data_dir
|
||||
from app.services.ext_data import ExtConfigStore
|
||||
|
||||
ext_store = ExtConfigStore(data_dir)
|
||||
configs = {c.id: c for c in ext_store.load_all()}
|
||||
|
||||
for config_id, field_name in ext_specs:
|
||||
view_name = f"ext_{config_id}"
|
||||
ext_col_name = f"{config_id}__{field_name}"
|
||||
try:
|
||||
ext_df = pl.from_arrow(db.query(
|
||||
f"SELECT symbol, \"{field_name}\" FROM {view_name}"
|
||||
).arrow())
|
||||
if not ext_df.is_empty() and "symbol" in ext_df.columns:
|
||||
ext_df = ext_df.rename({field_name: ext_col_name})
|
||||
df = df.join(ext_df.select(["symbol", ext_col_name]), on="symbol", how="left")
|
||||
ext_col_names.append(ext_col_name)
|
||||
except Exception:
|
||||
cfg = configs.get(config_id)
|
||||
if cfg:
|
||||
try:
|
||||
from app.api.ext_data import _parquet_glob
|
||||
glob = _parquet_glob(cfg, data_dir)
|
||||
ext_df = pl.read_parquet(glob)
|
||||
if not ext_df.is_empty() and "symbol" in ext_df.columns and field_name in ext_df.columns:
|
||||
ext_df = ext_df.select(["symbol", field_name]).rename({field_name: ext_col_name})
|
||||
df = df.join(ext_df, on="symbol", how="left")
|
||||
ext_col_names.append(ext_col_name)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 选择输出列
|
||||
cols = ["symbol", "name", "close", "change_pct", "boards", "status", consec_col, "sealed_status", "sealed_vol"] + ext_col_names
|
||||
df = df.select([c for c in cols if c in df.columns])
|
||||
# 排序: boards 降序, status 按主状态→炸/翘→断/止
|
||||
status_order = pl.when(pl.col("status") == status_main).then(0)
|
||||
status_order = status_order.when(pl.col("status") == status_broken).then(1)
|
||||
status_order = status_order.otherwise(2).alias("_status_order")
|
||||
df = df.with_columns(status_order).sort(["boards", "_status_order"], descending=[True, False]).drop("_status_order")
|
||||
|
||||
rows = df.to_dicts()
|
||||
for r in rows:
|
||||
for k, v in list(r.items()):
|
||||
if isinstance(v, float) and not math.isfinite(v):
|
||||
r[k] = None
|
||||
|
||||
# 按 boards 分组
|
||||
tiers: dict[int, list] = {}
|
||||
for r in rows:
|
||||
n = int(r.get("boards") or 0)
|
||||
tiers.setdefault(n, []).append(r)
|
||||
|
||||
tier_list = [
|
||||
{"boards": n, "count": len(stocks), "stocks": stocks}
|
||||
for n, stocks in sorted(tiers.items(), key=lambda x: -x[0])
|
||||
]
|
||||
|
||||
return {
|
||||
"as_of": str(as_of),
|
||||
"tiers": tier_list,
|
||||
"counts": {"up": count_up, "down": count_down},
|
||||
"counts_raw": {"up": count_up_raw, "down": count_down_raw},
|
||||
"sealed_ready": sealed_ready,
|
||||
"sealed_age": round(sealed_age, 0) if sealed_age is not None else None,
|
||||
"sealed_counts": {
|
||||
"real": sum(1 for t in tier_list for s in t.get("stocks", []) if s.get("sealed_status") == "real"),
|
||||
"fake": sum(1 for t in tier_list for s in t.get("stocks", []) if s.get("sealed_status") == "fake"),
|
||||
"pending": sum(1 for t in tier_list for s in t.get("stocks", []) if s.get("sealed_status") == "pending"),
|
||||
},
|
||||
"sealed_counts_up": sealed_counts_up,
|
||||
"sealed_counts_down": sealed_counts_down,
|
||||
}
|
||||
|
||||
|
||||
def _parse_ext_columns(ext_columns: str) -> list[tuple[str, str]]:
|
||||
"""解析 'config_id1.field1,config_id2.field2' 为 [(config_id, field_name), ...]。"""
|
||||
result = []
|
||||
for part in ext_columns.split(","):
|
||||
part = part.strip()
|
||||
if "." not in part:
|
||||
continue
|
||||
config_id, field_name = part.split(".", 1)
|
||||
config_id = config_id.strip()
|
||||
field_name = field_name.strip()
|
||||
if not config_id or not field_name:
|
||||
continue
|
||||
if not _EXT_IDENT_RE.match(config_id) or "\x00" in field_name:
|
||||
continue
|
||||
result.append((config_id, field_name))
|
||||
return result
|
||||
@@ -0,0 +1,854 @@
|
||||
"""设置 API — Key 配置 / 模式切换。
|
||||
|
||||
提供面向非开发者的 UI 配置入口,避免逼用户改 .env。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app import secrets_store
|
||||
from app.services.financial_sync import financial_scheduler
|
||||
from app.tickflow import client as tf_client
|
||||
from app.tickflow.policy import (
|
||||
detect_capabilities,
|
||||
extras_caps,
|
||||
missing_caps,
|
||||
probe_log,
|
||||
tier_label,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/settings", tags=["settings"])
|
||||
|
||||
# 默认端点 —— endpoints.json 列表第一项,UI"当前使用"始终对齐此项。
|
||||
# 注意:Free 模式 SDK 实际走 free-api(免费数据通道),但 UI 显示统一用默认节点。
|
||||
DEFAULT_PAID_ENDPOINT = "https://api.tickflow.org"
|
||||
|
||||
|
||||
class TickflowKeyIn(BaseModel):
|
||||
api_key: str
|
||||
|
||||
|
||||
def _sync_financial_scheduler(request: Request, capset) -> None:
|
||||
"""Key 变更后同步财务调度器状态,无需重启服务。"""
|
||||
try:
|
||||
financial_scheduler.update(request.app.state.repo.store.data_dir, capset)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("financial_scheduler update failed: %s", e)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def get_settings() -> dict:
|
||||
"""返回当前配置概况(Key 脱敏)。"""
|
||||
from app.config import settings
|
||||
from app.services import preferences
|
||||
|
||||
key = secrets_store.get_tickflow_key()
|
||||
return {
|
||||
"mode": tf_client.current_mode(),
|
||||
"tickflow_api_key_masked": secrets_store.mask(key),
|
||||
"has_tickflow_key": bool(key),
|
||||
"tier_label": tier_label(),
|
||||
"current_endpoint": tf_client.current_endpoint(),
|
||||
"probe_log": probe_log(),
|
||||
"missing_caps": missing_caps(),
|
||||
"extras_caps": extras_caps(),
|
||||
# 首次使用引导
|
||||
"onboarding_completed": preferences.get_onboarding_completed(),
|
||||
# AI 配置
|
||||
"ai_provider": secrets_store.get_ai_config("ai_provider", settings.ai_provider),
|
||||
"ai_base_url": secrets_store.get_ai_config("ai_base_url", settings.ai_base_url),
|
||||
"ai_api_key_masked": secrets_store.mask(secrets_store.get_ai_key()),
|
||||
"has_ai_key": bool(secrets_store.get_ai_key()),
|
||||
"ai_model": secrets_store.get_ai_config("ai_model", settings.ai_model),
|
||||
"ai_daily_token_budget": int(secrets_store.get_ai_config("ai_daily_token_budget", str(settings.ai_daily_token_budget)) or settings.ai_daily_token_budget),
|
||||
}
|
||||
|
||||
|
||||
class SwitchEndpointIn(BaseModel):
|
||||
url: str
|
||||
|
||||
|
||||
@router.post("/switch_endpoint")
|
||||
def switch_endpoint(req: SwitchEndpointIn, request: Request) -> dict:
|
||||
"""切换数据源端点并立即生效。
|
||||
|
||||
端点切换仅对付费档(starter+,走付费 API 节点)有意义;
|
||||
none/free 档运行在 free-api 服务器,无付费端点权限,禁止切换。
|
||||
"""
|
||||
# none/free 档没有付费端点权限,禁止切换
|
||||
if tf_client.current_mode() != "api_key":
|
||||
return {"ok": False, "error": "当前档位无法切换端点,仅付费套餐(Starter+)支持"}
|
||||
|
||||
url = req.url.strip().rstrip("/")
|
||||
if not url.startswith("https://"):
|
||||
return {"ok": False, "error": "仅支持 HTTPS 端点"}
|
||||
|
||||
# 持久化到 secrets.json
|
||||
secrets_store.save({"tickflow_base_url": url})
|
||||
# 重置客户端，下次调用自动用新端点
|
||||
tf_client.reset_clients()
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"current_endpoint": tf_client.current_endpoint(),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tickflow-key")
|
||||
def save_tickflow_key(req: TickflowKeyIn, request: Request) -> dict:
|
||||
"""保存数据源 API Key 并立即重新探测能力。
|
||||
|
||||
先探后存(关键改动,修复乱填 key 也会被持久化的问题):
|
||||
1. 临时用新 key 探测(付费端点),判定档位
|
||||
2. 判定为 none(连单只日K都拿不到)→ key 无效:不存,清除已存的,
|
||||
返回 {ok: false, reason: "invalid"},前端提示「Key 无效」
|
||||
3. 判定为 free(免费有效 key)→ 存 key,客户端切到 free-api 服务器
|
||||
4. 判定为 starter+ → 存 key,切到付费端点(现有逻辑)
|
||||
|
||||
端点联动:从无 key 升级到付费 key 时,残留的 free-api 端点不可用,
|
||||
故自动切到默认付费端点;free 档则清除自定义端点。
|
||||
"""
|
||||
from app.tickflow.policy import (
|
||||
base_tier_name, is_invalid_key,
|
||||
)
|
||||
|
||||
key = req.api_key.strip()
|
||||
if not key:
|
||||
return {"ok": False, "error": "key empty"}
|
||||
|
||||
# ===== 1) 临时存 key + 重置客户端,让探测走付费端点 =====
|
||||
secrets_store.save({"tickflow_api_key": key})
|
||||
tf_client.reset_clients()
|
||||
|
||||
# 立即重新探测(此时 client 已按档位判定,但首次探测必然走付费端点验证)
|
||||
capset = detect_capabilities(force=True)
|
||||
request.app.state.capabilities = capset
|
||||
_sync_financial_scheduler(request, capset)
|
||||
|
||||
# ===== 2) 判定为无效 key(连单只日K都拿不到)→ 不存,清除 =====
|
||||
if is_invalid_key() or base_tier_name() == "none":
|
||||
# 无效 key:清除刚存的,避免乱填被持久化;退回 none 档
|
||||
secrets_store.clear("tickflow_api_key", "tickflow_base_url")
|
||||
tf_client.reset_clients()
|
||||
capset = detect_capabilities(force=True)
|
||||
request.app.state.capabilities = capset
|
||||
_sync_financial_scheduler(request, capset)
|
||||
return {
|
||||
"ok": False,
|
||||
"reason": "invalid",
|
||||
"error": "Key 无效或已过期,请检查后重试",
|
||||
"mode": "none",
|
||||
"tier_label": tier_label(),
|
||||
"current_endpoint": tf_client.current_endpoint(),
|
||||
"probe_log": [],
|
||||
"capabilities_count": len(capset.all()),
|
||||
}
|
||||
|
||||
# ===== 3) free 档(免费有效 key)→ 存 key,切到 free-api 服务器 =====
|
||||
if base_tier_name() == "free":
|
||||
# 免费档运行时走 free-api 服务器,清除付费端点的自定义配置
|
||||
secrets_store.clear("tickflow_base_url")
|
||||
tf_client.reset_clients()
|
||||
return {
|
||||
"ok": True,
|
||||
"tickflow_api_key_masked": secrets_store.mask(key),
|
||||
"mode": "free",
|
||||
"tier_label": tier_label(),
|
||||
"current_endpoint": tf_client.current_endpoint(),
|
||||
"probe_log": [],
|
||||
"capabilities_count": len(capset.all()),
|
||||
}
|
||||
|
||||
# ===== 4) starter+ 付费档 → 确保走付费端点(现有逻辑) =====
|
||||
# 若之前是 none/free(无自定义付费端点),切到默认付费端点
|
||||
base = secrets_store.load().get("tickflow_base_url")
|
||||
if not base:
|
||||
secrets_store.save({"tickflow_base_url": DEFAULT_PAID_ENDPOINT})
|
||||
tf_client.reset_clients()
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"tickflow_api_key_masked": secrets_store.mask(key),
|
||||
"mode": "api_key",
|
||||
"tier_label": tier_label(),
|
||||
"current_endpoint": tf_client.current_endpoint(),
|
||||
"probe_log": [],
|
||||
"capabilities_count": len(capset.all()),
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/tickflow-key")
|
||||
def clear_tickflow_key(request: Request) -> dict:
|
||||
"""清除 Key,退回无档(none)。
|
||||
|
||||
同时清除 tickflow_base_url(测速切换的自定义端点),使客户端走 free-api
|
||||
服务器取历史日K;档位标签为 None(无档)。
|
||||
"""
|
||||
secrets_store.clear("tickflow_api_key", "tickflow_base_url")
|
||||
tf_client.reset_clients()
|
||||
|
||||
capset = detect_capabilities(force=True)
|
||||
request.app.state.capabilities = capset
|
||||
_sync_financial_scheduler(request, capset)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"mode": "none",
|
||||
"tier_label": tier_label(),
|
||||
"current_endpoint": tf_client.current_endpoint(),
|
||||
"capabilities_count": len(capset.all()),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/onboarding/complete")
|
||||
def complete_onboarding() -> dict:
|
||||
"""标记首次使用向导完成。
|
||||
|
||||
写入 preferences.json,前端守卫据此判断是否需要再次展示向导。
|
||||
跨设备/清缓存安全 —— 状态落在后端文件,不依赖浏览器本地存储。
|
||||
"""
|
||||
from app.services import preferences
|
||||
done = preferences.set_onboarding_completed(True)
|
||||
return {"ok": True, "onboarding_completed": done}
|
||||
|
||||
|
||||
class AiSettingsIn(BaseModel):
|
||||
provider: str = "openai_compat"
|
||||
base_url: str = ""
|
||||
api_key: str | None = None
|
||||
model: str = ""
|
||||
daily_token_budget: int = 500_000
|
||||
|
||||
|
||||
@router.post("/ai")
|
||||
def save_ai_settings(req: AiSettingsIn) -> dict:
|
||||
"""保存 AI 配置（全部持久化到 secrets.json）"""
|
||||
from app.config import settings
|
||||
|
||||
updates: dict = {}
|
||||
if req.provider:
|
||||
updates["ai_provider"] = req.provider
|
||||
settings.ai_provider = req.provider
|
||||
if req.base_url:
|
||||
updates["ai_base_url"] = req.base_url
|
||||
settings.ai_base_url = req.base_url
|
||||
if req.api_key is not None:
|
||||
if req.api_key:
|
||||
updates["ai_api_key"] = req.api_key
|
||||
settings.ai_api_key = req.api_key
|
||||
else:
|
||||
secrets_store.clear("ai_api_key")
|
||||
settings.ai_api_key = ""
|
||||
if req.model:
|
||||
updates["ai_model"] = req.model
|
||||
settings.ai_model = req.model
|
||||
updates["ai_daily_token_budget"] = req.daily_token_budget
|
||||
settings.ai_daily_token_budget = req.daily_token_budget
|
||||
|
||||
if updates:
|
||||
secrets_store.save(updates)
|
||||
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# ===== 偏好设置 =====
|
||||
|
||||
def _realtime_allowed() -> bool:
|
||||
"""当前档位是否允许实时行情(none/free 不允许)。"""
|
||||
from app.services.quote_service import QuoteService
|
||||
return QuoteService.is_realtime_allowed()
|
||||
|
||||
|
||||
class MinuteSyncPrefs(BaseModel):
|
||||
minute_sync_enabled: bool
|
||||
minute_sync_days: int = 5
|
||||
|
||||
|
||||
@router.get("/preferences")
|
||||
def get_preferences() -> dict:
|
||||
"""返回用户偏好设置。"""
|
||||
from app.services import preferences
|
||||
return {
|
||||
"realtime_quotes_enabled": preferences.get_realtime_quotes_enabled(),
|
||||
"realtime_allowed": _realtime_allowed(),
|
||||
"indices_nav_pinned": preferences.get_indices_nav_pinned(),
|
||||
"minute_sync_enabled": preferences.get_minute_sync_enabled(),
|
||||
"minute_sync_days": preferences.get_minute_sync_days(),
|
||||
"pipeline_schedule": preferences.get_pipeline_schedule(),
|
||||
"instruments_schedule": preferences.get_instruments_schedule(),
|
||||
"enriched_batch_size": preferences.get_enriched_batch_size(),
|
||||
"index_daily_batch_size": preferences.get_index_daily_batch_size(),
|
||||
"watchlist_columns": preferences.get_watchlist_columns(),
|
||||
"screener_result_columns": preferences.get_screener_result_columns(),
|
||||
"sse_refresh_pages": preferences.get_sse_refresh_pages(),
|
||||
"strategy_monitor_enabled": preferences.get_strategy_monitor_enabled(),
|
||||
"strategy_monitor_ids": preferences.get_strategy_monitor_ids(),
|
||||
"system_notify_enabled": preferences.get_system_notify_enabled(),
|
||||
"sidebar_index_symbols": preferences.get_sidebar_index_symbols(),
|
||||
"nav_order": preferences.get_nav_order(),
|
||||
"nav_hidden": preferences.get_nav_hidden(),
|
||||
"screener_auto_run": preferences.get_screener_auto_run(),
|
||||
"limit_ladder_monitor_enabled": preferences.get_limit_ladder_monitor_enabled(),
|
||||
"depth_polling_interval": preferences.get_depth_polling_interval(),
|
||||
"depth_finalize_time": preferences.get_depth_finalize_time(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/preferences/watchlist-columns")
|
||||
def get_watchlist_columns() -> dict:
|
||||
"""返回自选列表列配置。"""
|
||||
from app.services import preferences
|
||||
cols = preferences.get_watchlist_columns()
|
||||
return {"columns": cols}
|
||||
|
||||
|
||||
class NavOrderIn(BaseModel):
|
||||
nav_order: list[str]
|
||||
|
||||
|
||||
class NavHiddenIn(BaseModel):
|
||||
nav_hidden: list[str]
|
||||
|
||||
|
||||
@router.put("/preferences/nav-order")
|
||||
def update_nav_order(req: NavOrderIn) -> dict:
|
||||
"""保存左侧菜单排序（内置页面 path + 扩展分析菜单 id 的有序列表）。"""
|
||||
from app.services import preferences
|
||||
saved = preferences.set_nav_order(req.nav_order)
|
||||
return {"nav_order": saved}
|
||||
|
||||
|
||||
@router.put("/preferences/nav-hidden")
|
||||
def update_nav_hidden(req: NavHiddenIn) -> dict:
|
||||
"""保存左侧菜单隐藏项。"""
|
||||
from app.services import preferences
|
||||
saved = preferences.set_nav_hidden(req.nav_hidden)
|
||||
return {"nav_hidden": saved}
|
||||
|
||||
|
||||
@router.put("/preferences/watchlist-columns")
|
||||
def update_watchlist_columns(req: dict) -> dict:
|
||||
"""保存自选列表列配置。"""
|
||||
from app.services import preferences
|
||||
columns = req.get("columns", [])
|
||||
saved = preferences.set_watchlist_columns(columns)
|
||||
return {"columns": saved}
|
||||
|
||||
|
||||
@router.get("/preferences/screener-result-columns")
|
||||
def get_screener_result_columns() -> dict:
|
||||
"""返回策略结果列表列配置。"""
|
||||
from app.services import preferences
|
||||
cols = preferences.get_screener_result_columns()
|
||||
return {"columns": cols}
|
||||
|
||||
|
||||
@router.put("/preferences/screener-result-columns")
|
||||
def update_screener_result_columns(req: dict) -> dict:
|
||||
"""保存策略结果列表列配置。"""
|
||||
from app.services import preferences
|
||||
columns = req.get("columns", [])
|
||||
saved = preferences.set_screener_result_columns(columns)
|
||||
return {"columns": saved}
|
||||
|
||||
|
||||
@router.put("/preferences/minute-sync")
|
||||
def update_minute_sync(req: MinuteSyncPrefs) -> dict:
|
||||
"""保存分钟 K 同步偏好。"""
|
||||
from app.services import preferences
|
||||
days = max(1, min(30, req.minute_sync_days))
|
||||
preferences.save({
|
||||
"minute_sync_enabled": req.minute_sync_enabled,
|
||||
"minute_sync_days": days,
|
||||
})
|
||||
return {
|
||||
"minute_sync_enabled": req.minute_sync_enabled,
|
||||
"minute_sync_days": days,
|
||||
}
|
||||
|
||||
|
||||
class RealtimeQuotesPrefs(BaseModel):
|
||||
realtime_quotes_enabled: bool
|
||||
|
||||
|
||||
@router.put("/preferences/realtime-quotes")
|
||||
def update_realtime_quotes(req: RealtimeQuotesPrefs, request: Request) -> dict:
|
||||
"""保存全局实时行情开关。
|
||||
|
||||
none/free 档无实时行情权限:拒绝开启,persist 为关闭并返回 allowed=False,
|
||||
前端据此把开关置灰 / 回弹。
|
||||
"""
|
||||
from app.services import preferences
|
||||
qs = getattr(request.app.state, "quote_service", None)
|
||||
|
||||
allowed = qs.is_realtime_allowed() if qs else True
|
||||
if req.realtime_quotes_enabled and not allowed:
|
||||
# 当前档位不允许开启实时行情 — 强制关闭
|
||||
preferences.save({"realtime_quotes_enabled": False})
|
||||
if qs:
|
||||
qs.disable()
|
||||
return {"realtime_quotes_enabled": False, "realtime_allowed": False}
|
||||
|
||||
preferences.save({"realtime_quotes_enabled": req.realtime_quotes_enabled})
|
||||
if qs:
|
||||
if req.realtime_quotes_enabled:
|
||||
qs.enable()
|
||||
else:
|
||||
qs.disable()
|
||||
|
||||
return {"realtime_quotes_enabled": req.realtime_quotes_enabled, "realtime_allowed": allowed}
|
||||
|
||||
|
||||
class IndicesNavPinnedPrefs(BaseModel):
|
||||
indices_nav_pinned: bool
|
||||
|
||||
|
||||
@router.put("/preferences/indices-nav-pinned")
|
||||
def update_indices_nav_pinned(req: IndicesNavPinnedPrefs) -> dict:
|
||||
"""保存侧栏指数报价卡片固定显示开关。
|
||||
ON=常驻显示；OFF=跟随实时行情开关（仅实时开时显示）。"""
|
||||
from app.services import preferences
|
||||
preferences.save({"indices_nav_pinned": req.indices_nav_pinned})
|
||||
return {"indices_nav_pinned": req.indices_nav_pinned}
|
||||
|
||||
|
||||
class RealtimeMonitorConfigIn(BaseModel):
|
||||
sse_refresh_pages: dict[str, bool] | None = None
|
||||
strategy_monitor_enabled: bool | None = None
|
||||
strategy_monitor_ids: list[str] | None = None
|
||||
sidebar_index_symbols: list[str] | None = None
|
||||
screener_auto_run: bool | None = None
|
||||
|
||||
|
||||
@router.put("/preferences/realtime-monitor")
|
||||
def update_realtime_monitor_config(req: RealtimeMonitorConfigIn, request: Request) -> dict:
|
||||
"""更新实时监控配置。策略监控统一迁移为 MonitorRule,由监控引擎评估。"""
|
||||
from app.services import preferences
|
||||
|
||||
cfg = req.model_dump(exclude_none=True)
|
||||
result = preferences.set_realtime_monitor_config(cfg)
|
||||
|
||||
# 策略监控开关/池变化 → 同步迁移为 type=strategy 规则 + reload 引擎
|
||||
if req.strategy_monitor_ids is not None or req.strategy_monitor_enabled is not None:
|
||||
monitor_engine = getattr(request.app.state, "monitor_engine", None)
|
||||
strategy_engine = getattr(request.app.state, "strategy_engine", None)
|
||||
data_dir = request.app.state.repo.store.data_dir
|
||||
if monitor_engine is not None and strategy_engine is not None:
|
||||
from app.strategy import monitor_rules as mr_store
|
||||
try:
|
||||
if preferences.get_strategy_monitor_enabled():
|
||||
ids = preferences.get_strategy_monitor_ids()
|
||||
names = {s.id: s.name for s in strategy_engine.list_strategies()}
|
||||
mr_store.migrate_strategy_monitors(data_dir, ids, names)
|
||||
else:
|
||||
# 关闭策略监控: 停用所有策略规则
|
||||
mr_store.migrate_strategy_monitors(data_dir, [], {})
|
||||
# reload 规则到引擎
|
||||
monitor_engine.set_rules(mr_store.load_all(data_dir))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class QuoteIntervalIn(BaseModel):
|
||||
interval: float
|
||||
|
||||
|
||||
class SystemNotifyPrefsIn(BaseModel):
|
||||
enabled: bool
|
||||
|
||||
|
||||
@router.put("/preferences/system-notify")
|
||||
def update_system_notify(req: SystemNotifyPrefsIn) -> dict:
|
||||
"""系统通知开关 — 开启后监控告警同时推送到操作系统通知中心。
|
||||
|
||||
纯偏好, 无副作用 (不像策略监控要迁移规则), 直接落盘即可。
|
||||
quote_service 在每轮告警评估时读此开关决定是否发系统通知。
|
||||
"""
|
||||
from app.services import preferences
|
||||
saved = preferences.set_system_notify_enabled(req.enabled)
|
||||
return {"system_notify_enabled": saved}
|
||||
|
||||
|
||||
@router.put("/preferences/quote-interval")
|
||||
def update_quote_interval(req: QuoteIntervalIn, request: Request) -> dict:
|
||||
"""更新行情轮询间隔。按档位自动 clamp。"""
|
||||
qs = getattr(request.app.state, "quote_service", None)
|
||||
if not qs:
|
||||
return {"interval": req.interval, "min_interval": qs.get_min_interval(), "max_interval": 60.0}
|
||||
clamped = qs.set_interval(req.interval)
|
||||
return {
|
||||
"interval": clamped,
|
||||
"min_interval": qs.get_min_interval(),
|
||||
"max_interval": qs.MAX_INTERVAL,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/preferences/quote-interval")
|
||||
def get_quote_interval(request: Request) -> dict:
|
||||
"""获取当前行情轮询间隔和档位限制。"""
|
||||
qs = getattr(request.app.state, "quote_service", None)
|
||||
if not qs:
|
||||
return {"interval": 10.0, "min_interval": 5.0, "max_interval": 60.0}
|
||||
return {
|
||||
"interval": qs._interval,
|
||||
"min_interval": qs.get_min_interval(),
|
||||
"max_interval": qs.MAX_INTERVAL,
|
||||
}
|
||||
|
||||
|
||||
class TestEndpointIn(BaseModel):
|
||||
url: str
|
||||
# 测试轮数;不传时取 endpoints.json 的 testRounds(默认 5)
|
||||
rounds: int | None = None
|
||||
|
||||
|
||||
# 官方端点发现清单 —— 前端浏览器无法直接跨域拉取数据源官网 /endpoints.json
|
||||
# (无 CORS 头),因此由后端代理。缓存 5 分钟,失败时回退到内置列表。
|
||||
ENDPOINTS_URL = "https://tickflow.org/endpoints.json"
|
||||
ENDPOINTS_TTL = 300.0 # 秒
|
||||
|
||||
# 回退列表 —— 与官方 endpoints.json 的 endpoints[] 字段对齐。
|
||||
# 当远程拉取失败时使用,保证 UI 永远有内容可显示。
|
||||
_FALLBACK_ENDPOINTS: list[dict] = [
|
||||
{
|
||||
"id": "default",
|
||||
"url": "https://api.tickflow.org",
|
||||
"label": "默认端点",
|
||||
"region": "auto",
|
||||
"description": "默认端点",
|
||||
"premium": False,
|
||||
},
|
||||
{
|
||||
"id": "hk",
|
||||
"url": "https://hk-api.tickflow.org",
|
||||
"label": "香港端点",
|
||||
"region": "ap-east-1",
|
||||
"description": "备用端点，部分地区访问更稳定",
|
||||
"premium": False,
|
||||
},
|
||||
{
|
||||
"id": "sg",
|
||||
"url": "https://sg-api.tickflow.org",
|
||||
"label": "新加坡端点",
|
||||
"region": "ap-southeast-1",
|
||||
"description": "备用端点，亚太地区访问更稳定",
|
||||
"premium": False,
|
||||
},
|
||||
{
|
||||
"id": "us",
|
||||
"url": "https://us-api.tickflow.org",
|
||||
"label": "美国端点",
|
||||
"region": "us-east-1",
|
||||
"description": "备用端点，欧美地区访问更稳定",
|
||||
"premium": False,
|
||||
},
|
||||
{
|
||||
"id": "cn",
|
||||
"url": "https://139.196.55.234:50443",
|
||||
"label": "中国大陆端点（Beta）",
|
||||
"region": "cn-east-1",
|
||||
"description": "备用端点，中国大陆地区访问更稳定，目前处于测试阶段，谨慎使用",
|
||||
"premium": False,
|
||||
},
|
||||
{
|
||||
"id": "cn-premium",
|
||||
"url": "https://106.15.238.72:50443",
|
||||
"label": "中国大陆专线端点",
|
||||
"region": "cn-east-1",
|
||||
"description": "专线加速端点，需要专线加速权限（该权限包含在 Expert 及以上套餐中，也可通过自定义组合单独开通）",
|
||||
"premium": True,
|
||||
},
|
||||
]
|
||||
|
||||
# 进程内缓存:{ "ts": float, "data": dict }
|
||||
_endpoints_cache: dict = {"ts": 0.0, "data": None}
|
||||
|
||||
|
||||
@router.get("/endpoints")
|
||||
def list_endpoints() -> dict:
|
||||
"""代理拉取数据源官网 /endpoints.json 并返回规范化端点列表。
|
||||
|
||||
前端无法跨域直连该 URL(无 CORS 头),故由本接口代理。带 8s 超时、
|
||||
5 分钟内存缓存,远程失败时回退到内置列表,保证 UI 始终有内容。
|
||||
返回结构与原始 endpoints.json 一致(透传 schema/version 等元信息)。
|
||||
"""
|
||||
import httpx
|
||||
|
||||
now = time.monotonic()
|
||||
cached = _endpoints_cache.get("data")
|
||||
if cached is not None and (now - _endpoints_cache["ts"]) < ENDPOINTS_TTL:
|
||||
return cached
|
||||
|
||||
source = "remote"
|
||||
data: dict | None = None
|
||||
try:
|
||||
resp = httpx.get(ENDPOINTS_URL, timeout=8.0, follow_redirects=True)
|
||||
if resp.status_code == 200:
|
||||
parsed = resp.json()
|
||||
eps = parsed.get("endpoints")
|
||||
# 校验:必须是列表且每项含必要字段,否则视为无效
|
||||
if isinstance(eps, list) and all(
|
||||
isinstance(e, dict) and "url" in e for e in eps
|
||||
):
|
||||
data = {
|
||||
"version": parsed.get("version", 1),
|
||||
"description": parsed.get(
|
||||
"description", "API 端点配置"
|
||||
),
|
||||
"healthPath": parsed.get("healthPath", "/health"),
|
||||
"testRounds": parsed.get("testRounds", 5),
|
||||
"endpoints": eps,
|
||||
}
|
||||
except (httpx.HTTPError, ValueError):
|
||||
logger.warning("拉取 endpoints.json 失败，使用内置回退列表", exc_info=True)
|
||||
|
||||
if data is None:
|
||||
source = "fallback"
|
||||
data = {
|
||||
"version": 1,
|
||||
"description": "API 端点配置",
|
||||
"healthPath": "/health",
|
||||
"testRounds": 5,
|
||||
"endpoints": _FALLBACK_ENDPOINTS,
|
||||
}
|
||||
|
||||
# 标记数据来源,便于前端提示(回退时显示"内置列表")。
|
||||
data["source"] = source
|
||||
_endpoints_cache["ts"] = now
|
||||
_endpoints_cache["data"] = data
|
||||
return data
|
||||
|
||||
|
||||
async def _http_ping(url: str, timeout: float = 10.0) -> float | None:
|
||||
"""单次异步 GET 请求并返回延迟(ms),失败返回 None。
|
||||
|
||||
对齐官方 latency_test.py:用 /health 轻量端点测真实网络延迟,
|
||||
不携带 API Key(/health 公开)。异步实现,保证多端点并行测速不阻塞。
|
||||
"""
|
||||
import httpx
|
||||
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
resp = await client.get(url)
|
||||
dt = (time.perf_counter() - t0) * 1000
|
||||
# 只把 <400 视为成功;4xx/5xx 也算"不可达"
|
||||
if resp.status_code < 400:
|
||||
return round(dt, 2)
|
||||
return None
|
||||
except (httpx.TimeoutException, httpx.ConnectError, httpx.HTTPError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/test_endpoint")
|
||||
async def test_endpoint(req: TestEndpointIn) -> dict:
|
||||
"""测试端点网络延迟:对 /health 多轮探测取中位数。
|
||||
|
||||
参考官方 latency_test.py:
|
||||
- 路径用 /health(公开、轻量),反映真实网络延迟而非业务接口耗时
|
||||
- 多轮探测(默认 5 轮,取自 endpoints.json 的 testRounds),间隔 0.3s
|
||||
- 返回 median/min/max/success,前端显示中位数
|
||||
- 异步实现,保证"全部测速"时多端点真正并行
|
||||
"""
|
||||
import asyncio
|
||||
import statistics
|
||||
|
||||
base = req.url.rstrip("/")
|
||||
rounds = max(1, min(10, req.rounds or _endpoints_cache.get("data", {}).get("testRounds", 5)))
|
||||
health_url = base + "/health"
|
||||
|
||||
latencies: list[float] = []
|
||||
for _ in range(rounds):
|
||||
ms = await _http_ping(health_url)
|
||||
if ms is not None:
|
||||
latencies.append(ms)
|
||||
# 官方脚本间隔 0.3s;末轮无需等待
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
success = len(latencies)
|
||||
if success == 0:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "不可达",
|
||||
"url": req.url,
|
||||
"rounds": rounds,
|
||||
"success": 0,
|
||||
"median_ms": None,
|
||||
"min_ms": None,
|
||||
"max_ms": None,
|
||||
}
|
||||
|
||||
median = round(statistics.median(latencies), 2)
|
||||
return {
|
||||
"ok": True,
|
||||
"url": req.url,
|
||||
"rounds": rounds,
|
||||
"success": success,
|
||||
"median_ms": median,
|
||||
"min_ms": round(min(latencies), 2),
|
||||
"max_ms": round(max(latencies), 2),
|
||||
# 兼容旧字段:取中位数作为代表延迟
|
||||
"latency_ms": median,
|
||||
}
|
||||
|
||||
|
||||
class PipelineScheduleIn(BaseModel):
|
||||
hour: int
|
||||
minute: int
|
||||
|
||||
|
||||
@router.put("/preferences/pipeline-schedule")
|
||||
def update_pipeline_schedule(req: PipelineScheduleIn, request: Request) -> dict:
|
||||
"""保存盘后管道调度时间并立即 reschedule。"""
|
||||
from app.services import preferences
|
||||
sched = preferences.set_pipeline_schedule(req.hour, req.minute)
|
||||
|
||||
# 动态 reschedule
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
scheduler = getattr(request.app.state, "scheduler", None)
|
||||
if scheduler:
|
||||
scheduler.reschedule_job(
|
||||
"daily_pipeline",
|
||||
trigger=CronTrigger(
|
||||
day_of_week="mon-fri",
|
||||
hour=sched["hour"],
|
||||
minute=sched["minute"],
|
||||
timezone="Asia/Shanghai",
|
||||
),
|
||||
)
|
||||
logger.info("pipeline rescheduled to %02d:%02d mon-fri", sched["hour"], sched["minute"])
|
||||
|
||||
return sched
|
||||
|
||||
|
||||
@router.put("/preferences/instruments-schedule")
|
||||
def update_instruments_schedule(req: PipelineScheduleIn, request: Request) -> dict:
|
||||
"""保存盘前标的维表调度时间并立即 reschedule。"""
|
||||
from app.services import preferences
|
||||
sched = preferences.set_instruments_schedule(req.hour, req.minute)
|
||||
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
scheduler = getattr(request.app.state, "scheduler", None)
|
||||
if scheduler:
|
||||
scheduler.reschedule_job(
|
||||
"pre_market_instruments",
|
||||
trigger=CronTrigger(
|
||||
day_of_week="mon-fri",
|
||||
hour=sched["hour"],
|
||||
minute=sched["minute"],
|
||||
timezone="Asia/Shanghai",
|
||||
),
|
||||
)
|
||||
return sched
|
||||
|
||||
|
||||
class EnrichedBatchSizeIn(BaseModel):
|
||||
size: int
|
||||
|
||||
|
||||
@router.put("/preferences/enriched-batch-size")
|
||||
def update_enriched_batch_size(req: EnrichedBatchSizeIn) -> dict:
|
||||
"""保存 enriched 全量计算批次大小。"""
|
||||
from app.services import preferences
|
||||
size = preferences.set_enriched_batch_size(req.size)
|
||||
return {"enriched_batch_size": size}
|
||||
|
||||
|
||||
class IndexDailyBatchSizeIn(BaseModel):
|
||||
size: int
|
||||
|
||||
|
||||
@router.put("/preferences/index-daily-batch-size")
|
||||
def update_index_daily_batch_size(req: IndexDailyBatchSizeIn) -> dict:
|
||||
"""保存指数日 K 同步批次大小。"""
|
||||
from app.services import preferences
|
||||
size = preferences.set_index_daily_batch_size(req.size)
|
||||
return {"index_daily_batch_size": size}
|
||||
|
||||
|
||||
# ── 五档盘口 sealed 配置 ──────────────────────────────
|
||||
|
||||
class LimitLadderMonitorIn(BaseModel):
|
||||
enabled: bool
|
||||
|
||||
|
||||
@router.put("/preferences/limit-ladder-monitor")
|
||||
def update_limit_ladder_monitor(req: LimitLadderMonitorIn, request: Request) -> dict:
|
||||
"""连板梯队 5 档监控开关。开启→启动 depth 轮询, 关闭→停止。"""
|
||||
from app.services import preferences
|
||||
preferences.save({"limit_ladder_monitor_enabled": req.enabled})
|
||||
|
||||
# 立即应用: 启停 depth 轮询线程
|
||||
depth_svc = getattr(request.app.state, "depth_service", None)
|
||||
if depth_svc:
|
||||
depth_svc.apply_monitor_toggle(req.enabled)
|
||||
|
||||
return {"limit_ladder_monitor_enabled": req.enabled}
|
||||
|
||||
|
||||
@router.post("/preferences/limit-ladder-monitor/run")
|
||||
def run_limit_ladder_fix(request: Request) -> dict:
|
||||
"""立即手动修正一次真假板(拉取五档盘口 + 更新缓存)。需 Pro+。"""
|
||||
from app.tickflow.capabilities import Cap
|
||||
capset = request.app.state.capabilities
|
||||
capset.require(Cap.DEPTH5_BATCH) # 无能力抛 CapabilityDenied(403)
|
||||
|
||||
depth_svc = getattr(request.app.state, "depth_service", None)
|
||||
if not depth_svc:
|
||||
raise HTTPException(status_code=503, detail="depth 服务未初始化")
|
||||
return depth_svc.run_once()
|
||||
|
||||
|
||||
class DepthPollingIntervalIn(BaseModel):
|
||||
interval: float
|
||||
|
||||
|
||||
@router.put("/preferences/depth-polling-interval")
|
||||
def update_depth_polling_interval(req: DepthPollingIntervalIn, request: Request) -> dict:
|
||||
"""保存五档盘口盘中轮询间隔(秒)。需 Pro+。"""
|
||||
from app.tickflow.capabilities import Cap
|
||||
request.app.state.capabilities.require(Cap.DEPTH5_BATCH)
|
||||
|
||||
from app.services import preferences
|
||||
interval = preferences.set_depth_polling_interval(req.interval)
|
||||
return {"depth_polling_interval": interval}
|
||||
|
||||
|
||||
class DepthFinalizeTimeIn(BaseModel):
|
||||
hour: int
|
||||
minute: int
|
||||
|
||||
|
||||
@router.put("/preferences/depth-finalize-time")
|
||||
def update_depth_finalize_time(req: DepthFinalizeTimeIn, request: Request) -> dict:
|
||||
"""保存盘后 sealed 定版时间(范围15:01~18:00)并立即 reschedule。需 Pro+。"""
|
||||
from app.tickflow.capabilities import Cap
|
||||
request.app.state.capabilities.require(Cap.DEPTH5_BATCH)
|
||||
|
||||
from app.services import preferences
|
||||
sched = preferences.set_depth_finalize_time(req.hour, req.minute)
|
||||
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
scheduler = getattr(request.app.state, "scheduler", None)
|
||||
if scheduler:
|
||||
scheduler.reschedule_job(
|
||||
"depth_finalize",
|
||||
trigger=CronTrigger(
|
||||
day_of_week="mon-fri",
|
||||
hour=sched["hour"],
|
||||
minute=sched["minute"],
|
||||
timezone="Asia/Shanghai",
|
||||
),
|
||||
)
|
||||
logger.info("depth_finalize rescheduled to %02d:%02d mon-fri", sched["hour"], sched["minute"])
|
||||
|
||||
return sched
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""自定义信号 API 路由 — HTTP 请求 → 调用 custom_signals 模块 → 返回响应。
|
||||
|
||||
只做胶水：校验 → 持久化 → 失效缓存。不含表达式编译逻辑。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.strategy import custom_signals
|
||||
|
||||
router = APIRouter(prefix="/api/custom-signals", tags=["custom-signals"])
|
||||
|
||||
|
||||
def _data_dir(request: Request) -> Path:
|
||||
return request.app.state.repo.store.data_dir
|
||||
|
||||
|
||||
def _invalidate() -> None:
|
||||
"""失效 pipeline 的自定义信号缓存，下次计算重新加载。"""
|
||||
from app.indicators.pipeline import invalidate_custom_signals
|
||||
invalidate_custom_signals()
|
||||
|
||||
|
||||
class ConditionModel(BaseModel):
|
||||
left: str # 字段名（须在白名单）
|
||||
op: str # > >= < <= == !=
|
||||
right: str # "field:xxx" 或数字字符串
|
||||
|
||||
|
||||
class SignalModel(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
kind: str # entry | exit | both
|
||||
conditions: list[ConditionModel]
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
# ── 字段选项 / 运算符 ───────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/options")
|
||||
def get_options():
|
||||
"""返回可选字段与运算符，供前端下拉框使用。"""
|
||||
# 字段带中文标签（取自 ENRICHED_COLUMNS，回退为字段名本身）
|
||||
from app.indicators.pipeline import ENRICHED_COLUMNS
|
||||
|
||||
fields = [
|
||||
{"key": f, "label": ENRICHED_COLUMNS.get(f, f)}
|
||||
for f in sorted(custom_signals.ALLOWED_FIELDS)
|
||||
]
|
||||
return {
|
||||
"fields": fields,
|
||||
"operators": [">", ">=", "<", "<=", "==", "!="],
|
||||
"kinds": [
|
||||
{"key": "entry", "label": "买入"},
|
||||
{"key": "exit", "label": "卖出"},
|
||||
{"key": "both", "label": "买卖通用"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# ── 列表 ───────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_signals(request: Request):
|
||||
sigs = custom_signals.load_all(_data_dir(request))
|
||||
return {"signals": sigs}
|
||||
|
||||
|
||||
# ── 新建 / 更新 ────────────────────────────────────────
|
||||
|
||||
|
||||
@router.post("")
|
||||
def save_signal(req: SignalModel, request: Request):
|
||||
sig = req.model_dump()
|
||||
try:
|
||||
custom_signals.validate(sig)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
custom_signals.save_one(_data_dir(request), sig)
|
||||
_invalidate()
|
||||
return {"ok": True, "signal": sig}
|
||||
|
||||
|
||||
# ── 删除 ───────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.delete("/{signal_id}")
|
||||
def delete_signal(signal_id: str, request: Request):
|
||||
if not custom_signals.ID_RE.match(signal_id):
|
||||
raise HTTPException(status_code=400, detail="信号 id 非法")
|
||||
deleted = custom_signals.delete_one(_data_dir(request), signal_id)
|
||||
if not deleted:
|
||||
raise HTTPException(status_code=404, detail="信号不存在")
|
||||
_invalidate()
|
||||
return {"ok": True}
|
||||
@@ -0,0 +1,440 @@
|
||||
"""策略 API 路由 — HTTP 请求 → 调用策略模块 → 返回响应。
|
||||
|
||||
只做胶水，不含业务逻辑。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import asdict
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.strategy import config as strategy_config
|
||||
from app.strategy.engine import StrategyEngine, StrategyDef
|
||||
from app.strategy.ai_generator import AIStrategyGenerator
|
||||
from app.strategy.prompt_builder import build_step1, build_step2
|
||||
from app.strategy.monitor import StrategyMonitorService, StrategyAlert
|
||||
|
||||
router = APIRouter(prefix="/api/strategies", tags=["strategies"])
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _get_engine(request: Request) -> StrategyEngine:
|
||||
engine = getattr(request.app.state, "strategy_engine", None)
|
||||
if not engine:
|
||||
raise HTTPException(status_code=503, detail="策略引擎未初始化")
|
||||
return engine
|
||||
|
||||
|
||||
def _get_monitor(request: Request) -> StrategyMonitorService:
|
||||
mon = getattr(request.app.state, "strategy_monitor", None)
|
||||
if not mon:
|
||||
raise HTTPException(status_code=503, detail="策略监控未初始化")
|
||||
return mon
|
||||
|
||||
|
||||
def _data_dir(request: Request) -> Path:
|
||||
return request.app.state.repo.store.data_dir
|
||||
|
||||
|
||||
def _safe(result_dict: dict) -> dict:
|
||||
rows = result_dict.get("rows", [])
|
||||
for r in rows:
|
||||
for k, v in list(r.items()):
|
||||
if isinstance(v, float) and not math.isfinite(v):
|
||||
r[k] = None
|
||||
return result_dict
|
||||
|
||||
|
||||
def _strategy_detail(s: StrategyDef, overrides: dict | None = None) -> dict:
|
||||
"""策略详情（含用户覆盖）"""
|
||||
bf = {**s.basic_filter}
|
||||
scoring = dict(s.meta.get("scoring", {}))
|
||||
params_defaults = {p["id"]: p["default"] for p in s.meta.get("params", [])}
|
||||
|
||||
if overrides:
|
||||
if overrides.get("basic_filter"):
|
||||
bf.update(overrides["basic_filter"])
|
||||
if overrides.get("scoring"):
|
||||
scoring.update(overrides["scoring"])
|
||||
# 用户保存的参数覆盖默认值: 合并进 params_defaults, 前端据此回显
|
||||
if overrides.get("params"):
|
||||
params_defaults.update(overrides["params"])
|
||||
|
||||
# 名称/描述可被用户覆盖
|
||||
name = overrides.get("name", s.meta.get("name", "")) if overrides else s.meta.get("name", "")
|
||||
description = overrides.get("description", s.meta.get("description", "")) if overrides else s.meta.get("description", "")
|
||||
|
||||
return {
|
||||
"id": s.meta["id"],
|
||||
"name": name or s.meta.get("name", ""),
|
||||
"description": description or s.meta.get("description", ""),
|
||||
"tags": s.meta.get("tags", []),
|
||||
"source": s.source,
|
||||
"version": s.meta.get("version", "1.0.0"),
|
||||
"basic_filter": bf,
|
||||
"params": s.meta.get("params", []),
|
||||
"params_defaults": params_defaults,
|
||||
"scoring": scoring,
|
||||
"entry_signals": s.entry_signals,
|
||||
"exit_signals": s.exit_signals,
|
||||
"stop_loss": overrides.get("stop_loss", s.stop_loss) if overrides else s.stop_loss,
|
||||
"trailing_stop": getattr(s, "trailing_stop", None),
|
||||
"trailing_take_profit_activate": getattr(s, "trailing_take_profit_activate", None),
|
||||
"trailing_take_profit_drawdown": getattr(s, "trailing_take_profit_drawdown", None),
|
||||
"max_hold_days": overrides.get("max_hold_days", s.max_hold_days) if overrides else s.max_hold_days,
|
||||
"alerts": s.alerts,
|
||||
"order_by": s.meta.get("order_by", "score"),
|
||||
"descending": s.meta.get("descending", True),
|
||||
"limit": s.meta.get("limit", 30),
|
||||
"display_limit": overrides.get("display_limit") if overrides and "display_limit" in overrides else None,
|
||||
}
|
||||
|
||||
|
||||
# ── Request Models ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
class RunRequest(BaseModel):
|
||||
strategy_id: str
|
||||
as_of: date | None = None
|
||||
pool: list[str] | None = None
|
||||
params: dict | None = None
|
||||
|
||||
|
||||
class RunAllRequest(BaseModel):
|
||||
as_of: date | None = None
|
||||
|
||||
|
||||
class SaveConfigRequest(BaseModel):
|
||||
strategy_id: str
|
||||
overrides: dict
|
||||
|
||||
|
||||
class AIGenerateRequest(BaseModel):
|
||||
prompt: str
|
||||
|
||||
|
||||
class AISaveRequest(BaseModel):
|
||||
code: str
|
||||
strategy_id: str
|
||||
|
||||
|
||||
class MonitorStartRequest(BaseModel):
|
||||
strategy_id: str
|
||||
|
||||
|
||||
# ── 列表 / 详情 ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_strategies(request: Request):
|
||||
engine = _get_engine(request)
|
||||
data_dir = _data_dir(request)
|
||||
all_overrides = strategy_config.list_overrides(data_dir)
|
||||
|
||||
result = []
|
||||
for meta in engine.list_strategies():
|
||||
sid = meta["id"]
|
||||
s = engine.get(sid)
|
||||
overrides = all_overrides.get(sid)
|
||||
result.append(_strategy_detail(s, overrides))
|
||||
return {"strategies": result}
|
||||
|
||||
|
||||
@router.get("/{strategy_id}")
|
||||
def get_strategy(strategy_id: str, request: Request):
|
||||
engine = _get_engine(request)
|
||||
try:
|
||||
s = engine.get(strategy_id)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e)) from e
|
||||
overrides = strategy_config.load_override(_data_dir(request), strategy_id)
|
||||
return _strategy_detail(s, overrides or None)
|
||||
|
||||
|
||||
# ── 执行选股 ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.post("/run")
|
||||
def run_strategy(req: RunRequest, request: Request):
|
||||
engine = _get_engine(request)
|
||||
data_dir = _data_dir(request)
|
||||
|
||||
# 读取用户覆盖配置
|
||||
overrides = strategy_config.load_override(data_dir, req.strategy_id)
|
||||
params = req.params or {}
|
||||
# 合并用户保存的策略参数
|
||||
if overrides.get("params"):
|
||||
merged = dict(overrides["params"])
|
||||
merged.update(params) # 请求里的优先
|
||||
params = merged
|
||||
|
||||
# 确定日期
|
||||
as_of = req.as_of
|
||||
if not as_of:
|
||||
from app.services.screener import ScreenerService
|
||||
svc = ScreenerService(request.app.state.repo)
|
||||
as_of = svc.latest_date()
|
||||
if not as_of:
|
||||
raise HTTPException(status_code=400, detail="无可用数据日期")
|
||||
|
||||
try:
|
||||
result = engine.run(
|
||||
req.strategy_id, as_of,
|
||||
pool=req.pool,
|
||||
params=params,
|
||||
overrides=overrides or None,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=404, detail=str(e)) from e
|
||||
|
||||
return _safe(asdict(result))
|
||||
|
||||
|
||||
@router.post("/run-all")
|
||||
def run_all(req: RunAllRequest, request: Request):
|
||||
engine = _get_engine(request)
|
||||
data_dir = _data_dir(request)
|
||||
|
||||
as_of = req.as_of
|
||||
if not as_of:
|
||||
from app.services.screener import ScreenerService
|
||||
svc = ScreenerService(request.app.state.repo)
|
||||
as_of = svc.latest_date()
|
||||
if not as_of:
|
||||
return {"as_of": None, "results": {}}
|
||||
|
||||
all_overrides = strategy_config.list_overrides(data_dir)
|
||||
results: dict[str, dict] = {}
|
||||
for sid, result in engine.run_all(as_of, overrides_map=all_overrides).items():
|
||||
results[sid] = {"total": result.total, "as_of": str(as_of)}
|
||||
|
||||
return {"as_of": str(as_of), "results": results}
|
||||
|
||||
|
||||
# ── 配置持久化 ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.post("/config")
|
||||
def save_config(req: SaveConfigRequest, request: Request):
|
||||
engine = _get_engine(request)
|
||||
if not engine.has(req.strategy_id):
|
||||
raise HTTPException(status_code=404, detail=f"策略 {req.strategy_id} 不存在")
|
||||
|
||||
# 剥离与策略默认值相同的字段，只保存用户真正修改过的值
|
||||
overrides = _strip_defaults(req.strategy_id, req.overrides, engine)
|
||||
|
||||
strategy_config.save_override(_data_dir(request), req.strategy_id, overrides)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
def _strip_defaults(strategy_id: str, overrides: dict, engine) -> dict:
|
||||
"""剥离与策略默认值相同的字段，避免默认值被固化到 override 中。
|
||||
|
||||
核心问题: 前端把策略的默认 basic_filter 全量发回后端保存，
|
||||
导致隐含的默认过滤条件 (如 market_cap_min, amount_min) 被写入 override 文件。
|
||||
即使前端 UI 不展示这些字段，它们仍会在策略运行时生效。
|
||||
"""
|
||||
s = engine.get(strategy_id)
|
||||
result = dict(overrides)
|
||||
|
||||
# 处理 basic_filter: 只保留与策略默认值不同的键
|
||||
bf = result.get("basic_filter")
|
||||
if bf and isinstance(bf, dict):
|
||||
default_bf = s.basic_filter if s else {}
|
||||
stripped_bf = {}
|
||||
for k, v in bf.items():
|
||||
default_val = default_bf.get(k)
|
||||
# 保留与默认值不同的键，以及没有默认值的键
|
||||
if k not in default_bf or v != default_val:
|
||||
stripped_bf[k] = v
|
||||
if stripped_bf:
|
||||
result["basic_filter"] = stripped_bf
|
||||
else:
|
||||
del result["basic_filter"]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@router.delete("/config/{strategy_id}")
|
||||
def reset_config(strategy_id: str, request: Request):
|
||||
strategy_config.delete_override(_data_dir(request), strategy_id)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# ── AI 生成 ───────────────────────────────────────────────────────────
|
||||
|
||||
class BuildRequest(BaseModel):
|
||||
"""两步策略构建请求"""
|
||||
step: int # 1 / 2
|
||||
# step1 字段
|
||||
name: str = ""
|
||||
description: str = ""
|
||||
direction: str = "long"
|
||||
rules: str = ""
|
||||
strategy_id: str = ""
|
||||
# step2 字段
|
||||
current_code: str = ""
|
||||
instruction: str = ""
|
||||
|
||||
|
||||
@router.get("/ai/status")
|
||||
def ai_status(request: Request):
|
||||
"""检查 AI 配置状态"""
|
||||
from app.config import settings
|
||||
from app import secrets_store
|
||||
has_key = bool(secrets_store.get_ai_key())
|
||||
has_model = bool(settings.ai_model)
|
||||
return {"configured": has_key and has_model, "has_key": has_key, "has_model": has_model}
|
||||
|
||||
|
||||
@router.get("/{strategy_id}/source")
|
||||
def get_strategy_source(strategy_id: str, request: Request):
|
||||
"""获取策略源文件内容（用于 AI 修改）"""
|
||||
from pathlib import Path
|
||||
|
||||
# 先查 StrategyEngine 获取文件路径
|
||||
engine = _get_engine(request)
|
||||
try:
|
||||
s = engine.get(strategy_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=404, detail=f"策略 {strategy_id} 不存在")
|
||||
|
||||
path = s.file_path
|
||||
if not path or not path.exists():
|
||||
raise HTTPException(status_code=404, detail="策略源文件不存在")
|
||||
|
||||
return {"code": path.read_text(encoding="utf-8"), "source": s.source}
|
||||
|
||||
|
||||
@router.post("/ai/test")
|
||||
async def ai_test(request: Request):
|
||||
"""测试 AI 连通性 — 发送简单请求验证 Key 和模型"""
|
||||
from app.config import settings
|
||||
from app import secrets_store
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
ai_key = secrets_store.get_ai_key()
|
||||
if not ai_key:
|
||||
return {"ok": False, "error": "未配置 API Key"}
|
||||
|
||||
try:
|
||||
client = AsyncOpenAI(api_key=ai_key, base_url=settings.ai_base_url)
|
||||
resp = await client.chat.completions.create(
|
||||
model=settings.ai_model,
|
||||
messages=[{"role": "user", "content": "回复 OK"}],
|
||||
max_tokens=5,
|
||||
timeout=15,
|
||||
)
|
||||
return {"ok": True, "model": resp.model, "usage": {"prompt": resp.usage.prompt_tokens, "completion": resp.usage.completion_tokens} if resp.usage else None}
|
||||
except Exception as e:
|
||||
return {"ok": False, "error": str(e)}
|
||||
|
||||
|
||||
@router.post("/build")
|
||||
async def build_strategy(req: BuildRequest, request: Request):
|
||||
"""两步策略构建。
|
||||
step1: name + description + direction + rules → 完整策略
|
||||
step2: current_code + instruction → 修改任意部分
|
||||
"""
|
||||
gen = AIStrategyGenerator()
|
||||
|
||||
if req.step == 1:
|
||||
prompt = build_step1(req.name, req.description, req.direction, req.rules, req.strategy_id)
|
||||
elif req.step == 2:
|
||||
prompt = build_step2(req.current_code, req.instruction)
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail=f"无效步骤: {req.step}")
|
||||
|
||||
try:
|
||||
result = await gen.generate(prompt)
|
||||
except RuntimeError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@router.post("/ai/generate")
|
||||
async def ai_generate(req: AIGenerateRequest, request: Request):
|
||||
try:
|
||||
gen = AIStrategyGenerator()
|
||||
result = await gen.generate(req.prompt)
|
||||
except RuntimeError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"AI生成失败: {e}") from e
|
||||
return result
|
||||
|
||||
|
||||
@router.post("/ai/save")
|
||||
async def ai_save(req: AISaveRequest, request: Request):
|
||||
data_dir = _data_dir(request)
|
||||
out_dir = data_dir / "strategies" / "ai"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = out_dir / f"{req.strategy_id}.py"
|
||||
previous_code = path.read_text(encoding="utf-8") if path.exists() else None
|
||||
path.write_text(req.code, encoding="utf-8")
|
||||
|
||||
# 热重载，并确认保存的策略真的被引擎加载。
|
||||
engine = _get_engine(request)
|
||||
engine.reload()
|
||||
if not engine.has(req.strategy_id):
|
||||
if previous_code is None:
|
||||
path.unlink(missing_ok=True)
|
||||
else:
|
||||
path.write_text(previous_code, encoding="utf-8")
|
||||
engine.reload()
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"策略保存成功但加载失败: {req.strategy_id}，请检查代码语法和 META.id 是否一致",
|
||||
)
|
||||
return {"ok": True, "path": str(path)}
|
||||
|
||||
|
||||
@router.delete("/{strategy_id}")
|
||||
def delete_strategy(strategy_id: str, request: Request):
|
||||
"""删除自定义策略 — 清除 .py 文件 + overrides + 热重载。内置策略不可删除。"""
|
||||
from pathlib import Path
|
||||
|
||||
engine = _get_engine(request)
|
||||
try:
|
||||
s = engine.get(strategy_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=404, detail=f"策略 {strategy_id} 不存在")
|
||||
|
||||
if s.source == "builtin":
|
||||
raise HTTPException(status_code=403, detail="内置策略不可删除")
|
||||
|
||||
# 删除策略文件
|
||||
if s.file_path and s.file_path.exists():
|
||||
s.file_path.unlink()
|
||||
|
||||
# 删除 overrides
|
||||
data_dir = _data_dir(request)
|
||||
override_path = data_dir / "user_data" / "strategy_overrides" / f"{strategy_id}.json"
|
||||
if override_path.exists():
|
||||
override_path.unlink()
|
||||
|
||||
# 热重载
|
||||
engine.reload()
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# ── 监控 ─────────────────────────────────────────────────────────────
|
||||
# 注: 策略监控已统一迁移到 MonitorRuleEngine (监控通知页), 旧的 start/stop/status
|
||||
# 路由已移除。StrategyMonitorService 类保留 (其 _check_signals 被 MonitorRuleEngine 复用)。
|
||||
|
||||
|
||||
# ── 热重载 ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.post("/reload")
|
||||
def reload_strategies(request: Request):
|
||||
engine = _get_engine(request)
|
||||
engine.reload()
|
||||
return {"ok": True, "count": len(engine.list_strategies())}
|
||||
@@ -0,0 +1,202 @@
|
||||
"""自选股 API。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
from datetime import date
|
||||
|
||||
import polars as pl
|
||||
from fastapi import APIRouter, Query, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.services import watchlist
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/watchlist", tags=["watchlist"])
|
||||
|
||||
|
||||
class AddRequest(BaseModel):
|
||||
symbol: str
|
||||
note: str = ""
|
||||
|
||||
|
||||
class BatchAddRequest(BaseModel):
|
||||
symbols: list[str]
|
||||
note: str = ""
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_all():
|
||||
return {"symbols": watchlist.list_symbols()}
|
||||
|
||||
|
||||
@router.post("")
|
||||
def add_one(req: AddRequest):
|
||||
rows = watchlist.add(req.symbol, req.note)
|
||||
return {"symbols": rows}
|
||||
|
||||
|
||||
@router.post("/batch")
|
||||
def add_batch(req: BatchAddRequest):
|
||||
for sym in req.symbols:
|
||||
watchlist.add(sym, req.note)
|
||||
return {"symbols": watchlist.list_symbols(), "added": len(req.symbols)}
|
||||
|
||||
|
||||
@router.delete("/{symbol}")
|
||||
def remove_one(symbol: str):
|
||||
rows = watchlist.remove(symbol)
|
||||
return {"symbols": rows}
|
||||
|
||||
|
||||
@router.delete("")
|
||||
def clear_all():
|
||||
"""清空自选列表。"""
|
||||
count = watchlist.clear()
|
||||
return {"removed": count}
|
||||
|
||||
|
||||
# 自选页需要的列
|
||||
_WATCHLIST_COLS = [
|
||||
"symbol", "close", "change_pct", "change_amount", "amount",
|
||||
"turnover_rate",
|
||||
"amplitude", "annual_vol_20d",
|
||||
"vol_ratio_5d",
|
||||
"ma5", "ma10", "ma20", "ma60",
|
||||
"vol_ma5", "vol_ma10",
|
||||
"high_60d", "low_60d",
|
||||
"rsi_6", "rsi_14", "rsi_24",
|
||||
"macd_dif", "macd_dea", "macd_hist",
|
||||
"kdj_k", "kdj_d", "kdj_j",
|
||||
"boll_upper", "boll_lower",
|
||||
"atr_14",
|
||||
"momentum_5d", "momentum_10d", "momentum_20d", "momentum_30d", "momentum_60d",
|
||||
"consecutive_limit_ups", "consecutive_limit_downs",
|
||||
"signal_limit_up", "signal_limit_down", "signal_volume_surge",
|
||||
"signal_ma_golden_5_20", "signal_macd_golden", "signal_n_day_high",
|
||||
"signal_boll_breakout_upper", "signal_ma20_breakout",
|
||||
"signal_ma_dead_5_20", "signal_macd_dead", "signal_n_day_low",
|
||||
"signal_boll_breakdown_lower", "signal_ma20_breakdown",
|
||||
]
|
||||
|
||||
|
||||
@router.get("/enriched")
|
||||
def watchlist_enriched(
|
||||
request: Request,
|
||||
ext_columns: str | None = Query(None, description="逗号分隔的 ext 列: config_id.field_name"),
|
||||
):
|
||||
"""自选股 enriched 数据 — 直接从 enriched 最新日读取, 无即时计算。
|
||||
|
||||
ext_columns 参数示例: "industry_rating.score,fund_flow.net_inflow"
|
||||
会动态 LEFT JOIN 对应的 ext_{config_id} DuckDB view。
|
||||
"""
|
||||
t0 = time.perf_counter()
|
||||
|
||||
repo = request.app.state.repo
|
||||
symbols = [r["symbol"] for r in watchlist.list_symbols()]
|
||||
if not symbols:
|
||||
return {"rows": [], "as_of": None, "elapsed_ms": 0}
|
||||
|
||||
df_e, cache_date = repo.get_enriched_latest()
|
||||
if df_e.is_empty():
|
||||
return {"rows": [], "as_of": None, "elapsed_ms": 0}
|
||||
|
||||
# 按 symbol 过滤
|
||||
df = df_e.filter(pl.col("symbol").is_in(symbols))
|
||||
if df.is_empty():
|
||||
return {"rows": [], "as_of": str(cache_date) if cache_date else None, "elapsed_ms": 0}
|
||||
|
||||
# JOIN instruments 取 name + float_shares
|
||||
df_i = repo.get_instruments()
|
||||
if not df_i.is_empty() and "name" in df_i.columns:
|
||||
inst_cols = [c for c in ["symbol", "name", "float_shares"] if c in df_i.columns]
|
||||
df = df.join(df_i.select(inst_cols), on="symbol", how="left")
|
||||
|
||||
# 选择内置需要的列
|
||||
keep = [c for c in _WATCHLIST_COLS + ["name", "float_shares"] if c in df.columns]
|
||||
df = df.select(keep)
|
||||
|
||||
# 动态 JOIN 扩展数据表
|
||||
ext_specs = _parse_ext_columns(ext_columns) if ext_columns else []
|
||||
if ext_specs:
|
||||
db = repo.store.db
|
||||
data_dir = repo.store.data_dir
|
||||
from app.services.ext_data import ExtConfigStore
|
||||
from app.api.ext_data import _read_ext_dataframe
|
||||
|
||||
ext_store = ExtConfigStore(data_dir)
|
||||
configs = {c.id: c for c in ext_store.load_all()}
|
||||
|
||||
for config_id, field_name in ext_specs:
|
||||
view_name = f"ext_{config_id}"
|
||||
ext_col_name = f"{config_id}__{field_name}"
|
||||
try:
|
||||
# 扩展时序数据必须只取最新分区；否则一个 symbol 会按历史分区数被 JOIN 放大。
|
||||
cfg = configs.get(config_id)
|
||||
if cfg:
|
||||
ext_df, _ = _read_ext_dataframe(cfg, data_dir)
|
||||
else:
|
||||
ext_df = pl.from_arrow(db.query(
|
||||
f"SELECT symbol, \"{field_name}\" FROM {view_name}"
|
||||
).arrow())
|
||||
if not ext_df.is_empty() and "symbol" in ext_df.columns:
|
||||
ext_df = (
|
||||
ext_df
|
||||
.select(["symbol", field_name])
|
||||
.unique(subset=["symbol"], keep="last")
|
||||
.rename({field_name: ext_col_name})
|
||||
)
|
||||
df = df.join(ext_df.select(["symbol", ext_col_name]), on="symbol", how="left")
|
||||
except Exception:
|
||||
# view 不存在或字段不存在，尝试直接读 parquet
|
||||
cfg = configs.get(config_id)
|
||||
if cfg:
|
||||
try:
|
||||
ext_df, _ = _read_ext_dataframe(cfg, data_dir)
|
||||
if not ext_df.is_empty() and "symbol" in ext_df.columns and field_name in ext_df.columns:
|
||||
ext_df = (
|
||||
ext_df
|
||||
.select(["symbol", field_name])
|
||||
.unique(subset=["symbol"], keep="last")
|
||||
.rename({field_name: ext_col_name})
|
||||
)
|
||||
df = df.join(ext_df, on="symbol", how="left")
|
||||
except Exception as e2:
|
||||
logger.debug("ext join fallback failed for %s.%s: %s", config_id, field_name, e2)
|
||||
|
||||
# sanitize NaN / Inf
|
||||
float_cols = [c for c in df.columns if df[c].dtype.is_float()]
|
||||
if float_cols:
|
||||
df = df.with_columns([
|
||||
pl.when(pl.col(c).is_nan() | pl.col(c).is_infinite())
|
||||
.then(None)
|
||||
.otherwise(pl.col(c))
|
||||
.alias(c)
|
||||
for c in float_cols
|
||||
])
|
||||
|
||||
# 按自选添加顺序（新加的在前）重排行
|
||||
order_map = {s: i for i, s in enumerate(symbols)}
|
||||
df = df.with_columns(pl.col("symbol").map_elements(lambda s: order_map.get(s, len(symbols)), return_dtype=pl.Int32).alias("_sort_order"))
|
||||
df = df.sort("_sort_order").drop("_sort_order")
|
||||
|
||||
rows = df.to_dicts()
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
return {"rows": rows, "as_of": str(cache_date) if cache_date else None, "elapsed_ms": elapsed}
|
||||
|
||||
|
||||
def _parse_ext_columns(ext_columns: str) -> list[tuple[str, str]]:
|
||||
"""解析 'config_id1.field1,config_id2.field2' 为 [(config_id, field_name), ...]"""
|
||||
result = []
|
||||
for part in ext_columns.split(","):
|
||||
part = part.strip()
|
||||
if "." not in part:
|
||||
continue
|
||||
config_id, field_name = part.split(".", 1)
|
||||
config_id = config_id.strip()
|
||||
field_name = field_name.strip()
|
||||
if config_id and field_name:
|
||||
result.append((config_id, field_name))
|
||||
return result
|
||||
@@ -0,0 +1,189 @@
|
||||
"""访问门控 —— 支持管理员令牌 + 动态 UUID 两种凭证。
|
||||
|
||||
部署方式(优先级从高到低):
|
||||
1. ADMIN_TOKEN: 管理员初始令牌(如 admin7226132)。验证通过后进入管理页,
|
||||
可创建普通 UUID 供合伙人使用,管理员本身也可访问全部功能。
|
||||
2. 动态 UUID: 管理员通过 /admin/uuids 创建的访问 UUID,持久化在
|
||||
data/user_data/access_uuids.json。
|
||||
3. ACCESS_UUID: 遗留单共享 UUID,保持向后兼容。
|
||||
|
||||
以上全部留空则门控不启用。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import logging
|
||||
import secrets
|
||||
import time
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.config import settings
|
||||
from app import uuid_store
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AuthRole(str, Enum):
|
||||
ADMIN = "admin"
|
||||
USER = "user"
|
||||
|
||||
|
||||
# 内存中的令牌缓存:{token: {"role": role, "expires_at": float}}
|
||||
_token_cache: dict[str, dict[str, Any]] = {}
|
||||
|
||||
# 令牌默认有效期:7 天
|
||||
TOKEN_TTL_SECONDS = 7 * 24 * 60 * 60
|
||||
|
||||
|
||||
class VerifyIn(BaseModel):
|
||||
credential: str
|
||||
|
||||
|
||||
class VerifyOut(BaseModel):
|
||||
valid: bool
|
||||
role: str | None = None
|
||||
token: str | None = None
|
||||
|
||||
|
||||
class AuthStatusOut(BaseModel):
|
||||
enabled: bool
|
||||
verified: bool
|
||||
role: str | None = None
|
||||
|
||||
|
||||
class UuidRecordOut(BaseModel):
|
||||
uuid: str
|
||||
label: str
|
||||
enabled: bool
|
||||
created_at: int
|
||||
|
||||
|
||||
class UuidCreateIn(BaseModel):
|
||||
label: str = ""
|
||||
|
||||
|
||||
def access_control_enabled() -> bool:
|
||||
"""是否启用了任何访问门控。"""
|
||||
return bool(settings.admin_token) or bool(settings.access_uuid)
|
||||
|
||||
|
||||
def admin_mode_enabled() -> bool:
|
||||
"""是否启用了管理员令牌模式。"""
|
||||
return bool(settings.admin_token)
|
||||
|
||||
|
||||
def _constant_time_compare(a: str, b: str) -> bool:
|
||||
"""常量时间字符串比较,降低时序攻击风险。"""
|
||||
return hmac.compare_digest(a.encode(), b.encode())
|
||||
|
||||
|
||||
def verify_admin_token(credential: str | None) -> bool:
|
||||
"""校验是否为管理员令牌。"""
|
||||
if not credential or not settings.admin_token:
|
||||
return False
|
||||
return _constant_time_compare(credential.strip(), settings.admin_token.strip())
|
||||
|
||||
|
||||
def verify_uuid(credential: str | None) -> bool:
|
||||
"""校验输入是否为有效访问 UUID(遗留单 UUID 或动态 UUID)。"""
|
||||
if not credential:
|
||||
return False
|
||||
stripped = credential.strip()
|
||||
# 动态 UUID
|
||||
if uuid_store.exists(stripped):
|
||||
return True
|
||||
# 遗留单共享 UUID
|
||||
if settings.access_uuid and _constant_time_compare(stripped, settings.access_uuid.strip()):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def verify_credential(credential: str | None) -> AuthRole | None:
|
||||
"""校验任意凭证,返回对应角色;无效返回 None。"""
|
||||
if not credential:
|
||||
return None
|
||||
if verify_admin_token(credential):
|
||||
return AuthRole.ADMIN
|
||||
if verify_uuid(credential):
|
||||
return AuthRole.USER
|
||||
return None
|
||||
|
||||
|
||||
def create_access_token(role: AuthRole) -> str:
|
||||
"""生成一个随机访问令牌并缓存。"""
|
||||
token = secrets.token_urlsafe(32)
|
||||
_token_cache[token] = {"role": role, "expires_at": time.time() + TOKEN_TTL_SECONDS}
|
||||
return token
|
||||
|
||||
|
||||
def validate_access_token(token: str | None) -> AuthRole | None:
|
||||
"""校验令牌是否有效,返回角色。"""
|
||||
if not token:
|
||||
return None
|
||||
cached = _token_cache.get(token)
|
||||
if cached is None:
|
||||
return None
|
||||
expires_at = cached.get("expires_at", 0)
|
||||
if expires_at > 0 and time.time() > expires_at:
|
||||
_token_cache.pop(token, None)
|
||||
return None
|
||||
return cached.get("role")
|
||||
|
||||
|
||||
def revoke_access_token(token: str | None) -> None:
|
||||
"""使指定令牌失效。"""
|
||||
if token:
|
||||
_token_cache.pop(token, None)
|
||||
|
||||
|
||||
def get_access_token_from_request(request: Request) -> str | None:
|
||||
"""从请求头或 query 参数中提取访问令牌。"""
|
||||
header = request.headers.get("X-Access-Token")
|
||||
if header:
|
||||
return header.strip()
|
||||
return request.query_params.get("access_token")
|
||||
|
||||
|
||||
def require_access(request: Request, allowed_roles: set[AuthRole] | None = None) -> AuthRole:
|
||||
"""FastAPI 依赖:未通过校验时抛出 401。
|
||||
|
||||
allowed_roles: 仅允许指定角色访问;None 表示 admin/user 均可。
|
||||
"""
|
||||
if not access_control_enabled():
|
||||
return AuthRole.ADMIN # 未启用门控时视为最高权限
|
||||
token = get_access_token_from_request(request)
|
||||
role = validate_access_token(token)
|
||||
if role is None:
|
||||
raise HTTPException(status_code=401, detail="访问令牌无效或已过期")
|
||||
if allowed_roles is not None and role not in allowed_roles:
|
||||
raise HTTPException(status_code=403, detail="权限不足")
|
||||
return role
|
||||
|
||||
|
||||
def require_admin(request: Request) -> AuthRole:
|
||||
"""FastAPI 依赖:仅管理员可访问。"""
|
||||
return require_access(request, allowed_roles={AuthRole.ADMIN})
|
||||
|
||||
|
||||
def is_public_path(path: str) -> bool:
|
||||
"""判断请求路径是否属于白名单(无需校验)。"""
|
||||
public_prefixes = (
|
||||
"/health",
|
||||
"/api/auth/",
|
||||
"/assets/",
|
||||
"/index.html",
|
||||
"/favicon.ico",
|
||||
"/robots.txt",
|
||||
"/manifest.json",
|
||||
)
|
||||
lowered = path.lower()
|
||||
return lowered.startswith(public_prefixes) or lowered == "/"
|
||||
|
||||
|
||||
def is_admin_path(path: str) -> bool:
|
||||
"""判断是否为管理员接口路径。"""
|
||||
return path.lower().startswith("/api/admin/")
|
||||
@@ -0,0 +1,4 @@
|
||||
"""回测模块 — 因子回测 + 策略回测 + 信号回测。
|
||||
|
||||
架构: BacktestEngine (共享) → FactorBacktestService / StrategyBacktestService
|
||||
"""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,480 @@
|
||||
"""因子回测服务 — IC/IR 分析 + 分层回测 + 多空组合。
|
||||
|
||||
纯 Polars 向量化实现，无 pandas 依赖。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, timedelta
|
||||
from typing import Literal
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
from app.backtest.engine import BacktestEngine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 可用因子列 (从 ENRICHED_COLUMNS 过滤出数值型指标)
|
||||
FACTOR_COLUMNS: list[dict] = [
|
||||
{"id": "momentum_5d", "label": "5日动量", "group": "动量", "desc": "5日涨跌幅，正值表示上涨趋势"},
|
||||
{"id": "momentum_10d", "label": "10日动量", "group": "动量", "desc": "10日涨跌幅，中短期趋势指标"},
|
||||
{"id": "momentum_20d", "label": "20日动量", "group": "动量", "desc": "月度涨跌幅，常用因子"},
|
||||
{"id": "momentum_30d", "label": "30日动量", "group": "动量", "desc": "30日涨跌幅"},
|
||||
{"id": "momentum_60d", "label": "60日动量", "group": "动量", "desc": "季度涨跌幅，中期动量"},
|
||||
{"id": "rsi_6", "label": "RSI(6)", "group": "超买超卖", "desc": "6日相对强弱指标，敏感度高"},
|
||||
{"id": "rsi_14", "label": "RSI(14)", "group": "超买超卖", "desc": "14日相对强弱指标，经典周期"},
|
||||
{"id": "rsi_24", "label": "RSI(24)", "group": "超买超卖", "desc": "24日相对强弱指标"},
|
||||
{"id": "annual_vol_20d","label": "20日波动率", "group": "波动率", "desc": "20日年化波动率"},
|
||||
{"id": "atr_14", "label": "ATR(14)", "group": "波动率", "desc": "14日平均真实波幅"},
|
||||
{"id": "vol_ratio_5d", "label": "量比(5日)", "group": "量价", "desc": "当日成交量 / 5日均量"},
|
||||
{"id": "turnover_rate", "label": "换手率", "group": "量价", "desc": "当日换手率"},
|
||||
{"id": "macd_hist", "label": "MACD柱", "group": "趋势", "desc": "MACD柱状图值"},
|
||||
{"id": "kdj_k", "label": "KDJ-K", "group": "趋势", "desc": "KDJ指标K值"},
|
||||
{"id": "change_pct", "label": "日涨跌幅", "group": "基础", "desc": "当日涨跌幅"},
|
||||
{"id": "amplitude", "label": "日振幅", "group": "基础", "desc": "当日振幅 (最高-最低)/昨收"},
|
||||
]
|
||||
|
||||
FACTOR_WARMUP_DAYS = 120
|
||||
|
||||
|
||||
@dataclass
|
||||
class FactorConfig:
|
||||
factor_name: str
|
||||
symbols: list[str] | None
|
||||
start: date
|
||||
end: date
|
||||
n_groups: int = 5
|
||||
rebalance: Literal["daily", "weekly", "monthly"] = "monthly"
|
||||
weight: Literal["equal", "factor_weight"] = "equal"
|
||||
fees_pct: float = 0.0002
|
||||
slippage_bps: float = 5.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class GroupStats:
|
||||
group: int
|
||||
label: str
|
||||
total_return: float
|
||||
annual_return: float
|
||||
max_drawdown: float
|
||||
sharpe: float
|
||||
win_rate: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class FactorResult:
|
||||
run_id: str
|
||||
config: dict
|
||||
# IC 分析
|
||||
ic_mean: float | None = None
|
||||
ic_std: float | None = None
|
||||
ir: float | None = None
|
||||
ic_win_rate: float | None = None
|
||||
ic_series: list[dict] = field(default_factory=list)
|
||||
# 分层
|
||||
group_stats: list[dict] = field(default_factory=list)
|
||||
group_nav: list[dict] = field(default_factory=list)
|
||||
# 多空
|
||||
long_short_stats: dict = field(default_factory=dict)
|
||||
long_short_nav: list[dict] = field(default_factory=list)
|
||||
# 元信息
|
||||
elapsed_ms: float = 0.0
|
||||
n_symbols: int = 0
|
||||
n_dates: int = 0
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class FactorBacktestService:
|
||||
def __init__(self, engine: BacktestEngine) -> None:
|
||||
self.engine = engine
|
||||
|
||||
def run(self, config: FactorConfig) -> FactorResult:
|
||||
t0 = time.perf_counter()
|
||||
run_id = uuid.uuid4().hex[:10]
|
||||
|
||||
def _err(msg: str) -> FactorResult:
|
||||
return FactorResult(
|
||||
run_id=run_id,
|
||||
config=self._config_to_dict(config),
|
||||
error=msg,
|
||||
elapsed_ms=(time.perf_counter() - t0) * 1000,
|
||||
)
|
||||
|
||||
# 加载基础面板: 当前 enriched parquet 只持久化基础列, 指标因子可能需要运行时计算。
|
||||
panel_columns = ["symbol", "date", "open", "high", "low", "close", "volume", "turnover_rate"]
|
||||
if config.factor_name not in panel_columns:
|
||||
panel_columns.append(config.factor_name)
|
||||
load_start = config.start
|
||||
if config.factor_name not in {"turnover_rate"}:
|
||||
load_start = config.start - timedelta(days=FACTOR_WARMUP_DAYS)
|
||||
|
||||
panel = self.engine.load_panel(
|
||||
config.symbols,
|
||||
load_start,
|
||||
config.end,
|
||||
columns=panel_columns,
|
||||
)
|
||||
if panel.is_empty():
|
||||
return _err("无数据，请检查日期范围或先运行盘后管道")
|
||||
|
||||
factor_col = config.factor_name
|
||||
if factor_col not in panel.columns:
|
||||
panel = self._compute_missing_factor(panel, factor_col)
|
||||
if factor_col not in panel.columns:
|
||||
return _err(f"因子列 '{factor_col}' 不存在于 enriched 数据中, 且无法从基础行情计算")
|
||||
if "close" not in panel.columns:
|
||||
return _err("enriched 数据缺少收盘价 close")
|
||||
panel = panel.select(["symbol", "date", "close", factor_col])
|
||||
panel = panel.filter((pl.col("date") >= config.start) & (pl.col("date") <= config.end))
|
||||
|
||||
# 过滤有效行
|
||||
panel = panel.filter(
|
||||
pl.col(factor_col).is_not_null()
|
||||
& pl.col("close").is_not_null()
|
||||
& (pl.col("close") > 0)
|
||||
)
|
||||
if panel.is_empty():
|
||||
return _err("过滤后无有效数据")
|
||||
|
||||
n_symbols = panel["symbol"].n_unique()
|
||||
n_dates = panel["date"].n_unique()
|
||||
|
||||
# 计算下期收益
|
||||
# 根据调仓频率计算不同周期的 forward return
|
||||
if config.rebalance == "daily":
|
||||
panel = panel.with_columns(
|
||||
(pl.col("close").shift(-1).over("symbol") / pl.col("close") - 1)
|
||||
.alias("_next_return")
|
||||
)
|
||||
else:
|
||||
# weekly/monthly: 计算到下个调仓日的收益
|
||||
panel = self._calc_period_return(panel, config.rebalance)
|
||||
|
||||
# ── 1. IC 分析 ──
|
||||
ic_df = self._calc_ic(panel, factor_col)
|
||||
ic_series = [
|
||||
{"date": str(row["date"]), "ic": round(float(row["ic"]), 4)}
|
||||
for row in ic_df.iter_rows(named=True)
|
||||
if row["ic"] is not None and not np.isnan(float(row["ic"]))
|
||||
]
|
||||
ic_values = [r["ic"] for r in ic_series]
|
||||
ic_mean = float(np.mean(ic_values)) if ic_values else None
|
||||
ic_std = float(np.std(ic_values)) if ic_values else None
|
||||
ir = (ic_mean / ic_std) if (ic_mean is not None and ic_std and ic_std > 1e-8) else None
|
||||
ic_win_rate = (sum(1 for v in ic_values if v > 0) / len(ic_values)) if ic_values else None
|
||||
|
||||
# ── 2. 分层回测 ──
|
||||
panel = self._add_groups(panel, factor_col, config.n_groups)
|
||||
group_nav = self._calc_group_nav(panel, config)
|
||||
group_stats = self._calc_group_stats(group_nav, config.start, config.end)
|
||||
|
||||
# ── 3. 多空组合 ──
|
||||
long_short_nav, long_short_stats = self._calc_long_short(group_nav, config)
|
||||
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
return FactorResult(
|
||||
run_id=run_id,
|
||||
config=self._config_to_dict(config),
|
||||
ic_mean=round(ic_mean, 4) if ic_mean is not None else None,
|
||||
ic_std=round(ic_std, 4) if ic_std is not None else None,
|
||||
ir=round(ir, 4) if ir is not None else None,
|
||||
ic_win_rate=round(ic_win_rate, 4) if ic_win_rate is not None else None,
|
||||
ic_series=ic_series,
|
||||
group_stats=group_stats,
|
||||
group_nav=group_nav,
|
||||
long_short_stats=long_short_stats,
|
||||
long_short_nav=long_short_nav,
|
||||
elapsed_ms=round(elapsed, 1),
|
||||
n_symbols=n_symbols,
|
||||
n_dates=n_dates,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _compute_missing_factor(panel: pl.DataFrame, factor_col: str) -> pl.DataFrame:
|
||||
required = {"symbol", "date", "open", "high", "low", "close", "volume"}
|
||||
if not required.issubset(panel.columns):
|
||||
missing = sorted(required - set(panel.columns))
|
||||
logger.warning("factor %s cannot be computed, missing columns: %s", factor_col, missing)
|
||||
return panel
|
||||
|
||||
from app.indicators.pipeline import compute_indicators
|
||||
|
||||
computed = compute_indicators(panel)
|
||||
if factor_col not in computed.columns:
|
||||
return panel
|
||||
return computed.select(["symbol", "date", "close", factor_col])
|
||||
|
||||
# ── IC 计算 ──
|
||||
|
||||
@staticmethod
|
||||
def _calc_ic(panel: pl.DataFrame, factor_col: str) -> pl.DataFrame:
|
||||
"""计算截面 Rank IC (因子值 rank vs 下期收益 rank 的相关系数)。"""
|
||||
return (
|
||||
panel.filter(pl.col("_next_return").is_not_null())
|
||||
.group_by("date")
|
||||
.agg(
|
||||
pl.corr(
|
||||
pl.col(factor_col).rank(method="random"),
|
||||
pl.col("_next_return").rank(method="random"),
|
||||
).alias("ic")
|
||||
)
|
||||
.sort("date")
|
||||
)
|
||||
|
||||
# ── 调仓期收益 ──
|
||||
|
||||
@staticmethod
|
||||
def _calc_period_return(panel: pl.DataFrame, rebalance: str) -> pl.DataFrame:
|
||||
"""计算到下个调仓日的收益。
|
||||
|
||||
weekly: 下周一的 open / 今日 close - 1
|
||||
monthly: 下月首个交易日的 open / 今日 close - 1
|
||||
只在调仓日标记行有效，其他行为 null。
|
||||
"""
|
||||
import datetime as _dt
|
||||
|
||||
all_dates = sorted(panel["date"].unique().to_list())
|
||||
date_set = set(all_dates)
|
||||
|
||||
if rebalance == "weekly":
|
||||
# 调仓日 = 每周一
|
||||
rebalance_dates = set()
|
||||
for d in all_dates:
|
||||
if hasattr(d, "weekday"):
|
||||
wd = d.weekday()
|
||||
else:
|
||||
wd = _dt.date.fromisoformat(str(d)).weekday()
|
||||
if wd == 0: # Monday
|
||||
rebalance_dates.add(d)
|
||||
else: # monthly
|
||||
# 调仓日 = 每月首个交易日
|
||||
seen_months: set[str] = set()
|
||||
rebalance_dates = set()
|
||||
for d in sorted(all_dates):
|
||||
m = str(d)[:7] # "YYYY-MM"
|
||||
if m not in seen_months:
|
||||
seen_months.add(m)
|
||||
rebalance_dates.add(d)
|
||||
|
||||
if not rebalance_dates:
|
||||
panel = panel.with_columns(pl.lit(None).cast(pl.Float64).alias("_next_return"))
|
||||
return panel
|
||||
|
||||
# 对每个调仓日，找到下一个调仓日
|
||||
sorted_rebalance = sorted(rebalance_dates)
|
||||
next_rebalance_map: dict = {}
|
||||
for i, d in enumerate(sorted_rebalance):
|
||||
if i + 1 < len(sorted_rebalance):
|
||||
next_rebalance_map[d] = sorted_rebalance[i + 1]
|
||||
# 最后一个调仓日没有下一个，不计算收益
|
||||
|
||||
# 构建 (date, symbol) → next_rebalance_date 的 close 价格映射
|
||||
# 简化: 用下个调仓日的 close / 当前 close
|
||||
panel = panel.sort(["symbol", "date"])
|
||||
dates_col = panel["date"].to_list()
|
||||
close_col = panel["close"].to_list()
|
||||
symbol_col = panel["symbol"].to_list()
|
||||
|
||||
# 先找下个调仓日的 close
|
||||
# 建立 (date, symbol) → close 的快速查找
|
||||
price_map: dict[tuple, float] = {}
|
||||
for i in range(len(dates_col)):
|
||||
price_map[(str(dates_col[i]), symbol_col[i])] = close_col[i]
|
||||
|
||||
next_returns = [None] * len(panel)
|
||||
for i in range(len(panel)):
|
||||
d = dates_col[i]
|
||||
d_val = d if isinstance(d, _dt.date) else _dt.date.fromisoformat(str(d))
|
||||
if d not in rebalance_dates:
|
||||
continue
|
||||
next_d = next_rebalance_map.get(d)
|
||||
if next_d is None:
|
||||
continue
|
||||
next_d_str = str(next_d)[:10]
|
||||
d_str = str(d)[:10]
|
||||
sym = symbol_col[i]
|
||||
next_close = price_map.get((next_d_str, sym))
|
||||
cur_close = close_col[i]
|
||||
if next_close is not None and cur_close and cur_close > 0:
|
||||
next_returns[i] = (next_close / cur_close - 1.0)
|
||||
|
||||
panel = panel.with_columns(
|
||||
pl.Series("_next_return", next_returns, dtype=pl.Float64)
|
||||
)
|
||||
return panel
|
||||
|
||||
# ── 分组 ──
|
||||
|
||||
@staticmethod
|
||||
def _add_groups(panel: pl.DataFrame, factor_col: str, n_groups: int) -> pl.DataFrame:
|
||||
"""截面分位数分组。"""
|
||||
return panel.with_columns(
|
||||
pl.col(factor_col)
|
||||
.qcut(n_groups, labels=[f"Q{i+1}" for i in range(n_groups)])
|
||||
.over("date")
|
||||
.alias("_group")
|
||||
)
|
||||
|
||||
# ── 分组净值 ──
|
||||
|
||||
@staticmethod
|
||||
def _calc_group_nav(panel: pl.DataFrame, config: FactorConfig) -> list[dict]:
|
||||
"""计算分组净值曲线 — 只在调仓日更新净值。"""
|
||||
# 只保留有下期收益的行 (= 调仓日)
|
||||
group_ret = (
|
||||
panel.filter(pl.col("_next_return").is_not_null() & pl.col("_group").is_not_null())
|
||||
.group_by(["date", "_group"])
|
||||
.agg(pl.col("_next_return").mean().alias("group_return"))
|
||||
)
|
||||
|
||||
# pivot: date × group
|
||||
pivot = group_ret.pivot(index="date", columns="_group", values="group_return").sort("date")
|
||||
|
||||
if pivot.is_empty():
|
||||
return []
|
||||
|
||||
group_cols = [c for c in pivot.columns if c != "date"]
|
||||
|
||||
# 累乘净值曲线
|
||||
result: list[dict] = []
|
||||
nav_values: dict[str, float] = {c: 1.0 for c in group_cols}
|
||||
|
||||
for row in pivot.iter_rows(named=True):
|
||||
entry: dict = {"date": str(row["date"])[:10]}
|
||||
for c in group_cols:
|
||||
ret = float(row[c]) if row[c] is not None else 0.0
|
||||
nav_values[c] *= (1 + ret)
|
||||
entry[c] = round(nav_values[c], 4)
|
||||
result.append(entry)
|
||||
|
||||
return result
|
||||
|
||||
# ── 分组统计 ──
|
||||
|
||||
@staticmethod
|
||||
def _calc_group_stats(
|
||||
group_nav: list[dict], start: date, end: date,
|
||||
) -> list[dict]:
|
||||
if not group_nav:
|
||||
return []
|
||||
|
||||
group_cols = [k for k in group_nav[0] if k != "date"]
|
||||
n_days = max((end - start).days, 1)
|
||||
years = n_days / 365.25
|
||||
|
||||
stats = []
|
||||
for i, c in enumerate(sorted(group_cols)):
|
||||
values = [r[c] for r in group_nav if r.get(c) is not None]
|
||||
if not values:
|
||||
continue
|
||||
total_return = values[-1] - 1.0
|
||||
annual_return = (values[-1]) ** (1 / max(years, 0.01)) - 1 if values[-1] > 0 else 0.0
|
||||
|
||||
# 最大回撤
|
||||
peak = 1.0
|
||||
max_dd = 0.0
|
||||
for v in values:
|
||||
peak = max(peak, v)
|
||||
dd = (v - peak) / peak
|
||||
max_dd = min(max_dd, dd)
|
||||
|
||||
# 日收益序列
|
||||
daily_rets = []
|
||||
for j in range(1, len(values)):
|
||||
if values[j - 1] > 0:
|
||||
daily_rets.append(values[j] / values[j - 1] - 1)
|
||||
|
||||
# 夏普
|
||||
if daily_rets:
|
||||
arr = np.array(daily_rets)
|
||||
sharpe = float(np.mean(arr) / np.std(arr)) * np.sqrt(252) if np.std(arr) > 0 else 0.0
|
||||
win_rate = float(np.mean(arr > 0))
|
||||
else:
|
||||
sharpe = 0.0
|
||||
win_rate = 0.0
|
||||
|
||||
stats.append({
|
||||
"group": i + 1,
|
||||
"label": c,
|
||||
"total_return": round(total_return, 4),
|
||||
"annual_return": round(annual_return, 4),
|
||||
"max_drawdown": round(max_dd, 4),
|
||||
"sharpe": round(sharpe, 2),
|
||||
"win_rate": round(win_rate, 4),
|
||||
})
|
||||
|
||||
return stats
|
||||
|
||||
# ── 多空组合 ──
|
||||
|
||||
@staticmethod
|
||||
def _calc_long_short(
|
||||
group_nav: list[dict], config: FactorConfig,
|
||||
) -> tuple[list[dict], dict]:
|
||||
"""多空组合: 做多最高组 + 做空最低组。"""
|
||||
if not group_nav:
|
||||
return [], {}
|
||||
|
||||
group_cols = sorted([k for k in group_nav[0] if k != "date"])
|
||||
if len(group_cols) < 2:
|
||||
return [], {}
|
||||
|
||||
top_col = group_cols[-1] # Q5 (最高)
|
||||
bottom_col = group_cols[0] # Q1 (最低)
|
||||
|
||||
# 独立计算 top 和 bottom 的日收益，然后合成
|
||||
ls_value = 1.0
|
||||
prev_top = 1.0
|
||||
prev_bot = 1.0
|
||||
peak = 1.0
|
||||
max_dd = 0.0
|
||||
ls_nav: list[dict] = []
|
||||
|
||||
for row in group_nav:
|
||||
top_nav = float(row.get(top_col, 1.0)) if row.get(top_col) is not None else 1.0
|
||||
bot_nav = float(row.get(bottom_col, 1.0)) if row.get(bottom_col) is not None else 1.0
|
||||
|
||||
# top 组收益 (做多)
|
||||
top_ret = (top_nav / prev_top - 1) if prev_top > 0 else 0.0
|
||||
# bottom 组收益 (做空 = 取反)
|
||||
bot_ret = -(bot_nav / prev_bot - 1) if prev_bot > 0 else 0.0
|
||||
# 多空组合收益
|
||||
ls_ret = (top_ret + bot_ret) / 2 # 各分配 50% 资金
|
||||
ls_value *= (1 + ls_ret)
|
||||
|
||||
prev_top = top_nav
|
||||
prev_bot = bot_nav
|
||||
|
||||
peak = max(peak, ls_value)
|
||||
dd = (ls_value - peak) / peak if peak > 0 else 0.0
|
||||
max_dd = min(max_dd, dd)
|
||||
|
||||
ls_nav.append({"date": row["date"], "value": round(ls_value, 4)})
|
||||
|
||||
total_ret = ls_value - 1.0
|
||||
ls_stats = {
|
||||
"total_return": round(total_ret, 4),
|
||||
"max_drawdown": round(max_dd, 4),
|
||||
"top_group": top_col,
|
||||
"bottom_group": bottom_col,
|
||||
}
|
||||
|
||||
return ls_nav, ls_stats
|
||||
|
||||
@staticmethod
|
||||
def _config_to_dict(c: FactorConfig) -> dict:
|
||||
return {
|
||||
"factor_name": c.factor_name,
|
||||
"symbols": c.symbols,
|
||||
"start": str(c.start),
|
||||
"end": str(c.end),
|
||||
"n_groups": c.n_groups,
|
||||
"rebalance": c.rebalance,
|
||||
"weight": c.weight,
|
||||
"fees_pct": c.fees_pct,
|
||||
"slippage_bps": c.slippage_bps,
|
||||
}
|
||||
@@ -0,0 +1,714 @@
|
||||
"""策略回测服务 — 复用 StrategyDef 体系做全周期回测。
|
||||
|
||||
核心优化: 向量化 filter_fn，不逐日调用 StrategyEngine.run()。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, timedelta
|
||||
from typing import Callable, Literal
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
from app.backtest.engine import BacktestEngine, MatcherConfig, SimResult
|
||||
from app.strategy.engine import StrategyEngine, StrategyDef
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BENCHMARK_SYMBOL = "000001.SH"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StrategyBacktestConfig:
|
||||
strategy_id: str
|
||||
symbols: list[str] | None
|
||||
start: date
|
||||
end: date
|
||||
params: dict | None = None
|
||||
overrides: dict | None = None
|
||||
# matching 为向后兼容入口; 显式传 entry_fill/exit_fill 时以二者为准。
|
||||
matching: Literal["close_t", "open_t+1"] = "open_t+1"
|
||||
entry_fill: Literal["close_t", "open_t+1"] | None = None
|
||||
exit_fill: Literal["close_t", "open_t+1"] | None = None
|
||||
fees_pct: float = 0.0002
|
||||
slippage_bps: float = 5.0
|
||||
max_positions: int = 10
|
||||
max_exposure_pct: float = 1.0
|
||||
initial_capital: float = 1_000_000.0
|
||||
position_sizing: Literal["equal", "score_weight"] = "equal"
|
||||
mode: Literal["position", "full"] = "position"
|
||||
holding_days: int = 5
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.entry_fill is None:
|
||||
self.entry_fill = self.matching
|
||||
if self.exit_fill is None:
|
||||
self.exit_fill = self.matching
|
||||
|
||||
|
||||
@dataclass
|
||||
class StrategyBacktestResult:
|
||||
run_id: str
|
||||
config: dict
|
||||
stats: dict = field(default_factory=dict)
|
||||
equity_curve: list[dict] = field(default_factory=list)
|
||||
drawdown_curve: list[dict] = field(default_factory=list)
|
||||
benchmark_curve: list[dict] = field(default_factory=list)
|
||||
trades: list[dict] = field(default_factory=list)
|
||||
per_symbol_stats: list[dict] = field(default_factory=list)
|
||||
strategy_info: dict = field(default_factory=dict)
|
||||
elapsed_ms: float = 0.0
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class StrategyBacktestService:
|
||||
def __init__(
|
||||
self,
|
||||
engine: BacktestEngine,
|
||||
strategy_engine: StrategyEngine,
|
||||
) -> None:
|
||||
self.engine = engine
|
||||
self.strategy_engine = strategy_engine
|
||||
|
||||
def run(
|
||||
self,
|
||||
config: StrategyBacktestConfig,
|
||||
progress_cb: "Callable[[dict], None] | None" = None,
|
||||
cancel_event: "threading.Event | None" = None,
|
||||
) -> StrategyBacktestResult:
|
||||
t0 = time.perf_counter()
|
||||
run_id = uuid.uuid4().hex[:10]
|
||||
|
||||
def _err(msg: str) -> StrategyBacktestResult:
|
||||
return StrategyBacktestResult(
|
||||
run_id=run_id,
|
||||
config=self._config_to_dict(config),
|
||||
error=msg,
|
||||
elapsed_ms=(time.perf_counter() - t0) * 1000,
|
||||
)
|
||||
|
||||
# 获取策略定义
|
||||
try:
|
||||
s = self.strategy_engine.get(config.strategy_id)
|
||||
except ValueError as e:
|
||||
return _err(str(e))
|
||||
|
||||
params = self._normalize_params(config.params or {}, s)
|
||||
overrides = config.overrides or {}
|
||||
basic_filter = self._effective_basic_filter(s, overrides)
|
||||
entry_signals = self._effective_signals(overrides, "entry_signals", s.entry_signals)
|
||||
exit_signals = self._effective_signals(overrides, "exit_signals", s.exit_signals)
|
||||
stop_loss = self._override_value(overrides, "stop_loss", s.stop_loss)
|
||||
trailing_stop = self._normalize_pct(
|
||||
self._override_value(overrides, "trailing_stop", getattr(s, "trailing_stop", None)),
|
||||
0.005,
|
||||
0.5,
|
||||
)
|
||||
trailing_take_profit_activate = self._normalize_pct(
|
||||
self._override_value(overrides, "trailing_take_profit_activate", getattr(s, "trailing_take_profit_activate", None)),
|
||||
0.01,
|
||||
2.0,
|
||||
)
|
||||
trailing_take_profit_drawdown = self._normalize_pct(
|
||||
self._override_value(overrides, "trailing_take_profit_drawdown", getattr(s, "trailing_take_profit_drawdown", None)),
|
||||
0.005,
|
||||
0.5,
|
||||
)
|
||||
if trailing_take_profit_activate is not None and trailing_take_profit_drawdown is not None:
|
||||
trailing_take_profit_drawdown = min(trailing_take_profit_drawdown, trailing_take_profit_activate)
|
||||
max_hold_days = self._override_value(overrides, "max_hold_days", s.max_hold_days)
|
||||
score_min, score_max = self._normalize_score_range(
|
||||
overrides.get("score_min"),
|
||||
overrides.get("score_max"),
|
||||
)
|
||||
|
||||
timing_ms: dict[str, float] = {}
|
||||
|
||||
# 加载面板 (含 warmup + 全量指标 + 信号)。warmup 只用于指标/形态计算, 不参与正式交易。
|
||||
warmup_days = max(120, int(max(s.lookback_days or 1, 1) * 1.5))
|
||||
load_start = config.start - timedelta(days=warmup_days)
|
||||
|
||||
# 全量模式: entries 只在正式区间触发, exits 需要 end 之后的尾部数据继续执行策略卖点。
|
||||
# 若策略有 max_hold_days, 用它决定尾部窗口；否则 holding_days 只作为兜底观察上限。
|
||||
full_horizon_days = int(max_hold_days or config.holding_days or 5)
|
||||
full_horizon_days = max(full_horizon_days, 1)
|
||||
load_end = config.end
|
||||
if config.mode == "full":
|
||||
fwd_buffer = full_horizon_days + 5 # 多取几天, 容错停牌缺口/open_t+1
|
||||
load_end = config.end + timedelta(days=fwd_buffer * 2) # 日历日放宽, 确保覆盖 N 个交易日
|
||||
|
||||
t_load = time.perf_counter()
|
||||
panel = self.engine.load_panel(config.symbols, load_start, load_end)
|
||||
timing_ms["load_panel"] = round((time.perf_counter() - t_load) * 1000, 1)
|
||||
if panel.is_empty():
|
||||
return _err("无数据，请检查日期范围或先运行盘后管道")
|
||||
|
||||
formal_range = self._date_range_mask(panel, config.start, config.end)
|
||||
if not formal_range.any():
|
||||
return _err("正式回测区间内无数据")
|
||||
|
||||
t_signal = time.perf_counter()
|
||||
|
||||
# basic_filter 只影响买入候选, 不能删除行情 panel, 否则持仓 mark / 卖出 / full forward return 都会失真。
|
||||
basic_mask = pl.Series("_basic", [True] * len(panel), dtype=pl.Boolean)
|
||||
if basic_filter and basic_filter.get("enabled", True):
|
||||
expr = StrategyEngine._basic_filter_expr(panel, basic_filter)
|
||||
if expr is not None:
|
||||
try:
|
||||
basic_mask = panel.select(expr.alias("_basic"))["_basic"].fill_null(False).cast(pl.Boolean)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("basic_filter mask failed: %s", e)
|
||||
return _err(f"基础过滤计算失败: {e}")
|
||||
|
||||
# 策略候选层用于评分归一化；entry_signals 只是买点层, 不参与 score universe。
|
||||
candidate_filter_mask = self._build_candidate_filter_mask(panel, s, params)
|
||||
candidate_mask = basic_mask & candidate_filter_mask
|
||||
panel = self._apply_score(panel, s, overrides, universe_mask=candidate_mask)
|
||||
|
||||
entry_mask = self._build_entry_mask_from_candidate(panel, candidate_mask, s, entry_signals)
|
||||
entry_mask = entry_mask & formal_range
|
||||
raw_exit_mask = self._build_signal_mask(panel, exit_signals, "_exit")
|
||||
exit_mask = raw_exit_mask & (self._date_range_mask(panel, config.start, load_end) if config.mode == "full" else formal_range)
|
||||
timing_ms["signals_score"] = round((time.perf_counter() - t_signal) * 1000, 1)
|
||||
|
||||
if not entry_mask.any():
|
||||
return _err("在指定区间内未产生买入信号")
|
||||
|
||||
# warmup 之后才交给撮合；full mode 保留 end 之后前瞻段用于 shift(-N)。
|
||||
sim_end = load_end if config.mode == "full" else config.end
|
||||
sim_range = self._date_range_mask(panel, config.start, sim_end)
|
||||
sim_panel = panel.filter(sim_range)
|
||||
sim_entry_mask = entry_mask.filter(sim_range)
|
||||
sim_exit_mask = exit_mask.filter(sim_range)
|
||||
if sim_panel.is_empty():
|
||||
return _err("正式回测区间内无数据")
|
||||
|
||||
t_sim = time.perf_counter()
|
||||
matcher_config = MatcherConfig(
|
||||
matching=config.matching,
|
||||
entry_fill=config.entry_fill,
|
||||
exit_fill=config.exit_fill,
|
||||
fees_pct=config.fees_pct,
|
||||
slippage_bps=config.slippage_bps,
|
||||
stop_loss_pct=stop_loss,
|
||||
trailing_stop_pct=trailing_stop,
|
||||
trailing_take_profit_activate_pct=trailing_take_profit_activate,
|
||||
trailing_take_profit_drawdown_pct=trailing_take_profit_drawdown,
|
||||
max_hold_days=max_hold_days,
|
||||
max_positions=config.max_positions,
|
||||
max_exposure_pct=config.max_exposure_pct,
|
||||
score_min=score_min,
|
||||
score_max=score_max,
|
||||
initial_capital=config.initial_capital,
|
||||
position_sizing=config.position_sizing,
|
||||
)
|
||||
# 撮合 — full 为全候选独立执行；position 为账户级仓位模拟。
|
||||
if config.mode == "full":
|
||||
result = self.engine.simulate_independent_candidates(
|
||||
sim_panel,
|
||||
sim_entry_mask,
|
||||
sim_exit_mask,
|
||||
matcher_config,
|
||||
progress_cb,
|
||||
cancel_event,
|
||||
)
|
||||
else:
|
||||
result = self.engine.simulate_portfolio(sim_panel, sim_entry_mask, sim_exit_mask, matcher_config, progress_cb, cancel_event)
|
||||
timing_ms["simulate"] = round((time.perf_counter() - t_sim) * 1000, 1)
|
||||
|
||||
# 检查是否被取消
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
return StrategyBacktestResult(
|
||||
run_id=run_id,
|
||||
config=self._config_to_dict(config),
|
||||
error="cancelled",
|
||||
elapsed_ms=round((time.perf_counter() - t0) * 1000, 1),
|
||||
)
|
||||
|
||||
if result.stats.get("error"):
|
||||
return _err(result.stats["error"])
|
||||
|
||||
timing_ms["total"] = round((time.perf_counter() - t0) * 1000, 1)
|
||||
result.stats["timing_ms"] = timing_ms
|
||||
result.stats["panel_rows"] = int(sim_panel.height)
|
||||
|
||||
benchmark_curve = self._build_benchmark_curve(config.start, config.end)
|
||||
|
||||
# 构建策略信息
|
||||
strategy_info = {
|
||||
"id": s.meta.get("id", config.strategy_id),
|
||||
"name": s.meta.get("name", config.strategy_id),
|
||||
"description": s.meta.get("description", ""),
|
||||
"entry_signals": entry_signals,
|
||||
"exit_signals": exit_signals,
|
||||
"stop_loss": stop_loss,
|
||||
"trailing_stop": trailing_stop,
|
||||
"trailing_take_profit_activate": trailing_take_profit_activate,
|
||||
"trailing_take_profit_drawdown": trailing_take_profit_drawdown,
|
||||
"max_hold_days": max_hold_days,
|
||||
"full_horizon_days": full_horizon_days,
|
||||
"score_min": score_min,
|
||||
"score_max": score_max,
|
||||
"source": s.source,
|
||||
}
|
||||
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
|
||||
return StrategyBacktestResult(
|
||||
run_id=run_id,
|
||||
config=self._config_to_dict(config),
|
||||
stats=result.stats,
|
||||
equity_curve=result.equity_curve,
|
||||
drawdown_curve=result.drawdown_curve,
|
||||
benchmark_curve=benchmark_curve,
|
||||
trades=[self._trade_to_dict(t) for t in result.trades],
|
||||
per_symbol_stats=result.per_symbol_stats,
|
||||
strategy_info=strategy_info,
|
||||
elapsed_ms=round(elapsed, 1),
|
||||
)
|
||||
|
||||
# ── 全量模拟 (选股能力统计, 不建组合不算净值) ──
|
||||
|
||||
def _run_full_simulation(
|
||||
self,
|
||||
panel: pl.DataFrame,
|
||||
entry_mask: pl.Series,
|
||||
holding_days: int,
|
||||
) -> SimResult:
|
||||
"""对 entry_mask 命中的全部候选, 算持有 N 天后的前瞻收益统计。
|
||||
|
||||
不受 max_positions/资金约束, 反映策略选股能力本身。
|
||||
equity_curve 复用为"累计日均超额收益曲线"(基准归零)。
|
||||
"""
|
||||
n = holding_days if holding_days and holding_days > 0 else 5
|
||||
|
||||
df = panel.with_columns([
|
||||
entry_mask.cast(pl.Boolean).alias("_is_candidate"),
|
||||
(pl.col("close").shift(-n).over("symbol") / pl.col("close") - 1).alias("_fwd_return"),
|
||||
]).filter(
|
||||
pl.col("_is_candidate")
|
||||
& pl.col("_fwd_return").is_not_null()
|
||||
& pl.col("_fwd_return").is_not_nan()
|
||||
)
|
||||
|
||||
if df.is_empty():
|
||||
return self.engine._empty_result()
|
||||
|
||||
fwd = df["_fwd_return"].to_numpy()
|
||||
wins = fwd[fwd > 0]
|
||||
losses = fwd[fwd <= 0]
|
||||
avg_win = float(wins.mean()) if wins.size else 0.0
|
||||
avg_loss = abs(float(losses.mean())) if losses.size else 0.0
|
||||
|
||||
# 按日聚合: 当日候选的平均前瞻收益
|
||||
daily = (
|
||||
df.group_by("date").agg(
|
||||
pl.col("_fwd_return").mean().alias("avg_ret"),
|
||||
pl.col("_fwd_return").count().alias("n_cand"),
|
||||
).sort("date")
|
||||
)
|
||||
|
||||
# 累计超额曲线: 每日复利平均收益 (基准归零, 故 equity 即累计策略收益)
|
||||
equity_curve: list[dict] = []
|
||||
equity = 1.0
|
||||
peak = 1.0
|
||||
drawdown_curve: list[dict] = []
|
||||
for row in daily.iter_rows(named=True):
|
||||
ret = float(row["avg_ret"] or 0.0)
|
||||
equity *= (1 + ret)
|
||||
peak = max(peak, equity)
|
||||
dd = (equity - peak) / peak if peak > 0 else 0.0
|
||||
d_str = str(row["date"])[:10]
|
||||
equity_curve.append({
|
||||
"date": d_str,
|
||||
"value": round(equity, 4),
|
||||
"positions": int(row["n_cand"]),
|
||||
})
|
||||
drawdown_curve.append({"date": d_str, "value": round(dd, 4)})
|
||||
|
||||
# 同期上证收益 (用 benchmark close 算)
|
||||
benchmark_curve = self._build_benchmark_curve(
|
||||
daily["date"].min(), daily["date"].max()
|
||||
)
|
||||
benchmark_return = 0.0
|
||||
if benchmark_curve:
|
||||
closes = [b["close"] for b in benchmark_curve if b.get("close")]
|
||||
if len(closes) >= 2 and closes[0] > 0:
|
||||
benchmark_return = closes[-1] / closes[0] - 1
|
||||
|
||||
total_return = equity - 1.0
|
||||
max_dd = min((d["value"] for d in drawdown_curve), default=0.0)
|
||||
|
||||
# 日收益序列算 Sharpe (年化)
|
||||
daily_rets = daily["avg_ret"].to_numpy()
|
||||
sharpe = (
|
||||
float(daily_rets.mean() / daily_rets.std() * np.sqrt(252))
|
||||
if daily_rets.size > 1 and daily_rets.std() > 0 else 0.0
|
||||
)
|
||||
|
||||
# 收益分布直方图: 按 [-20%, +20%] 分 21 档 (每档 2%), 超出归入首尾档
|
||||
lo, hi, nbins = -0.20, 0.20, 20
|
||||
clipped = np.clip(fwd, lo, hi)
|
||||
counts, edges = np.histogram(clipped, bins=nbins, range=(lo, hi))
|
||||
dist = [
|
||||
{
|
||||
"range": f"{(edges[i]*100):+.0f}~{(edges[i+1]*100):+.0f}%",
|
||||
"count": int(counts[i]),
|
||||
"ratio": round(float(counts[i] / fwd.size), 4) if fwd.size else 0.0,
|
||||
}
|
||||
for i in range(nbins)
|
||||
]
|
||||
|
||||
stats = {
|
||||
"mode": "full",
|
||||
"n_candidates": int(fwd.size),
|
||||
"n_days": int(daily.height),
|
||||
"avg_daily_candidates": round(float(daily["n_cand"].mean()), 1),
|
||||
"avg_return": round(float(fwd.mean()), 4),
|
||||
"median_return": round(float(np.median(fwd)), 4),
|
||||
"win_rate": round(float(wins.size / fwd.size), 4) if fwd.size else 0.0,
|
||||
"profit_factor": round(avg_win / avg_loss, 2) if avg_loss > 0 else None,
|
||||
"best": round(float(fwd.max()), 4),
|
||||
"worst": round(float(fwd.min()), 4),
|
||||
"total_return": round(float(total_return), 4),
|
||||
"max_drawdown": round(float(max_dd), 4),
|
||||
"sharpe": round(sharpe, 2),
|
||||
"benchmark_return": round(float(benchmark_return), 4),
|
||||
"excess": round(float(total_return - benchmark_return), 4),
|
||||
"return_distribution": dist,
|
||||
}
|
||||
|
||||
return SimResult(
|
||||
equity_curve=equity_curve,
|
||||
drawdown_curve=drawdown_curve,
|
||||
trades=[],
|
||||
per_symbol_stats=[],
|
||||
stats=stats,
|
||||
)
|
||||
|
||||
# ── 向量化信号生成 ──
|
||||
|
||||
@staticmethod
|
||||
def _date_range_mask(panel: pl.DataFrame, start: date, end: date) -> pl.Series:
|
||||
return panel.select(
|
||||
((pl.col("date") >= start) & (pl.col("date") <= end)).alias("_range")
|
||||
)["_range"].fill_null(False).cast(pl.Boolean)
|
||||
|
||||
def _build_candidate_filter_mask(
|
||||
self,
|
||||
panel: pl.DataFrame,
|
||||
s: StrategyDef,
|
||||
params: dict,
|
||||
) -> pl.Series:
|
||||
"""生成策略候选层 mask。filter_history/filter 决定候选池, 不包含 entry_signals。"""
|
||||
false_mask = pl.Series("_candidate_filter", [False] * len(panel), dtype=pl.Boolean)
|
||||
true_mask = pl.Series("_candidate_filter", [True] * len(panel), dtype=pl.Boolean)
|
||||
|
||||
history_failed = False
|
||||
# 优先: filter_history_fn 策略 (涨停/反包等多日形态, 与选股路径共用同一逻辑)
|
||||
if s.filter_history_fn:
|
||||
try:
|
||||
hit_df = s.filter_history_fn(panel, params)
|
||||
if hit_df is None or hit_df.is_empty():
|
||||
return false_mask
|
||||
# 命中行 (symbol,date) → 转 panel 等长布尔 mask
|
||||
hits = hit_df.select(["symbol", "date"]).unique()
|
||||
marked = (
|
||||
panel.select(["symbol", "date"])
|
||||
.join(
|
||||
hits.with_columns(pl.lit(True).alias("_hit")),
|
||||
on=["symbol", "date"],
|
||||
how="left",
|
||||
)
|
||||
)
|
||||
return marked["_hit"].fill_null(False).cast(pl.Boolean)
|
||||
except Exception as e:
|
||||
history_failed = True
|
||||
logger.warning("strategy filter_history_fn failed: %s", e)
|
||||
# 失败则回退到 filter_fn (若存在)
|
||||
|
||||
# 策略 filter_fn: 候选层 (filter_history 不可用或失败时)
|
||||
if s.filter_fn:
|
||||
try:
|
||||
expr = s.filter_fn(panel, params)
|
||||
if expr is not None:
|
||||
result = panel.select(expr.alias("_candidate_filter"))
|
||||
if not result.is_empty():
|
||||
return result["_candidate_filter"].fill_null(False).cast(pl.Boolean)
|
||||
except Exception as e:
|
||||
logger.warning("strategy filter_fn failed: %s", e)
|
||||
return false_mask
|
||||
|
||||
if history_failed:
|
||||
return false_mask
|
||||
|
||||
# 没有策略候选层时, 由 entry_signals 直接决定买点。
|
||||
return true_mask
|
||||
|
||||
def _build_entry_mask_from_candidate(
|
||||
self,
|
||||
panel: pl.DataFrame,
|
||||
candidate_mask: pl.Series,
|
||||
s: StrategyDef,
|
||||
entry_signals: list[str],
|
||||
) -> pl.Series:
|
||||
"""向量化生成买入掩码：候选层 AND 买点层；无买点时只用策略候选层。"""
|
||||
signal_mask = self._build_signal_mask(panel, entry_signals, "_entry_signal")
|
||||
if entry_signals:
|
||||
return candidate_mask & signal_mask
|
||||
if s.filter_history_fn or s.filter_fn:
|
||||
return candidate_mask
|
||||
return pl.Series("_entry", [False] * len(panel), dtype=pl.Boolean)
|
||||
|
||||
def _build_entry_mask(
|
||||
self,
|
||||
panel: pl.DataFrame,
|
||||
s: StrategyDef,
|
||||
params: dict,
|
||||
entry_signals: list[str],
|
||||
) -> pl.Series:
|
||||
"""兼容旧调用: 候选层 AND 买点层。"""
|
||||
candidate_mask = self._build_candidate_filter_mask(panel, s, params)
|
||||
return self._build_entry_mask_from_candidate(panel, candidate_mask, s, entry_signals)
|
||||
|
||||
@staticmethod
|
||||
def _build_signal_mask(panel: pl.DataFrame, signals: list[str], name: str) -> pl.Series:
|
||||
"""向量化合并信号列，多个信号 OR。支持内置 signal_ 与自定义 csg_ 前缀。"""
|
||||
masks: list[pl.Series] = []
|
||||
for sig in signals:
|
||||
# csg_ (自定义信号) 直接用；否则按 signal_ 解析
|
||||
col = sig if (sig.startswith("signal_") or sig.startswith("csg_")) else f"signal_{sig}"
|
||||
if col in panel.columns:
|
||||
masks.append(panel[col].fill_null(False).cast(pl.Boolean))
|
||||
|
||||
if not masks:
|
||||
return pl.Series(name, [False] * len(panel), dtype=pl.Boolean)
|
||||
|
||||
combined = masks[0]
|
||||
for m in masks[1:]:
|
||||
combined = combined | m
|
||||
return combined
|
||||
|
||||
def _build_benchmark_curve(self, start: date, end: date) -> list[dict]:
|
||||
try:
|
||||
df = self.engine.repo.get_index_daily(BENCHMARK_SYMBOL, start, end, columns=["date", "close"])
|
||||
except Exception as e:
|
||||
logger.warning("load benchmark %s failed: %s", BENCHMARK_SYMBOL, e)
|
||||
return []
|
||||
|
||||
if df.is_empty() or "close" not in df.columns:
|
||||
return []
|
||||
|
||||
df = df.filter(pl.col("close").is_not_null() & (pl.col("close") > 0)).sort("date")
|
||||
if df.is_empty():
|
||||
return []
|
||||
|
||||
return [
|
||||
{
|
||||
"date": str(row["date"])[:10],
|
||||
"value": round(float(row["close"]), 4),
|
||||
"close": round(float(row["close"]), 4),
|
||||
"name": "上证指数",
|
||||
"symbol": BENCHMARK_SYMBOL,
|
||||
}
|
||||
for row in df.iter_rows(named=True)
|
||||
if row["close"] is not None
|
||||
]
|
||||
|
||||
# ── 工具 ──
|
||||
|
||||
@staticmethod
|
||||
def _effective_basic_filter(s: StrategyDef, overrides: dict) -> dict:
|
||||
basic_filter = dict(s.basic_filter or {})
|
||||
override_filter = overrides.get("basic_filter")
|
||||
if isinstance(override_filter, dict):
|
||||
basic_filter.update(override_filter)
|
||||
return basic_filter
|
||||
|
||||
@staticmethod
|
||||
def _effective_signals(overrides: dict, key: str, default: list[str]) -> list[str]:
|
||||
value = overrides.get(key)
|
||||
if isinstance(value, list):
|
||||
return [str(v) for v in value if v]
|
||||
return list(default or [])
|
||||
|
||||
@staticmethod
|
||||
def _override_value(overrides: dict, key: str, default):
|
||||
if key in overrides:
|
||||
return overrides.get(key)
|
||||
return default
|
||||
|
||||
@staticmethod
|
||||
def _normalize_pct(value, min_value: float, max_value: float) -> float | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
try:
|
||||
pct = abs(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return min(max(pct, min_value), max_value)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_score_range(min_value, max_value) -> tuple[float | None, float | None]:
|
||||
def _bound(value) -> float | None:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
try:
|
||||
score = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not np.isfinite(score):
|
||||
return None
|
||||
return min(max(score, 0.0), 100.0)
|
||||
|
||||
score_min = _bound(min_value)
|
||||
score_max = _bound(max_value)
|
||||
if score_min is not None and score_max is not None and score_min > score_max:
|
||||
score_min, score_max = score_max, score_min
|
||||
return score_min, score_max
|
||||
|
||||
@staticmethod
|
||||
def _normalize_params(params: dict, s: StrategyDef) -> dict:
|
||||
normalized = dict(params)
|
||||
for param in s.meta.get("params", []):
|
||||
pid = param.get("id")
|
||||
if not pid:
|
||||
continue
|
||||
value = normalized.get(pid, param.get("default"))
|
||||
p_type = param.get("type")
|
||||
if p_type in {"float", "int"}:
|
||||
try:
|
||||
num = float(value)
|
||||
except (TypeError, ValueError):
|
||||
num = float(param.get("default", 0) or 0)
|
||||
if param.get("min") is not None:
|
||||
num = max(num, float(param["min"]))
|
||||
if param.get("max") is not None:
|
||||
num = min(num, float(param["max"]))
|
||||
normalized[pid] = int(num) if p_type == "int" else num
|
||||
elif p_type == "select" and param.get("options"):
|
||||
normalized[pid] = value if value in param["options"] else param.get("default")
|
||||
elif p_type == "bool":
|
||||
if isinstance(value, bool):
|
||||
normalized[pid] = value
|
||||
elif isinstance(value, str):
|
||||
normalized[pid] = value.lower() == "true"
|
||||
else:
|
||||
normalized[pid] = bool(param.get("default", False))
|
||||
else:
|
||||
normalized[pid] = value
|
||||
return normalized
|
||||
|
||||
@staticmethod
|
||||
def _trade_to_dict(t) -> dict:
|
||||
return {
|
||||
"symbol": t.symbol,
|
||||
"name": t.name,
|
||||
"entry_date": str(t.entry_date) if isinstance(t.entry_date, date) else str(t.entry_date),
|
||||
"exit_date": str(t.exit_date) if isinstance(t.exit_date, date) else str(t.exit_date),
|
||||
"entry_price": t.entry_price,
|
||||
"exit_price": t.exit_price,
|
||||
"pnl_pct": t.pnl_pct,
|
||||
"duration": t.duration,
|
||||
"exit_reason": t.exit_reason,
|
||||
"shares": t.shares,
|
||||
"lots": t.lots,
|
||||
"position_pct": t.position_pct,
|
||||
"entry_value": t.entry_value,
|
||||
"exit_value": t.exit_value,
|
||||
"pnl_amount": t.pnl_amount,
|
||||
"entry_score": getattr(t, "entry_score", None),
|
||||
"entry_signal_date": str(t.entry_signal_date) if getattr(t, "entry_signal_date", None) is not None else None,
|
||||
"exit_signal_date": str(t.exit_signal_date) if getattr(t, "exit_signal_date", None) is not None else None,
|
||||
"blocked_exit_days": getattr(t, "blocked_exit_days", 0),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _config_to_dict(c: StrategyBacktestConfig) -> dict:
|
||||
score_min, score_max = StrategyBacktestService._normalize_score_range(
|
||||
(c.overrides or {}).get("score_min"),
|
||||
(c.overrides or {}).get("score_max"),
|
||||
)
|
||||
return {
|
||||
"strategy_id": c.strategy_id,
|
||||
"symbols": c.symbols,
|
||||
"start": str(c.start),
|
||||
"end": str(c.end),
|
||||
"params": c.params,
|
||||
"overrides": c.overrides,
|
||||
"score_min": score_min,
|
||||
"score_max": score_max,
|
||||
"matching": c.matching,
|
||||
"entry_fill": c.entry_fill,
|
||||
"exit_fill": c.exit_fill,
|
||||
"fees_pct": c.fees_pct,
|
||||
"slippage_bps": c.slippage_bps,
|
||||
"max_positions": c.max_positions,
|
||||
"max_exposure_pct": c.max_exposure_pct,
|
||||
"initial_capital": c.initial_capital,
|
||||
"position_sizing": c.position_sizing,
|
||||
"mode": c.mode,
|
||||
"holding_days": c.holding_days,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _apply_score(
|
||||
panel: pl.DataFrame,
|
||||
s: StrategyDef,
|
||||
overrides: dict | None,
|
||||
universe_mask: pl.Series | None = None,
|
||||
) -> pl.DataFrame:
|
||||
scoring = s.meta.get("scoring", {})
|
||||
scoring_overrides = (overrides or {}).get("scoring")
|
||||
if scoring_overrides:
|
||||
scoring = {**scoring, **scoring_overrides}
|
||||
|
||||
work = panel
|
||||
has_universe = universe_mask is not None and len(universe_mask) == len(panel)
|
||||
if has_universe:
|
||||
work = work.with_columns(universe_mask.rename("_score_universe"))
|
||||
|
||||
def _value_in_universe(col: str) -> pl.Expr:
|
||||
if has_universe:
|
||||
return pl.when(pl.col("_score_universe")).then(pl.col(col)).otherwise(None)
|
||||
return pl.col(col)
|
||||
|
||||
def _finish(df: pl.DataFrame) -> pl.DataFrame:
|
||||
return df.drop("_score_universe") if "_score_universe" in df.columns else df
|
||||
|
||||
if scoring:
|
||||
total_weight = sum(scoring.values())
|
||||
if total_weight > 0:
|
||||
score_parts: list[pl.Expr] = []
|
||||
for col, weight in scoring.items():
|
||||
if col not in work.columns:
|
||||
continue
|
||||
w = weight / total_weight
|
||||
value = _value_in_universe(col)
|
||||
col_min = value.min().over("date")
|
||||
col_max = value.max().over("date")
|
||||
col_range = col_max - col_min
|
||||
normalized = pl.when(col_range > 0).then(
|
||||
(pl.col(col) - col_min) / col_range
|
||||
).otherwise(pl.lit(0.5))
|
||||
if has_universe:
|
||||
normalized = pl.when(pl.col("_score_universe")).then(normalized).otherwise(0.0)
|
||||
score_parts.append(normalized * w)
|
||||
if score_parts:
|
||||
score_expr = score_parts[0]
|
||||
for part in score_parts[1:]:
|
||||
score_expr = score_expr + part
|
||||
return _finish(work.with_columns((score_expr * 100).fill_null(0).alias("score")))
|
||||
|
||||
order_by = s.meta.get("order_by")
|
||||
if order_by and order_by != "score" and order_by in work.columns:
|
||||
direction = 1 if s.meta.get("descending", True) else -1
|
||||
score_expr = pl.col(order_by).fill_null(0) * direction
|
||||
if has_universe:
|
||||
score_expr = pl.when(pl.col("_score_universe")).then(score_expr).otherwise(0.0)
|
||||
return _finish(work.with_columns(score_expr.alias("score")))
|
||||
return _finish(work.with_columns(pl.lit(0.0).alias("score")))
|
||||
@@ -0,0 +1,50 @@
|
||||
"""全局配置 — 硬编码,个人工具不依赖 .env。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=None, # 不读取 .env
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
# 数据源 API Key(个人工具直接写死;如分享代码请改为空字符串或从环境变量注入)
|
||||
tickflow_api_key: str = "tk_94a20304993f45b5b0e376b9767597cc"
|
||||
|
||||
# AI(可选,留空即关闭)
|
||||
ai_provider: str = "openai_compat"
|
||||
ai_base_url: str = "https://api.deepseek.com/v1"
|
||||
ai_api_key: str = ""
|
||||
ai_model: str = "deepseek-chat"
|
||||
ai_daily_token_budget: int = 500_000
|
||||
|
||||
# Server
|
||||
host: str = "0.0.0.0"
|
||||
port: int = 3018
|
||||
log_level: str = "INFO"
|
||||
backtest_range_guard: bool = False
|
||||
|
||||
# 访问门控:留空则不启用,部署时通过 ACCESS_UUID 环境变量注入
|
||||
# 优先级:ADMIN_TOKEN > 动态 UUID > ACCESS_UUID
|
||||
access_uuid: str = ""
|
||||
# 管理员初始令牌,硬编码以便开箱即用;如需更安全可改为空字符串并从环境变量 ADMIN_TOKEN 注入
|
||||
admin_token: str = "admin7226132"
|
||||
|
||||
# 路径 — 硬编码为 Docker 容器内路径,确保数据持久化
|
||||
data_dir: Path = Path("/app/data")
|
||||
tiers_yaml: Path = Path("/app/tiers.yaml")
|
||||
static_dir: Path = Path("/app/static")
|
||||
|
||||
@property
|
||||
def use_free_mode(self) -> bool:
|
||||
"""是否走 Free 模式。优先看 secrets.json,其次看 .env。"""
|
||||
from app import secrets_store
|
||||
return not secrets_store.get_tickflow_key()
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1 @@
|
||||
"""技术指标 — Polars 实现(§7.5 / §7.7)。"""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1 @@
|
||||
"""APScheduler 任务。"""
|
||||
@@ -0,0 +1,532 @@
|
||||
"""盘后管道 + 盘前维表同步。
|
||||
|
||||
调度:
|
||||
09:10 盘前 — 同步标的维表 instruments (全量覆盖)
|
||||
15:30 盘后 — 日K同步 + 增量除权因子 + enriched 计算 + 刷新视图
|
||||
|
||||
盘后同步策略:
|
||||
日 K: QuoteService 交易时段已实时落盘 → 有数据时跳过 batch,首次拉 1 年区间
|
||||
除权因子: 从已有数据最新日期的下一天开始增量获取,避免重复拉取和计算
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
from app.indicators.pipeline import run_pipeline
|
||||
from app.config import settings
|
||||
from app.services import index_sync, instrument_sync, kline_sync
|
||||
from app.tickflow.capabilities import Cap, CapabilitySet
|
||||
from app.tickflow.pools import DEMO_SYMBOLS, get_pool
|
||||
from app.tickflow.repository import KlineRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ProgressCb = Callable[..., None]
|
||||
|
||||
|
||||
def _noop(stage: str, pct: int, msg: str, **kwargs) -> None: # noqa: ARG001
|
||||
pass
|
||||
|
||||
|
||||
def _invalidate(table: str | None = None) -> None:
|
||||
"""stage 写完调用,让 /api/data/status 只重算被影响的那张表。"""
|
||||
from app.api.data import invalidate_data_cache
|
||||
invalidate_data_cache(table)
|
||||
|
||||
|
||||
def _resolve_universe(capset: CapabilitySet) -> list[str]:
|
||||
"""解析标的池 — 以 CN_Equity_A (沪深京A股 ~5522只) 为主。
|
||||
|
||||
有 batch 能力 → 直接拉 CN_Equity_A universe
|
||||
其他用户 → 用 instruments parquet + watchlist 兜底
|
||||
"""
|
||||
if capset.has(Cap.KLINE_DAILY_BATCH):
|
||||
try:
|
||||
all_a = get_pool("CN_Equity_A", refresh=True)
|
||||
if all_a:
|
||||
return sorted(all_a)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("CN_Equity_A pool unavailable, fallback: %s", e)
|
||||
|
||||
# Free 用户兜底: instruments parquet + watchlist + demo
|
||||
base: set[str] = set(DEMO_SYMBOLS)
|
||||
base.update(get_pool("watchlist"))
|
||||
d = Path(settings.data_dir)
|
||||
inst_path = d / "instruments" / "instruments.parquet"
|
||||
if inst_path.exists():
|
||||
try:
|
||||
inst = pl.read_parquet(inst_path, columns=["symbol"])
|
||||
base.update(inst["symbol"].to_list())
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("instruments supplement failed: %s", e)
|
||||
return sorted(base)
|
||||
|
||||
|
||||
def run_instruments_sync(repo: KlineRepository) -> dict:
|
||||
"""盘前同步标的维表。"""
|
||||
rows = instrument_sync.sync_instruments(repo.store.data_dir)
|
||||
_refresh_instruments_view(repo)
|
||||
_invalidate("instruments")
|
||||
return {"instruments_rows": rows}
|
||||
|
||||
|
||||
def run_now(
|
||||
repo: KlineRepository,
|
||||
capset: CapabilitySet,
|
||||
on_progress: ProgressCb | None = None,
|
||||
) -> dict:
|
||||
"""立即执行一次盘后管道,支持进度回调。
|
||||
|
||||
跳过的 stage **不 emit**,避免前端把"无 capability"的卡片错误标记为 active/done。
|
||||
result 里带 skipped_stages 列表供前端展示。
|
||||
"""
|
||||
emit = on_progress or _noop
|
||||
skipped: list[str] = []
|
||||
|
||||
# Step 0: 先同步标的维表, 再解析标的池 — 确保标的池基于最新 instruments
|
||||
emit("sync_instruments", 2, "同步标的维表…")
|
||||
inst_rows = instrument_sync.sync_instruments(repo.store.data_dir)
|
||||
if inst_rows > 0:
|
||||
_refresh_instruments_view(repo)
|
||||
emit("sync_instruments", 8, f"标的维表同步完成,{inst_rows} 只标的")
|
||||
_invalidate("instruments")
|
||||
|
||||
emit("resolve_universe", 9, "解析标的池…")
|
||||
universe = _resolve_universe(capset)
|
||||
emit("resolve_universe", 10, f"标的池规模:{len(universe)} 只")
|
||||
|
||||
# Step 1: 日 K 同步
|
||||
# 今天有数据 → 实时行情接口拉一次覆写（1请求全市场）
|
||||
# 今天没数据 → batch K-line API 补齐
|
||||
# 无任何数据 → batch K-line API 拉首次 1 年
|
||||
from datetime import date as _date, timedelta as _td, datetime as _dt
|
||||
latest_daily = repo.latest_daily_date()
|
||||
today = _date.today()
|
||||
today_exists = latest_daily and latest_daily >= today
|
||||
new_daily_days = 0
|
||||
|
||||
if today_exists:
|
||||
# 今天有数据（QuoteService 已落盘）→ 实时行情覆写，确保最新
|
||||
emit("sync_daily", 12, f"获取日K [{today} ~ {today}] 实时行情…")
|
||||
written_daily = kline_sync.sync_daily_by_quotes(repo)
|
||||
new_daily_days = 1
|
||||
emit("sync_daily", 45, f"日K 完成,{written_daily} 只标的")
|
||||
logger.info("sync_daily: [%s ~ %s] live quotes, %d symbols", today, today, written_daily)
|
||||
elif latest_daily:
|
||||
# 有历史但今天没数据 → batch 补齐缺口
|
||||
start_date = latest_daily
|
||||
emit("sync_daily", 12, f"获取日K [{start_date} ~ {today}]…")
|
||||
logger.info("sync_daily: [%s ~ %s] gap fill", start_date, today)
|
||||
|
||||
def _daily_chunk_progress(cur: int, tot: int) -> None:
|
||||
emit("sync_daily", 12 + int(33 * cur / tot),
|
||||
f"日K 批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
|
||||
written_daily = kline_sync.sync_and_persist_daily_batch(
|
||||
universe, repo, capset,
|
||||
start_date=_dt.combine(start_date, _dt.min.time()),
|
||||
end_date=_dt.combine(today, _dt.min.time()),
|
||||
on_chunk_done=_daily_chunk_progress,
|
||||
)
|
||||
gap_days = (today - start_date).days
|
||||
new_daily_days = gap_days
|
||||
emit("sync_daily", 45, f"日K 完成,覆盖 {gap_days} 天")
|
||||
logger.info("sync_daily: [%s ~ %s] done, %d days", start_date, today, gap_days)
|
||||
else:
|
||||
# 首次：无任何数据 → batch 拉 1 年
|
||||
start_date = today - _td(days=365)
|
||||
emit("sync_daily", 12, f"获取日K [{start_date} ~ {today}]…")
|
||||
logger.info("sync_daily: [%s ~ %s] initial fetch", start_date, today)
|
||||
|
||||
def _daily_chunk_progress(cur: int, tot: int) -> None:
|
||||
emit("sync_daily", 12 + int(33 * cur / tot),
|
||||
f"日K 批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
|
||||
written_daily = kline_sync.sync_and_persist_daily_batch(
|
||||
universe, repo, capset,
|
||||
start_date=_dt.combine(start_date, _dt.min.time()),
|
||||
end_date=_dt.combine(today, _dt.min.time()),
|
||||
on_chunk_done=_daily_chunk_progress,
|
||||
)
|
||||
new_daily_days = 365
|
||||
emit("sync_daily", 45, "日K 完成")
|
||||
logger.info("sync_daily: [%s ~ %s] done", start_date, today)
|
||||
_invalidate("daily")
|
||||
|
||||
# Step 1.5: 增量同步除权因子 — 从已有数据最新日期的下一天开始获取
|
||||
written_adj = 0
|
||||
affected_symbols: list[str] = []
|
||||
if capset.has(Cap.ADJ_FACTOR):
|
||||
from datetime import datetime, timedelta
|
||||
adj_end = datetime.now()
|
||||
# 从已有除权因子数据的最新日期开始获取，避免重复拉取
|
||||
adj_factor_path = repo.store.data_dir / "adj_factor" / "all.parquet"
|
||||
fallback_start = adj_end - timedelta(days=30)
|
||||
if adj_factor_path.exists():
|
||||
try:
|
||||
from datetime import date as date_cls
|
||||
max_date = pl.scan_parquet(adj_factor_path).select(
|
||||
pl.col("trade_date").max()
|
||||
).collect().item()
|
||||
if max_date is not None:
|
||||
# trade_date 可能是 date / datetime / string 类型
|
||||
if isinstance(max_date, str):
|
||||
td = date_cls.fromisoformat(max_date)
|
||||
elif isinstance(max_date, datetime):
|
||||
td = max_date.date()
|
||||
else:
|
||||
td = max_date
|
||||
adj_start = datetime.combine(td, datetime.min.time())
|
||||
else:
|
||||
adj_start = fallback_start
|
||||
except Exception:
|
||||
adj_start = fallback_start
|
||||
else:
|
||||
adj_start = fallback_start
|
||||
adj_start_str = adj_start.strftime("%Y-%m-%d")
|
||||
adj_end_str = adj_end.strftime("%Y-%m-%d")
|
||||
emit("sync_adj", 50, f"获取除权因子 [{adj_start_str} ~ {adj_end_str}]…")
|
||||
logger.info("sync_adj: [%s ~ %s] start", adj_start_str, adj_end_str)
|
||||
|
||||
def _adj_chunk_progress(cur: int, tot: int) -> None:
|
||||
emit("sync_adj", 50 + int(10 * cur / tot),
|
||||
f"除权因子批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
|
||||
written_adj, affected_symbols = kline_sync.sync_adj_factor(
|
||||
universe, repo, capset,
|
||||
start_time=adj_start, end_time=adj_end,
|
||||
on_chunk_done=_adj_chunk_progress,
|
||||
)
|
||||
if affected_symbols:
|
||||
_refresh_single_view(repo, "adj_factor")
|
||||
emit("sync_adj", 60, f"除权因子完成,新增 {len(affected_symbols)} 只个股")
|
||||
logger.info("sync_adj: [%s ~ %s] done, %d symbols", adj_start_str, adj_end_str, len(affected_symbols))
|
||||
else:
|
||||
emit("sync_adj", 60, "除权因子完成,无新增")
|
||||
logger.info("sync_adj: [%s ~ %s] no new factors", adj_start_str, adj_end_str)
|
||||
_invalidate("adj_factor")
|
||||
else:
|
||||
skipped.append("sync_adj")
|
||||
logger.info("sync_adj skipped: no ADJ_FACTOR capability")
|
||||
|
||||
# Step 2: 计算 enriched
|
||||
# 判断策略:
|
||||
# - 首次 (enriched 目录不存在) → 全量
|
||||
# - 往前扩展历史 (新日期 < enriched 已有最早日期) → 全量
|
||||
# 前面的除权因子会改变累积因子链,影响后面所有日期的复权价格
|
||||
# - 往后新增日期 (新日期 > enriched 已有最晚日期)
|
||||
# → 增量补新区块(所有标的) + 受除权影响个股全日期重算
|
||||
# - 无新日期 + 有新除权因子 → 增量: 只重算受影响个股的全部日期
|
||||
# - 无新日期 + 无变化 → 跳过
|
||||
enriched_dir = repo.store.data_dir / "kline_daily_enriched"
|
||||
enriched_exists = enriched_dir.exists() and any(enriched_dir.glob("date=*"))
|
||||
daily_dir = repo.store.data_dir / "kline_daily"
|
||||
daily_days = len(list(daily_dir.glob("date=*"))) if daily_dir.exists() else 0
|
||||
prev_enriched_days = len(list(enriched_dir.glob("date=*"))) if enriched_exists else 0
|
||||
|
||||
# 判断新日期方向: 找 daily 和 enriched 的日期集合做比较
|
||||
forward_incremental = False
|
||||
backward_extension = False
|
||||
|
||||
if daily_days > prev_enriched_days and enriched_exists:
|
||||
daily_dates = sorted(d.stem.split("=")[1] for d in daily_dir.glob("date=*"))
|
||||
enriched_dates = sorted(d.stem.split("=")[1] for d in enriched_dir.glob("date=*"))
|
||||
earliest_enriched = enriched_dates[0]
|
||||
latest_enriched = enriched_dates[-1]
|
||||
new_dates = set(daily_dates) - set(enriched_dates)
|
||||
if new_dates:
|
||||
# 有新日期早于 enriched 最早日期 → 往前扩展
|
||||
if any(d < earliest_enriched for d in new_dates):
|
||||
backward_extension = True
|
||||
# 有新日期晚于 enriched 最晚日期 → 往后新增
|
||||
if any(d > latest_enriched for d in new_dates):
|
||||
forward_incremental = True
|
||||
|
||||
def _enriched_batch_progress(cur: int, tot: int) -> None:
|
||||
emit("compute_enriched", 65 + int(23 * cur / tot),
|
||||
f"计算指标 批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
|
||||
|
||||
if not enriched_exists or backward_extension:
|
||||
# 首次 或 往前扩展 → 全量
|
||||
emit("compute_enriched", 65, "全量计算 enriched…")
|
||||
logger.info("compute_enriched: full rebuild (first=%s, backward=%s, daily=%d, enriched=%d)",
|
||||
not enriched_exists, backward_extension, daily_days, prev_enriched_days)
|
||||
written_enriched = run_pipeline(on_batch_done=_enriched_batch_progress)
|
||||
new_enriched_days = len(list(enriched_dir.glob("date=*")))
|
||||
emit("compute_enriched", 88, f"enriched 完成,覆盖 {new_enriched_days} 天")
|
||||
logger.info("compute_enriched: full rebuild done, %d days", new_enriched_days)
|
||||
elif forward_incremental:
|
||||
# 往后新增日期: 增量补新区块 + 受影响个股全日期重算
|
||||
symbols_to_recompute = list(set(affected_symbols)) if affected_symbols else []
|
||||
emit("compute_enriched", 65,
|
||||
f"增量计算 enriched (新日期 + {len(symbols_to_recompute)} 只个股重算)…"
|
||||
if symbols_to_recompute else "增量计算 enriched (新日期)…")
|
||||
logger.info("compute_enriched: forward incremental, %d symbols to recompute",
|
||||
len(symbols_to_recompute))
|
||||
written_enriched = run_pipeline(
|
||||
new_dates_only=True,
|
||||
symbols=symbols_to_recompute or None,
|
||||
on_batch_done=_enriched_batch_progress,
|
||||
)
|
||||
new_enriched_days = len(list(enriched_dir.glob("date=*")))
|
||||
emit("compute_enriched", 88, f"enriched 完成,覆盖 {new_enriched_days} 天")
|
||||
logger.info("compute_enriched: forward incremental done, %d days", new_enriched_days)
|
||||
elif affected_symbols:
|
||||
# 无新日期,仅除权因子变更 → 只重算受影响个股的全部日期
|
||||
emit("compute_enriched", 65, f"增量计算 enriched ({len(affected_symbols)} 只个股)…")
|
||||
logger.info("compute_enriched: adj_factor incremental, %d symbols", len(affected_symbols))
|
||||
written_enriched = run_pipeline(symbols=affected_symbols, on_batch_done=_enriched_batch_progress)
|
||||
emit("compute_enriched", 88, f"enriched 完成,{len(affected_symbols)} 只个股")
|
||||
else:
|
||||
written_enriched = 0
|
||||
logger.info("compute_enriched: skip (no new daily, no adj_factor changes)")
|
||||
_refresh_single_view(repo, "kline_enriched")
|
||||
_invalidate("enriched")
|
||||
|
||||
# Step 2.3: 指数同步 — 独立 kline_index_* 存储，不进入股票选股/策略链路。
|
||||
written_index_daily = 0
|
||||
index_count = 0
|
||||
if capset.has(Cap.KLINE_DAILY_BATCH):
|
||||
emit("sync_index", 88, "同步指数列表与日K…")
|
||||
try:
|
||||
index_count = index_sync.sync_index_instruments(repo)
|
||||
index_dir = repo.store.data_dir / "kline_index_enriched"
|
||||
index_dates = sorted(
|
||||
d.name[5:] for d in index_dir.glob("date=*")
|
||||
if d.is_dir() and d.name.startswith("date=")
|
||||
) if index_dir.exists() else []
|
||||
index_start = _date.fromisoformat(index_dates[-1]) if index_dates else today - _td(days=365)
|
||||
written_index_daily = index_sync.sync_and_persist_index_daily(
|
||||
repo,
|
||||
capset,
|
||||
start_date=_dt.combine(index_start, _dt.min.time()),
|
||||
end_date=_dt.combine(today, _dt.min.time()),
|
||||
)
|
||||
repo.refresh_index_views()
|
||||
_invalidate("index_instruments")
|
||||
_invalidate("index_daily")
|
||||
_invalidate("index_enriched")
|
||||
emit("sync_index", 89, f"指数完成,{index_count} 只指数,{written_index_daily} 行日K")
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("sync_index failed: %s", e)
|
||||
emit("sync_index", 89, f"指数同步失败:{e}")
|
||||
else:
|
||||
skipped.append("sync_index")
|
||||
|
||||
# Step 2.5: 分钟 K 同步(可选) — 未启用或无 capability 时静默跳过(不 emit)
|
||||
from app.services import preferences
|
||||
minute_on = preferences.get_minute_sync_enabled()
|
||||
minute_days = preferences.get_minute_sync_days()
|
||||
written_minute = 0
|
||||
if minute_on and capset.has(Cap.KLINE_MINUTE_BATCH):
|
||||
minute_start = today - _td(days=minute_days)
|
||||
emit("sync_minute", 90, f"获取分钟K [{minute_start} ~ {today}]…")
|
||||
logger.info("sync_minute: [%s ~ %s] start", minute_start, today)
|
||||
minute_symbols = _resolve_minute_symbols(capset)
|
||||
def _minute_chunk_progress(cur: int, tot: int) -> None:
|
||||
emit("sync_minute", 90 + int(3 * cur / tot),
|
||||
f"分钟K 批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
|
||||
written_minute = kline_sync.sync_and_persist_minute(
|
||||
minute_symbols, repo, capset, days=minute_days,
|
||||
on_chunk_done=_minute_chunk_progress,
|
||||
)
|
||||
minute_dir = repo.store.data_dir / "kline_minute"
|
||||
minute_cover_days = len(list(minute_dir.glob("date=*"))) if minute_dir.exists() else 0
|
||||
emit("sync_minute", 93, f"分钟K完成,覆盖 {minute_cover_days} 天")
|
||||
logger.info("sync_minute: [%s ~ %s] done, %d days", minute_start, today, minute_cover_days)
|
||||
_invalidate("minute")
|
||||
else:
|
||||
skipped.append("sync_minute")
|
||||
if minute_on:
|
||||
logger.info("sync_minute skipped: no KLINE_MINUTE_BATCH capability")
|
||||
else:
|
||||
logger.info("sync_minute skipped: user disabled")
|
||||
|
||||
# Step 3: 刷新视图
|
||||
emit("refresh_views", 95, "刷新 DuckDB 视图…")
|
||||
_refresh_views(repo)
|
||||
|
||||
emit("done", 100, "完成")
|
||||
_invalidate(None) # 兜底:全清
|
||||
|
||||
return {
|
||||
"universe_size": len(universe),
|
||||
"daily_days": new_daily_days,
|
||||
"adj_factor_symbols": len(affected_symbols),
|
||||
"enriched_days": written_enriched,
|
||||
"index_count": index_count,
|
||||
"index_daily_rows": written_index_daily,
|
||||
"minute_rows": written_minute,
|
||||
"skipped_stages": skipped,
|
||||
}
|
||||
|
||||
|
||||
def _refresh_views(repo: KlineRepository) -> None:
|
||||
"""刷新所有 DuckDB 视图。"""
|
||||
d = repo.store.data_dir.as_posix()
|
||||
views = {
|
||||
"kline_daily": f"{d}/kline_daily/**/*.parquet",
|
||||
"kline_enriched": f"{d}/kline_daily_enriched/**/*.parquet",
|
||||
"kline_index_daily": f"{d}/kline_index_daily/**/*.parquet",
|
||||
"kline_index_enriched": f"{d}/kline_index_enriched/**/*.parquet",
|
||||
"kline_minute": f"{d}/kline_minute/**/*.parquet",
|
||||
"adj_factor": f"{d}/adj_factor/**/*.parquet",
|
||||
"instruments": f"{d}/instruments/**/*.parquet",
|
||||
"instruments_index": f"{d}/instruments_index/**/*.parquet",
|
||||
}
|
||||
for name, path in views.items():
|
||||
try:
|
||||
repo.db.execute(
|
||||
f"CREATE OR REPLACE VIEW {name} AS "
|
||||
f"SELECT * FROM read_parquet('{path}', union_by_name=true)"
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("refresh view %s failed: %s", name, e)
|
||||
|
||||
|
||||
def _refresh_single_view(repo: KlineRepository, name: str) -> None:
|
||||
"""刷新单个 DuckDB 视图。"""
|
||||
d = repo.store.data_dir.as_posix()
|
||||
paths = {
|
||||
"kline_daily": f"{d}/kline_daily/**/*.parquet",
|
||||
"kline_enriched": f"{d}/kline_daily_enriched/**/*.parquet",
|
||||
"kline_index_daily": f"{d}/kline_index_daily/**/*.parquet",
|
||||
"kline_index_enriched": f"{d}/kline_index_enriched/**/*.parquet",
|
||||
"kline_minute": f"{d}/kline_minute/**/*.parquet",
|
||||
"adj_factor": f"{d}/adj_factor/**/*.parquet",
|
||||
"instruments": f"{d}/instruments/**/*.parquet",
|
||||
"instruments_index": f"{d}/instruments_index/**/*.parquet",
|
||||
}
|
||||
path = paths.get(name)
|
||||
if not path:
|
||||
return
|
||||
try:
|
||||
repo.db.execute(
|
||||
f"CREATE OR REPLACE VIEW {name} AS "
|
||||
f"SELECT * FROM read_parquet('{path}', union_by_name=true)"
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("refresh view %s failed: %s", name, e)
|
||||
|
||||
|
||||
def _resolve_minute_symbols(capset: CapabilitySet) -> list[str]:
|
||||
"""分钟 K 同步标的 — 与日K共用同一标的池。"""
|
||||
return _resolve_universe(capset)
|
||||
|
||||
|
||||
def _refresh_instruments_view(repo: KlineRepository) -> None:
|
||||
"""单独刷新 instruments 视图。"""
|
||||
d = repo.store.data_dir.as_posix()
|
||||
try:
|
||||
repo.db.execute(
|
||||
f"CREATE OR REPLACE VIEW instruments AS "
|
||||
f"SELECT * FROM read_parquet('{d}/instruments/**/*.parquet', union_by_name=true)"
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("refresh instruments view failed: %s", e)
|
||||
|
||||
|
||||
def _run_tracked(fn, job_label: str) -> None:
|
||||
"""调度触发时包装 JobStore 跟踪，确保同步历史有记录。"""
|
||||
from app.services.pipeline_jobs import job_store
|
||||
|
||||
job_id = job_store.create()
|
||||
job_store.start(job_id)
|
||||
|
||||
def progress(stage: str, pct: int, msg: str, stage_pct: int | None = None,
|
||||
skip_log: bool = False) -> None:
|
||||
job_store.progress(job_id, stage, pct, msg, stage_pct=stage_pct, skip_log=skip_log)
|
||||
|
||||
try:
|
||||
result = fn(on_progress=progress)
|
||||
job_store.succeed(job_id, result)
|
||||
logger.info("scheduled %s completed: job_id=%s", job_label, job_id)
|
||||
except Exception:
|
||||
logger.exception("scheduled %s failed: job_id=%s", job_label, job_id)
|
||||
job_store.fail(job_id, f"scheduled {job_label} failed")
|
||||
|
||||
|
||||
def start_scheduler(repo: KlineRepository, capset: CapabilitySet) -> AsyncIOScheduler:
|
||||
"""启动调度器。
|
||||
|
||||
工作日 09:10 — 同步标的维表
|
||||
工作日 HH:MM — 盘后管道（时间由用户偏好决定，默认 15:30）
|
||||
"""
|
||||
from app.services import preferences
|
||||
sched = preferences.get_pipeline_schedule()
|
||||
inst_sched = preferences.get_instruments_schedule()
|
||||
|
||||
scheduler = AsyncIOScheduler(timezone="Asia/Shanghai")
|
||||
|
||||
# 盘前: 同步 instruments（时间由偏好决定）
|
||||
def _instruments_task(on_progress=None):
|
||||
emit = on_progress or _noop
|
||||
emit("sync_instruments", 0, "同步标的维表…")
|
||||
result = run_instruments_sync(repo)
|
||||
emit("done", 100, f"标的维表同步完成,{result.get('instruments_rows', 0)} 只标的")
|
||||
return result
|
||||
|
||||
scheduler.add_job(
|
||||
lambda: _run_tracked(_instruments_task, "instruments_sync"),
|
||||
trigger=CronTrigger(day_of_week="mon-fri",
|
||||
hour=inst_sched["hour"], minute=inst_sched["minute"],
|
||||
timezone="Asia/Shanghai"),
|
||||
id="pre_market_instruments",
|
||||
misfire_grace_time=1800,
|
||||
replace_existing=True,
|
||||
)
|
||||
|
||||
# 盘后: 日 K + enriched（时间由偏好决定）
|
||||
scheduler.add_job(
|
||||
lambda: _run_tracked(
|
||||
lambda on_progress=None: run_now(repo, capset, on_progress=on_progress),
|
||||
"daily_pipeline",
|
||||
),
|
||||
trigger=CronTrigger(day_of_week="mon-fri",
|
||||
hour=sched["hour"], minute=sched["minute"],
|
||||
timezone="Asia/Shanghai"),
|
||||
id="daily_pipeline",
|
||||
misfire_grace_time=3600,
|
||||
replace_existing=True,
|
||||
)
|
||||
|
||||
# 盘后: 五档盘口 sealed 定版(时间由偏好决定, 默认15:02, 范围15:01~18:00)
|
||||
depth_sched = preferences.get_depth_finalize_time()
|
||||
|
||||
def _depth_finalize():
|
||||
depth_svc = getattr(_get_app_state(), "depth_service", None) if _get_app_state() else None
|
||||
if depth_svc:
|
||||
depth_svc.finalize()
|
||||
|
||||
scheduler.add_job(
|
||||
_depth_finalize,
|
||||
trigger=CronTrigger(day_of_week="mon-fri",
|
||||
hour=depth_sched["hour"], minute=depth_sched["minute"],
|
||||
timezone="Asia/Shanghai"),
|
||||
id="depth_finalize",
|
||||
misfire_grace_time=3600,
|
||||
replace_existing=True,
|
||||
)
|
||||
|
||||
scheduler.start()
|
||||
logger.info("scheduler started; instruments@%02d:%02d, pipeline@%02d:%02d, depth@%02d:%02d mon-fri",
|
||||
inst_sched["hour"], inst_sched["minute"], sched["hour"], sched["minute"],
|
||||
depth_sched["hour"], depth_sched["minute"])
|
||||
return scheduler
|
||||
|
||||
|
||||
# app_state 延迟引用(start_scheduler 在 lifespan 早期调用, app.state 可能还没就绪)
|
||||
_app_state_ref = None
|
||||
|
||||
|
||||
def set_app_state(app_state) -> None:
|
||||
"""lifespan 注册 app.state 引用, 供 scheduled job 访问 depth_service 等单例。"""
|
||||
global _app_state_ref
|
||||
_app_state_ref = app_state
|
||||
|
||||
|
||||
def _get_app_state():
|
||||
return _app_state_ref
|
||||
@@ -0,0 +1,265 @@
|
||||
"""FastAPI 入口。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from app import __version__
|
||||
from app import auth as auth_module
|
||||
from app.api import analysis, auth, backtest, data, ext_data, financials, indices, intraday, kline, monitor_rules, alerts, overview, pipeline, screener, settings as settings_api, signals, strategy, watchlist
|
||||
from app.api.routes import router as core_router
|
||||
from app.config import settings
|
||||
from app.jobs import daily_pipeline
|
||||
from app.services.quote_service import QuoteService
|
||||
from app.tickflow import client as tf_client
|
||||
from app.tickflow.policy import detect_capabilities
|
||||
from app.tickflow.repository import DataStore, KlineRepository
|
||||
|
||||
logging.basicConfig(
|
||||
level=settings.log_level,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
logger.info(
|
||||
"Stock Panel v%s starting (mode=%s)",
|
||||
__version__, tf_client.current_mode(),
|
||||
)
|
||||
|
||||
# 数据层
|
||||
store = DataStore()
|
||||
repo = KlineRepository(store)
|
||||
app.state.datastore = store
|
||||
app.state.repo = repo
|
||||
|
||||
# Polars 缓存预热
|
||||
repo.refresh_cache()
|
||||
|
||||
# 能力探测
|
||||
capset = detect_capabilities()
|
||||
app.state.capabilities = capset
|
||||
logger.info("ready; %d capabilities active", len(capset.all()))
|
||||
|
||||
# 全局行情服务
|
||||
qs = QuoteService()
|
||||
app.state.quote_service = qs
|
||||
qs.set_repo(repo)
|
||||
qs.boot_check()
|
||||
|
||||
# QuoteService 需要访问 strategy_monitor 等单例
|
||||
# 先创建 strategy_monitor，再注入 app.state
|
||||
from app.strategy.monitor import StrategyMonitorService
|
||||
strategy_monitor = StrategyMonitorService()
|
||||
app.state.strategy_monitor = strategy_monitor
|
||||
qs.set_app_state(app.state)
|
||||
|
||||
# 五档盘口 sealed 服务(真假涨停/跌停, 独立旁路线)
|
||||
from app.services.depth_service import DepthService
|
||||
depth_service = DepthService()
|
||||
depth_service.set_repo(repo)
|
||||
depth_service.set_app_state(app.state)
|
||||
app.state.depth_service = depth_service
|
||||
|
||||
# 启动调度器(若 enriched 数据为空,首次启动可手动 POST /api/pipeline/run)
|
||||
try:
|
||||
daily_pipeline.set_app_state(app.state) # 供 depth_finalize job 访问 depth_service
|
||||
scheduler = daily_pipeline.start_scheduler(repo, capset)
|
||||
app.state.scheduler = scheduler
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("scheduler not started: %s", e)
|
||||
app.state.scheduler = None
|
||||
|
||||
# depth sealed: 启动补跑(当天文件不存在) + 盘中轮询(有能力时)
|
||||
try:
|
||||
depth_service.boot_check()
|
||||
depth_service.start_polling()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth_service init failed: %s", e)
|
||||
|
||||
# 扩展数据定时拉取
|
||||
from app.services.ext_pull import pull_scheduler
|
||||
pull_scheduler.start(store.data_dir)
|
||||
pull_scheduler.refresh(store.data_dir)
|
||||
app.state.pull_scheduler = pull_scheduler
|
||||
|
||||
# 财务数据独立调度 (需 Expert 套餐)
|
||||
from app.services.financial_sync import financial_scheduler
|
||||
financial_scheduler.start(store.data_dir, capset)
|
||||
app.state.financial_scheduler = financial_scheduler
|
||||
|
||||
# 策略引擎
|
||||
from app.strategy.engine import StrategyEngine
|
||||
from app.strategy.monitor import StrategyMonitorService
|
||||
from app.services.screener import ScreenerService
|
||||
|
||||
_screener_svc = ScreenerService(repo)
|
||||
strategy_dirs = [
|
||||
Path(__file__).resolve().parent / "strategy" / "builtin",
|
||||
store.data_dir / "strategies" / "custom",
|
||||
store.data_dir / "strategies" / "ai",
|
||||
]
|
||||
strategy_engine = StrategyEngine(
|
||||
enriched_loader=_screener_svc._load_enriched_for_date,
|
||||
enriched_history_loader=_screener_svc._load_enriched_history,
|
||||
strategy_dirs=strategy_dirs,
|
||||
)
|
||||
app.state.strategy_engine = strategy_engine
|
||||
logger.info("strategy engine loaded: %d strategies", len(strategy_engine.list_strategies()))
|
||||
|
||||
# 通用监控规则引擎: 启动时 reload 规则到内存态 (修复重启后告警失效)
|
||||
from app.strategy.monitor import MonitorRuleEngine
|
||||
from app.strategy import monitor_rules as mr_store
|
||||
from app.services import preferences
|
||||
monitor_engine = MonitorRuleEngine()
|
||||
monitor_engine.set_strategy_engine(strategy_engine)
|
||||
monitor_engine.set_data_dir(store.data_dir)
|
||||
|
||||
# 自动迁移: 把旧 strategy_monitor_ids 同步为 type=strategy 规则 (统一到监控页)
|
||||
try:
|
||||
if preferences.get_strategy_monitor_enabled():
|
||||
ids = preferences.get_strategy_monitor_ids()
|
||||
if ids:
|
||||
names = {s.id: s.name for s in strategy_engine.list_strategies()}
|
||||
mr_store.migrate_strategy_monitors(store.data_dir, ids, names)
|
||||
logger.info("strategy monitor migrated: %d strategies", len(ids))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("strategy monitor migration failed: %s", e)
|
||||
|
||||
try:
|
||||
rules = mr_store.load_all(store.data_dir)
|
||||
monitor_engine.set_rules(rules)
|
||||
logger.info("monitor engine loaded: %d rules", monitor_engine.rule_count)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("monitor engine load failed: %s", e)
|
||||
app.state.monitor_engine = monitor_engine
|
||||
|
||||
yield
|
||||
|
||||
if app.state.scheduler:
|
||||
app.state.scheduler.shutdown(wait=False)
|
||||
ps = getattr(app.state, "pull_scheduler", None)
|
||||
if ps:
|
||||
ps.stop()
|
||||
fsc = getattr(app.state, "financial_scheduler", None)
|
||||
if fsc:
|
||||
fsc.stop()
|
||||
qs = getattr(app.state, "quote_service", None)
|
||||
if qs:
|
||||
qs.stop()
|
||||
dsvc = getattr(app.state, "depth_service", None)
|
||||
if dsvc:
|
||||
dsvc.stop_polling()
|
||||
logger.info("shutdown")
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="Stock Panel",
|
||||
version=__version__,
|
||||
description="A 股选股 + 监控 + 回测面板",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
# CORS: 允许局域网访问 (自托管场景, 放开所有来源)
|
||||
# 注: allow_credentials=True 与 allow_origins=['*'] 不能共存 (浏览器规范),
|
||||
# 本项目认证走 header (API Key), 不依赖 cookie, 故关闭 credentials 换取通配来源。
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=False,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
# 访问门控中间件:仅对 /api/* 路径校验,静态资源和前端路由放行,
|
||||
# 由前端 AccessGuard 控制 UI 展示。
|
||||
@app.middleware("http")
|
||||
async def access_uuid_middleware(request: Request, call_next):
|
||||
if auth_module.access_control_enabled():
|
||||
path = request.url.path
|
||||
# 白名单直接放行
|
||||
if not auth_module.is_public_path(path):
|
||||
token = auth_module.get_access_token_from_request(request)
|
||||
role = auth_module.validate_access_token(token)
|
||||
# 管理员接口需 admin 角色
|
||||
if auth_module.is_admin_path(path):
|
||||
if role != auth_module.AuthRole.ADMIN:
|
||||
return JSONResponse(
|
||||
status_code=403 if role else 401,
|
||||
content={"detail": "需要管理员权限"},
|
||||
)
|
||||
# 其它 API 调用需任意有效角色
|
||||
elif path.startswith("/api/"):
|
||||
if role is None:
|
||||
return JSONResponse(
|
||||
status_code=401,
|
||||
content={"detail": "访问令牌无效或已过期,请先验证"},
|
||||
)
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# 路由
|
||||
app.include_router(core_router)
|
||||
app.include_router(auth.router)
|
||||
app.include_router(auth.admin_router)
|
||||
app.include_router(kline.router)
|
||||
app.include_router(watchlist.router)
|
||||
app.include_router(screener.router)
|
||||
app.include_router(backtest.router)
|
||||
app.include_router(intraday.router)
|
||||
app.include_router(indices.router)
|
||||
app.include_router(overview.router)
|
||||
app.include_router(analysis.router)
|
||||
app.include_router(pipeline.router)
|
||||
app.include_router(data.router)
|
||||
app.include_router(ext_data.router)
|
||||
app.include_router(financials.router)
|
||||
app.include_router(settings_api.router)
|
||||
app.include_router(strategy.router)
|
||||
app.include_router(signals.router)
|
||||
app.include_router(monitor_rules.router)
|
||||
app.include_router(alerts.router)
|
||||
|
||||
|
||||
# 能力门控异常 → 403(而非默认 500)
|
||||
# 业务代码用 capset.require(Cap.X) 断言能力,缺失时抛 CapabilityDenied;
|
||||
# 若不注册 handler 会冒泡成 500 Internal Server Error,对前端不友好且语义错误。
|
||||
from app.tickflow.capabilities import CapabilityDenied
|
||||
|
||||
|
||||
@app.exception_handler(CapabilityDenied)
|
||||
async def capability_denied_handler(request: Request, exc: CapabilityDenied) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=403,
|
||||
content={"detail": str(exc), "suggestion": exc.suggestion},
|
||||
)
|
||||
|
||||
# 生产期静态文件(前端 dist)
|
||||
_static = Path(settings.static_dir)
|
||||
if _static.exists():
|
||||
if (_static / "assets").exists():
|
||||
app.mount("/assets", StaticFiles(directory=_static / "assets"), name="assets")
|
||||
|
||||
@app.get("/{full_path:path}", include_in_schema=False)
|
||||
def spa_fallback(full_path: str): # noqa: ARG001
|
||||
"""所有未匹配路径回退到 index.html — React Router 接管。
|
||||
|
||||
index.html 禁止缓存 (Cache-Control: no-store), 确保浏览器每次拿到
|
||||
最新版本引用的 JS/CSS 文件名 (assets 带 hash, 可长缓存)。
|
||||
"""
|
||||
index = _static / "index.html"
|
||||
if index.exists():
|
||||
return FileResponse(
|
||||
index,
|
||||
headers={"Cache-Control": "no-store, must-revalidate"},
|
||||
)
|
||||
return {"error": "frontend not built"}
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Key / 凭据本地存储(§14)。
|
||||
|
||||
存储位置:`data/user_data/secrets.json`,权限 0600。
|
||||
优先级:secrets.json > .env > 空(Free 模式)。
|
||||
|
||||
UI 改 Key 时只动这个文件,不动 .env。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _path() -> Path:
|
||||
from app.config import settings
|
||||
p = settings.data_dir / "user_data" / "secrets.json"
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def load() -> dict:
|
||||
p = _path()
|
||||
if p.exists():
|
||||
try:
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("secrets.json malformed: %s", e)
|
||||
return {}
|
||||
|
||||
|
||||
def save(updates: dict) -> dict:
|
||||
"""合并写入(不会清掉未提及的字段)。返回新内容。"""
|
||||
current = load()
|
||||
current.update({k: v for k, v in updates.items() if v is not None})
|
||||
p = _path()
|
||||
p.write_text(json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
try:
|
||||
os.chmod(p, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
return current
|
||||
|
||||
|
||||
def clear(*keys: str) -> dict:
|
||||
"""清掉指定字段(留空清全部)。"""
|
||||
p = _path()
|
||||
if not p.exists():
|
||||
return {}
|
||||
if not keys:
|
||||
p.unlink()
|
||||
return {}
|
||||
current = load()
|
||||
for k in keys:
|
||||
current.pop(k, None)
|
||||
p.write_text(json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
return current
|
||||
|
||||
|
||||
def get_tickflow_key() -> str:
|
||||
"""取当前数据源 Key:secrets.json 优先,否则 .env。"""
|
||||
val = load().get("tickflow_api_key")
|
||||
if val:
|
||||
return val
|
||||
from app.config import settings
|
||||
return settings.tickflow_api_key or ""
|
||||
|
||||
|
||||
def get_ai_key() -> str:
|
||||
"""取当前 AI Key:secrets.json 优先,否则 .env。"""
|
||||
val = load().get("ai_api_key")
|
||||
if val:
|
||||
return val
|
||||
from app.config import settings
|
||||
return settings.ai_api_key or ""
|
||||
|
||||
|
||||
def get_ai_config(key: str, default: str = "") -> str:
|
||||
"""取 AI 配置项:secrets.json 优先,否则 config。"""
|
||||
val = load().get(key)
|
||||
if val:
|
||||
return val
|
||||
from app.config import settings
|
||||
return getattr(settings, key, default) or default
|
||||
|
||||
|
||||
def mask(key: str, prefix: int = 4, suffix: int = 4) -> str:
|
||||
"""脱敏显示。"""
|
||||
if not key:
|
||||
return ""
|
||||
if len(key) <= prefix + suffix:
|
||||
return "•" * len(key)
|
||||
return f"{key[:prefix]}{'•' * 6}{key[-suffix:]}"
|
||||
@@ -0,0 +1 @@
|
||||
"""业务服务层。"""
|
||||
@@ -0,0 +1,209 @@
|
||||
"""告警触发记录存储 — JSONL 追加写 + 滚动清理。
|
||||
|
||||
职责:
|
||||
- 把每次触发的 AlertEvent 追加写入 data/user_data/alerts.jsonl
|
||||
- 提供查询 (按来源/类型过滤、时间倒序、限量)
|
||||
- 滚动清理: 保留近 N 天 + 上限 M 条 (取交集)
|
||||
|
||||
设计:
|
||||
- JSONL 每行一个 JSON 对象,便于增量追加和流式读取
|
||||
- 清理策略: 追加后按需 prune (按 ts 删旧),避免文件无限膨胀
|
||||
- 读时全量加载到内存过滤 (记录量受上限约束, 5000 条量级无压力)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 保留策略
|
||||
MAX_DAYS = 7
|
||||
MAX_RECORDS = 5000
|
||||
# 每隔多少次写入触发一次清理 (避免每次写都 prune)
|
||||
PRUNE_EVERY = 20
|
||||
|
||||
_lock = threading.Lock()
|
||||
_write_count = 0
|
||||
|
||||
|
||||
def _path(data_dir: Path) -> Path:
|
||||
p = data_dir / "user_data" / "alerts.jsonl"
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def append(data_dir: Path, event: dict) -> None:
|
||||
"""追加一条触发记录。event 应含 ts(毫秒)、rule_id、source 等字段。"""
|
||||
line = json.dumps(event, ensure_ascii=False)
|
||||
with _lock:
|
||||
p = _path(data_dir)
|
||||
with p.open("a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
global _write_count
|
||||
_write_count += 1
|
||||
if _write_count >= PRUNE_EVERY:
|
||||
_write_count = 0
|
||||
_prune_locked(p)
|
||||
|
||||
|
||||
def append_many(data_dir: Path, events: list[dict]) -> None:
|
||||
"""批量追加。"""
|
||||
if not events:
|
||||
return
|
||||
with _lock:
|
||||
p = _path(data_dir)
|
||||
with p.open("a", encoding="utf-8") as f:
|
||||
for ev in events:
|
||||
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
|
||||
global _write_count
|
||||
_write_count += len(events)
|
||||
if _write_count >= PRUNE_EVERY:
|
||||
_write_count = 0
|
||||
_prune_locked(p)
|
||||
|
||||
|
||||
def list_recent(
|
||||
data_dir: Path,
|
||||
days: int = MAX_DAYS,
|
||||
limit: int = MAX_RECORDS,
|
||||
source: str | None = None,
|
||||
type: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""读取近 N 天记录,按时间倒序,支持按 source/type 过滤。"""
|
||||
import time
|
||||
cutoff = (time.time() - days * 86400) * 1000 # 毫秒
|
||||
out: list[dict] = []
|
||||
p = _path(data_dir)
|
||||
if not p.exists():
|
||||
return []
|
||||
try:
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if ev.get("ts", 0) < cutoff:
|
||||
continue
|
||||
if source and ev.get("source") != source:
|
||||
continue
|
||||
if type and ev.get("type") != type:
|
||||
continue
|
||||
out.append(ev)
|
||||
except Exception as e:
|
||||
logger.warning("alert_store read failed: %s", e)
|
||||
return []
|
||||
# 时间倒序 + 截断
|
||||
out.sort(key=lambda x: x.get("ts", 0), reverse=True)
|
||||
return out[:limit]
|
||||
|
||||
|
||||
def clear(data_dir: Path) -> int:
|
||||
"""清空全部记录,返回清除的条数。"""
|
||||
with _lock:
|
||||
p = _path(data_dir)
|
||||
if not p.exists():
|
||||
return 0
|
||||
count = 0
|
||||
try:
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
count = sum(1 for line in f if line.strip())
|
||||
except Exception:
|
||||
pass
|
||||
p.write_text("", encoding="utf-8")
|
||||
return count
|
||||
|
||||
|
||||
def delete_one(data_dir: Path, ts: int) -> bool:
|
||||
"""删除指定 ts 的单条记录,返回是否删除成功。
|
||||
|
||||
JSONL 无主键, 用 ts(毫秒时间戳) 作为标识。
|
||||
若存在多条同 ts, 只删第一条。
|
||||
"""
|
||||
with _lock:
|
||||
p = _path(data_dir)
|
||||
if not p.exists():
|
||||
return False
|
||||
kept: list[dict] = []
|
||||
deleted = False
|
||||
try:
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if not deleted and ev.get("ts") == ts:
|
||||
deleted = True
|
||||
continue
|
||||
kept.append(ev)
|
||||
except Exception as e:
|
||||
logger.warning("alert_store delete_one read failed: %s", e)
|
||||
return False
|
||||
if not deleted:
|
||||
return False
|
||||
try:
|
||||
with p.open("w", encoding="utf-8") as f:
|
||||
for ev in kept:
|
||||
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
|
||||
except Exception as e:
|
||||
logger.warning("alert_store delete_one write failed: %s", e)
|
||||
return False
|
||||
return True
|
||||
return count
|
||||
|
||||
|
||||
def count(data_dir: Path) -> int:
|
||||
"""返回当前记录总数。"""
|
||||
p = _path(data_dir)
|
||||
if not p.exists():
|
||||
return 0
|
||||
try:
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
return sum(1 for line in f if line.strip())
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def _prune_locked(p: Path) -> None:
|
||||
"""(调用方需持锁) 保留近 MAX_DAYS 天 + 上限 MAX_RECORDS 条。"""
|
||||
import time
|
||||
cutoff = (time.time() - MAX_DAYS * 86400) * 1000
|
||||
kept: list[dict] = []
|
||||
try:
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if ev.get("ts", 0) >= cutoff:
|
||||
kept.append(ev)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
except Exception as e:
|
||||
logger.warning("alert_store prune read failed: %s", e)
|
||||
return
|
||||
# 上限截断 (保留最新的)
|
||||
if len(kept) > MAX_RECORDS:
|
||||
kept.sort(key=lambda x: x.get("ts", 0))
|
||||
kept = kept[-MAX_RECORDS:]
|
||||
# 重写文件
|
||||
try:
|
||||
with p.open("w", encoding="utf-8") as f:
|
||||
for ev in kept:
|
||||
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
|
||||
except Exception as e:
|
||||
logger.warning("alert_store prune write failed: %s", e)
|
||||
@@ -0,0 +1,392 @@
|
||||
"""回测服务(§6.7)。
|
||||
|
||||
包 vectorbt — 全项目唯一一处出现 pandas。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from typing import Literal
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import polars as pl
|
||||
|
||||
from app.config import settings
|
||||
from app.tickflow.repository import KlineRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# vectorbt 是 optional extras(见 pyproject.toml).未装时只有 backtest 不可用,其他功能正常.
|
||||
_vbt = None
|
||||
_vbt_unavailable_reason: str | None = None
|
||||
|
||||
|
||||
class VectorbtUnavailable(RuntimeError):
|
||||
"""vectorbt 未安装 — 提示用户 `uv sync --extra backtest`."""
|
||||
|
||||
|
||||
def _get_vbt():
|
||||
global _vbt, _vbt_unavailable_reason
|
||||
if _vbt is not None:
|
||||
return _vbt
|
||||
if _vbt_unavailable_reason is not None:
|
||||
raise VectorbtUnavailable(_vbt_unavailable_reason)
|
||||
try:
|
||||
import vectorbt as vbt
|
||||
_vbt = vbt
|
||||
return _vbt
|
||||
except ImportError as e:
|
||||
_vbt_unavailable_reason = (
|
||||
"vectorbt 未安装 — 它是回测的可选依赖.macOS Intel 用户先 `brew install cmake` "
|
||||
"然后 `uv sync --extra backtest`"
|
||||
)
|
||||
logger.warning("vectorbt unavailable: %s", e)
|
||||
raise VectorbtUnavailable(_vbt_unavailable_reason) from e
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
"""供 API 层快速检测."""
|
||||
try:
|
||||
_get_vbt()
|
||||
return True
|
||||
except VectorbtUnavailable:
|
||||
return False
|
||||
|
||||
|
||||
SignalKind = Literal[
|
||||
"macd_golden", "macd_dead",
|
||||
"ma_golden_5_20", "ma_dead_5_20",
|
||||
"ma_golden_20_60",
|
||||
"ma20_breakout", "ma20_breakdown",
|
||||
"n_day_high", "n_day_low",
|
||||
"boll_breakout_upper", "boll_breakdown_lower",
|
||||
"volume_surge",
|
||||
"rsi_oversold", "rsi_overbought",
|
||||
"stop_loss", "trailing_stop", "max_hold",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class BacktestConfig:
|
||||
symbols: list[str]
|
||||
start: date
|
||||
end: date
|
||||
# 买入信号(任一触发即买)
|
||||
entries: list[str] = field(default_factory=list)
|
||||
# 卖出信号(任一触发即卖)
|
||||
exits: list[str] = field(default_factory=list)
|
||||
# 其他参数
|
||||
stop_loss_pct: float | None = None # 例 -0.05 = -5%
|
||||
max_hold_days: int | None = None
|
||||
fees_pct: float = 0.0002 # 万二佣金
|
||||
slippage_bps: float = 5 # 5 bps
|
||||
# 撮合
|
||||
matching: Literal["close_t", "open_t+1"] = "close_t"
|
||||
rsi_oversold_threshold: float = 30
|
||||
rsi_overbought_threshold: float = 70
|
||||
|
||||
|
||||
@dataclass
|
||||
class BacktestResult:
|
||||
run_id: str
|
||||
config: dict
|
||||
stats: dict
|
||||
equity_curve: list[dict] # [{date, value}]
|
||||
trades: list[dict] # [{symbol, entry_date, exit_date, pnl_pct, ...}]
|
||||
per_symbol_stats: list[dict] # 每只股票的统计
|
||||
|
||||
|
||||
# enriched 表里的信号列名映射
|
||||
_SIGNAL_COLS: dict[SignalKind, str] = {
|
||||
"macd_golden": "signal_macd_golden",
|
||||
"macd_dead": "signal_macd_dead",
|
||||
"ma_golden_5_20": "signal_ma_golden_5_20",
|
||||
"ma_dead_5_20": "signal_ma_dead_5_20",
|
||||
"ma_golden_20_60": "signal_ma_golden_20_60",
|
||||
"ma20_breakout": "signal_ma20_breakout",
|
||||
"ma20_breakdown": "signal_ma20_breakdown",
|
||||
"n_day_high": "signal_n_day_high",
|
||||
"n_day_low": "signal_n_day_low",
|
||||
"boll_breakout_upper": "signal_boll_breakout_upper",
|
||||
"boll_breakdown_lower": "signal_boll_breakdown_lower",
|
||||
"volume_surge": "signal_volume_surge",
|
||||
}
|
||||
|
||||
|
||||
class BacktestService:
|
||||
def __init__(self, repo: KlineRepository) -> None:
|
||||
self.repo = repo
|
||||
|
||||
def _load_panel(
|
||||
self,
|
||||
symbols: list[str],
|
||||
start: date,
|
||||
end: date,
|
||||
) -> pd.DataFrame:
|
||||
"""加载 [date × symbol] 价格面板 — Polars scan_parquet + 即时计算指标。
|
||||
|
||||
**全项目唯一从 Polars 转 pandas 的边界**(§7.4 / ADR-19)。
|
||||
"""
|
||||
try:
|
||||
enriched_glob = str(self.repo.store.data_dir / "kline_daily_enriched" / "**" / "*.parquet")
|
||||
df = (
|
||||
pl.scan_parquet(enriched_glob)
|
||||
.filter(
|
||||
(pl.col("symbol").is_in(symbols))
|
||||
& (pl.col("date") >= start)
|
||||
& (pl.col("date") <= end)
|
||||
)
|
||||
.sort(["date", "symbol"])
|
||||
.collect()
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("backtest load failed: %s", e)
|
||||
return pd.DataFrame()
|
||||
|
||||
if df.is_empty():
|
||||
return pd.DataFrame()
|
||||
|
||||
# 即时计算指标 + 信号
|
||||
from app.indicators.pipeline import compute_all
|
||||
df = compute_all(df)
|
||||
|
||||
# 选择需要的列
|
||||
needed_cols = [
|
||||
"date", "symbol", "open", "high", "low", "close", "volume",
|
||||
"rsi_14", "signal_macd_golden", "signal_macd_dead",
|
||||
"signal_ma_golden_5_20", "signal_ma_dead_5_20",
|
||||
"signal_ma_golden_20_60",
|
||||
"signal_ma20_breakout", "signal_ma20_breakdown",
|
||||
"signal_n_day_high", "signal_n_day_low",
|
||||
"signal_boll_breakout_upper", "signal_boll_breakdown_lower",
|
||||
"signal_volume_surge",
|
||||
]
|
||||
existing = [c for c in needed_cols if c in df.columns]
|
||||
df = df.select(existing)
|
||||
|
||||
# to_pandas 边界
|
||||
return df.to_pandas(use_pyarrow_extension_array=False)
|
||||
|
||||
def _build_signal_matrix(
|
||||
self,
|
||||
panel: pd.DataFrame,
|
||||
kinds: list[str],
|
||||
config: BacktestConfig,
|
||||
) -> pd.DataFrame:
|
||||
"""从面板构造 [date × symbol] 的布尔信号矩阵。"""
|
||||
if not kinds or panel.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
# pivot 成 [date × symbol] 形式
|
||||
result = None
|
||||
for kind in kinds:
|
||||
mat = None
|
||||
if kind in _SIGNAL_COLS:
|
||||
col = _SIGNAL_COLS[kind]
|
||||
mat = panel.pivot(index="date", columns="symbol", values=col).fillna(False).astype(bool)
|
||||
elif kind == "rsi_oversold":
|
||||
mat = (panel.pivot(index="date", columns="symbol", values="rsi_14")
|
||||
< config.rsi_oversold_threshold)
|
||||
elif kind == "rsi_overbought":
|
||||
mat = (panel.pivot(index="date", columns="symbol", values="rsi_14")
|
||||
> config.rsi_overbought_threshold)
|
||||
# stop_loss / trailing / max_hold 通过 vectorbt 参数处理,不参与信号矩阵
|
||||
|
||||
if mat is not None:
|
||||
result = mat if result is None else (result | mat)
|
||||
return result if result is not None else pd.DataFrame()
|
||||
|
||||
def run(self, config: BacktestConfig) -> BacktestResult:
|
||||
vbt = _get_vbt()
|
||||
run_id = uuid.uuid4().hex[:10]
|
||||
|
||||
panel = self._load_panel(config.symbols, config.start, config.end)
|
||||
if panel.empty:
|
||||
return BacktestResult(
|
||||
run_id=run_id,
|
||||
config=_config_to_dict(config),
|
||||
stats={"error": "no data"},
|
||||
equity_curve=[],
|
||||
trades=[],
|
||||
per_symbol_stats=[],
|
||||
)
|
||||
|
||||
# 价格面板
|
||||
close = panel.pivot(index="date", columns="symbol", values="close")
|
||||
|
||||
# 信号矩阵
|
||||
entries = self._build_signal_matrix(panel, config.entries, config)
|
||||
exits = self._build_signal_matrix(panel, config.exits, config)
|
||||
|
||||
# 对齐 index/columns
|
||||
if not entries.empty:
|
||||
entries = entries.reindex_like(close).fillna(False).astype(bool)
|
||||
else:
|
||||
entries = pd.DataFrame(False, index=close.index, columns=close.columns)
|
||||
if not exits.empty:
|
||||
exits = exits.reindex_like(close).fillna(False).astype(bool)
|
||||
else:
|
||||
exits = pd.DataFrame(False, index=close.index, columns=close.columns)
|
||||
|
||||
if not entries.any().any():
|
||||
return BacktestResult(
|
||||
run_id=run_id,
|
||||
config=_config_to_dict(config),
|
||||
stats={"error": "no buy signals"},
|
||||
equity_curve=[],
|
||||
trades=[],
|
||||
per_symbol_stats=[],
|
||||
)
|
||||
|
||||
# T+1 适配:vectorbt 默认信号当根 K 撮合
|
||||
# close_t 撮合:维持默认
|
||||
# open_t+1 撮合:shift 信号 1 根 + 用 open 作为价
|
||||
if config.matching == "open_t+1":
|
||||
entries = entries.shift(1).fillna(False).astype(bool)
|
||||
exits = exits.shift(1).fillna(False).astype(bool)
|
||||
price = panel.pivot(index="date", columns="symbol", values="open")
|
||||
else:
|
||||
price = close
|
||||
|
||||
# 跑回测
|
||||
try:
|
||||
pf_kwargs = dict(
|
||||
close=close,
|
||||
entries=entries,
|
||||
exits=exits,
|
||||
price=price,
|
||||
fees=config.fees_pct,
|
||||
slippage=config.slippage_bps / 10000.0,
|
||||
freq="1D",
|
||||
)
|
||||
if config.stop_loss_pct is not None:
|
||||
pf_kwargs["sl_stop"] = abs(config.stop_loss_pct)
|
||||
if config.max_hold_days is not None:
|
||||
# vectorbt 没有内置 max-hold;用时间退出近似:
|
||||
# 在 max_hold_days 后强制 exit
|
||||
exits_idx = entries.copy()
|
||||
for col in entries.columns:
|
||||
entry_rows = np.where(entries[col].values)[0]
|
||||
for i in entry_rows:
|
||||
end_i = min(i + config.max_hold_days, len(entries) - 1)
|
||||
if end_i > i:
|
||||
exits_idx.iloc[end_i][col] = True
|
||||
pf_kwargs["exits"] = (exits | exits_idx).astype(bool)
|
||||
|
||||
pf = vbt.Portfolio.from_signals(**pf_kwargs)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.exception("vectorbt backtest failed")
|
||||
return BacktestResult(
|
||||
run_id=run_id,
|
||||
config=_config_to_dict(config),
|
||||
stats={"error": str(e)},
|
||||
equity_curve=[],
|
||||
trades=[],
|
||||
per_symbol_stats=[],
|
||||
)
|
||||
|
||||
# 提取结果
|
||||
try:
|
||||
stats_series = pf.stats(silence_warnings=True)
|
||||
if isinstance(stats_series, pd.DataFrame):
|
||||
# 多列时取 agg
|
||||
stats_dict = stats_series.mean(numeric_only=True).to_dict()
|
||||
else:
|
||||
stats_dict = stats_series.to_dict()
|
||||
except Exception: # noqa: BLE001
|
||||
stats_dict = {}
|
||||
|
||||
# 净值曲线(组合平均)
|
||||
equity = pf.value().mean(axis=1) if isinstance(pf.value(), pd.DataFrame) else pf.value()
|
||||
equity_curve = [
|
||||
{"date": str(idx.date() if hasattr(idx, "date") else idx), "value": float(v)}
|
||||
for idx, v in equity.items() if pd.notna(v)
|
||||
]
|
||||
|
||||
# 交易记录
|
||||
try:
|
||||
trades_df = pf.trades.records_readable
|
||||
trades = trades_df.to_dict(orient="records") if not trades_df.empty else []
|
||||
# 字段名美化
|
||||
trades = [
|
||||
{
|
||||
"symbol": t.get("Column", t.get("Symbol", "")),
|
||||
"entry_date": str(t.get("Entry Timestamp", t.get("Entry Date", ""))),
|
||||
"exit_date": str(t.get("Exit Timestamp", t.get("Exit Date", ""))),
|
||||
"entry_price": float(t.get("Avg Entry Price", t.get("Avg. Entry Price", 0))),
|
||||
"exit_price": float(t.get("Avg Exit Price", t.get("Avg. Exit Price", 0))),
|
||||
"pnl_pct": float(t.get("Return", t.get("PnL %", 0))),
|
||||
"duration": str(t.get("Duration", "")),
|
||||
}
|
||||
for t in trades
|
||||
]
|
||||
except Exception: # noqa: BLE001
|
||||
trades = []
|
||||
|
||||
# 每标的统计
|
||||
per_symbol = []
|
||||
try:
|
||||
total_ret = pf.total_return()
|
||||
if isinstance(total_ret, pd.Series):
|
||||
for sym, ret in total_ret.items():
|
||||
if pd.notna(ret):
|
||||
per_symbol.append({"symbol": sym, "total_return": float(ret)})
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
result = BacktestResult(
|
||||
run_id=run_id,
|
||||
config=_config_to_dict(config),
|
||||
stats={k: _json_safe(v) for k, v in stats_dict.items()},
|
||||
equity_curve=equity_curve,
|
||||
trades=trades,
|
||||
per_symbol_stats=per_symbol,
|
||||
)
|
||||
|
||||
# 落盘
|
||||
self._persist(result)
|
||||
return result
|
||||
|
||||
def _persist(self, result: BacktestResult) -> None:
|
||||
out_dir = settings.data_dir / "backtest_results"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
# 用 polars 写一份汇总
|
||||
summary = pl.DataFrame({
|
||||
"run_id": [result.run_id],
|
||||
"stats_json": [str(result.stats)],
|
||||
"n_trades": [len(result.trades)],
|
||||
})
|
||||
summary.write_parquet(out_dir / f"run_id={result.run_id}.parquet")
|
||||
|
||||
def get_result(self, run_id: str) -> BacktestResult | None:
|
||||
# Phase 1:只保留近似落盘,完整结果保存在内存的近期 cache 中
|
||||
# 简化:重新 run 比缓存复杂结果代价小,暂不实现 get_result
|
||||
return None
|
||||
|
||||
|
||||
def _config_to_dict(c: BacktestConfig) -> dict:
|
||||
return {
|
||||
"symbols": c.symbols,
|
||||
"start": str(c.start),
|
||||
"end": str(c.end),
|
||||
"entries": c.entries,
|
||||
"exits": c.exits,
|
||||
"stop_loss_pct": c.stop_loss_pct,
|
||||
"max_hold_days": c.max_hold_days,
|
||||
"fees_pct": c.fees_pct,
|
||||
"slippage_bps": c.slippage_bps,
|
||||
"matching": c.matching,
|
||||
}
|
||||
|
||||
|
||||
def _json_safe(v):
|
||||
if isinstance(v, (int, float, str, bool)) or v is None:
|
||||
return v
|
||||
if isinstance(v, (np.floating, np.integer)):
|
||||
return float(v) if not np.isnan(float(v)) else None
|
||||
if hasattr(v, "isoformat"):
|
||||
return v.isoformat()
|
||||
return str(v)
|
||||
@@ -0,0 +1,575 @@
|
||||
"""五档盘口 sealed(真假涨停/跌停) 服务 — 独立旁路线。
|
||||
|
||||
架构(完全解耦):
|
||||
- 只读 enriched(拿涨跌停名单), 不写回 enriched(14列不动)
|
||||
- sealed 存独立 parquet(data/depth5/date=xxx/part.parquet)
|
||||
- limit_ladder API 查询时 LEFT JOIN(同 ext_columns 机制)
|
||||
- signal_limit_up 永远是"价格涨停", sealed 是叠加的真假判定层
|
||||
|
||||
数据流:
|
||||
盘中轮询线程(交易时段, 独立 sleep, 不绑行情轮询):
|
||||
读 enriched 内存缓存(线程安全) → 涨跌停名单 → tf.depth.batch
|
||||
→ 算 sealed → 更新内存缓存(不落盘) → sealed_ready=True
|
||||
盘后定版 job(可配置时间, 默认15:02):
|
||||
最后拉一次 → 落盘 depth5 parquet(定版)
|
||||
|
||||
三层防护节流("设过大设上限, 设过小设最小值"):
|
||||
① 套餐范围 clamp: Pro 10~120s, Expert 3~300s
|
||||
② 限速安全 clamp: safe = 60/((rpm*0.8)/batches), 涨跌停多就自动放慢
|
||||
③ 系统接管通知: 用户设置会超限时, 推 toast 告知已自动调整
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
from datetime import date, datetime, time as dt_time
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# 套餐 → (轮询间隔下限s, 上限s)
|
||||
TIER_INTERVAL_RANGE: dict[str, tuple[float, float]] = {
|
||||
"pro": (10.0, 120.0),
|
||||
"expert": (3.0, 300.0),
|
||||
}
|
||||
# 兜底: 其他有 DEPTH5_BATCH 的套餐按 pro 范围
|
||||
DEFAULT_RANGE = (10.0, 120.0)
|
||||
|
||||
# 限速余量: 只用 rpm 的 80%, 给系统其他 depth 调用留空间
|
||||
RPM_MARGIN = 0.8
|
||||
# 间隔硬下限/上限(任何套餐)
|
||||
INTERVAL_HARD_MIN = 10.0
|
||||
INTERVAL_HARD_MAX = 300.0
|
||||
|
||||
|
||||
class DepthService:
|
||||
"""五档盘口 sealed 服务 — 单例。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._running = False
|
||||
self._thread: threading.Thread | None = None
|
||||
self._repo = None # 延迟注入(KlineRepository)
|
||||
self._app_state = None # 延迟注入(FastAPI app.state)
|
||||
|
||||
# 内存缓存: {symbol: SealedEntry}
|
||||
# SealedEntry = {sealed_up, sealed_down, ask1_vol, bid1_vol, status, fetched_ts}
|
||||
self._sealed_cache: dict[str, dict] = {}
|
||||
self._sealed_ready = False
|
||||
self._sealed_date: date | None = None # sealed 数据对应的交易日(可能是昨天,如休市)
|
||||
self._sealed_fetched_ts: float = 0.0 # 上次拉取的 perf_counter
|
||||
self._sealed_fetched_at: float = 0.0 # 上次拉取的 wall-clock 时间戳
|
||||
self._persisted_date: date | None = None # 已落盘的日期
|
||||
|
||||
# 系统接管状态(防通知刷屏)
|
||||
self._last_taken_over: bool | None = None
|
||||
self._last_user_interval: float | None = None
|
||||
|
||||
# ================================================================
|
||||
# 注入
|
||||
# ================================================================
|
||||
|
||||
def set_repo(self, repo) -> None:
|
||||
self._repo = repo
|
||||
|
||||
def set_app_state(self, app_state) -> None:
|
||||
self._app_state = app_state
|
||||
|
||||
# ================================================================
|
||||
# 生命周期
|
||||
# ================================================================
|
||||
|
||||
def boot_check(self) -> None:
|
||||
"""启动补跑: 当天 depth5 文件不存在则 finalize 一次; 已存在则恢复内存缓存。"""
|
||||
if not self._has_capability():
|
||||
logger.info("depth sealed: 无 DEPTH5_BATCH 能力, 跳过启动补跑")
|
||||
return
|
||||
today = date.today()
|
||||
if self._persisted_for_date(today):
|
||||
# parquet 已存在: 恢复内存缓存(避免重启后每次查询都读 parquet)
|
||||
self._restore_from_parquet(today)
|
||||
return
|
||||
logger.info("depth sealed: 启动补跑今天定版")
|
||||
try:
|
||||
self.finalize()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth sealed 启动补跑失败: %s", e)
|
||||
|
||||
def _restore_from_parquet(self, d: date) -> None:
|
||||
"""从 parquet 恢复内存缓存(服务重启后)。"""
|
||||
if not self._repo:
|
||||
return
|
||||
out = self._repo.store.data_dir / "depth5" / f"date={d.isoformat()}" / "part.parquet"
|
||||
if not out.exists():
|
||||
return
|
||||
try:
|
||||
df = pl.read_parquet(out)
|
||||
cache: dict[str, dict] = {}
|
||||
for row in df.to_dicts():
|
||||
sym = row.get("symbol")
|
||||
if not sym:
|
||||
continue
|
||||
cache[sym] = {
|
||||
"sealed_up": row.get("sealed_up"),
|
||||
"sealed_down": row.get("sealed_down"),
|
||||
"ask1_vol": row.get("ask1_vol"),
|
||||
"bid1_vol": row.get("bid1_vol"),
|
||||
"status": row.get("status"),
|
||||
"fetched_ts": row.get("fetched_at"),
|
||||
}
|
||||
with self._lock:
|
||||
self._sealed_cache = cache
|
||||
self._sealed_ready = True
|
||||
self._sealed_date = d
|
||||
self._persisted_date = d
|
||||
logger.info("depth sealed: 从 parquet 恢复 %d 只 (日期=%s)", len(cache), d)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth sealed 从 parquet 恢复失败: %s", e)
|
||||
|
||||
def start_polling(self) -> None:
|
||||
"""启动盘中轮询线程(连板梯队监控开启 + 有能力 + 交易时段)。"""
|
||||
if self._running:
|
||||
return
|
||||
if not self._has_capability():
|
||||
return
|
||||
from app.services import preferences
|
||||
if not preferences.get_limit_ladder_monitor_enabled():
|
||||
return
|
||||
self._running = True
|
||||
self._thread = threading.Thread(target=self._poll_loop, daemon=True)
|
||||
self._thread.start()
|
||||
logger.info("depth sealed 盘中轮询已启动")
|
||||
|
||||
def stop_polling(self) -> None:
|
||||
"""停止盘中轮询线程。"""
|
||||
self._running = False
|
||||
if self._thread:
|
||||
self._thread.join(timeout=10)
|
||||
self._thread = None
|
||||
logger.info("depth sealed 盘中轮询已停止")
|
||||
|
||||
def apply_monitor_toggle(self, enabled: bool) -> None:
|
||||
"""连板梯队监控开关切换时调用: 开启→启动轮询, 关闭→停止轮询。"""
|
||||
if enabled:
|
||||
self.start_polling()
|
||||
else:
|
||||
self.stop_polling()
|
||||
|
||||
def run_once(self) -> dict:
|
||||
"""手动触发一次修正(立即拉取 depth + 更新内存缓存)。
|
||||
|
||||
不受监控开关限制 — 用户可随时手动修正一次。
|
||||
返回 {"ok": bool, "count": int, "msg": str}
|
||||
"""
|
||||
if not self._has_capability():
|
||||
return {"ok": False, "count": 0, "msg": "无五档盘口能力(需 Pro+)"}
|
||||
try:
|
||||
self._fetch_and_seal(persist=True) # 落盘, 刷新页面不丢
|
||||
with self._lock:
|
||||
count = len(self._sealed_cache)
|
||||
return {"ok": True, "count": count, "msg": f"已修正 {count} 只"}
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth run_once 失败: %s", e)
|
||||
return {"ok": False, "count": 0, "msg": f"修正失败: {e}"}
|
||||
|
||||
# ================================================================
|
||||
# 核心拉取
|
||||
# ================================================================
|
||||
|
||||
def _fetch_and_seal(self, persist: bool = False) -> None:
|
||||
"""拉一次 depth.batch, 算 sealed, 更新内存缓存(可选落盘)。
|
||||
|
||||
persist=True: 盘后定版, 写 depth5 parquet
|
||||
persist=False: 盘中轮询, 只更新内存缓存
|
||||
"""
|
||||
if not self._repo:
|
||||
return
|
||||
|
||||
# 只读 enriched 内存缓存(线程安全, 避免和 quote_service 写盘竞态)
|
||||
enriched, enriched_date = self._repo.get_enriched_latest()
|
||||
if enriched.is_empty():
|
||||
return
|
||||
|
||||
# 筛涨跌停名单(用 fill_null 防止列缺失)
|
||||
syms_up: list[str] = []
|
||||
syms_down: list[str] = []
|
||||
if "signal_limit_up" in enriched.columns:
|
||||
syms_up = enriched.filter(
|
||||
pl.col("signal_limit_up").fill_null(False)
|
||||
)["symbol"].to_list()
|
||||
if "signal_limit_down" in enriched.columns:
|
||||
syms_down = enriched.filter(
|
||||
pl.col("signal_limit_down").fill_null(False)
|
||||
)["symbol"].to_list()
|
||||
|
||||
all_syms = list(dict.fromkeys(syms_up + syms_down)) # 去重保序
|
||||
if not all_syms:
|
||||
logger.debug("depth sealed: 当日无涨跌停股, 跳过")
|
||||
return
|
||||
|
||||
# 拉 depth(涨跌停一次拉, 按 capset batch 切片)
|
||||
depth_data = self._call_depth_batch(all_syms)
|
||||
if not depth_data:
|
||||
logger.warning("depth sealed: depth.batch 返回空")
|
||||
return
|
||||
|
||||
up_set = set(syms_up)
|
||||
down_set = set(syms_down)
|
||||
now_perf = time.perf_counter()
|
||||
now_wall = time.time()
|
||||
|
||||
new_cache: dict[str, dict] = {}
|
||||
for sym, d in depth_data.items():
|
||||
ask_vols = d.get("ask_volumes") or []
|
||||
bid_vols = d.get("bid_volumes") or []
|
||||
ask1 = ask_vols[0] if ask_vols else None
|
||||
bid1 = bid_vols[0] if bid_vols else None
|
||||
# depth 返回的 timestamp(毫秒 epoch), 回退到当前 wall-clock
|
||||
depth_ts = d.get("timestamp")
|
||||
fetched = (depth_ts / 1000.0) if isinstance(depth_ts, (int, float)) and depth_ts else now_wall
|
||||
entry = {
|
||||
# 涨停真封: 涨停价上卖一(主动卖压)为 0
|
||||
"sealed_up": (ask1 == 0) if sym in up_set and ask1 is not None else None,
|
||||
# 跌停真封: 跌停价上买一为 0
|
||||
"sealed_down": (bid1 == 0) if sym in down_set and bid1 is not None else None,
|
||||
"ask1_vol": ask1,
|
||||
"bid1_vol": bid1,
|
||||
"status": "limit_down" if sym in down_set and sym not in up_set else "limit_up",
|
||||
"fetched_ts": fetched,
|
||||
}
|
||||
new_cache[sym] = entry
|
||||
|
||||
with self._lock:
|
||||
self._sealed_cache = new_cache
|
||||
self._sealed_ready = True
|
||||
self._sealed_date = enriched_date # 记录数据对应的交易日(可能是昨天,如休市)
|
||||
self._sealed_fetched_ts = now_perf
|
||||
self._sealed_fetched_at = now_wall
|
||||
|
||||
logger.info("depth sealed: 拉取 %d 只 (涨停%d/跌停%d) 日期=%s%s",
|
||||
len(new_cache), len(syms_up), len(syms_down),
|
||||
enriched_date, " → 落盘" if persist else "")
|
||||
|
||||
# 缓存已更新: 通知 SSE 推 depth_updated, 触发连板梯队刷新封单数据。
|
||||
self._notify_depth_updated(len(new_cache))
|
||||
|
||||
if persist and enriched_date:
|
||||
self._persist(enriched_date)
|
||||
|
||||
def _call_depth_batch(self, symbols: list[str]) -> dict:
|
||||
"""调 tf.depth.batch, 按 capset 的 batch 切片 + 节流。返回 {symbol: MarketDepth}。"""
|
||||
from app.tickflow.client import get_client
|
||||
tf = get_client()
|
||||
|
||||
capset = self._get_capset()
|
||||
lim = capset.limits(__import__("app.tickflow.capabilities", fromlist=["Cap"]).Cap.DEPTH5_BATCH)
|
||||
batch_size = (lim.batch if lim and lim.batch else 100)
|
||||
rpm = (lim.rpm if lim and lim.rpm else 30)
|
||||
# 批间隔 = 60/rpm(匀速)
|
||||
inter_batch = 60.0 / rpm if rpm > 0 else 2.0
|
||||
|
||||
result: dict = {}
|
||||
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
|
||||
for i, chunk in enumerate(chunks):
|
||||
if i > 0:
|
||||
time.sleep(inter_batch)
|
||||
try:
|
||||
# SDK 的 batch 内部已按 batch_size 切, 这里再切一层防单请求过大
|
||||
data = tf.depth.batch(chunk)
|
||||
if isinstance(data, dict):
|
||||
result.update(data)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth.batch 第 %d 批失败(%d 只): %s", i + 1, len(chunk), e)
|
||||
# 单批失败不影响其他批
|
||||
return result
|
||||
|
||||
def finalize(self) -> None:
|
||||
"""盘后定版: 拉一次 + 落盘。"""
|
||||
if not self._has_capability():
|
||||
return
|
||||
self._fetch_and_seal(persist=True)
|
||||
|
||||
# ================================================================
|
||||
# 落盘
|
||||
# ================================================================
|
||||
|
||||
def _persist(self, today: date) -> None:
|
||||
"""把内存缓存写 depth5/date=今天/part.parquet。"""
|
||||
with self._lock:
|
||||
cache = dict(self._sealed_cache)
|
||||
if not cache:
|
||||
return
|
||||
|
||||
rows = []
|
||||
for sym, e in cache.items():
|
||||
rows.append({
|
||||
"symbol": sym,
|
||||
"sealed_up": e.get("sealed_up"),
|
||||
"sealed_down": e.get("sealed_down"),
|
||||
"ask1_vol": e.get("ask1_vol"),
|
||||
"bid1_vol": e.get("bid1_vol"),
|
||||
"status": e.get("status"),
|
||||
"fetched_at": e.get("fetched_ts"),
|
||||
})
|
||||
df = pl.DataFrame(rows)
|
||||
ds = today.isoformat()
|
||||
out = self._repo.store.data_dir / "depth5" / f"date={ds}" / "part.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.write_parquet(out)
|
||||
self._persisted_date = today
|
||||
logger.info("depth sealed 落盘: %d 行 → %s", df.height, out)
|
||||
|
||||
def _persisted_for_date(self, d: date) -> bool:
|
||||
"""检查某日 depth5 文件是否已存在。"""
|
||||
if not self._repo:
|
||||
return False
|
||||
out = self._repo.store.data_dir / "depth5" / f"date={d.isoformat()}" / "part.parquet"
|
||||
return out.exists()
|
||||
|
||||
# ================================================================
|
||||
# 查询(供 limit_ladder API 用)
|
||||
# ================================================================
|
||||
|
||||
def get_sealed_map(self, target_date: date, is_down: bool) -> dict:
|
||||
"""返回 {symbol: {sealed, vol, ready, age}} 供 JOIN。
|
||||
|
||||
优先内存缓存(盘中), 回退 parquet(历史/盘后)。
|
||||
sealed: bool | None (None=待确认或降级)
|
||||
vol: 封单量(int) | None
|
||||
ready: sealed 数据是否就绪(False→降级标识)
|
||||
age: 距上次拉取秒数(盘后定版为 None)
|
||||
"""
|
||||
# 内存缓存(sealed 数据对应的交易日 = target_date 时才用)
|
||||
if self._sealed_date and target_date == self._sealed_date and self._sealed_ready and self._sealed_cache:
|
||||
return self._read_from_memory(is_down)
|
||||
# parquet(历史或盘后定版)
|
||||
return self._read_from_parquet(target_date, is_down)
|
||||
|
||||
def _read_from_memory(self, is_down: bool) -> dict:
|
||||
sealed_key = "sealed_down" if is_down else "sealed_up"
|
||||
# 封单量: 涨停=买一量(涨停价买单堆积), 跌停=卖一量(跌停价卖单堆积)
|
||||
vol_key = "ask1_vol" if is_down else "bid1_vol"
|
||||
now = time.perf_counter()
|
||||
with self._lock:
|
||||
cache = dict(self._sealed_cache)
|
||||
fetched_ts = self._sealed_fetched_ts
|
||||
age = (now - fetched_ts) if fetched_ts else 0.0
|
||||
result = {}
|
||||
for sym, e in cache.items():
|
||||
result[sym] = {
|
||||
"sealed": e.get(sealed_key),
|
||||
"vol": e.get(vol_key),
|
||||
"ready": True,
|
||||
"age": age,
|
||||
}
|
||||
return result
|
||||
|
||||
def _read_from_parquet(self, target_date: date, is_down: bool) -> dict:
|
||||
if not self._repo:
|
||||
return {}
|
||||
out = self._repo.store.data_dir / "depth5" / f"date={target_date.isoformat()}" / "part.parquet"
|
||||
if not out.exists():
|
||||
return {}
|
||||
try:
|
||||
df = pl.read_parquet(out)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth5 parquet 读取失败: %s", e)
|
||||
return {}
|
||||
sealed_key = "sealed_down" if is_down else "sealed_up"
|
||||
# 封单量: 涨停=买一量, 跌停=卖一量
|
||||
vol_key = "ask1_vol" if is_down else "bid1_vol"
|
||||
result = {}
|
||||
for row in df.to_dicts():
|
||||
sym = row.get("symbol")
|
||||
if not sym:
|
||||
continue
|
||||
result[sym] = {
|
||||
"sealed": row.get(sealed_key),
|
||||
"vol": row.get(vol_key),
|
||||
"ready": True,
|
||||
"age": None, # 盘后定版, 无 age
|
||||
}
|
||||
return result
|
||||
|
||||
def is_sealed_ready(self, target_date: date) -> bool:
|
||||
"""sealed 数据是否就绪(供前端降级判定)。"""
|
||||
# 内存缓存对应的数据日 == 查询日 → 看内存就绪状态
|
||||
if self._sealed_date and target_date == self._sealed_date:
|
||||
return self._sealed_ready
|
||||
# 其他日期: 有 parquet 就 ready
|
||||
return self._persisted_for_date(target_date)
|
||||
|
||||
def get_sealed_age(self, target_date: date) -> float | None:
|
||||
"""返回 sealed 数据 age(秒), 盘后定版为 None。"""
|
||||
if self._sealed_date and target_date == self._sealed_date and self._sealed_ready and self._sealed_fetched_ts:
|
||||
return time.perf_counter() - self._sealed_fetched_ts
|
||||
return None
|
||||
|
||||
# ================================================================
|
||||
# 盘中轮询线程
|
||||
# ================================================================
|
||||
|
||||
def _poll_loop(self) -> None:
|
||||
"""盘中轮询: 按 capset 自适应间隔拉 depth, 更新内存缓存。"""
|
||||
while self._running:
|
||||
try:
|
||||
if self._is_trading_hours():
|
||||
self._poll_once()
|
||||
else:
|
||||
logger.debug("depth sealed: 非交易时段, 跳过")
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth sealed 轮询异常: %s", e)
|
||||
|
||||
# 等待下一轮(用 _running 检查保证能及时退出)
|
||||
interval = self._current_sleep_interval()
|
||||
waited = 0.0
|
||||
while self._running and waited < interval:
|
||||
time.sleep(0.5)
|
||||
waited += 0.5
|
||||
|
||||
def _poll_once(self) -> None:
|
||||
"""单次轮询: 算间隔(三层防护) → 拉取 → 检测系统接管通知。"""
|
||||
# 数当前涨跌停股
|
||||
n = self._count_limit_stocks()
|
||||
if n == 0:
|
||||
return
|
||||
|
||||
interval, taken_over, user_interval = self._compute_interval(n)
|
||||
|
||||
# 系统接管通知(状态切换时才推, 防刷屏)
|
||||
if taken_over and (self._last_taken_over is False or self._last_user_interval != user_interval):
|
||||
self._notify_takeover(n, user_interval, interval)
|
||||
self._last_taken_over = taken_over
|
||||
self._last_user_interval = user_interval
|
||||
|
||||
self._fetch_and_seal(persist=False)
|
||||
|
||||
def _current_sleep_interval(self) -> float:
|
||||
"""计算当前 sleep 间隔(供 _poll_loop 等待用)。"""
|
||||
n = self._count_limit_stocks()
|
||||
if n == 0:
|
||||
return 30.0 # 无涨跌停, 慢轮询
|
||||
interval, _, _ = self._compute_interval(n)
|
||||
return interval
|
||||
|
||||
# ================================================================
|
||||
# 三层防护节流
|
||||
# ================================================================
|
||||
|
||||
def _compute_interval(self, n_symbols: int) -> tuple[float, bool, float]:
|
||||
"""三层防护计算实际轮询间隔。
|
||||
|
||||
返回 (actual_interval, taken_over, user_interval)
|
||||
- actual_interval: 实际使用的间隔(秒)
|
||||
- taken_over: 是否被系统接管(用户设置会超限)
|
||||
- user_interval: 用户设置(经套餐 clamp 后)的间隔
|
||||
"""
|
||||
from app.services import preferences
|
||||
from app.tickflow.policy import tier_label
|
||||
|
||||
capset = self._get_capset()
|
||||
lim = capset.limits(__import__("app.tickflow.capabilities", fromlist=["Cap"]).Cap.DEPTH5_BATCH)
|
||||
batch_size = (lim.batch if lim and lim.batch else 100)
|
||||
rpm = (lim.rpm if lim and lim.rpm else 30)
|
||||
|
||||
# ① 套餐范围 clamp
|
||||
tier = tier_label().split()[0].split("+")[0].strip().lower()
|
||||
lo, hi = TIER_INTERVAL_RANGE.get(tier, DEFAULT_RANGE)
|
||||
raw_user = preferences.get_depth_polling_interval()
|
||||
user_interval = max(lo, min(hi, raw_user))
|
||||
|
||||
# ② 限速安全 clamp
|
||||
batches = max(1, math.ceil(n_symbols / batch_size))
|
||||
usable_rpm = rpm * RPM_MARGIN
|
||||
calls_per_min = usable_rpm / batches if batches > 0 else usable_rpm
|
||||
safe_interval = 60.0 / calls_per_min if calls_per_min > 0 else INTERVAL_HARD_MAX
|
||||
|
||||
# 实际: 取用户设置和安全的较大值
|
||||
actual = max(user_interval, safe_interval)
|
||||
# 硬上下限
|
||||
actual = max(INTERVAL_HARD_MIN, min(actual, INTERVAL_HARD_MAX))
|
||||
taken_over = safe_interval > user_interval
|
||||
|
||||
return actual, taken_over, user_interval
|
||||
|
||||
def _count_limit_stocks(self) -> int:
|
||||
"""数当前涨跌停股总数(供节流计算)。"""
|
||||
if not self._repo:
|
||||
return 0
|
||||
enriched, _ = self._repo.get_enriched_latest()
|
||||
if enriched.is_empty():
|
||||
return 0
|
||||
n = 0
|
||||
if "signal_limit_up" in enriched.columns:
|
||||
n += enriched.filter(pl.col("signal_limit_up").fill_null(False)).height
|
||||
if "signal_limit_down" in enriched.columns:
|
||||
n += enriched.filter(pl.col("signal_limit_down").fill_null(False)).height
|
||||
return n
|
||||
|
||||
# ================================================================
|
||||
# 通知
|
||||
# ================================================================
|
||||
|
||||
def _notify_takeover(self, n_stocks: int, user_interval: float, actual_interval: float) -> None:
|
||||
"""系统接管通知: 复用 quote_service 的 _pending_alerts 通道。"""
|
||||
if not self._app_state:
|
||||
return
|
||||
qs = getattr(self._app_state, "quote_service", None)
|
||||
if not qs:
|
||||
return
|
||||
msg = (f"五档轮询: 当前涨跌停 {n_stocks} 只, 您设置的 {user_interval:.0f} 秒间隔会超限, "
|
||||
f"系统已自动调整为 {actual_interval:.0f} 秒")
|
||||
alert = {
|
||||
"source": "depth",
|
||||
"type": "takeover",
|
||||
"message": msg,
|
||||
}
|
||||
try:
|
||||
with qs._lock:
|
||||
qs._pending_alerts.append(alert)
|
||||
qs._alert_event.set()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("depth 接管通知推送失败: %s", e)
|
||||
|
||||
def _notify_depth_updated(self, count: int) -> None:
|
||||
"""修正完成通知: set quote_service._depth_update_event, SSE 推 depth_updated 刷新连板梯队。"""
|
||||
if not self._app_state:
|
||||
return
|
||||
qs = getattr(self._app_state, "quote_service", None)
|
||||
if not qs:
|
||||
return
|
||||
try:
|
||||
qs.notify_depth_updated()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("depth 更新通知推送失败: %s", e)
|
||||
|
||||
# ================================================================
|
||||
# 工具
|
||||
# ================================================================
|
||||
|
||||
def _has_capability(self) -> bool:
|
||||
capset = self._get_capset()
|
||||
from app.tickflow.capabilities import Cap
|
||||
return capset.has(Cap.DEPTH5_BATCH)
|
||||
|
||||
def _get_capset(self):
|
||||
"""获取当前 capset(优先 app.state, 回退 detect)。"""
|
||||
if self._app_state:
|
||||
cs = getattr(self._app_state, "capabilities", None)
|
||||
if cs:
|
||||
return cs
|
||||
from app.tickflow.policy import detect_capabilities
|
||||
return detect_capabilities()
|
||||
|
||||
@staticmethod
|
||||
def _is_trading_hours() -> bool:
|
||||
now = datetime.now()
|
||||
t = now.time()
|
||||
morning = dt_time(9, 25) <= t <= dt_time(11, 35)
|
||||
afternoon = dt_time(12, 55) <= t <= dt_time(15, 5)
|
||||
return now.weekday() < 5 and (morning or afternoon)
|
||||
@@ -0,0 +1,514 @@
|
||||
"""扩展数据服务 — 配置管理 + 文件解析 + Parquet 存储。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import polars as pl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 配置模型
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ExtField:
|
||||
"""扩展字段定义。"""
|
||||
__slots__ = ("name", "dtype", "label")
|
||||
|
||||
def __init__(self, name: str, dtype: str = "string", label: str = "") -> None:
|
||||
self.name = name
|
||||
self.dtype = dtype # string | int | float | bool
|
||||
self.label = label or name
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {"name": self.name, "dtype": self.dtype, "label": self.label}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict) -> ExtField:
|
||||
return cls(d["name"], d.get("dtype", "string"), d.get("label", ""))
|
||||
|
||||
|
||||
class PullConfig:
|
||||
"""定时拉取配置。"""
|
||||
__slots__ = (
|
||||
"url", "method", "headers", "body", "response_path",
|
||||
"field_map", "schedule_minutes", "enabled",
|
||||
"last_run", "last_status", "last_message", "last_rows",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str = "",
|
||||
method: str = "GET",
|
||||
headers: dict[str, str] | None = None,
|
||||
body: str | None = None,
|
||||
response_path: str = "",
|
||||
field_map: dict[str, str] | None = None,
|
||||
schedule_minutes: int = 1440,
|
||||
enabled: bool = False,
|
||||
last_run: str | None = None,
|
||||
last_status: str | None = None,
|
||||
last_message: str | None = None,
|
||||
last_rows: int | None = None,
|
||||
) -> None:
|
||||
self.url = url
|
||||
self.method = method # GET | POST
|
||||
self.headers = headers or {}
|
||||
self.body = body # JSON string (POST body template)
|
||||
self.response_path = response_path # dot-path to rows array, e.g. "data.list"
|
||||
self.field_map = field_map or {} # external_name → config_field_name
|
||||
self.schedule_minutes = schedule_minutes
|
||||
self.enabled = enabled
|
||||
self.last_run = last_run
|
||||
self.last_status = last_status # "success" | "error"
|
||||
self.last_message = last_message
|
||||
self.last_rows = last_rows
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"url": self.url,
|
||||
"method": self.method,
|
||||
"headers": self.headers,
|
||||
"body": self.body,
|
||||
"response_path": self.response_path,
|
||||
"field_map": self.field_map,
|
||||
"schedule_minutes": self.schedule_minutes,
|
||||
"enabled": self.enabled,
|
||||
"last_run": self.last_run,
|
||||
"last_status": self.last_status,
|
||||
"last_message": self.last_message,
|
||||
"last_rows": self.last_rows,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict) -> PullConfig:
|
||||
if not d:
|
||||
return cls()
|
||||
return cls(
|
||||
url=d.get("url", ""),
|
||||
method=d.get("method", "GET"),
|
||||
headers=d.get("headers"),
|
||||
body=d.get("body"),
|
||||
response_path=d.get("response_path", ""),
|
||||
field_map=d.get("field_map"),
|
||||
schedule_minutes=d.get("schedule_minutes", 1440),
|
||||
enabled=d.get("enabled", False),
|
||||
last_run=d.get("last_run"),
|
||||
last_status=d.get("last_status"),
|
||||
last_message=d.get("last_message"),
|
||||
last_rows=d.get("last_rows"),
|
||||
)
|
||||
|
||||
|
||||
class ExtConfig:
|
||||
"""一个扩展数据源的完整配置。"""
|
||||
__slots__ = (
|
||||
"id", "label", "mode", "fields", "description",
|
||||
"symbol_map", "code_map",
|
||||
"created_at", "updated_at", "pull",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
id: str,
|
||||
label: str,
|
||||
mode: Literal["snapshot", "timeseries"],
|
||||
fields: list[ExtField],
|
||||
description: str = "",
|
||||
symbol_map: dict | None = None,
|
||||
code_map: dict | None = None,
|
||||
created_at: str | None = None,
|
||||
updated_at: str | None = None,
|
||||
pull: PullConfig | None = None,
|
||||
) -> None:
|
||||
self.id = id
|
||||
self.label = label
|
||||
self.mode = mode
|
||||
self.fields = fields
|
||||
self.description = description
|
||||
# 映射关系: {"type": "mapped", "col": "原始列名"} 或 {"type": "computed", "from": "symbol|code", "method": "strip_exchange|append_exchange"}
|
||||
self.symbol_map = symbol_map or {}
|
||||
self.code_map = code_map or {}
|
||||
self.created_at = created_at or datetime.now().isoformat()
|
||||
self.updated_at = updated_at or datetime.now().isoformat()
|
||||
self.pull = pull
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
d = {
|
||||
"id": self.id,
|
||||
"label": self.label,
|
||||
"mode": self.mode,
|
||||
"fields": [f.to_dict() for f in self.fields],
|
||||
"description": self.description,
|
||||
"symbol_map": self.symbol_map,
|
||||
"code_map": self.code_map,
|
||||
"created_at": self.created_at,
|
||||
"updated_at": self.updated_at,
|
||||
}
|
||||
if self.pull:
|
||||
d["pull"] = self.pull.to_dict()
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict) -> ExtConfig:
|
||||
return cls(
|
||||
id=d["id"],
|
||||
label=d["label"],
|
||||
mode=d["mode"],
|
||||
fields=[ExtField.from_dict(f) for f in d.get("fields", [])],
|
||||
description=d.get("description", ""),
|
||||
symbol_map=d.get("symbol_map"),
|
||||
code_map=d.get("code_map"),
|
||||
created_at=d.get("created_at"),
|
||||
updated_at=d.get("updated_at"),
|
||||
pull=PullConfig.from_dict(d["pull"]) if d.get("pull") else None,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 配置持久化
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ExtConfigStore:
|
||||
"""扩展数据配置文件读写 — 每个表独立目录 data/ext/{config_id}/config.json。"""
|
||||
|
||||
def __init__(self, data_dir: Path) -> None:
|
||||
self._base = data_dir / "ext_data"
|
||||
|
||||
def _config_path(self, config_id: str) -> Path:
|
||||
return self._base / config_id / "config.json"
|
||||
|
||||
def load_all(self) -> list[ExtConfig]:
|
||||
# 兼容旧版: 如果目录为空且旧配置文件存在则迁移
|
||||
if not self._base.exists() or not any(self._base.iterdir()):
|
||||
old = self._base.parent / "ext_configs.json"
|
||||
if not old.exists():
|
||||
old = self._base.parent / "ext_configs.json.bak"
|
||||
if old.exists():
|
||||
self._migrate_legacy(old)
|
||||
if not self._base.exists():
|
||||
return []
|
||||
configs = []
|
||||
for d in sorted(self._base.iterdir()):
|
||||
cp = d / "config.json"
|
||||
if d.is_dir() and cp.exists():
|
||||
try:
|
||||
raw = json.loads(cp.read_text(encoding="utf-8"))
|
||||
configs.append(ExtConfig.from_dict(raw))
|
||||
except Exception as e:
|
||||
logger.warning("扩展表配置解析失败 %s: %s", cp, e)
|
||||
return configs
|
||||
|
||||
def get(self, config_id: str) -> ExtConfig | None:
|
||||
cp = self._config_path(config_id)
|
||||
if not cp.exists():
|
||||
return None
|
||||
try:
|
||||
raw = json.loads(cp.read_text(encoding="utf-8"))
|
||||
return ExtConfig.from_dict(raw)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def upsert(self, config: ExtConfig) -> None:
|
||||
config.updated_at = datetime.now().isoformat()
|
||||
cp = self._config_path(config.id)
|
||||
cp.parent.mkdir(parents=True, exist_ok=True)
|
||||
cp.write_text(
|
||||
json.dumps(config.to_dict(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def delete(self, config_id: str) -> bool:
|
||||
import shutil
|
||||
cp = self._config_path(config_id)
|
||||
if not cp.exists():
|
||||
return False
|
||||
shutil.rmtree(cp.parent, ignore_errors=True)
|
||||
return True
|
||||
|
||||
def _migrate_legacy(self, old_path: Path) -> None:
|
||||
"""一次性迁移旧版 ext_configs.json 到独立目录结构。"""
|
||||
try:
|
||||
raw = json.loads(old_path.read_text(encoding="utf-8"))
|
||||
configs = [ExtConfig.from_dict(d) for d in raw]
|
||||
for c in configs:
|
||||
cp = self._config_path(c.id)
|
||||
cp.parent.mkdir(parents=True, exist_ok=True)
|
||||
cp.write_text(
|
||||
json.dumps(c.to_dict(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
# 迁移完成后重命名旧文件作为备份
|
||||
backup = old_path.with_suffix(".json.bak")
|
||||
old_path.rename(backup)
|
||||
logger.info("ext_configs.json 已迁移至 ext/ (备份: %s)", backup.name)
|
||||
except Exception as e:
|
||||
logger.warning("ext_configs 迁移失败: %s", e)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSV / Excel 解析 → Parquet 写入
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_POLARS_DTYPE_MAP = {
|
||||
"string": pl.Utf8,
|
||||
"int": pl.Int64,
|
||||
"float": pl.Float64,
|
||||
"bool": pl.Boolean,
|
||||
}
|
||||
|
||||
|
||||
def build_code_lookup(data_dir: Path) -> dict[str, str]:
|
||||
"""从 instruments 维表构建 code → symbol 映射。"""
|
||||
path = data_dir / "instruments" / "instruments.parquet"
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
df = pl.read_parquet(path, columns=["code", "symbol"])
|
||||
return dict(zip(df["code"].to_list(), df["symbol"].to_list()))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def normalize_symbol(series: pl.Series, lookup: dict[str, str] | None = None) -> pl.Series:
|
||||
"""将 symbol 列标准化为 代码.交易所 格式。
|
||||
|
||||
优先使用 instruments 维表查找 code → symbol，确保 100% 准确。
|
||||
查不到时按规则兜底：6开头 → .SH，其余 → .SZ。
|
||||
"""
|
||||
_lookup = lookup or {}
|
||||
|
||||
def _fix_one(val: str) -> str:
|
||||
if not val:
|
||||
return val
|
||||
val = val.strip()
|
||||
# 已经是标准格式（含 .），直接返回
|
||||
if "." in val:
|
||||
return val
|
||||
# 纯6位数字代码 → 优先查维表
|
||||
if len(val) == 6 and val.isdigit():
|
||||
mapped = _lookup.get(val)
|
||||
if mapped:
|
||||
return mapped
|
||||
# 兜底规则
|
||||
if val.startswith(("6",)):
|
||||
return f"{val}.SH"
|
||||
else:
|
||||
return f"{val}.SZ"
|
||||
return val
|
||||
|
||||
return series.map_elements(_fix_one, return_dtype=pl.Utf8)
|
||||
|
||||
|
||||
def ensure_utf8_csv(file_path: Path) -> Path:
|
||||
"""确保 CSV 文件以 UTF-8 编码可读，非 UTF-8（如 GBK/GB18030）则转换。
|
||||
|
||||
国内行情软件（同花顺/东财/通达信）和 Windows 中文 Excel 导出的 CSV 多为
|
||||
GBK 系编码，Polars 的 read_csv 默认按 UTF-8 解析会抛 "invalid utf-8 sequence"。
|
||||
这里在交给 Polars 前做一次编码规范化。
|
||||
|
||||
返回值：若已是 UTF-8 则返回原路径；否则在同目录写一个 *.utf8 文件并返回它
|
||||
（调用方用临时目录，随目录一起清理）。
|
||||
"""
|
||||
raw = file_path.read_bytes()
|
||||
# BOM 处理：UTF-8-SIG 等带 BOM 文件直接交给 Polars（它认识 BOM）
|
||||
try:
|
||||
raw.decode("utf-8")
|
||||
return file_path # 已是合法 UTF-8
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
# 依次尝试常见中文编码，第一个能完整解码的即为命中
|
||||
for enc in ("gb18030", "gbk", "gb2312", "big5"):
|
||||
try:
|
||||
text = raw.decode(enc)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
out_path = file_path.with_suffix(file_path.suffix + ".utf8")
|
||||
out_path.write_text(text, encoding="utf-8")
|
||||
logger.info("CSV 编码转换 %s → %s (%s)", file_path.name, out_path.name, enc)
|
||||
return out_path
|
||||
# 都无法解码：返回原路径，让 Polars 抛出更精确的原始错误
|
||||
return file_path
|
||||
|
||||
|
||||
def parse_upload_file(file_path: Path, symbol_col: str = "symbol", data_dir: Path | None = None) -> pl.DataFrame:
|
||||
"""解析上传的 CSV / Excel 文件为 Polars DataFrame。"""
|
||||
suffix = file_path.suffix.lower()
|
||||
if suffix == ".csv":
|
||||
df = pl.read_csv(ensure_utf8_csv(file_path), infer_schema_length=10000)
|
||||
elif suffix in (".xlsx", ".xls"):
|
||||
df = pl.read_excel(file_path)
|
||||
else:
|
||||
raise ValueError(f"不支持的文件格式: {suffix}")
|
||||
|
||||
if symbol_col not in df.columns:
|
||||
# 尝试模糊匹配
|
||||
candidates = [c for c in df.columns if c.lower() in ("symbol", "code", "代码", "标的")]
|
||||
if candidates:
|
||||
df = df.rename({candidates[0]: symbol_col})
|
||||
else:
|
||||
raise ValueError(f"未找到标的代码列 (symbol)，可选列: {df.columns}")
|
||||
|
||||
# 确保 symbol 列为字符串并标准化
|
||||
lookup = build_code_lookup(data_dir) if data_dir else None
|
||||
df = df.with_columns(normalize_symbol(df[symbol_col].cast(pl.Utf8), lookup))
|
||||
return df
|
||||
|
||||
|
||||
def cast_df_to_schema(df: pl.DataFrame, fields: list[ExtField]) -> pl.DataFrame:
|
||||
"""按配置的字段类型转换 DataFrame 列类型。"""
|
||||
for f in fields:
|
||||
if f.name in df.columns:
|
||||
target = _POLARS_DTYPE_MAP.get(f.dtype, pl.Utf8)
|
||||
df = df.with_columns(pl.col(f.name).cast(target))
|
||||
return df
|
||||
|
||||
|
||||
def _config_dir(config_id: str, data_dir: Path) -> Path:
|
||||
"""返回扩展配置的根目录 data/ext_data/{config_id}/。"""
|
||||
return data_dir / "ext_data" / config_id
|
||||
|
||||
|
||||
def write_ext_parquet(
|
||||
df: pl.DataFrame,
|
||||
config: ExtConfig,
|
||||
data_dir: Path,
|
||||
snapshot_date: date | None = None,
|
||||
) -> int:
|
||||
"""将 DataFrame 写入扩展数据 Parquet。
|
||||
|
||||
目录结构:
|
||||
- snapshot: data/ext_data/{id}/part.parquet（与 config.json 同级，覆盖写）
|
||||
- timeseries: data/ext_data/{id}/timeseries/date=xxx/part.parquet（按日分区）
|
||||
|
||||
Returns:
|
||||
写入行数。
|
||||
"""
|
||||
snap = snapshot_date or date.today()
|
||||
cfg_dir = _config_dir(config.id, data_dir)
|
||||
|
||||
# 标准化 symbol 列: 用维表查找 → 准确匹配交易所
|
||||
if "symbol" in df.columns:
|
||||
lookup = build_code_lookup(data_dir)
|
||||
df = df.with_columns(normalize_symbol(df["symbol"], lookup))
|
||||
|
||||
if config.mode == "snapshot":
|
||||
# 快照: 与 config.json 同级，直接覆盖
|
||||
cfg_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_path = cfg_dir / "part.parquet"
|
||||
|
||||
# 如果已有文件，合并去重后覆盖
|
||||
if out_path.exists():
|
||||
try:
|
||||
existing = pl.read_parquet(out_path)
|
||||
key = "symbol" if "symbol" in df.columns else df.columns[0]
|
||||
df = pl.concat([existing, df]).unique(subset=[key], keep="last")
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
# 时序: timeseries/ 下按日期分区
|
||||
out_dir = cfg_dir / "timeseries" / f"date={snap}"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_path = out_dir / "part.parquet"
|
||||
|
||||
# 如果已有文件，合并去重
|
||||
if out_path.exists():
|
||||
try:
|
||||
existing = pl.read_parquet(out_path)
|
||||
key = "symbol" if "symbol" in df.columns else df.columns[0]
|
||||
df = pl.concat([existing, df]).unique(subset=[key], keep="last")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
df = cast_df_to_schema(df, config.fields)
|
||||
df.write_parquet(out_path)
|
||||
logger.info("扩展表写入: %s → %s (%d 行)", config.id, out_path, len(df))
|
||||
return len(df)
|
||||
|
||||
|
||||
def delete_ext_parquet(config_id: str, data_dir: Path) -> None:
|
||||
"""删除扩展数据源关联的所有 Parquet 数据（保留 config.json）。
|
||||
|
||||
- snapshot: 删除 ext_data/{id}/part.parquet
|
||||
- timeseries: 删除 ext_data/{id}/timeseries/ 目录
|
||||
"""
|
||||
cfg_dir = _config_dir(config_id, data_dir)
|
||||
# 删除快照文件
|
||||
snap = cfg_dir / "part.parquet"
|
||||
if snap.exists():
|
||||
snap.unlink()
|
||||
# 删除时序目录
|
||||
ts_dir = cfg_dir / "timeseries"
|
||||
if ts_dir.exists():
|
||||
import shutil
|
||||
shutil.rmtree(ts_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def fix_symbol_format(config: ExtConfig, data_dir: Path) -> int:
|
||||
"""扫描该扩展配置的所有 Parquet 文件，将 symbol 列标准化为 代码.交易所 格式。
|
||||
|
||||
- snapshot: 扫描 ext_data/{id}/part.parquet
|
||||
- timeseries: 扫描 ext_data/{id}/timeseries/date=xxx/part.parquet
|
||||
|
||||
Returns:
|
||||
修复的文件数。
|
||||
"""
|
||||
cfg_dir = _config_dir(config.id, data_dir)
|
||||
if not cfg_dir.exists():
|
||||
return 0
|
||||
|
||||
# 收集需要扫描的 parquet 文件列表
|
||||
parquet_files: list[Path] = []
|
||||
if config.mode == "snapshot":
|
||||
p = cfg_dir / "part.parquet"
|
||||
if p.exists():
|
||||
parquet_files.append(p)
|
||||
else:
|
||||
ts_dir = cfg_dir / "timeseries"
|
||||
if ts_dir.exists():
|
||||
for part_dir in sorted(ts_dir.iterdir()):
|
||||
if not part_dir.is_dir() or not part_dir.name.startswith("date="):
|
||||
continue
|
||||
p = part_dir / "part.parquet"
|
||||
if p.exists():
|
||||
parquet_files.append(p)
|
||||
|
||||
fixed = 0
|
||||
lookup = build_code_lookup(data_dir)
|
||||
for parquet_path in parquet_files:
|
||||
try:
|
||||
df = pl.read_parquet(parquet_path)
|
||||
if "symbol" not in df.columns:
|
||||
continue
|
||||
old = df["symbol"].to_list()
|
||||
df = df.with_columns(normalize_symbol(df["symbol"], lookup))
|
||||
new = df["symbol"].to_list()
|
||||
if old != new:
|
||||
df.write_parquet(parquet_path)
|
||||
fixed += 1
|
||||
logger.info("代码格式修复: %s/%s (%d 行)", config.id, parquet_path.parent.name, len(df))
|
||||
except Exception as e:
|
||||
logger.warning("代码格式修复跳过 %s: %s", parquet_path, e)
|
||||
|
||||
return fixed
|
||||
|
||||
|
||||
def rows_to_parquet(
|
||||
rows: list[dict],
|
||||
config: ExtConfig,
|
||||
data_dir: Path,
|
||||
snapshot_date: date | None = None,
|
||||
) -> int:
|
||||
"""将 JSON 行列表转为 DataFrame 写入 Parquet，复用 write_ext_parquet 的存储逻辑。
|
||||
|
||||
Returns:
|
||||
写入行数。
|
||||
"""
|
||||
df = pl.DataFrame(rows)
|
||||
if "symbol" in df.columns:
|
||||
df = df.with_columns(pl.col("symbol").cast(pl.Utf8))
|
||||
return write_ext_parquet(df, config, data_dir, snapshot_date=snapshot_date)
|
||||
@@ -0,0 +1,216 @@
|
||||
"""扩展数据定时拉取引擎 — 从外部 API 拉取数据写入 Parquet。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from datetime import date, datetime, timezone
|
||||
from functools import reduce
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.services.ext_data import (
|
||||
ExtConfig,
|
||||
ExtConfigStore,
|
||||
PullConfig,
|
||||
rows_to_parquet,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 响应解析
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _extract_rows(data: Any, path: str) -> list[dict]:
|
||||
"""按 dot-path 从 JSON 响应中提取行数组。
|
||||
|
||||
例: path="data.list" → response["data"]["list"]
|
||||
如果 path 为空，直接将 data 视为数组。
|
||||
"""
|
||||
if not path:
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
raise ValueError("response_path 为空但响应不是数组")
|
||||
|
||||
keys = path.split(".")
|
||||
current = data
|
||||
for key in keys:
|
||||
if isinstance(current, dict):
|
||||
if key not in current:
|
||||
raise ValueError(f"响应中不存在路径 '{path}'，缺失键 '{key}'")
|
||||
current = current[key]
|
||||
elif isinstance(current, list):
|
||||
try:
|
||||
current = current[int(key)]
|
||||
except (ValueError, IndexError) as e:
|
||||
raise ValueError(f"响应路径 '{path}' 解析失败: {e}") from e
|
||||
else:
|
||||
raise ValueError(f"响应路径 '{path}' 中间值不是 dict/list: {type(current)}")
|
||||
|
||||
if not isinstance(current, list):
|
||||
raise ValueError(f"路径 '{path}' 指向的不是数组，而是 {type(current)}")
|
||||
|
||||
return current
|
||||
|
||||
|
||||
def _apply_field_map(rows: list[dict], field_map: dict[str, str]) -> list[dict]:
|
||||
"""将外部字段名映射为内部配置字段名。field_map: {外部名: 内部名}。"""
|
||||
if not field_map:
|
||||
return rows
|
||||
mapped = []
|
||||
for row in rows:
|
||||
new_row: dict = {}
|
||||
for k, v in row.items():
|
||||
mapped_key = field_map.get(k, k)
|
||||
new_row[mapped_key] = v
|
||||
mapped.append(new_row)
|
||||
return mapped
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 拉取执行
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def fetch_and_ingest(
|
||||
config: ExtConfig,
|
||||
data_dir,
|
||||
) -> tuple[int, str]:
|
||||
"""执行一次拉取: 请求外部 API → 解析响应 → 写入 Parquet。
|
||||
|
||||
Returns:
|
||||
(rows_written, date_str)
|
||||
"""
|
||||
pull = config.pull
|
||||
if not pull or not pull.url:
|
||||
raise ValueError("拉取未配置或 URL 为空")
|
||||
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
headers = pull.headers or {}
|
||||
kwargs: dict[str, Any] = {"headers": headers}
|
||||
|
||||
if pull.method.upper() == "POST" and pull.body:
|
||||
kwargs["content"] = pull.body
|
||||
if "content-type" not in {k.lower() for k in headers}:
|
||||
kwargs["headers"]["Content-Type"] = "application/json"
|
||||
|
||||
resp = await client.request(pull.method.upper(), pull.url, **kwargs)
|
||||
resp.raise_for_status()
|
||||
|
||||
# 解析 JSON
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
raise ValueError(f"响应不是有效 JSON: {e}") from e
|
||||
|
||||
# 提取行
|
||||
rows = _extract_rows(data, pull.response_path)
|
||||
if not rows:
|
||||
raise ValueError("提取到的行数为 0")
|
||||
|
||||
# 字段映射
|
||||
rows = _apply_field_map(rows, pull.field_map)
|
||||
|
||||
# 校验 symbol 列
|
||||
if rows and "symbol" not in rows[0]:
|
||||
raise ValueError("数据行中缺少 symbol 字段，请配置 field_map 映射")
|
||||
|
||||
# 写入
|
||||
snap = date.today()
|
||||
n = rows_to_parquet(rows, config, data_dir, snapshot_date=snap)
|
||||
return n, snap.isoformat()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 调度器
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class PullScheduler:
|
||||
"""后台调度器：为每个启用了 pull 的 ExtConfig 维护定时任务。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._tasks: dict[str, asyncio.Task] = {}
|
||||
self._running = False
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def start(self, data_dir) -> None:
|
||||
"""启动调度（在 lifespan startup 调用）。"""
|
||||
self._running = True
|
||||
self._data_dir = data_dir
|
||||
logger.info("PullScheduler started")
|
||||
|
||||
def stop(self) -> None:
|
||||
"""停止所有任务。"""
|
||||
self._running = False
|
||||
for task in self._tasks.values():
|
||||
task.cancel()
|
||||
self._tasks.clear()
|
||||
logger.info("PullScheduler stopped")
|
||||
|
||||
def refresh(self, data_dir) -> None:
|
||||
"""重新加载配置，更新调度任务（增/删/改）。"""
|
||||
self._data_dir = data_dir
|
||||
store = ExtConfigStore(data_dir)
|
||||
configs = store.load_all()
|
||||
|
||||
active_ids: set[str] = set()
|
||||
|
||||
for config in configs:
|
||||
if not config.pull or not config.pull.enabled or not config.pull.url:
|
||||
continue
|
||||
active_ids.add(config.id)
|
||||
if config.id not in self._tasks:
|
||||
# 新增调度
|
||||
task = asyncio.create_task(self._run_loop(config))
|
||||
self._tasks[config.id] = task
|
||||
logger.info("PullScheduler: scheduled %s (every %d min)", config.id, config.pull.schedule_minutes)
|
||||
|
||||
# 移除不再活跃的
|
||||
for cid in list(self._tasks):
|
||||
if cid not in active_ids:
|
||||
self._tasks[cid].cancel()
|
||||
del self._tasks[cid]
|
||||
logger.info("PullScheduler: removed %s", cid)
|
||||
|
||||
async def _run_loop(self, config: ExtConfig) -> None:
|
||||
"""单个配置的定时拉取循环。"""
|
||||
try:
|
||||
while self._running:
|
||||
pull = config.pull
|
||||
if not pull:
|
||||
break
|
||||
interval = max(pull.schedule_minutes * 60, 60) # 至少 60s
|
||||
await asyncio.sleep(interval)
|
||||
if not self._running:
|
||||
break
|
||||
try:
|
||||
# 重新加载最新配置（用户可能中途修改）
|
||||
store = ExtConfigStore(self._data_dir)
|
||||
fresh = store.get(config.id)
|
||||
if not fresh or not fresh.pull or not fresh.pull.enabled:
|
||||
break
|
||||
n, d = await fetch_and_ingest(fresh, self._data_dir)
|
||||
fresh.pull.last_run = datetime.now(timezone.utc).isoformat()
|
||||
fresh.pull.last_status = "success"
|
||||
fresh.pull.last_message = f"{n} rows @ {d}"
|
||||
fresh.pull.last_rows = n
|
||||
store.upsert(fresh)
|
||||
logger.info("PullScheduler: %s success, %d rows", config.id, n)
|
||||
except Exception as e:
|
||||
store = ExtConfigStore(self._data_dir)
|
||||
fresh = store.get(config.id)
|
||||
if fresh and fresh.pull:
|
||||
fresh.pull.last_run = datetime.now(timezone.utc).isoformat()
|
||||
fresh.pull.last_status = "error"
|
||||
fresh.pull.last_message = str(e)[:200]
|
||||
store.upsert(fresh)
|
||||
logger.warning("PullScheduler: %s error: %s", config.id, e)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
# 全局单例
|
||||
pull_scheduler = PullScheduler()
|
||||
@@ -0,0 +1,224 @@
|
||||
"""向前扩展历史数据 — 完全独立于 daily_pipeline 的盘后管道。
|
||||
|
||||
用户从日 K 卡片手动触发,指定往前补的时长 (x 天/月/年)。
|
||||
流程:
|
||||
1. 获取当前最早日期
|
||||
2. 向前拉日 K batch (start = 最早日期 - offset, end = 最早日期)
|
||||
3. 向前拉除权因子 (同范围)
|
||||
4. 全量重算 enriched
|
||||
5. 刷新视图 + 缓存
|
||||
|
||||
⚠️ 本模块不导入 daily_pipeline 的任何函数,只复用基础设施:
|
||||
- kline_sync.sync_and_persist_daily_batch / sync_adj_factor
|
||||
- indicators.pipeline.run_pipeline
|
||||
- pipeline_jobs.JobStore
|
||||
- tickflow.repository.KlineRepository
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from datetime import date, datetime, timedelta
|
||||
|
||||
from app.services import kline_sync
|
||||
from app.services.pipeline_jobs import job_store
|
||||
from app.tickflow.capabilities import Cap, CapabilitySet
|
||||
from app.tickflow.repository import KlineRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _noop(stage: str, pct: int, msg: str, **kwargs) -> None: # noqa: ARG001
|
||||
pass
|
||||
|
||||
|
||||
def _invalidate(table: str | None = None) -> None:
|
||||
from app.api.data import invalidate_data_cache
|
||||
invalidate_data_cache(table)
|
||||
|
||||
|
||||
def _resolve_universe(capset: CapabilitySet) -> list[str]:
|
||||
"""解析标的池 — 与 daily_pipeline 独立的副本。"""
|
||||
if capset.has(Cap.KLINE_DAILY_BATCH):
|
||||
try:
|
||||
from app.tickflow.pools import get_pool
|
||||
all_a = get_pool("CN_Equity_A", refresh=True)
|
||||
if all_a:
|
||||
return sorted(all_a)
|
||||
except Exception as e:
|
||||
logger.warning("CN_Equity_A pool unavailable: %s", e)
|
||||
|
||||
from app.tickflow.pools import DEMO_SYMBOLS, get_pool as _get_pool
|
||||
from app.config import settings
|
||||
from pathlib import Path
|
||||
import polars as pl
|
||||
base: set[str] = set(DEMO_SYMBOLS)
|
||||
base.update(_get_pool("watchlist"))
|
||||
d = Path(settings.data_dir)
|
||||
inst_path = d / "instruments" / "instruments.parquet"
|
||||
if inst_path.exists():
|
||||
try:
|
||||
inst = pl.read_parquet(inst_path, columns=["symbol"])
|
||||
base.update(inst["symbol"].to_list())
|
||||
except Exception as e:
|
||||
logger.warning("instruments supplement failed: %s", e)
|
||||
return sorted(base)
|
||||
|
||||
|
||||
def _refresh_single_view(repo: KlineRepository, name: str) -> None:
|
||||
"""刷新单个 DuckDB 视图。"""
|
||||
d = repo.store.data_dir.as_posix()
|
||||
paths = {
|
||||
"kline_daily": f"{d}/kline_daily/**/*.parquet",
|
||||
"kline_enriched": f"{d}/kline_daily_enriched/**/*.parquet",
|
||||
"kline_minute": f"{d}/kline_minute/**/*.parquet",
|
||||
"adj_factor": f"{d}/adj_factor/**/*.parquet",
|
||||
"instruments": f"{d}/instruments/**/*.parquet",
|
||||
}
|
||||
path = paths.get(name)
|
||||
if not path:
|
||||
return
|
||||
try:
|
||||
repo.db.execute(
|
||||
f"CREATE OR REPLACE VIEW {name} AS "
|
||||
f"SELECT * FROM read_parquet('{path}', union_by_name=true)"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("refresh view %s failed: %s", name, e)
|
||||
|
||||
|
||||
def compute_offset(value: int, unit: str) -> timedelta:
|
||||
"""将用户输入的 value + unit 转成 timedelta。"""
|
||||
if unit == "day":
|
||||
return timedelta(days=value)
|
||||
elif unit == "month":
|
||||
return timedelta(days=value * 30)
|
||||
elif unit == "year":
|
||||
return timedelta(days=value * 365)
|
||||
else:
|
||||
raise ValueError(f"不支持的单位: {unit}")
|
||||
|
||||
|
||||
def run_extend_history(
|
||||
repo: KlineRepository,
|
||||
capset: CapabilitySet,
|
||||
value: int,
|
||||
unit: str,
|
||||
on_progress: Callable | None = None,
|
||||
) -> dict:
|
||||
"""向前扩展历史数据的主函数。
|
||||
|
||||
完全独立于 daily_pipeline.run_now(),不调用其任何逻辑。
|
||||
返回结果 dict 供 job_store 记录。
|
||||
"""
|
||||
emit = on_progress or _noop
|
||||
|
||||
# 0. 计算时间偏移
|
||||
offset = compute_offset(value, unit)
|
||||
today = date.today()
|
||||
|
||||
# 1. 获取当前最早日期
|
||||
emit("extend_history", 2, "检查当前数据范围…")
|
||||
earliest = repo.earliest_daily_date()
|
||||
|
||||
if not earliest:
|
||||
return {"error": "本地无日K数据,请先执行一次完整同步"}
|
||||
|
||||
new_start = earliest - offset
|
||||
# 不能超过今天
|
||||
if new_start >= earliest:
|
||||
return {"error": "扩展范围无效,请增大时间跨度"}
|
||||
|
||||
# 2. 解析标的池
|
||||
emit("extend_history", 5, "解析标的池…")
|
||||
universe = _resolve_universe(capset)
|
||||
if not universe:
|
||||
return {"error": "标的池为空"}
|
||||
emit("extend_history", 8, f"标的池: {len(universe)} 只")
|
||||
|
||||
start_str = new_start.strftime("%Y-%m-%d")
|
||||
end_str = earliest.strftime("%Y-%m-%d")
|
||||
|
||||
# 3. 拉日 K
|
||||
emit("extend_history", 10, f"获取日K [{start_str} ~ {end_str}]…")
|
||||
logger.info("extend_history: daily K [%s ~ %s], %d symbols", start_str, end_str, len(universe))
|
||||
|
||||
def _daily_chunk(cur: int, tot: int) -> None:
|
||||
emit("extend_history", 10 + int(35 * cur / tot),
|
||||
f"日K 批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
|
||||
|
||||
written_daily = kline_sync.sync_and_persist_daily_batch(
|
||||
universe, repo, capset,
|
||||
start_date=datetime.combine(new_start, datetime.min.time()),
|
||||
end_date=datetime.combine(earliest, datetime.min.time()),
|
||||
on_chunk_done=_daily_chunk,
|
||||
)
|
||||
emit("extend_history", 45, f"日K 完成,写入 {written_daily} 行")
|
||||
logger.info("extend_history: daily K done, %d rows", written_daily)
|
||||
_refresh_single_view(repo, "kline_daily")
|
||||
_invalidate("daily")
|
||||
|
||||
# 4. 拉除权因子 (新范围)
|
||||
written_adj = 0
|
||||
adj_start = datetime.combine(new_start, datetime.min.time())
|
||||
adj_end = datetime.combine(today, datetime.min.time())
|
||||
adj_start_str = new_start.strftime("%Y-%m-%d")
|
||||
adj_end_str = today.strftime("%Y-%m-%d")
|
||||
|
||||
if capset.has(Cap.ADJ_FACTOR):
|
||||
emit("extend_history", 48, f"获取除权因子 [{adj_start_str} ~ {adj_end_str}]…")
|
||||
logger.info("extend_history: adj_factor [%s ~ %s]", adj_start_str, adj_end_str)
|
||||
|
||||
def _adj_chunk(cur: int, tot: int) -> None:
|
||||
emit("extend_history", 48 + int(10 * cur / tot),
|
||||
f"除权因子批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
|
||||
|
||||
written_adj, _affected = kline_sync.sync_adj_factor(
|
||||
universe, repo, capset,
|
||||
start_time=adj_start, end_time=adj_end,
|
||||
on_chunk_done=_adj_chunk,
|
||||
)
|
||||
emit("extend_history", 60, f"除权因子完成,{written_adj} 行")
|
||||
logger.info("extend_history: adj_factor done, %d rows", written_adj)
|
||||
_refresh_single_view(repo, "adj_factor")
|
||||
_invalidate("adj_factor")
|
||||
else:
|
||||
emit("extend_history", 60, "除权因子跳过(无权限)")
|
||||
logger.info("extend_history: adj_factor skipped, no ADJ_FACTOR capability")
|
||||
|
||||
# 5. 全量重算 enriched
|
||||
emit("extend_history", 65, "全量计算 enriched…")
|
||||
logger.info("extend_history: full enriched rebuild start")
|
||||
|
||||
from app.indicators.pipeline import run_pipeline
|
||||
written_enriched = run_pipeline()
|
||||
|
||||
enriched_dir = repo.store.data_dir / "kline_daily_enriched"
|
||||
enriched_days = len(list(enriched_dir.glob("date=*"))) if enriched_dir.exists() else 0
|
||||
emit("extend_history", 92, f"enriched 完成,覆盖 {enriched_days} 天")
|
||||
logger.info("extend_history: enriched done, %d days", enriched_days)
|
||||
_refresh_single_view(repo, "kline_enriched")
|
||||
_invalidate("enriched")
|
||||
|
||||
# 6. 刷新视图
|
||||
emit("extend_history", 95, "刷新视图…")
|
||||
_refresh_single_view(repo, "kline_daily")
|
||||
_refresh_single_view(repo, "kline_enriched")
|
||||
_refresh_single_view(repo, "adj_factor")
|
||||
_invalidate(None)
|
||||
|
||||
# 7. 统计结果
|
||||
daily_dir = repo.store.data_dir / "kline_daily"
|
||||
daily_days = len(list(daily_dir.glob("date=*"))) if daily_dir.exists() else 0
|
||||
|
||||
emit("extend_history", 100, f"完成,已扩展至 {new_start}")
|
||||
|
||||
return {
|
||||
"earliest_before": earliest.isoformat(),
|
||||
"earliest_after": new_start.isoformat(),
|
||||
"daily_rows": written_daily,
|
||||
"daily_days": daily_days,
|
||||
"adj_factor_rows": written_adj,
|
||||
"enriched_days": enriched_days,
|
||||
"universe_size": len(universe),
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
"""财务数据独立同步服务。
|
||||
|
||||
解耦于 K-line 管道, 自有调度 + 自有存储。
|
||||
能力门控: Cap.FINANCIAL (Expert 套餐)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.tickflow.capabilities import Cap, CapabilitySet
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 每个 API 请求最多 100 个标的
|
||||
_BATCH_SIZE = 100
|
||||
|
||||
# 4 张财务表
|
||||
FINANCIAL_TABLES = ("metrics", "income", "balance_sheet", "cash_flow")
|
||||
|
||||
|
||||
# ================================================================
|
||||
# 同步函数
|
||||
# ================================================================
|
||||
|
||||
def _get_symbols(data_dir: Path) -> list[str]:
|
||||
"""从 instruments 表获取标的列表。"""
|
||||
inst_path = data_dir / "instruments" / "instruments.parquet"
|
||||
if not inst_path.exists():
|
||||
return []
|
||||
try:
|
||||
df = pl.read_parquet(inst_path, columns=["symbol"])
|
||||
return df["symbol"].to_list()
|
||||
except Exception as e:
|
||||
logger.warning("读取 instruments 失败: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
def _sync_table(
|
||||
table: str,
|
||||
symbols: list[str],
|
||||
data_dir: Path,
|
||||
capset: CapabilitySet,
|
||||
latest_only: bool = True,
|
||||
) -> int:
|
||||
"""同步单张财务表。返回写入的行数。"""
|
||||
if not capset.has(Cap.FINANCIAL):
|
||||
logger.info("sync_%s skipped: no FINANCIAL capability", table)
|
||||
return 0
|
||||
if not symbols:
|
||||
logger.warning("sync_%s skipped: no symbols", table)
|
||||
return 0
|
||||
|
||||
from app.tickflow.client import get_client
|
||||
tf = get_client()
|
||||
|
||||
# 分批拉取
|
||||
api_method = {
|
||||
"metrics": tf.financials.metrics,
|
||||
"income": tf.financials.income,
|
||||
"balance_sheet": tf.financials.balance_sheet,
|
||||
"cash_flow": tf.financials.cash_flow,
|
||||
}[table]
|
||||
|
||||
all_records: list[dict] = []
|
||||
total_batches = (len(symbols) + _BATCH_SIZE - 1) // _BATCH_SIZE
|
||||
|
||||
for i in range(0, len(symbols), _BATCH_SIZE):
|
||||
chunk = symbols[i : i + _BATCH_SIZE]
|
||||
batch_num = i // _BATCH_SIZE + 1
|
||||
try:
|
||||
data = api_method(chunk, latest=latest_only)
|
||||
# data 格式: { "600519.SH": [record, ...], ... }
|
||||
if isinstance(data, dict):
|
||||
for sym, records in data.items():
|
||||
if isinstance(records, list):
|
||||
for rec in records:
|
||||
if isinstance(rec, dict):
|
||||
rec["symbol"] = sym
|
||||
all_records.append(rec)
|
||||
logger.debug("sync_%s batch %d/%d: %d records", table, batch_num, total_batches, len(data) if isinstance(data, dict) else 0)
|
||||
except Exception as e:
|
||||
logger.warning("sync_%s batch %d/%d failed: %s", table, batch_num, total_batches, e)
|
||||
|
||||
if not all_records:
|
||||
return 0
|
||||
|
||||
df = pl.DataFrame(all_records)
|
||||
if df.is_empty():
|
||||
return 0
|
||||
|
||||
# 确保 symbol 列存在
|
||||
if "symbol" not in df.columns:
|
||||
return 0
|
||||
|
||||
# 写入 Parquet (全量覆盖)
|
||||
out_dir = data_dir / "financials" / table
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_file = out_dir / "part.parquet"
|
||||
df.write_parquet(out_file)
|
||||
|
||||
logger.info("sync_%s done: %d records written", table, len(df))
|
||||
return len(df)
|
||||
|
||||
|
||||
def sync_metrics(data_dir: Path, capset: CapabilitySet) -> int:
|
||||
"""同步核心财务指标 (metrics)。"""
|
||||
symbols = _get_symbols(data_dir)
|
||||
return _sync_table("metrics", symbols, data_dir, capset, latest_only=True)
|
||||
|
||||
|
||||
def sync_income(data_dir: Path, capset: CapabilitySet) -> int:
|
||||
"""同步利润表。"""
|
||||
symbols = _get_symbols(data_dir)
|
||||
return _sync_table("income", symbols, data_dir, capset, latest_only=True)
|
||||
|
||||
|
||||
def sync_balance_sheet(data_dir: Path, capset: CapabilitySet) -> int:
|
||||
"""同步资产负债表。"""
|
||||
symbols = _get_symbols(data_dir)
|
||||
return _sync_table("balance_sheet", symbols, data_dir, capset, latest_only=True)
|
||||
|
||||
|
||||
def sync_cash_flow(data_dir: Path, capset: CapabilitySet) -> int:
|
||||
"""同步现金流量表。"""
|
||||
symbols = _get_symbols(data_dir)
|
||||
return _sync_table("cash_flow", symbols, data_dir, capset, latest_only=True)
|
||||
|
||||
|
||||
def sync_all(data_dir: Path, capset: CapabilitySet) -> dict[str, int]:
|
||||
"""同步所有财务表。返回 {table: rows}。"""
|
||||
if not capset.has(Cap.FINANCIAL):
|
||||
logger.info("sync_all financials skipped: no FINANCIAL capability")
|
||||
return {}
|
||||
|
||||
symbols = _get_symbols(data_dir)
|
||||
results: dict[str, int] = {}
|
||||
for table in FINANCIAL_TABLES:
|
||||
results[table] = _sync_table(table, symbols, data_dir, capset, latest_only=True)
|
||||
|
||||
# 同步完成后注册 DuckDB 视图
|
||||
_refresh_financials_views(data_dir)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# ================================================================
|
||||
# DuckDB 视图
|
||||
# ================================================================
|
||||
|
||||
def _refresh_financials_views(data_dir: Path) -> None:
|
||||
"""刷新财务表 DuckDB 视图 (在 DataStore.db 上注册)。"""
|
||||
d = data_dir.as_posix()
|
||||
views = {
|
||||
"financials_metrics": f"{d}/financials/metrics/*.parquet",
|
||||
"financials_income": f"{d}/financials/income/*.parquet",
|
||||
"financials_balance_sheet": f"{d}/financials/balance_sheet/*.parquet",
|
||||
"financials_cash_flow": f"{d}/financials/cash_flow/*.parquet",
|
||||
}
|
||||
for name, path in views.items():
|
||||
out = data_dir / "financials" / name.replace("financials_", "") / "part.parquet"
|
||||
if not out.exists():
|
||||
continue
|
||||
# 视图注册需要由 DataStore 完成,这里只做日志
|
||||
logger.debug("financial parquet ready: %s (%d rows)", name, out.stat().st_size)
|
||||
|
||||
|
||||
def get_financial_df(data_dir: Path, table: str) -> pl.DataFrame:
|
||||
"""读取本地财务 Parquet。"""
|
||||
path = data_dir / "financials" / table / "part.parquet"
|
||||
if not path.exists():
|
||||
return pl.DataFrame()
|
||||
try:
|
||||
return pl.read_parquet(path)
|
||||
except Exception as e:
|
||||
logger.warning("读取 financials/%s 失败: %s", table, e)
|
||||
return pl.DataFrame()
|
||||
|
||||
|
||||
# ================================================================
|
||||
# 调度器
|
||||
# ================================================================
|
||||
|
||||
class FinancialScheduler:
|
||||
"""独立调度器: 每周同步 metrics, 每季度同步三张报表。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._task: asyncio.Task | None = None
|
||||
self._running = False
|
||||
self._data_dir: Path | None = None
|
||||
self._capset: CapabilitySet | None = None
|
||||
self._lock = threading.Lock()
|
||||
self._last_sync: dict[str, str] = {} # {table: iso_timestamp}
|
||||
# 手动同步(run_now)是否正在进行。前端据此显示"同步中"并防重复点击。
|
||||
self._is_syncing = False
|
||||
|
||||
def start(self, data_dir: Path, capset: CapabilitySet) -> None:
|
||||
if not capset.has(Cap.FINANCIAL):
|
||||
logger.info("FinancialScheduler skipped: no FINANCIAL capability")
|
||||
return
|
||||
self._data_dir = data_dir
|
||||
self._capset = capset
|
||||
self._running = True
|
||||
self._task = asyncio.create_task(self._run_loop())
|
||||
logger.info("FinancialScheduler started")
|
||||
|
||||
def stop(self) -> None:
|
||||
self._running = False
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
self._task = None
|
||||
logger.info("FinancialScheduler stopped")
|
||||
|
||||
def update(self, data_dir: Path, capset: CapabilitySet) -> None:
|
||||
"""运行时更新数据目录和能力集。
|
||||
|
||||
用户在设置页更换/清除 Key 后,能力集可能变化,无需重启服务即可让
|
||||
财务调度器生效或失效。
|
||||
"""
|
||||
had_financial = self._capset is not None and self._capset.has(Cap.FINANCIAL)
|
||||
has_financial = capset.has(Cap.FINANCIAL)
|
||||
self._data_dir = data_dir
|
||||
self._capset = capset
|
||||
|
||||
if has_financial and not self._running:
|
||||
self.start(data_dir, capset)
|
||||
elif had_financial and not has_financial and self._running:
|
||||
self.stop()
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
"""每周执行一次 metrics 同步。"""
|
||||
try:
|
||||
while self._running:
|
||||
# 首次启动等 60s, 之后每 7 天执行一次
|
||||
await asyncio.sleep(60)
|
||||
if not self._running:
|
||||
break
|
||||
|
||||
# 每周: 只同步 metrics
|
||||
try:
|
||||
rows = sync_metrics(self._data_dir, self._capset)
|
||||
self._last_sync["metrics"] = datetime.now(timezone.utc).isoformat()
|
||||
logger.info("FinancialScheduler: metrics synced, %d rows", rows)
|
||||
except Exception as e:
|
||||
logger.warning("FinancialScheduler: metrics sync failed: %s", e)
|
||||
|
||||
# 等待下一次 (7天)
|
||||
for _ in range(7 * 24 * 60): # 每分钟检查一次 _running
|
||||
if not self._running:
|
||||
break
|
||||
await asyncio.sleep(60)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
def run_now(self, table: str | None = None) -> dict[str, int]:
|
||||
"""手动触发同步。table=None 同步全部。
|
||||
|
||||
用 _is_syncing 标志防并发:若已有同步在进行,本次直接跳过,
|
||||
避免重复请求拖慢服务端 / 触发上游限流。
|
||||
"""
|
||||
if not self._capset or not self._capset.has(Cap.FINANCIAL):
|
||||
return {}
|
||||
with self._lock:
|
||||
if self._is_syncing:
|
||||
logger.info("financial sync skipped: already running")
|
||||
return {"_skipped": 1}
|
||||
self._is_syncing = True
|
||||
try:
|
||||
if table:
|
||||
fn = {
|
||||
"metrics": sync_metrics,
|
||||
"income": sync_income,
|
||||
"balance_sheet": sync_balance_sheet,
|
||||
"cash_flow": sync_cash_flow,
|
||||
}.get(table)
|
||||
if not fn:
|
||||
return {}
|
||||
rows = fn(self._data_dir, self._capset)
|
||||
self._last_sync[table] = datetime.now(timezone.utc).isoformat()
|
||||
return {table: rows}
|
||||
else:
|
||||
# 全部同步: 逐表执行, 每张完成立即更新 last_sync,
|
||||
# 让前端轮询 /status 能看到进度递增 (而非等全部完成才一次性更新)。
|
||||
symbols = _get_symbols(self._data_dir)
|
||||
result: dict[str, int] = {}
|
||||
for t in FINANCIAL_TABLES:
|
||||
result[t] = _sync_table(t, symbols, self._data_dir, self._capset, latest_only=True)
|
||||
self._last_sync[t] = datetime.now(timezone.utc).isoformat()
|
||||
_refresh_financials_views(self._data_dir)
|
||||
return result
|
||||
finally:
|
||||
with self._lock:
|
||||
self._is_syncing = False
|
||||
|
||||
@property
|
||||
def is_syncing(self) -> bool:
|
||||
"""手动同步是否正在进行(供 /status 返回,前端据此显示"同步中")。"""
|
||||
with self._lock:
|
||||
return self._is_syncing
|
||||
|
||||
@property
|
||||
def last_sync(self) -> dict[str, str]:
|
||||
return dict(self._last_sync)
|
||||
|
||||
|
||||
# 全局单例
|
||||
financial_scheduler = FinancialScheduler()
|
||||
@@ -0,0 +1,146 @@
|
||||
"""指数数据同步服务。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import gc
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.indicators.pipeline import compute_enriched
|
||||
from app.services import kline_sync, preferences
|
||||
from app.tickflow.capabilities import Cap, CapabilitySet
|
||||
from app.tickflow.client import get_client
|
||||
from app.tickflow.repository import KlineRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _quotes_to_index_instruments(resp) -> pl.DataFrame:
|
||||
"""将数据源 quotes 响应规范为指数 instruments。"""
|
||||
if resp is None:
|
||||
return pl.DataFrame()
|
||||
|
||||
if isinstance(resp, pl.DataFrame):
|
||||
df = resp
|
||||
elif hasattr(resp, "columns"):
|
||||
df = pl.from_pandas(resp.reset_index() if hasattr(resp, "reset_index") else resp)
|
||||
else:
|
||||
rows: list[dict] = []
|
||||
for q in resp or []:
|
||||
item = q if isinstance(q, dict) else {}
|
||||
ext = item.get("ext") or {}
|
||||
symbol = item.get("symbol")
|
||||
if not symbol:
|
||||
continue
|
||||
rows.append({
|
||||
"symbol": str(symbol),
|
||||
"name": ext.get("name") or item.get("name") or str(symbol),
|
||||
})
|
||||
df = pl.DataFrame(rows)
|
||||
|
||||
if df.is_empty() or "symbol" not in df.columns:
|
||||
return pl.DataFrame()
|
||||
|
||||
rename = {"ts_code": "symbol"}
|
||||
df = df.rename({k: v for k, v in rename.items() if k in df.columns})
|
||||
|
||||
if "name" not in df.columns:
|
||||
if "ext" in df.columns:
|
||||
df = df.with_columns(pl.col("symbol").cast(pl.Utf8).alias("name"))
|
||||
else:
|
||||
df = df.with_columns(pl.col("symbol").cast(pl.Utf8).alias("name"))
|
||||
|
||||
result = df.select([
|
||||
pl.col("symbol").cast(pl.Utf8),
|
||||
pl.col("name").cast(pl.Utf8),
|
||||
]).with_columns([
|
||||
pl.col("symbol").str.split(".").list.first().alias("code"),
|
||||
pl.lit("index").alias("asset_type"),
|
||||
])
|
||||
return result.unique(subset=["symbol"], keep="last").sort("symbol")
|
||||
|
||||
|
||||
def sync_index_instruments(repo: KlineRepository) -> int:
|
||||
"""同步 CN_Index 指数标的维表，返回指数数量。"""
|
||||
tf = get_client()
|
||||
resp = None
|
||||
errors: list[str] = []
|
||||
for kwargs in (
|
||||
{"universes": ["CN_Index"]},
|
||||
{"universes": ["CN_Index"], "as_dataframe": False},
|
||||
):
|
||||
try:
|
||||
resp = tf.quotes.get_by_universes(**kwargs)
|
||||
if resp is not None and len(resp) > 0:
|
||||
break
|
||||
except Exception as e: # noqa: BLE001
|
||||
errors.append(str(e))
|
||||
resp = None
|
||||
|
||||
if resp is None or len(resp) == 0:
|
||||
logger.warning("CN_Index universe returned empty: %s", "; ".join(errors))
|
||||
return 0
|
||||
|
||||
instruments = _quotes_to_index_instruments(resp)
|
||||
if instruments.is_empty():
|
||||
return 0
|
||||
repo.save_index_instruments(instruments)
|
||||
repo.refresh_index_views()
|
||||
return instruments.height
|
||||
|
||||
|
||||
def sync_and_persist_index_daily(
|
||||
repo: KlineRepository,
|
||||
capset: CapabilitySet,
|
||||
count: int | None = None,
|
||||
start_date: datetime | None = None,
|
||||
end_date: datetime | None = None,
|
||||
) -> int:
|
||||
"""同步指数日K到独立 parquet，并计算指数 enriched。"""
|
||||
if not capset.has(Cap.KLINE_DAILY_BATCH):
|
||||
return 0
|
||||
|
||||
instruments = repo.get_index_instruments()
|
||||
if instruments.is_empty():
|
||||
sync_index_instruments(repo)
|
||||
instruments = repo.get_index_instruments()
|
||||
if instruments.is_empty() or "symbol" not in instruments.columns:
|
||||
return 0
|
||||
|
||||
symbols = sorted(set(instruments["symbol"].to_list()))
|
||||
lim = capset.limits(Cap.KLINE_DAILY_BATCH)
|
||||
batch_size = preferences.get_index_daily_batch_size()
|
||||
if lim and lim.batch:
|
||||
batch_size = min(batch_size, lim.batch)
|
||||
rpm = lim.rpm if lim else None
|
||||
|
||||
end_time = end_date or datetime.now()
|
||||
start_time = start_date or (end_time - timedelta(days=365))
|
||||
|
||||
total_rows = 0
|
||||
interval = (60.0 / rpm) if rpm else 0
|
||||
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
|
||||
for i, chunk in enumerate(chunks):
|
||||
if i > 0 and interval > 0 and len(chunks) > rpm:
|
||||
import time
|
||||
time.sleep(interval)
|
||||
raw = kline_sync.sync_daily_batch(
|
||||
chunk,
|
||||
count=count,
|
||||
batch_size=None,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
if raw.is_empty():
|
||||
continue
|
||||
|
||||
repo.append_index_daily(raw)
|
||||
enriched = compute_enriched(raw, factors=None, instruments=None)
|
||||
repo.append_index_enriched(enriched)
|
||||
total_rows += raw.height
|
||||
logger.info("index daily synced: %d/%d chunks, +%d rows", i + 1, len(chunks), raw.height)
|
||||
del raw, enriched
|
||||
gc.collect()
|
||||
repo.refresh_index_views()
|
||||
return total_rows
|
||||
@@ -0,0 +1,122 @@
|
||||
"""标的维表同步服务。
|
||||
|
||||
盘前 9:10 调用 tf.exchanges.get_instruments("SH"/"SZ"/"BJ", type="stock")
|
||||
获取全量标的元数据，flatten ext 字段，写入 instruments.parquet。
|
||||
|
||||
Starter+ 盘后可用 quotes.get(universes) 顺便补充 name。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.tickflow.client import get_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_EXCHANGES = ["SH", "SZ", "BJ"]
|
||||
|
||||
|
||||
def _flatten_instruments(items: list[dict]) -> list[dict]:
|
||||
"""把 SDK 返回的 Instrument 列表 flatten 成扁平行。"""
|
||||
rows = []
|
||||
for item in items:
|
||||
row = {
|
||||
"symbol": item.get("symbol"),
|
||||
"name": item.get("name"),
|
||||
"code": item.get("code"),
|
||||
"exchange": item.get("exchange"),
|
||||
"region": item.get("region"),
|
||||
"type": item.get("type"),
|
||||
}
|
||||
ext = item.get("ext") or {}
|
||||
row["listing_date"] = ext.get("listing_date")
|
||||
row["total_shares"] = ext.get("total_shares")
|
||||
row["float_shares"] = ext.get("float_shares")
|
||||
row["tick_size"] = ext.get("tick_size")
|
||||
row["limit_up"] = ext.get("limit_up")
|
||||
row["limit_down"] = ext.get("limit_down")
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def sync_instruments(data_dir: Path) -> int:
|
||||
"""全量同步标的维表 → data/instruments/instruments.parquet。
|
||||
|
||||
返回写入的行数。
|
||||
"""
|
||||
tf = get_client()
|
||||
all_rows: list[dict] = []
|
||||
|
||||
for ex in _EXCHANGES:
|
||||
try:
|
||||
items = tf.exchanges.get_instruments(ex, instrument_type="stock")
|
||||
if items:
|
||||
all_rows.extend(_flatten_instruments(items))
|
||||
logger.info("instruments %s: %d stocks", ex, len(items))
|
||||
except Exception as e:
|
||||
logger.warning("get_instruments(%s) failed: %s", ex, e)
|
||||
|
||||
if not all_rows:
|
||||
return 0
|
||||
|
||||
df = pl.DataFrame(all_rows)
|
||||
df = df.with_columns(pl.lit(date.today()).alias("as_of"))
|
||||
|
||||
out = data_dir / "instruments" / "instruments.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.write_parquet(out)
|
||||
|
||||
logger.info("instruments synced: %d rows → %s", df.height, out)
|
||||
return df.height
|
||||
|
||||
|
||||
def enrich_names_from_quotes(
|
||||
data_dir: Path,
|
||||
quotes_data: list[dict],
|
||||
) -> int:
|
||||
"""从 quotes 响应中提取 name，更新 instruments 维表（兜底补充）。
|
||||
|
||||
盘后 quotes.get(universes) 返回的数据中包含 ext.name，
|
||||
用来补充 instruments 中可能缺失的 name。
|
||||
"""
|
||||
if not quotes_data:
|
||||
return 0
|
||||
|
||||
# 构建 symbol → name 映射
|
||||
name_map: dict[str, str] = {}
|
||||
for q in quotes_data:
|
||||
symbol = q.get("symbol", "")
|
||||
ext = q.get("ext") or {}
|
||||
name = ext.get("name") or q.get("name", "")
|
||||
if symbol and name:
|
||||
name_map[symbol] = name
|
||||
|
||||
if not name_map:
|
||||
return 0
|
||||
|
||||
inst_path = data_dir / "instruments" / "instruments.parquet"
|
||||
if not inst_path.exists():
|
||||
return 0
|
||||
|
||||
df = pl.read_parquet(inst_path)
|
||||
|
||||
# 只更新空 name 的行
|
||||
updates = pl.DataFrame({
|
||||
"symbol": list(name_map.keys()),
|
||||
"_new_name": list(name_map.values()),
|
||||
})
|
||||
df = df.join(updates, on="symbol", how="left")
|
||||
df = df.with_columns(
|
||||
pl.when(pl.col("name").is_null() | (pl.col("name") == ""))
|
||||
.then(pl.col("_new_name"))
|
||||
.otherwise(pl.col("name"))
|
||||
.alias("name"),
|
||||
).drop("_new_name")
|
||||
|
||||
df.write_parquet(inst_path)
|
||||
logger.info("instruments name enriched from quotes: %d names", len(name_map))
|
||||
return len(name_map)
|
||||
@@ -0,0 +1,617 @@
|
||||
"""日 K 同步服务(§7.7 Step 1)。
|
||||
|
||||
调度器在 capability 允许下,把符号集合的日 K 批量同步到本地 Parquet。
|
||||
策略:
|
||||
- 日 K 仅使用 `kline.daily.batch`
|
||||
- 除权因子仅使用 `adj_factor`
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.indicators.pipeline import filter_halt_days
|
||||
from app.tickflow.capabilities import Cap, CapabilitySet
|
||||
from app.tickflow.client import get_client
|
||||
from app.tickflow.repository import KlineRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# 标准列(无论 SDK 返回什么形状,我们把它规范成这套)
|
||||
CANONICAL_DAILY_COLS = [
|
||||
"symbol", "date", "open", "high", "low", "close", "volume", "amount",
|
||||
]
|
||||
|
||||
|
||||
def _normalize_daily(df_in, default_symbol: str | None = None) -> pl.DataFrame:
|
||||
"""把 SDK 返回的 pandas/任意 DataFrame 规范成 canonical 列。"""
|
||||
if df_in is None or len(df_in) == 0:
|
||||
return pl.DataFrame()
|
||||
|
||||
if not isinstance(df_in, pl.DataFrame):
|
||||
df = pl.from_pandas(df_in.reset_index() if hasattr(df_in, "reset_index") else df_in)
|
||||
else:
|
||||
df = df_in
|
||||
|
||||
# 兼容字段名差异
|
||||
rename_map = {
|
||||
"ts_code": "symbol",
|
||||
"trade_date": "date",
|
||||
"vol": "volume",
|
||||
"amt": "amount",
|
||||
"datetime": "date",
|
||||
}
|
||||
df = df.rename({k: v for k, v in rename_map.items() if k in df.columns})
|
||||
|
||||
if "symbol" not in df.columns and default_symbol is not None:
|
||||
df = df.with_columns(pl.lit(default_symbol).alias("symbol"))
|
||||
|
||||
# 类型规范
|
||||
if "date" in df.columns and df.schema["date"] != pl.Date:
|
||||
df = df.with_columns(pl.col("date").cast(pl.Date, strict=False))
|
||||
|
||||
for col in ("open", "high", "low", "close"):
|
||||
if col in df.columns:
|
||||
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
|
||||
for col in ("volume", "amount"):
|
||||
if col in df.columns:
|
||||
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
|
||||
|
||||
# 过滤停牌日 (open/high 为 0; close 可能被填充为前收盘价, 不能用全零判断)
|
||||
df = filter_halt_days(df)
|
||||
|
||||
# 只保留 canonical 列
|
||||
keep = [c for c in CANONICAL_DAILY_COLS if c in df.columns]
|
||||
return df.select(keep)
|
||||
|
||||
|
||||
def sync_daily_batch(symbols: list[str],
|
||||
count: int | None = None,
|
||||
batch_size: int | None = None,
|
||||
rpm: int | None = None,
|
||||
start_time: datetime | None = None,
|
||||
end_time: datetime | None = None,
|
||||
on_chunk_done: Callable[[int, int], None] | None = None) -> pl.DataFrame:
|
||||
"""批量拉取多股日 K。
|
||||
|
||||
优先使用 start_time / end_time 区间 + count=10000,确保覆盖完整时间段。
|
||||
仅传 count 时按条数回溯。
|
||||
"""
|
||||
tf = get_client()
|
||||
out: list[pl.DataFrame] = []
|
||||
interval = (60.0 / rpm) if rpm else 0
|
||||
|
||||
if batch_size is None:
|
||||
chunks = [symbols]
|
||||
else:
|
||||
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
if i > 0 and interval > 0 and len(chunks) > rpm:
|
||||
time.sleep(interval)
|
||||
try:
|
||||
if start_time and end_time:
|
||||
raw = tf.klines.batch(
|
||||
chunk, period="1d", adjust="none",
|
||||
start_time=_datetime_to_ms(start_time),
|
||||
end_time=_datetime_to_ms(end_time),
|
||||
count=10000,
|
||||
as_dataframe=True, show_progress=False,
|
||||
)
|
||||
else:
|
||||
raw = tf.klines.batch(chunk, period="1d", count=count or 250, adjust="none",
|
||||
as_dataframe=True, show_progress=False)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("batch fetch failed for %d symbols: %s", len(chunk), e)
|
||||
continue
|
||||
|
||||
# 兼容两种形态:dict[sym → df] 和扁平 df
|
||||
if isinstance(raw, dict):
|
||||
for sym, sub in raw.items():
|
||||
if sub is None or len(sub) == 0:
|
||||
continue
|
||||
out.append(_normalize_daily(sub, default_symbol=sym))
|
||||
elif raw is not None and len(raw) > 0:
|
||||
out.append(_normalize_daily(raw))
|
||||
|
||||
if on_chunk_done:
|
||||
on_chunk_done(i + 1, len(chunks))
|
||||
|
||||
if not out:
|
||||
return pl.DataFrame()
|
||||
return pl.concat(out, how="diagonal_relaxed")
|
||||
|
||||
|
||||
def sync_and_persist_daily_batch(
|
||||
symbols: list[str],
|
||||
repo: KlineRepository,
|
||||
capset: CapabilitySet,
|
||||
count: int | None = None,
|
||||
start_date: datetime | None = None,
|
||||
end_date: datetime | None = None,
|
||||
on_chunk_done: Callable[[int, int], None] | None = None,
|
||||
) -> int:
|
||||
"""批量同步日 K 并落到 Parquet。返回写入的行数。
|
||||
|
||||
start_date/end_date: 外部传入的时间范围(由 pipeline 根据已有数据计算)。
|
||||
未传入时默认拉最近 1 年。
|
||||
"""
|
||||
if not symbols or not capset.has(Cap.KLINE_DAILY_BATCH):
|
||||
return 0
|
||||
|
||||
lim = capset.limits(Cap.KLINE_DAILY_BATCH)
|
||||
batch_size = lim.batch if lim and lim.batch else 100
|
||||
rpm = lim.rpm if lim else None
|
||||
|
||||
end_time = end_date or datetime.now()
|
||||
start_time = start_date or (end_time - timedelta(days=365))
|
||||
|
||||
df = sync_daily_batch(
|
||||
symbols, count=count, batch_size=batch_size, rpm=rpm,
|
||||
start_time=start_time, end_time=end_time,
|
||||
on_chunk_done=on_chunk_done,
|
||||
)
|
||||
|
||||
if df.is_empty():
|
||||
return 0
|
||||
|
||||
repo.append_daily(df)
|
||||
|
||||
try:
|
||||
d = repo.store.data_dir.as_posix()
|
||||
repo.db.execute(
|
||||
f"""CREATE OR REPLACE VIEW kline_daily AS
|
||||
SELECT * FROM read_parquet('{d}/kline_daily/**/*.parquet', union_by_name=true)"""
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("refresh view failed: %s", e)
|
||||
|
||||
return df.height
|
||||
|
||||
|
||||
def sync_daily_by_quotes(repo: KlineRepository) -> int:
|
||||
"""用实时行情接口拉全市场当日数据,覆写 kline_daily 今天分区。
|
||||
|
||||
一个请求覆盖 ~5500 只股票,比 batch K-line 快几个数量级。
|
||||
返回写入的行数。
|
||||
"""
|
||||
from datetime import date as _date
|
||||
|
||||
from app.tickflow.client import get_client
|
||||
|
||||
tf = get_client()
|
||||
try:
|
||||
resp = tf.quotes.get_by_universes(universes=["CN_Equity_A"])
|
||||
except Exception as e:
|
||||
logger.warning("get_by_universes failed: %s", e)
|
||||
return 0
|
||||
|
||||
if not resp:
|
||||
logger.warning("get_by_universes returned empty")
|
||||
return 0
|
||||
|
||||
records = []
|
||||
for q in resp:
|
||||
ext = q.get("ext") or {}
|
||||
records.append({
|
||||
"symbol": q.get("symbol"),
|
||||
"open": q.get("open"),
|
||||
"high": q.get("high"),
|
||||
"low": q.get("low"),
|
||||
"close": q.get("last_price"),
|
||||
"volume": q.get("volume"),
|
||||
"amount": q.get("amount"),
|
||||
})
|
||||
|
||||
df = pl.DataFrame(records)
|
||||
if df.is_empty():
|
||||
return 0
|
||||
|
||||
today = _date.today()
|
||||
daily_df = df.with_columns(pl.lit(today).cast(pl.Date).alias("date"))
|
||||
|
||||
# 过滤停牌 (open/high 为 0; close 可能被填充为前收盘价, 不能用全零判断)
|
||||
daily_df = filter_halt_days(daily_df)
|
||||
|
||||
repo.flush_live_daily(daily_df)
|
||||
logger.info("sync_daily_by_quotes: %d symbols flushed for %s", daily_df.height, today)
|
||||
return daily_df.height
|
||||
|
||||
|
||||
def sync_adj_factor(symbols: list[str], repo: KlineRepository,
|
||||
capset: CapabilitySet,
|
||||
start_time: datetime | None = None,
|
||||
end_time: datetime | None = None,
|
||||
on_chunk_done: Callable[[int, int], None] | None = None) -> tuple[int, list[str]]:
|
||||
"""同步除权因子(Starter+)。SDK 接口:`tf.klines.ex_factors(symbols=...)`。
|
||||
|
||||
支持增量: 传 start_time/end_time 只拉取该时间范围内的新除权事件。
|
||||
返回 (写入行数, 受影响的 symbol 列表) — 供 enriched 局部重算使用。
|
||||
"""
|
||||
if not capset.has(Cap.ADJ_FACTOR) or not symbols:
|
||||
return 0, []
|
||||
|
||||
tf = get_client()
|
||||
lim = capset.limits(Cap.ADJ_FACTOR)
|
||||
batch_size = lim.batch if lim and lim.batch else 50
|
||||
rpm = lim.rpm if lim else 30
|
||||
interval = 60.0 / rpm if rpm else 0
|
||||
|
||||
# 构建 SDK 参数
|
||||
sdk_kwargs: dict = {"as_dataframe": True, "batch_size": batch_size, "show_progress": False}
|
||||
if start_time:
|
||||
sdk_kwargs["start_time"] = _datetime_to_ms(start_time)
|
||||
if end_time:
|
||||
sdk_kwargs["end_time"] = _datetime_to_ms(end_time)
|
||||
|
||||
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
|
||||
all_dfs: list[pl.DataFrame] = []
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
if i > 0 and interval > 0 and len(chunks) > rpm:
|
||||
time.sleep(interval)
|
||||
try:
|
||||
raw = tf.klines.ex_factors(chunk, **sdk_kwargs)
|
||||
if raw is not None and len(raw) > 0:
|
||||
all_dfs.append(pl.from_pandas(
|
||||
raw.reset_index() if hasattr(raw, "reset_index") else raw
|
||||
))
|
||||
logger.debug("adj_factor chunk %d/%d: %d symbols", i + 1, len(chunks), len(chunk))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("adj_factor chunk %d failed: %s", i + 1, e)
|
||||
|
||||
if on_chunk_done:
|
||||
on_chunk_done(i + 1, len(chunks))
|
||||
|
||||
if not all_dfs:
|
||||
return 0, []
|
||||
|
||||
new_data = pl.concat(all_dfs, how="diagonal_relaxed") if len(all_dfs) > 1 else all_dfs[0]
|
||||
|
||||
# 提取受影响的 symbol 列表(合并前)
|
||||
affected = new_data["symbol"].unique().to_list()
|
||||
|
||||
out = repo.store.data_dir / "adj_factor" / "all.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if out.exists():
|
||||
existing = pl.read_parquet(out)
|
||||
before = existing.height
|
||||
merged = pl.concat([existing, new_data]).unique(
|
||||
subset=["symbol", "trade_date"], keep="last",
|
||||
).sort(["symbol", "trade_date"])
|
||||
merged.write_parquet(out)
|
||||
added = merged.height - before
|
||||
logger.info("adj_factor merged: %d total (+%d new), %d/%d symbols",
|
||||
merged.height, added, new_data.height, len(symbols))
|
||||
return added, affected
|
||||
else:
|
||||
new_data.sort(["symbol", "trade_date"]).write_parquet(out)
|
||||
logger.info("adj_factor synced: %d rows (%d symbols)", new_data.height, len(symbols))
|
||||
return new_data.height, affected
|
||||
|
||||
|
||||
# ===== 分钟 K 同步 =====
|
||||
|
||||
CANONICAL_MINUTE_COLS = [
|
||||
"symbol", "datetime", "open", "high", "low", "close", "volume", "amount",
|
||||
]
|
||||
|
||||
|
||||
def _normalize_minute(df_in, default_symbol: str | None = None) -> pl.DataFrame:
|
||||
"""把 SDK 返回的分钟 K 数据规范成 canonical 列。"""
|
||||
if df_in is None or len(df_in) == 0:
|
||||
return pl.DataFrame()
|
||||
|
||||
if not isinstance(df_in, pl.DataFrame):
|
||||
df = pl.from_pandas(df_in.reset_index() if hasattr(df_in, "reset_index") else df_in)
|
||||
else:
|
||||
df = df_in
|
||||
|
||||
rename_map = {
|
||||
"ts_code": "symbol",
|
||||
"vol": "volume",
|
||||
"amt": "amount",
|
||||
}
|
||||
df = df.rename({k: v for k, v in rename_map.items() if k in df.columns})
|
||||
|
||||
# datetime 列:优先用 timestamp(毫秒精度),其次 trade_time
|
||||
if "timestamp" in df.columns:
|
||||
df = df.with_columns(
|
||||
pl.from_epoch("timestamp", time_unit="ms").alias("datetime"),
|
||||
).drop("timestamp")
|
||||
for drop_col in ("trade_time", "trade_date"):
|
||||
if drop_col in df.columns:
|
||||
df = df.drop(drop_col)
|
||||
elif "trade_time" in df.columns:
|
||||
df = df.rename({"trade_time": "datetime"})
|
||||
if "trade_date" in df.columns:
|
||||
df = df.drop("trade_date")
|
||||
elif "trade_date" in df.columns:
|
||||
df = df.rename({"trade_date": "datetime"})
|
||||
|
||||
if "symbol" not in df.columns and default_symbol is not None:
|
||||
df = df.with_columns(pl.lit(default_symbol).alias("symbol"))
|
||||
|
||||
# 类型规范:统一转 Datetime('us')
|
||||
if "datetime" in df.columns:
|
||||
dt_type = df.schema["datetime"]
|
||||
if not isinstance(dt_type, pl.Datetime) or dt_type.time_unit != "us":
|
||||
df = df.with_columns(pl.col("datetime").cast(pl.Datetime("us"), strict=False))
|
||||
|
||||
for col in ("open", "high", "low", "close"):
|
||||
if col in df.columns:
|
||||
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
|
||||
for col in ("volume", "amount"):
|
||||
if col in df.columns:
|
||||
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
|
||||
|
||||
keep = [c for c in CANONICAL_MINUTE_COLS if c in df.columns]
|
||||
return df.select(keep)
|
||||
|
||||
|
||||
def _datetime_to_ms(dt: datetime) -> int:
|
||||
"""datetime → 毫秒时间戳 (供 SDK start_time / end_time 使用)。"""
|
||||
return int(dt.timestamp() * 1000)
|
||||
|
||||
|
||||
def sync_minute_batch(
|
||||
symbols: list[str],
|
||||
start_time: datetime | None = None,
|
||||
end_time: datetime | None = None,
|
||||
count: int | None = None,
|
||||
batch_size: int | None = None,
|
||||
rpm: int | None = None,
|
||||
on_chunk_done: Callable[[int, int], None] | None = None,
|
||||
) -> pl.DataFrame:
|
||||
"""批量拉取多股分钟 K。
|
||||
|
||||
优先使用 start_time / end_time 区间, 确保所有标的覆盖同一时间段。
|
||||
count 仅作为 fallback 保留。
|
||||
on_chunk_done(current, total) 每个 chunk 完成后回调。
|
||||
"""
|
||||
tf = get_client()
|
||||
out: list[pl.DataFrame] = []
|
||||
interval = (60.0 / rpm) if rpm else 0
|
||||
|
||||
if batch_size is None:
|
||||
chunks = [symbols]
|
||||
else:
|
||||
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
if i > 0 and interval > 0 and len(chunks) > rpm:
|
||||
time.sleep(interval)
|
||||
try:
|
||||
if start_time and end_time:
|
||||
raw = tf.klines.batch(
|
||||
chunk, period="1m",
|
||||
start_time=_datetime_to_ms(start_time),
|
||||
end_time=_datetime_to_ms(end_time),
|
||||
count=10000,
|
||||
as_dataframe=True, show_progress=False,
|
||||
)
|
||||
else:
|
||||
raw = tf.klines.batch(chunk, period="1m", count=count or 1200,
|
||||
as_dataframe=True, show_progress=False)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("minute batch fetch failed for %d symbols: %s", len(chunk), e)
|
||||
continue
|
||||
|
||||
if isinstance(raw, dict):
|
||||
for sym, sub in raw.items():
|
||||
if sub is None or len(sub) == 0:
|
||||
continue
|
||||
out.append(_normalize_minute(sub, default_symbol=sym))
|
||||
elif raw is not None and len(raw) > 0:
|
||||
out.append(_normalize_minute(raw))
|
||||
|
||||
if on_chunk_done:
|
||||
on_chunk_done(i + 1, len(chunks))
|
||||
|
||||
if not out:
|
||||
return pl.DataFrame()
|
||||
return pl.concat(out, how="diagonal_relaxed")
|
||||
|
||||
|
||||
def fetch_minute_single(symbol: str, trade_date: date) -> pl.DataFrame:
|
||||
"""从数据源实时拉取单股单日分钟 K（不写入本地）。"""
|
||||
from datetime import datetime
|
||||
start_time = datetime(trade_date.year, trade_date.month, trade_date.day, 9, 25, 0)
|
||||
end_time = datetime(trade_date.year, trade_date.month, trade_date.day, 15, 5, 0)
|
||||
tf = get_client()
|
||||
try:
|
||||
raw = tf.klines.batch(
|
||||
[symbol], period="1m",
|
||||
start_time=_datetime_to_ms(start_time),
|
||||
end_time=_datetime_to_ms(end_time),
|
||||
count=10000,
|
||||
as_dataframe=True, show_progress=False,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("fetch_minute_single(%s, %s) failed: %s", symbol, trade_date, e)
|
||||
return pl.DataFrame()
|
||||
|
||||
if isinstance(raw, dict):
|
||||
sub = raw.get(symbol)
|
||||
return _normalize_minute(sub) if sub is not None and len(sub) > 0 else pl.DataFrame()
|
||||
if raw is not None and len(raw) > 0:
|
||||
return _normalize_minute(raw)
|
||||
return pl.DataFrame()
|
||||
|
||||
|
||||
def _latest_minute_datetime(repo: KlineRepository) -> datetime | None:
|
||||
"""本地分钟 K 数据的最新时间。"""
|
||||
try:
|
||||
res = repo.execute_one("SELECT max(datetime) FROM kline_minute")
|
||||
if res and res[0]:
|
||||
d = res[0]
|
||||
if isinstance(d, datetime):
|
||||
return d
|
||||
return datetime.fromisoformat(str(d))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _cleanup_null_datetime_minute(repo: KlineRepository) -> None:
|
||||
"""检测并清除 datetime 全为 null 的旧版分钟 K 数据(迁移用)。"""
|
||||
minute_dir = repo.store.data_dir / "kline_minute"
|
||||
if not minute_dir.exists():
|
||||
return
|
||||
try:
|
||||
row = repo.execute_one(
|
||||
"SELECT count(*) AS total, count(datetime) AS non_null FROM kline_minute"
|
||||
)
|
||||
if row and row[0] > 0 and (row[1] is None or row[1] == 0):
|
||||
# 全部 datetime 为 null — 清除所有分钟 K parquet
|
||||
n = 0
|
||||
for f in minute_dir.rglob("*.parquet"):
|
||||
f.unlink()
|
||||
n += 1
|
||||
logger.info("cleaned %d corrupted minute-K parquet files (null datetime)", n)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("minute cleanup check failed: %s", e)
|
||||
|
||||
|
||||
def _migrate_symbol_to_date_partition(repo: KlineRepository) -> None:
|
||||
"""将旧版 symbol= 分区迁移为 date= 分区。迁移完成后删除旧目录。"""
|
||||
minute_dir = repo.store.data_dir / "kline_minute"
|
||||
if not minute_dir.exists():
|
||||
return
|
||||
|
||||
old_dirs = [d for d in minute_dir.iterdir() if d.is_dir() and d.name.startswith("symbol=")]
|
||||
if not old_dirs:
|
||||
return
|
||||
|
||||
logger.info("migrating %d symbol-partitioned minute-K dirs to date partition…", len(old_dirs))
|
||||
|
||||
all_frames: list[pl.DataFrame] = []
|
||||
for sym_dir in old_dirs:
|
||||
for pq in sym_dir.glob("*.parquet"):
|
||||
try:
|
||||
df = pl.read_parquet(pq)
|
||||
if "datetime" in df.columns:
|
||||
df = df.filter(pl.col("datetime").is_not_null())
|
||||
if not df.is_empty():
|
||||
all_frames.append(df)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
if not all_frames:
|
||||
# 数据全部不可用，直接删旧目录
|
||||
for d in old_dirs:
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
for f in d.rglob("*"):
|
||||
if f.is_file():
|
||||
f.unlink()
|
||||
d.rmdir()
|
||||
return
|
||||
|
||||
combined = pl.concat(all_frames, how="diagonal_relaxed")
|
||||
combined = combined.unique(subset=["symbol", "datetime"], keep="last")
|
||||
|
||||
# 按日期写新分区
|
||||
combined = combined.with_columns(pl.col("datetime").dt.date().alias("_trade_date"))
|
||||
for day_df in combined.partition_by("_trade_date"):
|
||||
trade_date = day_df["_trade_date"][0]
|
||||
out = minute_dir / f"date={trade_date}" / "part.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
day_df = day_df.drop("_trade_date").sort("symbol", "datetime")
|
||||
day_df.write_parquet(out)
|
||||
|
||||
# 删旧目录
|
||||
for d in old_dirs:
|
||||
for f in d.rglob("*"):
|
||||
if f.is_file():
|
||||
f.unlink()
|
||||
# 移除空目录
|
||||
try:
|
||||
d.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
logger.info("minute-K migration done: %d rows migrated", combined.height)
|
||||
|
||||
|
||||
def sync_and_persist_minute(
|
||||
symbols: list[str],
|
||||
repo: KlineRepository,
|
||||
capset: CapabilitySet,
|
||||
days: int = 5,
|
||||
on_chunk_done: Callable[[int, int], None] | None = None,
|
||||
) -> int:
|
||||
"""同步分钟 K 并存到 Parquet(仅 raw,不前复权)。返回写入行数。
|
||||
|
||||
使用 start_time / end_time 区间拉取, 确保所有标的覆盖同一时间段。
|
||||
on_chunk_done(current, total) 每个 chunk 完成后回调。
|
||||
"""
|
||||
if not symbols or not capset.has(Cap.KLINE_MINUTE_BATCH):
|
||||
return 0
|
||||
|
||||
# 迁移:旧版 _normalize_minute 未转换 timestamp→datetime,导致全部 datetime 为 null
|
||||
# 检测到后直接清除(这些数据无法使用)
|
||||
_cleanup_null_datetime_minute(repo)
|
||||
|
||||
# 迁移:旧版按 symbol= 分区转为 date= 分区
|
||||
_migrate_symbol_to_date_partition(repo)
|
||||
|
||||
now = datetime.now()
|
||||
|
||||
# 计算时间区间: 首次拉取回溯 N 天, 增量从最后数据时间开始
|
||||
last_dt = _latest_minute_datetime(repo)
|
||||
if last_dt:
|
||||
start_time = last_dt
|
||||
else:
|
||||
start_time = now - timedelta(days=days)
|
||||
end_time = now
|
||||
|
||||
lim = capset.limits(Cap.KLINE_MINUTE_BATCH)
|
||||
batch_size = lim.batch if lim and lim.batch else 100
|
||||
rpm = lim.rpm if lim else 30
|
||||
|
||||
df = sync_minute_batch(symbols, start_time=start_time, end_time=end_time,
|
||||
batch_size=batch_size, rpm=rpm,
|
||||
on_chunk_done=on_chunk_done)
|
||||
if df.is_empty():
|
||||
return 0
|
||||
|
||||
# 按日期分区写: data/kline_minute/date={YYYY-MM-DD}/part.parquet
|
||||
df = df.with_columns(
|
||||
pl.col("datetime").dt.date().alias("_trade_date")
|
||||
)
|
||||
written = 0
|
||||
for day_df in df.partition_by("_trade_date"):
|
||||
trade_date = day_df["_trade_date"][0]
|
||||
out = repo.store.data_dir / "kline_minute" / f"date={trade_date}" / "part.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
if out.exists():
|
||||
existing = pl.read_parquet(out)
|
||||
if "datetime" in existing.columns:
|
||||
existing = existing.filter(pl.col("datetime").is_not_null())
|
||||
day_df = pl.concat([existing, day_df.drop("_trade_date")]).unique(
|
||||
subset=["symbol", "datetime"], keep="last",
|
||||
)
|
||||
else:
|
||||
day_df = day_df.drop("_trade_date")
|
||||
day_df = day_df.sort("symbol", "datetime")
|
||||
day_df.write_parquet(out)
|
||||
written += day_df.height
|
||||
|
||||
# 刷新视图
|
||||
try:
|
||||
d = repo.store.data_dir.as_posix()
|
||||
repo.db.execute(
|
||||
f"""CREATE OR REPLACE VIEW kline_minute AS
|
||||
SELECT * FROM read_parquet('{d}/kline_minute/**/*.parquet', union_by_name=true)"""
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("refresh kline_minute view failed: %s", e)
|
||||
|
||||
logger.info("minute K synced: %d rows (%d symbols)", written, len(symbols))
|
||||
return written
|
||||
@@ -0,0 +1,112 @@
|
||||
"""系统通知适配器 — 调用操作系统原生通知命令。
|
||||
|
||||
职责: 把后端产生的告警事件推送到操作系统通知中心。
|
||||
窗口最小化 / 被遮挡 / 后台运行时都能弹通知。
|
||||
|
||||
平台实现:
|
||||
- macOS: osascript (系统已内置)
|
||||
- Linux: notify-send (系统已内置)
|
||||
- Windows: 暂不支持原生通知中心 (无额外依赖实现)
|
||||
|
||||
设计: 失败静默降级, 绝不因通知失败阻断告警主流程 (落盘 / SSE 推送)。
|
||||
通知去重不在本层做, 复用 MonitorRuleEngine 的 cooldown 逻辑。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 单次通知最长字符 (避免某些平台截断报错)
|
||||
_MAX_LEN = 200
|
||||
|
||||
# 避免重复探测平台能力, 缓存一次
|
||||
_backend_cache: str | None = None
|
||||
|
||||
|
||||
def _detect_backend() -> str | None:
|
||||
"""探测当前平台可用的通知后端。"""
|
||||
global _backend_cache
|
||||
if _backend_cache is not None:
|
||||
return _backend_cache if _backend_cache != "none" else None
|
||||
|
||||
backend = None
|
||||
if sys.platform == "darwin":
|
||||
backend = "osascript"
|
||||
elif sys.platform.startswith("linux"):
|
||||
backend = "notify-send"
|
||||
|
||||
_backend_cache = backend or "none"
|
||||
return backend
|
||||
|
||||
|
||||
def _truncate(text: str) -> str:
|
||||
"""截断超长文本, 避免平台通知上限报错。"""
|
||||
text = (text or "").strip()
|
||||
return text[:_MAX_LEN] + ("…" if len(text) > _MAX_LEN else "")
|
||||
|
||||
|
||||
def notify(title: str, message: str, icon: Path | None = None) -> bool:
|
||||
"""推送一条系统通知。
|
||||
|
||||
Args:
|
||||
title: 通知标题
|
||||
message: 通知正文
|
||||
icon: 可选图标路径 (部分平台支持)
|
||||
|
||||
Returns:
|
||||
True=成功送达, False=失败或无可用后端。
|
||||
失败静默, 不抛异常 (通知是辅助通道, 不能阻断告警主流程)。
|
||||
"""
|
||||
title = _truncate(title)
|
||||
message = _truncate(message)
|
||||
if not title:
|
||||
return False
|
||||
|
||||
backend = _detect_backend()
|
||||
if backend is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
if backend == "osascript":
|
||||
return _notify_osascript(title, message)
|
||||
if backend == "notify-send":
|
||||
return _notify_notify_send(title, message, icon)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("系统通知失败 (%s): %s", backend, e)
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _notify_osascript(title: str, message: str) -> bool:
|
||||
"""macOS 通知 (osascript) — 调用系统 AppleScript。"""
|
||||
# 转义双引号, 避免 AppleScript 注入
|
||||
safe_title = title.replace('"', '\\"')
|
||||
safe_msg = message.replace('"', '\\"')
|
||||
script = (
|
||||
f'display notification "{safe_msg}" with title "{safe_title}"'
|
||||
)
|
||||
result = subprocess.run( # noqa: S603, S607
|
||||
["osascript", "-e", script],
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def _notify_notify_send(title: str, message: str, icon: Path | None) -> bool:
|
||||
"""Linux 通知 (notify-send) — freedesktop.org 标准。"""
|
||||
args = ["notify-send", title]
|
||||
if icon and Path(icon).exists():
|
||||
args.extend(["--icon", str(icon)])
|
||||
args.append(message)
|
||||
result = subprocess.run( # noqa: S603
|
||||
args,
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
return result.returncode == 0
|
||||
@@ -0,0 +1,250 @@
|
||||
"""异步盘后管道任务注册表 — 每个 job 独立 JSON 文件。
|
||||
|
||||
设计:
|
||||
- job_store/ 文件夹,每个 job 一个 {id}.json,最多保留 max_jobs 个文件
|
||||
- running/pending 状态的 job 仅存内存(高频读写)
|
||||
- succeeded/failed 后写入独立文件并从内存释放
|
||||
- 列表查询 = 内存中的活跃 job + 磁盘文件扫描,按时间排序
|
||||
- 单个查询 = 内存优先,没有则读磁盘
|
||||
- 创建新 job 前检查文件数量,>= max_jobs 时删除最老的文件
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JobStatus = Literal["pending", "running", "succeeded", "failed"]
|
||||
|
||||
|
||||
def _default_store_dir() -> Path:
|
||||
from app.config import settings
|
||||
return settings.data_dir / "job_store"
|
||||
|
||||
|
||||
_STORE_DIR = _default_store_dir()
|
||||
|
||||
|
||||
class JobStore:
|
||||
def __init__(self, max_jobs: int = 50, store_dir: Path = _STORE_DIR) -> None:
|
||||
self._max_jobs = max_jobs
|
||||
self._store_dir = store_dir
|
||||
self._active_jobs: dict[str, dict[str, Any]] = {} # running/pending
|
||||
self._active_id: str | None = None
|
||||
self._lock = threading.Lock()
|
||||
self._store_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ===== persistence =====
|
||||
|
||||
def _write_file(self, job: dict[str, Any]) -> None:
|
||||
"""将终态 job 写入独立 JSON 文件。"""
|
||||
path = self._store_dir / f"{job['id']}.json"
|
||||
try:
|
||||
path.write_text(
|
||||
json.dumps(job, ensure_ascii=False, indent=None),
|
||||
encoding="utf-8",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("failed to write job file %s", path)
|
||||
|
||||
def _read_file(self, job_id: str) -> dict[str, Any] | None:
|
||||
"""从磁盘读取单个 job 文件。"""
|
||||
path = self._store_dir / f"{job_id}.json"
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(path.read_text("utf-8"))
|
||||
except Exception:
|
||||
logger.warning("failed to read job file %s", path)
|
||||
return None
|
||||
|
||||
def _delete_oldest(self) -> None:
|
||||
"""删除最老的 job 文件,保持文件数量 < max_jobs。"""
|
||||
try:
|
||||
files = sorted(self._store_dir.glob("*.json"), key=lambda f: f.stat().st_mtime)
|
||||
except Exception:
|
||||
return
|
||||
while len(files) >= self._max_jobs:
|
||||
oldest = files.pop(0)
|
||||
try:
|
||||
oldest.unlink()
|
||||
except Exception:
|
||||
logger.warning("failed to delete old job file %s", oldest)
|
||||
|
||||
def _job_files_sorted(self) -> list[dict[str, Any]]:
|
||||
"""扫描磁盘上所有 job 文件,按 started_at 从新到旧排序。"""
|
||||
jobs: list[dict[str, Any]] = []
|
||||
for f in self._store_dir.glob("*.json"):
|
||||
try:
|
||||
jobs.append(json.loads(f.read_text("utf-8")))
|
||||
except Exception:
|
||||
continue
|
||||
jobs.sort(key=lambda j: j.get("started_at") or "", reverse=True)
|
||||
return jobs
|
||||
|
||||
# ===== lifecycle =====
|
||||
|
||||
def create(self) -> str:
|
||||
with self._lock:
|
||||
if self._active_id and self._active_jobs.get(self._active_id, {}).get("status") == "running":
|
||||
return self._active_id
|
||||
|
||||
job_id = uuid.uuid4().hex[:10]
|
||||
self._active_jobs[job_id] = {
|
||||
"id": job_id,
|
||||
"status": "pending",
|
||||
"stage": "init",
|
||||
"progress": 0,
|
||||
"stage_pct": 0,
|
||||
"log": [],
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"duration_s": None,
|
||||
"result": None,
|
||||
"error": None,
|
||||
}
|
||||
self._active_id = job_id
|
||||
return job_id
|
||||
|
||||
def start(self, job_id: str) -> None:
|
||||
with self._lock:
|
||||
j = self._active_jobs.get(job_id)
|
||||
if not j:
|
||||
return
|
||||
j["status"] = "running"
|
||||
j["started_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
|
||||
def succeed(self, job_id: str, result: Any) -> None:
|
||||
with self._lock:
|
||||
j = self._active_jobs.pop(job_id, None)
|
||||
if not j:
|
||||
return
|
||||
j["status"] = "succeeded"
|
||||
j["finished_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
j["progress"] = 100
|
||||
j["result"] = result
|
||||
j["duration_s"] = _duration_s(j)
|
||||
if self._active_id == job_id:
|
||||
self._active_id = None
|
||||
self._delete_oldest()
|
||||
self._write_file(j)
|
||||
|
||||
def fail(self, job_id: str, error: str) -> None:
|
||||
with self._lock:
|
||||
j = self._active_jobs.pop(job_id, None)
|
||||
if not j:
|
||||
return
|
||||
j["status"] = "failed"
|
||||
j["finished_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
j["error"] = error
|
||||
j["duration_s"] = _duration_s(j)
|
||||
if self._active_id == job_id:
|
||||
self._active_id = None
|
||||
self._delete_oldest()
|
||||
self._write_file(j)
|
||||
|
||||
# ===== progress =====
|
||||
|
||||
def progress(self, job_id: str, stage: str, pct: int, msg: str,
|
||||
stage_pct: int | None = None, skip_log: bool = False) -> None:
|
||||
with self._lock:
|
||||
j = self._active_jobs.get(job_id)
|
||||
if not j:
|
||||
return
|
||||
j["stage"] = stage
|
||||
j["progress"] = max(0, min(100, int(pct)))
|
||||
if stage_pct is not None:
|
||||
j["stage_pct"] = max(0, min(100, int(stage_pct)))
|
||||
elif j["stage"] != stage:
|
||||
j["stage_pct"] = 0
|
||||
entry = {
|
||||
"ts": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
"stage": stage,
|
||||
"msg": msg,
|
||||
}
|
||||
if skip_log:
|
||||
entry["_skip"] = True
|
||||
if skip_log and j["log"] and j["log"][-1].get("stage") == stage and j["log"][-1].get("_skip"):
|
||||
j["log"][-1] = entry
|
||||
else:
|
||||
j["log"].append(entry)
|
||||
if len(j["log"]) > 200:
|
||||
j["log"] = j["log"][-200:]
|
||||
|
||||
# ===== query =====
|
||||
|
||||
def get(self, job_id: str) -> dict[str, Any] | None:
|
||||
# 内存中的活跃 job 优先
|
||||
j = self._active_jobs.get(job_id)
|
||||
if j:
|
||||
return j
|
||||
# 否则从磁盘读
|
||||
return self._read_file(job_id)
|
||||
|
||||
def list_recent(self, limit: int = 20) -> list[dict[str, Any]]:
|
||||
# 合并: 内存中的活跃 job + 磁盘文件
|
||||
all_jobs: list[dict[str, Any]] = list(self._active_jobs.values())
|
||||
all_jobs.extend(self._job_files_sorted())
|
||||
# 按 started_at 从新到旧排序,去重(理论上不会有重复)
|
||||
seen: set[str] = set()
|
||||
result: list[dict[str, Any]] = []
|
||||
for j in sorted(all_jobs, key=lambda x: x.get("started_at") or "", reverse=True):
|
||||
jid = j["id"]
|
||||
if jid in seen:
|
||||
continue
|
||||
seen.add(jid)
|
||||
result.append(_summary(j))
|
||||
if len(result) >= limit:
|
||||
break
|
||||
return result
|
||||
|
||||
def active_id(self) -> str | None:
|
||||
return self._active_id
|
||||
|
||||
def clear(self) -> None:
|
||||
"""清空所有任务（内存 + 磁盘文件）。"""
|
||||
with self._lock:
|
||||
self._active_jobs.clear()
|
||||
self._active_id = None
|
||||
for f in self._store_dir.glob("*.json"):
|
||||
try:
|
||||
f.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _summary(j: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": j["id"],
|
||||
"status": j["status"],
|
||||
"stage": j["stage"],
|
||||
"progress": j["progress"],
|
||||
"stage_pct": j.get("stage_pct", 0),
|
||||
"started_at": j["started_at"],
|
||||
"finished_at": j["finished_at"],
|
||||
"duration_s": j["duration_s"],
|
||||
"result": j["result"],
|
||||
"error": j["error"],
|
||||
}
|
||||
|
||||
|
||||
def _duration_s(j: dict[str, Any]) -> float | None:
|
||||
if not j.get("started_at") or not j.get("finished_at"):
|
||||
return None
|
||||
try:
|
||||
s = datetime.fromisoformat(j["started_at"])
|
||||
e = datetime.fromisoformat(j["finished_at"])
|
||||
return round((e - s).total_seconds(), 2)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
# 进程内单例
|
||||
job_store = JobStore()
|
||||
@@ -0,0 +1,313 @@
|
||||
"""用户偏好设置持久化。
|
||||
|
||||
存储位置: data/user_data/preferences.json
|
||||
沿用 secrets_store 的 merge-write 模式,但不做 chmod 0600 (非敏感数据)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _path() -> Path:
|
||||
from app.config import settings
|
||||
p = settings.data_dir / "user_data" / "preferences.json"
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def load() -> dict:
|
||||
p = _path()
|
||||
if p.exists():
|
||||
try:
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("preferences.json malformed: %s", e)
|
||||
return {}
|
||||
|
||||
|
||||
def save(updates: dict) -> dict:
|
||||
"""合并写入。返回新内容。"""
|
||||
current = load()
|
||||
current.update(updates)
|
||||
_path().write_text(
|
||||
json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8",
|
||||
)
|
||||
return current
|
||||
|
||||
|
||||
def get_realtime_quotes_enabled() -> bool:
|
||||
return load().get("realtime_quotes_enabled", False)
|
||||
|
||||
|
||||
def get_indices_nav_pinned() -> bool:
|
||||
"""侧栏指数报价卡片是否固定显示。默认 True（常驻）。
|
||||
关闭后，卡片跟随实时行情开关（仅实时开时显示）。"""
|
||||
return load().get("indices_nav_pinned", True)
|
||||
|
||||
|
||||
def get_realtime_quote_interval() -> float:
|
||||
return load().get("realtime_quote_interval", 10.0)
|
||||
|
||||
|
||||
def set_realtime_quote_interval(interval: float) -> float:
|
||||
"""保存行情轮询间隔（不在此做 min/max 校验，由调用方按档位限制）。"""
|
||||
current = load()
|
||||
current["realtime_quote_interval"] = interval
|
||||
_path().write_text(
|
||||
json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8",
|
||||
)
|
||||
return interval
|
||||
|
||||
|
||||
def get_minute_sync_enabled() -> bool:
|
||||
return load().get("minute_sync_enabled", False)
|
||||
|
||||
|
||||
def get_minute_sync_days() -> int:
|
||||
return max(1, min(30, load().get("minute_sync_days", 5)))
|
||||
|
||||
|
||||
def get_pipeline_schedule() -> dict:
|
||||
"""返回盘后管道调度时间 {"hour": 15, "minute": 30}。"""
|
||||
d = load().get("pipeline_schedule", {"hour": 15, "minute": 30})
|
||||
return {"hour": d.get("hour", 15), "minute": d.get("minute", 30)}
|
||||
|
||||
|
||||
def set_pipeline_schedule(hour: int, minute: int) -> dict:
|
||||
h = max(0, min(23, hour))
|
||||
m = max(0, min(59, minute))
|
||||
# 盘后不早于 15:00
|
||||
if h * 60 + m < 15 * 60:
|
||||
h, m = 15, 0
|
||||
save({"pipeline_schedule": {"hour": h, "minute": m}})
|
||||
return {"hour": h, "minute": m}
|
||||
|
||||
|
||||
def get_instruments_schedule() -> dict:
|
||||
"""返回盘前标的维表调度时间 {"hour": 9, "minute": 10}。"""
|
||||
d = load().get("instruments_schedule", {"hour": 9, "minute": 10})
|
||||
return {"hour": d.get("hour", 9), "minute": d.get("minute", 10)}
|
||||
|
||||
|
||||
def set_instruments_schedule(hour: int, minute: int) -> dict:
|
||||
h = max(0, min(23, hour))
|
||||
m = max(0, min(59, minute))
|
||||
# 盘前不晚于 09:15
|
||||
if h * 60 + m > 9 * 60 + 15:
|
||||
h, m = 9, 15
|
||||
save({"instruments_schedule": {"hour": h, "minute": m}})
|
||||
return {"hour": h, "minute": m}
|
||||
|
||||
|
||||
def get_enriched_batch_size() -> int:
|
||||
"""返回 enriched 全量计算每批 symbol 数量。"""
|
||||
return max(1, min(10000, load().get("enriched_batch_size", 1000)))
|
||||
|
||||
|
||||
def set_enriched_batch_size(size: int) -> int:
|
||||
"""保存 enriched 全量计算批次大小。"""
|
||||
size = max(10, min(6000, size))
|
||||
save({"enriched_batch_size": size})
|
||||
return size
|
||||
|
||||
|
||||
def get_index_daily_batch_size() -> int:
|
||||
"""返回指数日 K 同步每批 symbol 数量。"""
|
||||
return max(1, min(10000, load().get("index_daily_batch_size", 100)))
|
||||
|
||||
|
||||
def set_index_daily_batch_size(size: int) -> int:
|
||||
"""保存指数日 K 同步批次大小。"""
|
||||
size = max(1, min(10000, size))
|
||||
save({"index_daily_batch_size": size})
|
||||
return size
|
||||
|
||||
|
||||
# ── 五档盘口 sealed(真假涨停) 配置 ──────────────────────
|
||||
|
||||
def get_limit_ladder_monitor_enabled() -> bool:
|
||||
"""连板梯队 5 档监控开关。关闭时 depth 不轮询(连板梯队降级显示)。"""
|
||||
return load().get("limit_ladder_monitor_enabled", False)
|
||||
|
||||
|
||||
def get_depth_polling_interval() -> float:
|
||||
"""depth 盘中轮询间隔(秒)。默认 20(Pro/Expert 都适用)。"""
|
||||
return float(load().get("depth_polling_interval", 20.0))
|
||||
|
||||
|
||||
def set_depth_polling_interval(interval: float) -> float:
|
||||
"""保存 depth 轮询间隔。套餐范围 clamp 由 depth_service 按档位做。"""
|
||||
interval = max(1.0, min(600.0, float(interval)))
|
||||
save({"depth_polling_interval": interval})
|
||||
return interval
|
||||
|
||||
|
||||
def get_depth_finalize_time() -> dict:
|
||||
"""盘后 sealed 定版时间 {"hour": 15, "minute": 2}。范围 15:01~18:00。"""
|
||||
d = load().get("depth_finalize_time", {"hour": 15, "minute": 2})
|
||||
return {"hour": d.get("hour", 15), "minute": d.get("minute", 2)}
|
||||
|
||||
|
||||
def set_depth_finalize_time(hour: int, minute: int) -> dict:
|
||||
"""保存盘后 sealed 定版时间,强制范围 15:01~18:00。"""
|
||||
h = max(0, min(23, hour))
|
||||
m = max(0, min(59, minute))
|
||||
# 下限 15:01, 上限 18:00
|
||||
if h * 60 + m < 15 * 60 + 1:
|
||||
h, m = 15, 1
|
||||
if h * 60 + m > 18 * 60:
|
||||
h, m = 18, 0
|
||||
save({"depth_finalize_time": {"hour": h, "minute": m}})
|
||||
return {"hour": h, "minute": m}
|
||||
|
||||
|
||||
|
||||
# ===== 实时监控 =====
|
||||
|
||||
# 页面 SSE 刷新配置: { "watchlist": true, "monitor": true, ... }
|
||||
# 可刷新的页面列表及其默认值
|
||||
SSE_REFRESH_PAGES_DEFAULT = {
|
||||
"watchlist": True,
|
||||
"limit-ladder": False,
|
||||
}
|
||||
|
||||
SIDEBAR_INDEX_SYMBOLS_DEFAULT = ["000001.SH", "399001.SZ", "399006.SZ", "000680.SH"]
|
||||
|
||||
|
||||
def get_sse_refresh_pages() -> dict[str, bool]:
|
||||
"""返回每个页面的 SSE 刷新开关。"""
|
||||
stored = load().get("sse_refresh_pages", {})
|
||||
# 合并默认值 (新增页面自动出现)
|
||||
result = dict(SSE_REFRESH_PAGES_DEFAULT)
|
||||
result.update(stored)
|
||||
return result
|
||||
|
||||
|
||||
def set_sse_refresh_pages(pages: dict[str, bool]) -> dict[str, bool]:
|
||||
"""保存页面 SSE 刷新配置。"""
|
||||
save({"sse_refresh_pages": pages})
|
||||
return get_sse_refresh_pages()
|
||||
|
||||
|
||||
def get_sidebar_index_symbols() -> list[str]:
|
||||
"""返回左侧菜单显示的指数代码。"""
|
||||
stored = load().get("sidebar_index_symbols", SIDEBAR_INDEX_SYMBOLS_DEFAULT)
|
||||
allowed = set(SIDEBAR_INDEX_SYMBOLS_DEFAULT)
|
||||
return [s for s in stored if s in allowed]
|
||||
|
||||
|
||||
def get_strategy_monitor_enabled() -> bool:
|
||||
"""策略告警评估总开关。"""
|
||||
return load().get("strategy_monitor_enabled", False)
|
||||
|
||||
|
||||
def get_system_notify_enabled() -> bool:
|
||||
"""系统通知开关 — 开启后监控告警同时推送到操作系统通知中心。"""
|
||||
return load().get("system_notify_enabled", False)
|
||||
|
||||
|
||||
def set_system_notify_enabled(enabled: bool) -> bool:
|
||||
"""保存系统通知开关。"""
|
||||
save({"system_notify_enabled": bool(enabled)})
|
||||
return bool(enabled)
|
||||
|
||||
|
||||
def get_screener_auto_run() -> bool:
|
||||
"""选股页进入时是否自动运行所有策略 (获取命中数)。默认开。"""
|
||||
return load().get("screener_auto_run", True)
|
||||
|
||||
|
||||
def get_strategy_monitor_ids() -> list[str]:
|
||||
"""返回监控池中的策略 ID。"""
|
||||
return load().get("strategy_monitor_ids", [])
|
||||
|
||||
|
||||
def set_realtime_monitor_config(cfg: dict) -> dict:
|
||||
"""批量更新实时监控配置。"""
|
||||
updates = {}
|
||||
if "sse_refresh_pages" in cfg:
|
||||
updates["sse_refresh_pages"] = cfg["sse_refresh_pages"]
|
||||
if "strategy_monitor_enabled" in cfg:
|
||||
updates["strategy_monitor_enabled"] = cfg["strategy_monitor_enabled"]
|
||||
if "strategy_monitor_ids" in cfg:
|
||||
updates["strategy_monitor_ids"] = cfg["strategy_monitor_ids"]
|
||||
if "sidebar_index_symbols" in cfg:
|
||||
allowed = set(SIDEBAR_INDEX_SYMBOLS_DEFAULT)
|
||||
updates["sidebar_index_symbols"] = [s for s in cfg["sidebar_index_symbols"] if s in allowed]
|
||||
if "screener_auto_run" in cfg:
|
||||
updates["screener_auto_run"] = bool(cfg["screener_auto_run"])
|
||||
if updates:
|
||||
save(updates)
|
||||
return get_realtime_monitor_config()
|
||||
|
||||
|
||||
def get_realtime_monitor_config() -> dict:
|
||||
"""返回完整的实时监控配置。"""
|
||||
return {
|
||||
"sse_refresh_pages": get_sse_refresh_pages(),
|
||||
"strategy_monitor_enabled": get_strategy_monitor_enabled(),
|
||||
"strategy_monitor_ids": get_strategy_monitor_ids(),
|
||||
"sidebar_index_symbols": get_sidebar_index_symbols(),
|
||||
"screener_auto_run": get_screener_auto_run(),
|
||||
}
|
||||
|
||||
|
||||
def get_nav_order() -> list[str]:
|
||||
"""返回左侧菜单的自定义排序（内置页面 path + 扩展分析菜单 id）。"""
|
||||
return load().get("nav_order", [])
|
||||
|
||||
|
||||
def set_nav_order(order: list[str]) -> list[str]:
|
||||
"""保存左侧菜单排序。"""
|
||||
save({"nav_order": order})
|
||||
return get_nav_order()
|
||||
|
||||
|
||||
def get_nav_hidden() -> list[str]:
|
||||
"""返回左侧菜单中隐藏的项 id 列表。"""
|
||||
return load().get("nav_hidden", [])
|
||||
|
||||
|
||||
def set_nav_hidden(hidden: list[str]) -> list[str]:
|
||||
"""保存左侧菜单隐藏项。"""
|
||||
save({"nav_hidden": hidden})
|
||||
return get_nav_hidden()
|
||||
|
||||
|
||||
def get_watchlist_columns() -> list[dict] | None:
|
||||
"""返回自选列表列配置。"""
|
||||
return load().get("watchlist_columns")
|
||||
|
||||
|
||||
def set_watchlist_columns(columns: list[dict]) -> list[dict]:
|
||||
"""保存自选列表列配置。"""
|
||||
save({"watchlist_columns": columns})
|
||||
return columns
|
||||
|
||||
|
||||
def get_screener_result_columns() -> list[dict] | None:
|
||||
"""返回策略结果列表列配置。"""
|
||||
return load().get("screener_result_columns")
|
||||
|
||||
|
||||
def set_screener_result_columns(columns: list[dict]) -> list[dict]:
|
||||
"""保存策略结果列表列配置。"""
|
||||
save({"screener_result_columns": columns})
|
||||
return columns
|
||||
|
||||
|
||||
# ===== 首次使用引导 =====
|
||||
|
||||
def get_onboarding_completed() -> bool:
|
||||
"""是否已完成首次使用向导。默认 False（新用户）。"""
|
||||
return bool(load().get("onboarding_completed", False))
|
||||
|
||||
|
||||
def set_onboarding_completed(done: bool = True) -> bool:
|
||||
"""标记首次使用向导完成状态。"""
|
||||
save({"onboarding_completed": bool(done)})
|
||||
return bool(done)
|
||||
@@ -0,0 +1,779 @@
|
||||
"""全局实时行情服务。
|
||||
|
||||
集中管理全市场行情拉取 + enriched 缓存，供盘中选股、自选股等所有模块复用。
|
||||
|
||||
架构:
|
||||
- 后台线程轮询数据源 get_by_universes(["CN_Equity_A", "CN_Index"])
|
||||
- 拉取行情 → 写 kline_daily (不复权) + 增量计算 enriched → 写盘 + 更新缓存
|
||||
- _enriched_cache 是唯一的盘中数据源 (OHLCV + 全套技术指标)
|
||||
- _live_agg_cache 是递推状态 (只加载一次, 盘中不变)
|
||||
|
||||
数据流 (每轮 ~15s):
|
||||
1. API 拉取 → raw_records (临时变量)
|
||||
2. raw_records → 写 kline_daily (不复权原始价格)
|
||||
3. raw_records → 更新 _enriched_cache 的 OHLCV
|
||||
4. 增量计算 enriched 指标 (~50ms)
|
||||
5. 写 kline_daily_enriched + 替换 _enriched_cache
|
||||
6. 通知 SSE
|
||||
|
||||
生命周期:
|
||||
- 服务启动时读取 preferences，若 enabled 则自动启动线程
|
||||
- 运行中可通过 API 切换开关
|
||||
- 关闭时停止线程
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from datetime import date, datetime, time as dt_time
|
||||
|
||||
import polars as pl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class QuoteService:
|
||||
"""全局实时行情服务 — 单例。"""
|
||||
|
||||
CORE_INDEX_SYMBOLS = ("000001.SH", "399001.SZ", "399006.SZ", "000680.SH")
|
||||
|
||||
# 档位 → 最小轮询间隔 (秒)
|
||||
TIER_MIN_INTERVAL = {
|
||||
"expert": 1.0,
|
||||
"pro": 2.0,
|
||||
"starter": 3.0,
|
||||
}
|
||||
DEFAULT_INTERVAL = 10.0
|
||||
MAX_INTERVAL = 60.0
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._running = False
|
||||
self._enabled = False # 全局开关 (持久化到 preferences)
|
||||
self._interval = self.DEFAULT_INTERVAL
|
||||
self._thread: threading.Thread | None = None
|
||||
self._repo = None # 延迟注入, 避免循环导入
|
||||
self._update_event = threading.Event() # SSE 通知: 行情更新后 set
|
||||
self._alert_event = threading.Event() # SSE 通知: 有告警时 set
|
||||
self._depth_update_event = threading.Event() # SSE 通知: depth 五档修正后 set (刷新连板梯队)
|
||||
self._pending_alerts: list[dict] = [] # 待推送的告警
|
||||
self._max_pending_alerts: int = 1000 # 背压上限: 超出丢弃最旧
|
||||
self._strategy_monitor = None # 延迟注入
|
||||
self._app_state = None # 延迟注入 (FastAPI app.state)
|
||||
|
||||
# 拉取元信息 (给 SSE / status 用)
|
||||
self._fetch_time: float = 0.0 # perf_counter (用于计算 quote_age_ms)
|
||||
self._fetch_ms: float = 0.0 # 拉取耗时 (毫秒)
|
||||
self._fetched_at: float = 0.0 # 拉取完成的 Unix 时间戳 (毫秒)
|
||||
self._symbol_count: int = 0
|
||||
self._index_symbol_count: int = 0
|
||||
self._index_quotes_cache: pl.DataFrame | None = None
|
||||
|
||||
# ================================================================
|
||||
# 生命周期
|
||||
# ================================================================
|
||||
|
||||
def start(self, interval: float = 0.0) -> None:
|
||||
"""启动后台行情轮询线程。"""
|
||||
if self._running:
|
||||
return
|
||||
if interval <= 0:
|
||||
from app.services import preferences
|
||||
interval = preferences.get_realtime_quote_interval()
|
||||
self._interval = self._clamp_interval(interval)
|
||||
self._running = True
|
||||
self._enabled = True
|
||||
self._thread = threading.Thread(target=self._poll_loop, daemon=True)
|
||||
self._thread.start()
|
||||
self._save_enabled(True)
|
||||
logger.info("行情服务已启动, 轮询间隔 %.1fs", self._interval)
|
||||
|
||||
def stop(self) -> None:
|
||||
"""停止后台行情轮询线程。"""
|
||||
self._running = False
|
||||
self._enabled = False
|
||||
if self._thread:
|
||||
self._thread.join(timeout=10)
|
||||
self._thread = None
|
||||
self._save_enabled(False)
|
||||
logger.info("行情服务已停止")
|
||||
|
||||
def enable(self) -> bool:
|
||||
"""开启自动行情 (不立即启动线程，等下一个交易时段)。
|
||||
|
||||
none/free 档无实时行情权限,拒绝开启并返回 False;
|
||||
starter+ 正常启动。返回值表示是否真正开启。
|
||||
"""
|
||||
if not self.is_realtime_allowed():
|
||||
logger.warning("实时行情开启被拒:当前档位(none/free)无实时行情权限")
|
||||
return False
|
||||
self._enabled = True
|
||||
self._save_enabled(True)
|
||||
if not self._running:
|
||||
from app.services import preferences
|
||||
self._interval = self._clamp_interval(preferences.get_realtime_quote_interval())
|
||||
self._running = True
|
||||
self._thread = threading.Thread(target=self._poll_loop, daemon=True)
|
||||
self._thread.start()
|
||||
logger.info("行情服务已启用, 轮询间隔 %.1fs", self._interval)
|
||||
|
||||
def disable(self) -> None:
|
||||
"""关闭自动行情。"""
|
||||
self.stop()
|
||||
logger.info("行情服务已关闭")
|
||||
|
||||
def boot_check(self) -> None:
|
||||
"""启动时检查 preferences，若 enabled 则自动启动。
|
||||
|
||||
none/free 档无实时行情权限:即使 preferences 标记为 enabled,
|
||||
也不启动,并同步 preferences 为关闭(避免 UI 误显示已开启)。
|
||||
"""
|
||||
from app.services import preferences
|
||||
if not self.is_realtime_allowed():
|
||||
if preferences.get_realtime_quotes_enabled():
|
||||
self._save_enabled(False)
|
||||
logger.info("实时行情未启动:当前档位(none/free)无实时行情权限")
|
||||
return
|
||||
if preferences.get_realtime_quotes_enabled():
|
||||
self.start()
|
||||
|
||||
def set_repo(self, repo) -> None:
|
||||
"""注入 KlineRepository, 用于实时落盘。"""
|
||||
self._repo = repo
|
||||
|
||||
def set_app_state(self, app_state) -> None:
|
||||
"""注入 FastAPI app.state, 用于获取 strategy_monitor 等单例。"""
|
||||
self._app_state = app_state
|
||||
|
||||
def set_interval(self, interval: float) -> float:
|
||||
"""运行时更新轮询间隔（立即生效）。"""
|
||||
clamped = self._clamp_interval(interval)
|
||||
self._interval = clamped
|
||||
from app.services import preferences
|
||||
preferences.set_realtime_quote_interval(clamped)
|
||||
logger.info("轮询间隔已更新为 %.1fs", clamped)
|
||||
return clamped
|
||||
|
||||
def get_min_interval(self) -> float:
|
||||
"""返回当前档位允许的最小间隔。"""
|
||||
return self._tier_min_interval()
|
||||
|
||||
def wait_for_update(self, timeout: float = 30.0) -> bool:
|
||||
"""阻塞等待下一次行情更新 (供 SSE 线程使用)。"""
|
||||
self._update_event.clear()
|
||||
return self._update_event.wait(timeout=timeout)
|
||||
|
||||
def wait_for_alert(self, timeout: float = 30.0) -> bool:
|
||||
"""阻塞等待告警 (供 SSE 线程使用)。"""
|
||||
self._alert_event.clear()
|
||||
return self._alert_event.wait(timeout=timeout)
|
||||
|
||||
def notify_depth_updated(self) -> None:
|
||||
"""五档盘口修正完成后调用: 通知 SSE 推送 depth_updated, 触发连板梯队刷新。
|
||||
|
||||
与行情/告警通道独立 — 只刷新连板梯队, 不连带刷新 watchlist 等。
|
||||
"""
|
||||
self._depth_update_event.set()
|
||||
|
||||
def wait_for_depth_update(self, timeout: float = 30.0) -> bool:
|
||||
"""阻塞等待 depth 修正 (供 SSE 线程使用)。"""
|
||||
self._depth_update_event.clear()
|
||||
return self._depth_update_event.wait(timeout=timeout)
|
||||
|
||||
def pop_alerts(self) -> list[dict]:
|
||||
"""取走所有待推送的告警 (线程安全)。"""
|
||||
with self._lock:
|
||||
alerts = self._pending_alerts
|
||||
self._pending_alerts = []
|
||||
return alerts
|
||||
|
||||
# ================================================================
|
||||
# 档位感知间隔限制
|
||||
# ================================================================
|
||||
|
||||
@staticmethod
|
||||
def _current_tier() -> str:
|
||||
"""获取当前档位名（小写）。"""
|
||||
from app.tickflow.policy import tier_label
|
||||
return tier_label().split()[0].split("+")[0].strip().lower()
|
||||
|
||||
@classmethod
|
||||
def is_realtime_allowed(cls) -> bool:
|
||||
"""当前档位是否允许使用实时行情。
|
||||
|
||||
none/free 档走 free-api 服务器,无实时行情权限 → 不允许;
|
||||
starter+ 付费档走付费端点,有实时行情 → 允许。
|
||||
"""
|
||||
return cls._current_tier() not in ("none", "free")
|
||||
|
||||
@classmethod
|
||||
def _tier_min_interval(cls) -> float:
|
||||
tier = cls._current_tier()
|
||||
return cls.TIER_MIN_INTERVAL.get(tier, cls.DEFAULT_INTERVAL)
|
||||
|
||||
def _clamp_interval(self, interval: float) -> float:
|
||||
return max(self._tier_min_interval(), min(self.MAX_INTERVAL, interval))
|
||||
|
||||
# ================================================================
|
||||
# 行情数据访问
|
||||
# ================================================================
|
||||
|
||||
def get_enriched_today(self) -> tuple[pl.DataFrame, date | None]:
|
||||
"""返回今天 enriched 数据 + 日期 (线程安全)。
|
||||
|
||||
所有页面统一通过此方法获取实时行情 + 技术指标。
|
||||
"""
|
||||
if not self._repo:
|
||||
return pl.DataFrame(), None
|
||||
return self._repo.get_enriched_latest()
|
||||
|
||||
def get_quotes_compat(self) -> pl.DataFrame:
|
||||
"""兼容接口: 返回行情 DataFrame (用于盘中选股等需要 last_price/prev_close 的场景)。
|
||||
|
||||
从 _enriched_cache 取 today 的数据, 只选行情基础列, 补上 last_price 别名。
|
||||
不返回指标列, 避免 JOIN live_agg 时列名冲突。
|
||||
"""
|
||||
df, _ = self.get_enriched_today()
|
||||
if df.is_empty():
|
||||
return df
|
||||
|
||||
# 只取盘中选股需要的行情基础列
|
||||
keep = [c for c in [
|
||||
"symbol", "close", "open", "high", "low", "volume", "amount",
|
||||
"prev_close", "change_pct", "change_amount", "amplitude", "turnover_rate",
|
||||
] if c in df.columns]
|
||||
df = df.select(keep)
|
||||
|
||||
# enriched 的 close 等价于 last_price
|
||||
if "close" in df.columns and "last_price" not in df.columns:
|
||||
df = df.with_columns(pl.col("close").alias("last_price"))
|
||||
return df
|
||||
|
||||
def get_index_quotes(self, symbols: list[str] | None = None) -> pl.DataFrame:
|
||||
"""返回实时指数行情缓存。不会触发数据源请求。"""
|
||||
with self._lock:
|
||||
df = self._index_quotes_cache.clone() if self._index_quotes_cache is not None else pl.DataFrame()
|
||||
if df.is_empty():
|
||||
return df
|
||||
if symbols:
|
||||
return df.filter(pl.col("symbol").is_in(symbols))
|
||||
return df
|
||||
|
||||
def status(self) -> dict:
|
||||
"""返回行情服务状态。"""
|
||||
age = (time.perf_counter() - self._fetch_time) * 1000 if self._fetch_time else -1
|
||||
return {
|
||||
"enabled": self._enabled,
|
||||
"running": self._running,
|
||||
"interval_s": self._interval,
|
||||
"symbol_count": self._symbol_count,
|
||||
"index_symbol_count": self._index_symbol_count,
|
||||
"quote_age_ms": round(age, 0) if age >= 0 else None,
|
||||
"is_trading_hours": self._is_trading_hours(),
|
||||
"last_fetch_ms": round(self._fetched_at, 0) if self._fetched_at else None,
|
||||
}
|
||||
|
||||
def refresh(self) -> dict:
|
||||
"""手动触发一次行情拉取。"""
|
||||
self._fetch_quotes()
|
||||
return self.status()
|
||||
|
||||
# ================================================================
|
||||
# 后台轮询
|
||||
# ================================================================
|
||||
|
||||
def _poll_loop(self) -> None:
|
||||
while self._running and self._enabled:
|
||||
try:
|
||||
if self._is_trading_hours():
|
||||
self._fetch_quotes()
|
||||
else:
|
||||
logger.debug("非交易时段, 跳过行情轮询")
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("行情轮询异常: %s", e)
|
||||
|
||||
waited = 0.0
|
||||
while self._running and self._enabled and waited < self._interval:
|
||||
time.sleep(0.5)
|
||||
waited += 0.5
|
||||
|
||||
def _fetch_quotes(self) -> None:
|
||||
"""拉取全市场行情 → 写 daily + 计算 enriched + 更新缓存。"""
|
||||
from app.tickflow.client import get_client
|
||||
|
||||
tf = get_client()
|
||||
t0 = time.perf_counter()
|
||||
now_ts = time.perf_counter()
|
||||
|
||||
try:
|
||||
all_index_symbols = set(self._repo.get_index_symbol_set()) if self._repo else set()
|
||||
all_index_symbols.update(self.CORE_INDEX_SYMBOLS)
|
||||
resp = tf.quotes.get_by_universes(universes=["CN_Equity_A", "CN_Index"])
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("行情拉取失败: %s", e)
|
||||
return
|
||||
|
||||
if not resp:
|
||||
logger.warning("行情数据为空")
|
||||
return
|
||||
|
||||
# ---- 解析 API 响应 (临时变量, 用完丢弃) ----
|
||||
records = []
|
||||
for q in resp:
|
||||
ext = q.get("ext") or {}
|
||||
last_price = q.get("last_price")
|
||||
prev_close = q.get("prev_close")
|
||||
change_amount = ext.get("change_amount")
|
||||
change_pct = ext.get("change_pct")
|
||||
if change_amount is None and last_price is not None and prev_close is not None:
|
||||
change_amount = float(last_price) - float(prev_close)
|
||||
if change_pct is None and change_amount is not None and prev_close not in (None, 0):
|
||||
change_pct = float(change_amount) / float(prev_close) * 100
|
||||
records.append({
|
||||
"symbol": q.get("symbol"),
|
||||
"name": q.get("name") or ext.get("name"),
|
||||
"last_price": last_price,
|
||||
"prev_close": prev_close,
|
||||
"open": q.get("open"),
|
||||
"high": q.get("high"),
|
||||
"low": q.get("low"),
|
||||
"volume": q.get("volume"),
|
||||
"amount": q.get("amount"),
|
||||
"change_pct": change_pct,
|
||||
"change_amount": change_amount,
|
||||
"amplitude": ext.get("amplitude"),
|
||||
"turnover_rate": ext.get("turnover_rate"),
|
||||
"timestamp": q.get("timestamp"),
|
||||
"session": q.get("session"),
|
||||
})
|
||||
|
||||
index_records = [r for r in records if r.get("symbol") in all_index_symbols]
|
||||
stock_records = [r for r in records if r.get("symbol") not in all_index_symbols]
|
||||
|
||||
fetch_ms = (time.perf_counter() - t0) * 1000
|
||||
fetched_at = time.time() * 1000
|
||||
|
||||
# ---- 更新元信息 ----
|
||||
with self._lock:
|
||||
self._fetch_time = now_ts
|
||||
self._fetch_ms = fetch_ms
|
||||
self._fetched_at = fetched_at
|
||||
self._symbol_count = len(stock_records)
|
||||
self._index_symbol_count = len(index_records)
|
||||
self._index_quotes_cache = self._build_index_quotes(index_records)
|
||||
|
||||
logger.info("行情刷新: %d 只股票, %d 只指数, 耗时 %.0fms", len(stock_records), len(index_records), fetch_ms)
|
||||
|
||||
# ---- 写 kline_daily (不复权原始价格, 只有 OHLCV) ----
|
||||
daily_df = self._build_daily(stock_records)
|
||||
if not daily_df.is_empty() and self._repo:
|
||||
try:
|
||||
self._repo.flush_live_daily(daily_df)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("日K写盘失败: %s", e)
|
||||
|
||||
# ---- 构建 API 直接值的补充表 (不写 daily, 只用于 enriched 计算) ----
|
||||
quote_extra = self._build_quote_extra(stock_records)
|
||||
|
||||
# ---- 增量计算 enriched + 写盘 + 更新缓存 ----
|
||||
if not daily_df.is_empty() and self._repo:
|
||||
self._flush_live_enriched(daily_df, quote_extra)
|
||||
|
||||
# ---- 通知 SSE ----
|
||||
self._update_event.set()
|
||||
|
||||
# ---- 策略监控 + 告警评估 ----
|
||||
self._evaluate_monitors(daily_df, quote_extra)
|
||||
|
||||
# ================================================================
|
||||
# 工具
|
||||
# ================================================================
|
||||
|
||||
@staticmethod
|
||||
def _build_daily(records: list[dict]) -> pl.DataFrame:
|
||||
"""将 API records 转为日K格式 DataFrame (只有 OHLCV, 写 kline_daily 用)。"""
|
||||
if not records:
|
||||
return pl.DataFrame()
|
||||
df = pl.DataFrame(records)
|
||||
cols_map = {
|
||||
"symbol": "symbol",
|
||||
"last_price": "close",
|
||||
"open": "open",
|
||||
"high": "high",
|
||||
"low": "low",
|
||||
"volume": "volume",
|
||||
"amount": "amount",
|
||||
}
|
||||
select_exprs = []
|
||||
for src, dst in cols_map.items():
|
||||
if src in df.columns:
|
||||
select_exprs.append(pl.col(src).alias(dst))
|
||||
if not select_exprs:
|
||||
return pl.DataFrame()
|
||||
result = df.select(select_exprs).with_columns(
|
||||
pl.lit(date.today()).cast(pl.Date).alias("date"),
|
||||
)
|
||||
# 修复: API 在非交易时段可能返回 open/high/low=0 或 null,
|
||||
# 导致蜡烛从 0 开始。用 close 填充这些异常值。
|
||||
for col in ("open", "high", "low"):
|
||||
if col in result.columns:
|
||||
result = result.with_columns(
|
||||
pl.when((pl.col(col) == 0) | pl.col(col).is_null())
|
||||
.then(pl.col("close"))
|
||||
.otherwise(pl.col(col))
|
||||
.alias(col)
|
||||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _build_quote_extra(records: list[dict]) -> pl.DataFrame:
|
||||
"""构建 API 直接提供的补充字段 (不写 daily, 只传给 enriched 计算)。
|
||||
|
||||
包含: prev_close, change_pct, change_amount, amplitude, turnover_rate。
|
||||
"""
|
||||
if not records:
|
||||
return pl.DataFrame()
|
||||
df = pl.DataFrame(records)
|
||||
keep = [c for c in [
|
||||
"symbol", "prev_close", "change_pct", "change_amount",
|
||||
"amplitude", "turnover_rate",
|
||||
] if c in df.columns]
|
||||
if not keep or "symbol" not in keep:
|
||||
return pl.DataFrame()
|
||||
return df.select(keep)
|
||||
|
||||
@staticmethod
|
||||
def _build_index_quotes(records: list[dict]) -> pl.DataFrame:
|
||||
"""构建指数实时行情缓存，不落股票 parquet。
|
||||
|
||||
注意: API 返回的 change_pct/amplitude 是小数 (0.0366 = 3.66%),
|
||||
统一转成百分比输出, 与 _fallback_index_quotes_from_daily 口径一致
|
||||
(前端指数侧不×100, 直接 toFixed(2)% 展示)。
|
||||
"""
|
||||
if not records:
|
||||
return pl.DataFrame()
|
||||
df = pl.DataFrame(records)
|
||||
keep = [c for c in [
|
||||
"symbol", "name", "last_price", "prev_close", "open", "high", "low",
|
||||
"volume", "amount", "change_pct", "change_amount", "amplitude", "timestamp", "session",
|
||||
] if c in df.columns]
|
||||
if not keep or "symbol" not in keep:
|
||||
return pl.DataFrame()
|
||||
df = df.select(keep)
|
||||
# change_pct / amplitude: 小数 → 百分比 (统一指数展示口径)
|
||||
for col in ("change_pct", "amplitude"):
|
||||
if col in df.columns:
|
||||
df = df.with_columns((pl.col(col).cast(pl.Float64) * 100).alias(col))
|
||||
if "last_price" in df.columns and "close" not in df.columns:
|
||||
df = df.with_columns(pl.col("last_price").alias("close"))
|
||||
return df
|
||||
|
||||
@staticmethod
|
||||
def _is_trading_hours() -> bool:
|
||||
now = datetime.now()
|
||||
t = now.time()
|
||||
morning = dt_time(9, 15) <= t <= dt_time(11, 35)
|
||||
afternoon = dt_time(12, 55) <= t <= dt_time(15, 5)
|
||||
return now.weekday() < 5 and (morning or afternoon)
|
||||
|
||||
@staticmethod
|
||||
def _save_enabled(enabled: bool) -> None:
|
||||
from app.services import preferences
|
||||
preferences.save({"realtime_quotes_enabled": enabled})
|
||||
|
||||
# ================================================================
|
||||
# 策略监控
|
||||
# ================================================================
|
||||
|
||||
def _evaluate_monitors(self, daily_df: pl.DataFrame, quote_extra: pl.DataFrame | None) -> None:
|
||||
"""行情更新后评估统一监控规则引擎,并刷新策略结果缓存。"""
|
||||
try:
|
||||
# 获取 enriched 数据 (刚算好的)
|
||||
enriched_today, enriched_date = self.get_enriched_today()
|
||||
if enriched_today.is_empty():
|
||||
return
|
||||
|
||||
all_alerts: list[dict] = []
|
||||
|
||||
# 通用监控规则评估 (统一引擎: signal/price/market/strategy)
|
||||
if self._app_state:
|
||||
engine = getattr(self._app_state, "monitor_engine", None)
|
||||
if engine and engine.rule_count > 0:
|
||||
# 预构建 symbol → name 映射 (enriched 已 drop name 列, 引擎触发时回填用)
|
||||
try:
|
||||
inst_df = self._app_state.repo.get_instruments()
|
||||
if not inst_df.is_empty() and "symbol" in inst_df.columns and "name" in inst_df.columns:
|
||||
engine.set_name_map({
|
||||
row["symbol"]: row["name"]
|
||||
for row in inst_df.select(["symbol", "name"]).iter_rows(named=True)
|
||||
if row.get("name")
|
||||
})
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("name_map 构建失败 (不影响监控): %s", e)
|
||||
rule_events = engine.evaluate(enriched_today)
|
||||
if rule_events:
|
||||
# 落盘到 alerts.jsonl
|
||||
try:
|
||||
from app.services import alert_store
|
||||
alert_store.append_many(
|
||||
self._app_state.repo.store.data_dir, rule_events,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("告警落盘失败: %s", e)
|
||||
# 转为 SSE 推送格式 (兼容旧 alert schema)
|
||||
for ev in rule_events:
|
||||
all_alerts.append({
|
||||
"source": ev["source"],
|
||||
"type": ev["type"],
|
||||
"rule_id": ev.get("rule_id"),
|
||||
"strategy_id": ev.get("rule_id") if ev["source"] == "strategy" else None,
|
||||
"symbol": ev["symbol"],
|
||||
"name": ev["name"],
|
||||
"message": ev["message"],
|
||||
"price": ev["price"],
|
||||
"change_pct": ev["change_pct"],
|
||||
"signals": ev["signals"],
|
||||
"severity": ev.get("severity", "info"),
|
||||
})
|
||||
|
||||
# 刷新策略结果缓存 (实时行情开启时，每轮行情更新后自动重算)
|
||||
if self._enabled and self._app_state:
|
||||
self._refresh_strategy_cache(enriched_today, enriched_date)
|
||||
|
||||
# 推入待推送队列 + 通知 SSE (含背压保护)
|
||||
if all_alerts:
|
||||
with self._lock:
|
||||
self._pending_alerts.extend(all_alerts)
|
||||
# 背压: 超出上限丢弃最旧
|
||||
if len(self._pending_alerts) > self._max_pending_alerts:
|
||||
overflow = len(self._pending_alerts) - self._max_pending_alerts
|
||||
self._pending_alerts = self._pending_alerts[overflow:]
|
||||
self._alert_event.set()
|
||||
logger.info("监控评估完成: %d 条通知", len(all_alerts))
|
||||
|
||||
# 系统通知 (可选通道, 由 preferences 开关控制)。
|
||||
# cooldown 去重已在 MonitorRuleEngine 做过, 这里只负责转发。
|
||||
self._maybe_send_system_notifications(all_alerts)
|
||||
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("监控评估失败: %s", e)
|
||||
|
||||
def _maybe_send_system_notifications(self, all_alerts: list[dict]) -> None:
|
||||
"""把告警转发到操作系统通知中心 (由 preferences 开关控制)。
|
||||
|
||||
- 开关关闭: 直接返回
|
||||
- 开关开启: 逐条发系统通知; 失败静默, 不阻断主流程
|
||||
- 去重: 复用 MonitorRuleEngine 的 cooldown, 此处不重复去重
|
||||
- 批量策略事件 (symbol="") 聚合为一条通知, 避免刷屏
|
||||
"""
|
||||
try:
|
||||
from app.services import preferences
|
||||
from app.services import notify_adapter
|
||||
|
||||
if not preferences.get_system_notify_enabled():
|
||||
return
|
||||
|
||||
for ev in all_alerts:
|
||||
# 通知标题: 用 source 分类 (策略/信号/价格/异动)
|
||||
source = ev.get("source", "")
|
||||
source_label = {
|
||||
"strategy": "策略", "signal": "信号",
|
||||
"price": "价格", "market": "异动",
|
||||
}.get(source, source or "通知")
|
||||
|
||||
name = ev.get("name") or ""
|
||||
symbol = ev.get("symbol") or ""
|
||||
message = ev.get("message") or ""
|
||||
|
||||
# 正文: 优先用现成 message, 拼上 symbol/name 让用户一眼定位
|
||||
if symbol:
|
||||
body = f"{symbol} {name} {message}".strip()
|
||||
else:
|
||||
body = message or name
|
||||
|
||||
title = f"Stock Panel · {source_label}"
|
||||
notify_adapter.notify(title, body)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("系统通知发送异常 (不影响告警主流程): %s", e)
|
||||
|
||||
def _refresh_strategy_cache(self, enriched_today: pl.DataFrame, enriched_date: date | None) -> None:
|
||||
"""利用已计算好的 enriched 数据，运行策略池并写入缓存。"""
|
||||
import math
|
||||
from dataclasses import asdict
|
||||
from app.services import strategy_cache
|
||||
from app.services.screener import PRESET_STRATEGIES, ScreenerService
|
||||
from app.strategy import config as strategy_config
|
||||
|
||||
try:
|
||||
if enriched_date is None:
|
||||
return
|
||||
as_of = enriched_date
|
||||
data_dir = self._repo.store.data_dir
|
||||
svc = ScreenerService(self._repo)
|
||||
engine = getattr(self._app_state, "strategy_engine", None)
|
||||
|
||||
# 确定要运行的策略: 策略监控池中的策略
|
||||
monitor_ids = self._get_monitor_pool_ids()
|
||||
if not monitor_ids:
|
||||
return
|
||||
|
||||
# 一次加载所有 override
|
||||
all_overrides = strategy_config.list_overrides(data_dir)
|
||||
|
||||
# 历史策略: 只在需要时加载
|
||||
shared_history = None
|
||||
history_strats = []
|
||||
if engine:
|
||||
id_set = set(monitor_ids)
|
||||
history_strats = [
|
||||
(sid, s) for sid, s in engine._strategies.items()
|
||||
if s.filter_history_fn and sid in id_set
|
||||
]
|
||||
if history_strats:
|
||||
max_lb = max(s.lookback_days for _, s in history_strats)
|
||||
shared_history = svc._load_enriched_history(as_of, max(1, max_lb))
|
||||
|
||||
results: dict[str, dict] = {}
|
||||
for sid in monitor_ids:
|
||||
try:
|
||||
overrides = all_overrides.get(sid, {})
|
||||
bf = overrides.get("basic_filter") if overrides else None
|
||||
dl = overrides.get("display_limit") if overrides else None
|
||||
if dl is None and overrides and "display_limit" in overrides:
|
||||
dl = 0
|
||||
|
||||
if sid in PRESET_STRATEGIES:
|
||||
r = svc.run_preset(sid, as_of=as_of, precomputed=enriched_today, basic_filter=bf, display_limit=dl)
|
||||
elif engine:
|
||||
r = engine.run(
|
||||
sid, as_of, overrides=overrides or None,
|
||||
precomputed=enriched_today, precomputed_history=shared_history,
|
||||
)
|
||||
if dl is not None and dl > 0:
|
||||
r.rows = r.rows[:dl]
|
||||
r.total = min(r.total, dl)
|
||||
else:
|
||||
continue
|
||||
|
||||
# sanitize NaN/Inf
|
||||
rows = []
|
||||
for row_dict in asdict(r).get("rows", []):
|
||||
for k, v in list(row_dict.items()):
|
||||
if isinstance(v, float) and not math.isfinite(v):
|
||||
row_dict[k] = None
|
||||
rows.append(row_dict)
|
||||
results[sid] = {"total": r.total, "as_of": str(as_of), "rows": rows}
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
|
||||
if results:
|
||||
strategy_cache.write_cache(data_dir, str(as_of), results)
|
||||
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("策略缓存刷新失败: %s", e)
|
||||
|
||||
def _get_monitor_pool_ids(self) -> list[str]:
|
||||
"""获取策略监控池中的策略 ID 列表。"""
|
||||
from app.services import preferences
|
||||
ids = preferences.get_strategy_monitor_ids()
|
||||
if not ids:
|
||||
return []
|
||||
return [sid for sid in ids if sid]
|
||||
|
||||
@staticmethod
|
||||
def _get_strategy_monitor():
|
||||
"""获取 StrategyMonitorService — 不再使用, 改用 _app_state 注入。"""
|
||||
return None
|
||||
|
||||
# ================================================================
|
||||
# enriched 增量计算
|
||||
# ================================================================
|
||||
|
||||
def _flush_live_enriched(self, daily_df: pl.DataFrame, quote_extra: pl.DataFrame = None) -> None:
|
||||
"""增量计算今天的 enriched: 用昨天的递推状态 + 今天 OHLCV → 只算今天 5500 行。
|
||||
|
||||
quote_extra: API 直接提供的补充字段 (prev_close, change_pct 等),
|
||||
不写 daily, 直接传给 compute_enriched_today 避免重复计算。
|
||||
"""
|
||||
try:
|
||||
today = date.today()
|
||||
t0 = time.perf_counter()
|
||||
|
||||
# ---- 尝试增量路径 ----
|
||||
live_agg = self._repo.get_live_agg()
|
||||
prev_enriched, prev_date = self._repo.get_enriched_latest()
|
||||
|
||||
use_incremental = (
|
||||
not live_agg.is_empty()
|
||||
and not prev_enriched.is_empty()
|
||||
and prev_date is not None
|
||||
)
|
||||
|
||||
if use_incremental:
|
||||
from app.indicators.pipeline import compute_enriched_today
|
||||
instruments = self._repo.get_instruments()
|
||||
# 将 API 直接提供的补充字段 JOIN 到 daily_df
|
||||
today_ohlcv = daily_df
|
||||
if quote_extra is not None and not quote_extra.is_empty():
|
||||
today_ohlcv = daily_df.join(quote_extra, on="symbol", how="left")
|
||||
enriched_today = compute_enriched_today(
|
||||
live_agg=live_agg,
|
||||
prev_enriched=prev_enriched,
|
||||
today_ohlcv=today_ohlcv,
|
||||
instruments=instruments,
|
||||
)
|
||||
if enriched_today.is_empty():
|
||||
logger.warning("增量计算结果为空, 回退到全量计算")
|
||||
use_incremental = False
|
||||
|
||||
# ---- 全量回退路径 ----
|
||||
if not use_incremental:
|
||||
from datetime import timedelta
|
||||
from app.indicators.pipeline import compute_enriched
|
||||
|
||||
logger.info("enriched 全量计算 (live_agg=%s, 上次日期=%s)",
|
||||
"ok" if not live_agg.is_empty() else "空", prev_date)
|
||||
|
||||
cutoff = today - timedelta(days=90)
|
||||
daily_glob = str(self._repo.store.data_dir / "kline_daily" / "**" / "*.parquet")
|
||||
ohlcv_cols = ["symbol", "date", "open", "high", "low", "close", "volume", "amount"]
|
||||
hist_df = (
|
||||
pl.scan_parquet(daily_glob)
|
||||
.filter(pl.col("date") >= cutoff)
|
||||
.sort(["symbol", "date"])
|
||||
.collect()
|
||||
)
|
||||
if hist_df.is_empty():
|
||||
return
|
||||
|
||||
hist_cols = [c for c in ohlcv_cols if c in hist_df.columns]
|
||||
hist_df = hist_df.select(hist_cols).filter(pl.col("date") != today)
|
||||
daily_ohlcv = daily_df.select([c for c in ohlcv_cols if c in daily_df.columns])
|
||||
full_df = pl.concat([hist_df, daily_ohlcv], how="diagonal_relaxed")
|
||||
full_df = full_df.sort(["symbol", "date"])
|
||||
|
||||
factor_path = self._repo.store.data_dir / "adj_factor" / "all.parquet"
|
||||
factors = pl.DataFrame()
|
||||
if factor_path.exists():
|
||||
try:
|
||||
factors = pl.read_parquet(factor_path)
|
||||
except Exception:
|
||||
pass
|
||||
instruments = self._repo.get_instruments()
|
||||
|
||||
enriched_full = compute_enriched(full_df, factors=factors, instruments=instruments)
|
||||
enriched_today = enriched_full.filter(pl.col("date") == today)
|
||||
|
||||
if enriched_today.is_empty():
|
||||
return
|
||||
|
||||
# ---- 写盘 + 更新缓存 ----
|
||||
self._repo.flush_live_enriched(enriched_today)
|
||||
|
||||
elapsed = time.perf_counter() - t0
|
||||
mode_label = "增量" if use_incremental else "全量"
|
||||
logger.info("enriched %s: %d 只, %s, 耗时 %.0fms",
|
||||
mode_label, len(enriched_today), today, elapsed * 1000)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("enriched 计算失败: %s", e)
|
||||
@@ -0,0 +1,593 @@
|
||||
"""Screener 服务(§6.3)。
|
||||
|
||||
性能优化:
|
||||
- enriched parquet 仅存 14 列基础数据, 指标和信号即时计算
|
||||
- preset 策略: 从内存缓存或即时计算获取完整指标, ~10-50ms
|
||||
- custom SQL: DuckDB (用户传 SQL WHERE 字符串), ~10-50ms
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, timedelta
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.tickflow.repository import KlineRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── 进程级历史数据缓存 (避免 run_all 每次重新扫描 parquet + 计算指标) ──
|
||||
_history_cache: dict[tuple[date, int], tuple[float, pl.DataFrame]] = {}
|
||||
_HISTORY_CACHE_TTL = 120.0 # 秒
|
||||
|
||||
|
||||
# 内置预设策略 — Polars 表达式方式
|
||||
PRESET_STRATEGIES: dict[str, dict] = {
|
||||
"trend_breakout": {
|
||||
"name": "趋势突破",
|
||||
"description": "MA60 上方 + 60 日新高 + 量能 ≥ 2 倍均量",
|
||||
"filter": (
|
||||
(pl.col("close") > pl.col("ma60"))
|
||||
& pl.col("signal_n_day_high").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= 2.0)
|
||||
),
|
||||
"order_by": "momentum_60d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"ma_golden_cross": {
|
||||
"name": "MA 金叉",
|
||||
"description": "MA5 上穿 MA20 当日触发,量能配合",
|
||||
"filter": (
|
||||
pl.col("signal_ma_golden_5_20").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= 1.2)
|
||||
& (pl.col("close") > pl.col("ma60"))
|
||||
),
|
||||
"order_by": "momentum_20d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"macd_golden": {
|
||||
"name": "MACD 金叉放量",
|
||||
"description": "MACD 金叉当日 + 量能放大",
|
||||
"filter": (
|
||||
pl.col("signal_macd_golden").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= 1.5)
|
||||
),
|
||||
"order_by": "momentum_60d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"volume_price_surge": {
|
||||
"name": "量价齐升",
|
||||
"description": "突破 MA20 + 放量 + 收阳",
|
||||
"filter": (
|
||||
pl.col("signal_ma20_breakout").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= 2.0)
|
||||
& (pl.col("close") > pl.col("open"))
|
||||
),
|
||||
"order_by": "vol_ratio_5d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"low_volatility_leader": {
|
||||
"name": "低波动龙头",
|
||||
"description": "20 日动量为正 + 年化波动 < 30% + MA20 上方",
|
||||
"filter": (
|
||||
(pl.col("momentum_20d") > 0)
|
||||
& (pl.col("annual_vol_20d") < 0.30)
|
||||
& (pl.col("close") > pl.col("ma20"))
|
||||
),
|
||||
"order_by": "momentum_60d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"broken_board_recovery": {
|
||||
"name": "断板反包",
|
||||
"description": "连板 ≥2 后断板 1-2 天，出现放量反包信号",
|
||||
"filter": (
|
||||
pl.col("signal_limit_up").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= 1.5)
|
||||
& (pl.col("change_pct") > 0.03)
|
||||
),
|
||||
"order_by": "change_pct",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"oversold_bounce": {
|
||||
"name": "超跌反弹",
|
||||
"description": "RSI14 < 30 超卖区 + 当日收阳 + 放量，抄底信号",
|
||||
"filter": (
|
||||
(pl.col("rsi_14") < 30)
|
||||
& (pl.col("close") > pl.col("open"))
|
||||
& (pl.col("vol_ratio_5d") >= 1.2)
|
||||
),
|
||||
"order_by": "rsi_14",
|
||||
"descending": False,
|
||||
"limit": 100,
|
||||
},
|
||||
"boll_breakout": {
|
||||
"name": "布林突破",
|
||||
"description": "突破布林上轨 + 放量，强势加速信号",
|
||||
"filter": (
|
||||
pl.col("signal_boll_breakout_upper").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= 1.5)
|
||||
),
|
||||
"order_by": "vol_ratio_5d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"bullish_alignment": {
|
||||
"name": "均线多头",
|
||||
"description": "MA5 > MA10 > MA20 > MA60 多头排列 + 短期动量为正",
|
||||
"filter": (
|
||||
(pl.col("ma5") > pl.col("ma10"))
|
||||
& (pl.col("ma10") > pl.col("ma20"))
|
||||
& (pl.col("ma20") > pl.col("ma60"))
|
||||
& (pl.col("momentum_20d") > 0)
|
||||
),
|
||||
"order_by": "momentum_60d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"consecutive_limit_ups": {
|
||||
"name": "连板股",
|
||||
"description": "当日涨停且连续涨停 ≥ 2 天，强势追涨",
|
||||
"filter": (
|
||||
pl.col("signal_limit_up").fill_null(False)
|
||||
& (pl.col("consecutive_limit_ups") >= 2)
|
||||
),
|
||||
"order_by": "consecutive_limit_ups",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"pullback_to_support": {
|
||||
"name": "缩量回踩",
|
||||
"description": "回踩 MA20 附近 + 缩量 + 中期趋势向上",
|
||||
"filter": (
|
||||
(pl.col("close") > pl.col("ma20") * 0.98)
|
||||
& (pl.col("close") < pl.col("ma20") * 1.02)
|
||||
& (pl.col("vol_ratio_5d") < 0.8)
|
||||
& (pl.col("close") > pl.col("ma60"))
|
||||
& (pl.col("momentum_20d") > 0)
|
||||
),
|
||||
"order_by": "momentum_60d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"n_day_low_reversal": {
|
||||
"name": "新低反转",
|
||||
"description": "触及 60 日新低后当日收阳放量，反转信号",
|
||||
"filter": (
|
||||
pl.col("signal_n_day_low").fill_null(False)
|
||||
& (pl.col("close") > pl.col("open"))
|
||||
& (pl.col("vol_ratio_5d") >= 1.5)
|
||||
),
|
||||
"order_by": "change_pct",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScreenerResult:
|
||||
as_of: date
|
||||
strategy: str | None
|
||||
rows: list[dict] = field(default_factory=list)
|
||||
total: int = 0
|
||||
elapsed_ms: float = 0.0
|
||||
|
||||
|
||||
class ScreenerService:
|
||||
def __init__(self, repo: KlineRepository) -> None:
|
||||
self.repo = repo
|
||||
|
||||
@staticmethod
|
||||
def clear_history_cache() -> None:
|
||||
"""清空进程级 _history_cache (TTL 缓存)。
|
||||
|
||||
清除数据后调用, 避免内存里的旧历史窗口残留导致策略/看板仍命中旧数据。
|
||||
"""
|
||||
_history_cache.clear()
|
||||
|
||||
def _load_enriched_for_date(self, target_date: date) -> pl.DataFrame:
|
||||
"""从 enriched parquet 读取指定日期的基础数据并即时计算完整指标+信号。
|
||||
|
||||
enriched parquet 仅存 14 列。读取后需要即时计算 ma/ema/macd/kdj/rsi/boll/momentum/signal 等列。
|
||||
对于最新日, 优先使用内存缓存 (已包含完整指标)。
|
||||
"""
|
||||
# 优先使用 repo 最新日缓存
|
||||
cache, cache_date = self.repo.get_enriched_latest()
|
||||
if cache is not None and not cache.is_empty() and cache_date == target_date:
|
||||
df = cache
|
||||
# JOIN instruments
|
||||
df_i = self.repo.get_instruments()
|
||||
if not df_i.is_empty():
|
||||
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"] if c in df_i.columns]
|
||||
if "name" not in df.columns:
|
||||
df = df.join(df_i.select(inst_cols), on="symbol", how="left")
|
||||
return df
|
||||
|
||||
# 尝试从 repo 级预计算历史缓存中提取目标日期
|
||||
cached_hist = self.repo.get_enriched_history(target_date, 1)
|
||||
if cached_hist is not None and not cached_hist.is_empty() and "date" in cached_hist.columns:
|
||||
df = cached_hist.filter(pl.col("date") == target_date)
|
||||
if not df.is_empty():
|
||||
logger.debug("_load_enriched_for_date: repo history cache for %s", target_date)
|
||||
# JOIN instruments
|
||||
df_i = self.repo.get_instruments()
|
||||
if not df_i.is_empty():
|
||||
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"] if c in df_i.columns]
|
||||
if "name" not in df.columns:
|
||||
df = df.join(df_i.select(inst_cols), on="symbol", how="left")
|
||||
return df
|
||||
|
||||
# 历史日期: 从 parquet 读取 14 列, 即时计算指标 (慢路径)
|
||||
enriched_dir = self.repo.store.data_dir / "kline_daily_enriched"
|
||||
ds = target_date.isoformat()
|
||||
target_parquet = enriched_dir / f"date={ds}" / "part.parquet"
|
||||
|
||||
if not target_parquet.exists():
|
||||
return pl.DataFrame()
|
||||
|
||||
try:
|
||||
df = pl.read_parquet(target_parquet)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("load_enriched_for_date failed: %s", e)
|
||||
return pl.DataFrame()
|
||||
|
||||
if df.is_empty():
|
||||
return df
|
||||
|
||||
# 即时计算指标: 需要加载历史窗口作 warmup
|
||||
df_full = self._compute_enriched_full(df, target_date)
|
||||
return df_full
|
||||
|
||||
def _compute_enriched_full(self, df_target: pl.DataFrame, target_date: date) -> pl.DataFrame:
|
||||
"""从 14 列基础数据即时计算完整 enriched (含全部指标和信号)。
|
||||
|
||||
读取历史数据作为指标计算的 warmup, 计算完成后只返回目标日期的行。
|
||||
"""
|
||||
from app.indicators.pipeline import compute_indicators, compute_signals, compute_limit_signals
|
||||
|
||||
# 加载 warmup 历史 (目标日期前 ~120 天)
|
||||
enriched_dir = self.repo.store.data_dir / "kline_daily_enriched"
|
||||
start = target_date - timedelta(days=150)
|
||||
read_cols = ["symbol", "date", "open", "high", "low", "close", "volume",
|
||||
"amount", "raw_close", "raw_high", "raw_low"]
|
||||
|
||||
try:
|
||||
lf = (
|
||||
pl.scan_parquet(str(enriched_dir / "**" / "*.parquet"))
|
||||
.filter(
|
||||
(pl.col("date") >= start)
|
||||
& (pl.col("date") <= target_date)
|
||||
)
|
||||
.sort(["symbol", "date"])
|
||||
)
|
||||
available = [c for c in read_cols if c in lf.schema]
|
||||
df_hist = lf.select(available).collect()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("warmup history load failed: %s", e)
|
||||
df_hist = df_target
|
||||
|
||||
if df_hist.is_empty():
|
||||
df_hist = df_target
|
||||
|
||||
# 计算指标
|
||||
df_full = compute_indicators(df_hist)
|
||||
df_full = compute_signals(df_full)
|
||||
|
||||
# 计算涨跌停信号 (需要 instruments)
|
||||
instruments = self.repo.get_instruments()
|
||||
if instruments is not None and not instruments.is_empty():
|
||||
df_full = compute_limit_signals(df_full, instruments)
|
||||
|
||||
# 只保留目标日期
|
||||
df_result = df_full.filter(pl.col("date") == target_date)
|
||||
|
||||
# JOIN instruments (name, total_shares, float_shares)
|
||||
if not instruments.is_empty():
|
||||
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"] if c in instruments.columns]
|
||||
if "name" not in df_result.columns:
|
||||
df_result = df_result.join(instruments.select(inst_cols), on="symbol", how="left")
|
||||
|
||||
return df_result
|
||||
|
||||
def _load_enriched_history(self, target_date: date, lookback_days: int) -> pl.DataFrame:
|
||||
"""读取目标日期之前的基础行情数据, 供历史窗口策略使用。
|
||||
|
||||
优先从 repo 内存缓存获取 (启动时已预计算), 命中时 0ms。
|
||||
缓存 miss 时走 scan_parquet + compute_indicators 慢路径。
|
||||
"""
|
||||
# 优先级 1: repo 级预计算缓存 (启动时 _refresh_enriched 已计算完整历史)
|
||||
t0 = time.perf_counter()
|
||||
cached = self.repo.get_enriched_history(target_date, lookback_days)
|
||||
if cached is not None and not cached.is_empty():
|
||||
# JOIN instruments (repo 缓存不含 name 等列)
|
||||
instruments = self.repo.get_instruments()
|
||||
if instruments is not None and not instruments.is_empty() and "name" not in cached.columns:
|
||||
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"]
|
||||
if c in instruments.columns]
|
||||
cached = cached.join(instruments.select(inst_cols), on="symbol", how="left")
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
logger.info("_load_enriched_history(%s, %d): repo cache hit, %.1fms, %d rows",
|
||||
target_date, lookback_days, elapsed, len(cached))
|
||||
return cached
|
||||
|
||||
# 优先级 2: 进程级 history_cache (之前的 TTL 缓存)
|
||||
cache_key = (target_date, lookback_days)
|
||||
now = time.monotonic()
|
||||
ttl_cached = _history_cache.get(cache_key)
|
||||
if ttl_cached is not None:
|
||||
ts, cached_df = ttl_cached
|
||||
if now - ts < _HISTORY_CACHE_TTL:
|
||||
logger.debug("history TTL cache hit: %s lookback=%d", target_date, lookback_days)
|
||||
return cached_df
|
||||
del _history_cache[cache_key]
|
||||
|
||||
# 优先级 3: scan_parquet + compute_indicators (慢路径, ~5s)
|
||||
logger.warning("_load_enriched_history cache miss, computing indicators (%s, %d)...",
|
||||
target_date, lookback_days)
|
||||
from app.indicators.pipeline import compute_indicators, compute_signals, compute_limit_signals
|
||||
|
||||
warmup = 60
|
||||
start = target_date - timedelta(days=min((lookback_days + warmup) * 2, 180))
|
||||
|
||||
enriched_dir = self.repo.store.data_dir / "kline_daily_enriched"
|
||||
read_cols = ["symbol", "date", "open", "high", "low", "close", "volume",
|
||||
"amount", "raw_close", "raw_high", "raw_low"]
|
||||
|
||||
try:
|
||||
lf = (
|
||||
pl.scan_parquet(str(enriched_dir / "**" / "*.parquet"))
|
||||
.filter((pl.col("date") >= start) & (pl.col("date") <= target_date))
|
||||
.sort(["symbol", "date"])
|
||||
)
|
||||
available = [c for c in read_cols if c in lf.collect_schema().names()]
|
||||
df_hist = lf.select(available).collect()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("load_enriched_history failed: %s", e)
|
||||
return pl.DataFrame()
|
||||
|
||||
if df_hist.is_empty():
|
||||
return pl.DataFrame()
|
||||
|
||||
df_full = compute_indicators(df_hist)
|
||||
df_full = compute_signals(df_full)
|
||||
|
||||
instruments = self.repo.get_instruments()
|
||||
if instruments is not None and not instruments.is_empty():
|
||||
df_full = compute_limit_signals(df_full, instruments)
|
||||
|
||||
if instruments is not None and not instruments.is_empty():
|
||||
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"] if c in instruments.columns]
|
||||
if "name" not in df_full.columns:
|
||||
df_full = df_full.join(instruments.select(inst_cols), on="symbol", how="left")
|
||||
|
||||
# 裁剪掉 warmup 部分, 只保留 lookback 范围 (减少 group_by 开销)
|
||||
lookback_start = target_date - timedelta(days=lookback_days)
|
||||
if "date" in df_full.columns:
|
||||
df_full = df_full.filter(pl.col("date") >= lookback_start)
|
||||
|
||||
df_full = df_full.sort(["symbol", "date"])
|
||||
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
logger.info("_load_enriched_history(%s, %d): computed in %.1fms, %d rows",
|
||||
target_date, lookback_days, elapsed, len(df_full))
|
||||
|
||||
_history_cache[cache_key] = (now, df_full)
|
||||
if len(_history_cache) > 10:
|
||||
expired = [k for k, (ts, _) in _history_cache.items() if now - ts > _HISTORY_CACHE_TTL]
|
||||
for k in expired:
|
||||
del _history_cache[k]
|
||||
|
||||
return df_full
|
||||
|
||||
def run(
|
||||
self,
|
||||
as_of: date,
|
||||
conditions: list[str],
|
||||
order_by: str | None = None,
|
||||
limit: int = 30,
|
||||
pool: list[str] | None = None,
|
||||
) -> ScreenerResult:
|
||||
"""自定义 SQL 条件选股。
|
||||
|
||||
先通过 Polars 即时计算完整指标, 再用 DuckDB 做 SQL WHERE 过滤。
|
||||
kline_enriched DuckDB 视图只有 14 列, 不能直接用于指标过滤。
|
||||
"""
|
||||
t0 = time.perf_counter()
|
||||
|
||||
if not conditions:
|
||||
return ScreenerResult(as_of=as_of, strategy=None)
|
||||
|
||||
# 从即时计算获取完整 enriched 数据
|
||||
df = self._load_enriched_for_date(as_of)
|
||||
if df.is_empty():
|
||||
return ScreenerResult(as_of=as_of, strategy=None)
|
||||
|
||||
# Pool 过滤
|
||||
if pool:
|
||||
df = df.filter(pl.col("symbol").is_in(pool))
|
||||
|
||||
# 用 DuckDB 做 SQL 过滤 (注册临时视图)
|
||||
try:
|
||||
import duckdb
|
||||
con = duckdb.connect(database=":memory:")
|
||||
con.register("enriched", df.to_arrow())
|
||||
where = " AND ".join(f"({c})" for c in conditions)
|
||||
sql = f"SELECT * FROM enriched WHERE {where}"
|
||||
if order_by:
|
||||
sql += f" ORDER BY {order_by}"
|
||||
if limit:
|
||||
sql += f" LIMIT {limit}"
|
||||
df_result = con.execute(sql).pl()
|
||||
con.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("screener SQL query failed: %s", e)
|
||||
df_result = pl.DataFrame()
|
||||
|
||||
rows = df_result.to_dicts() if not df_result.is_empty() else []
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
|
||||
return ScreenerResult(
|
||||
as_of=as_of,
|
||||
strategy=None,
|
||||
rows=rows,
|
||||
total=len(rows),
|
||||
elapsed_ms=elapsed,
|
||||
)
|
||||
|
||||
def run_preset(
|
||||
self,
|
||||
strategy_id: str,
|
||||
as_of: date,
|
||||
pool: list[str] | None = None,
|
||||
precomputed: pl.DataFrame | None = None,
|
||||
basic_filter: dict | None = None,
|
||||
display_limit: int | None = None,
|
||||
) -> ScreenerResult:
|
||||
"""预设策略选股 — 从 enriched 读取预计算好的指标列后过滤。
|
||||
|
||||
- precomputed 不为空: 直接复用（run_all 场景）
|
||||
- precomputed 为空: 从 enriched 读目标日期
|
||||
- basic_filter: 用户保存的基础参数过滤（boards、价格等）
|
||||
"""
|
||||
t0 = time.perf_counter()
|
||||
|
||||
strat = PRESET_STRATEGIES.get(strategy_id)
|
||||
if not strat:
|
||||
raise ValueError(f"unknown strategy: {strategy_id}")
|
||||
|
||||
if precomputed is not None and not precomputed.is_empty():
|
||||
df = precomputed
|
||||
else:
|
||||
df = self._load_enriched_for_date(as_of)
|
||||
if df.is_empty():
|
||||
return ScreenerResult(as_of=as_of, strategy=strategy_id)
|
||||
|
||||
# 应用用户基础参数过滤（boards、价格区间等）
|
||||
if basic_filter and basic_filter.get("enabled", True):
|
||||
df = self._apply_basic_filter(df, basic_filter)
|
||||
|
||||
# 应用策略过滤
|
||||
df = df.filter(strat["filter"])
|
||||
|
||||
# 应用 pool
|
||||
if pool:
|
||||
df = df.filter(pl.col("symbol").is_in(pool))
|
||||
|
||||
# 排序 + 限制
|
||||
order_col = strat["order_by"]
|
||||
if order_col in df.columns:
|
||||
df = df.sort(order_col, descending=strat.get("descending", True))
|
||||
|
||||
# display_limit: None=不限制, 0=全部, N=前N个
|
||||
if display_limit == 0:
|
||||
limit = None # 不限制
|
||||
elif display_limit is not None:
|
||||
limit = display_limit
|
||||
else:
|
||||
limit = None # 未配置时默认不限制
|
||||
if limit is not None and limit > 0:
|
||||
df = df.head(limit)
|
||||
|
||||
# 基于排序列生成 0-100 评分 (与 StrategyEngine 统一)
|
||||
if order_col in df.columns and not df.is_empty():
|
||||
col_vals = df[order_col].cast(pl.Float64)
|
||||
col_min = col_vals.min()
|
||||
col_max = col_vals.max()
|
||||
col_range = col_max - col_min
|
||||
if col_range and col_range > 0:
|
||||
normalized = (col_vals - col_min) / col_range
|
||||
else:
|
||||
normalized = pl.Series("norm", [0.5] * len(df))
|
||||
if not strat.get("descending", True):
|
||||
normalized = 1.0 - normalized
|
||||
df = df.with_columns((normalized * 100).alias("score"))
|
||||
|
||||
rows = df.to_dicts()
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
|
||||
# sanitize
|
||||
for r in rows:
|
||||
for k, v in list(r.items()):
|
||||
if isinstance(v, float) and (v != v or abs(v) == float("inf")):
|
||||
r[k] = None
|
||||
|
||||
return ScreenerResult(
|
||||
as_of=as_of,
|
||||
strategy=strategy_id,
|
||||
rows=rows,
|
||||
total=len(rows),
|
||||
elapsed_ms=elapsed,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _apply_basic_filter(df: pl.DataFrame, bf: dict) -> pl.DataFrame:
|
||||
"""应用用户基础参数过滤（boards、价格区间、市值等）"""
|
||||
exprs: list[pl.Expr] = []
|
||||
if bf.get("price_min") is not None:
|
||||
exprs.append(pl.col("close") >= bf["price_min"])
|
||||
if bf.get("price_max") is not None:
|
||||
exprs.append(pl.col("close") <= bf["price_max"])
|
||||
if bf.get("float_cap_min") is not None and "float_shares" in df.columns:
|
||||
exprs.append(pl.col("close") * pl.col("float_shares") >= bf["float_cap_min"])
|
||||
if bf.get("float_cap_max") is not None and "float_shares" in df.columns:
|
||||
exprs.append(pl.col("close") * pl.col("float_shares") <= bf["float_cap_max"])
|
||||
if bf.get("amount_min") is not None:
|
||||
exprs.append(pl.col("amount") >= bf["amount_min"])
|
||||
if bf.get("amount_max") is not None:
|
||||
exprs.append(pl.col("amount") <= bf["amount_max"])
|
||||
if bf.get("turnover_min") is not None and "turnover_rate" in df.columns:
|
||||
exprs.append(pl.col("turnover_rate") >= bf["turnover_min"])
|
||||
if bf.get("turnover_max") is not None and "turnover_rate" in df.columns:
|
||||
exprs.append(pl.col("turnover_rate") <= bf["turnover_max"])
|
||||
if bf.get("exclude_st") and "name" in df.columns:
|
||||
exprs.append(~pl.col("name").str.contains("(?i)ST|\\*ST|退"))
|
||||
# 板块过滤
|
||||
boards = bf.get("boards")
|
||||
if boards and isinstance(boards, list) and len(boards) > 0:
|
||||
board_exprs: list[pl.Expr] = []
|
||||
for b in boards:
|
||||
if b == "沪主板":
|
||||
board_exprs.append(pl.col("symbol").str.starts_with("60"))
|
||||
elif b == "深主板":
|
||||
board_exprs.append(
|
||||
pl.col("symbol").str.starts_with("00")
|
||||
| pl.col("symbol").str.starts_with("001")
|
||||
)
|
||||
elif b == "创业板":
|
||||
board_exprs.append(
|
||||
pl.col("symbol").str.starts_with("300")
|
||||
| pl.col("symbol").str.starts_with("301")
|
||||
)
|
||||
elif b == "科创板":
|
||||
board_exprs.append(pl.col("symbol").str.starts_with("688"))
|
||||
elif b == "北交所":
|
||||
board_exprs.append(pl.col("symbol").str.contains(r"\.BJ$"))
|
||||
if board_exprs:
|
||||
exprs.append(pl.any_horizontal(board_exprs))
|
||||
if exprs:
|
||||
return df.filter(pl.all_horizontal(exprs))
|
||||
return df
|
||||
|
||||
def latest_date(self) -> date | None:
|
||||
d = self.repo.enriched_latest_date()
|
||||
if d:
|
||||
return d
|
||||
# 回退 DuckDB
|
||||
try:
|
||||
res = self.repo.execute_one(
|
||||
"SELECT max(date) FROM kline_enriched",
|
||||
)
|
||||
if res and res[0]:
|
||||
d = res[0]
|
||||
return d if isinstance(d, date) else date.fromisoformat(str(d))
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
return None
|
||||
@@ -0,0 +1,150 @@
|
||||
"""策略结果缓存 — 写入本地文件，供策略页面秒加载。
|
||||
|
||||
缓存结构:
|
||||
{
|
||||
"as_of": "2024-01-15",
|
||||
"results": { strategy_id: { total, as_of, rows } },
|
||||
"today_ever_matched": { strategy_id: [symbol, ...] }, // 今日曾命中 symbol 并集
|
||||
"today_ever_rows": { strategy_id: { symbol: row_data } },// 今日曾命中的完整行数据
|
||||
"updated_at": 1705324800000 # Unix ms
|
||||
}
|
||||
|
||||
文件路径: data/user_data/strategy_cache.json
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _json_default(obj: Any) -> Any:
|
||||
"""处理 date/datetime 等 JSON 不认识的类型。"""
|
||||
if isinstance(obj, date):
|
||||
return obj.isoformat()
|
||||
if isinstance(obj, datetime):
|
||||
return obj.isoformat()
|
||||
raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable")
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CACHE_FILENAME = "strategy_cache.json"
|
||||
|
||||
|
||||
def _cache_path(data_dir: Path) -> Path:
|
||||
return data_dir / "user_data" / _CACHE_FILENAME
|
||||
|
||||
|
||||
def _enriched_parquet_path(data_dir: Path, as_of: str) -> Path:
|
||||
"""返回 enriched parquet 文件路径。"""
|
||||
return data_dir / "kline_daily_enriched" / f"date={as_of}" / "part.parquet"
|
||||
|
||||
|
||||
def _get_enriched_mtime(data_dir: Path, as_of: str) -> float | None:
|
||||
"""返回 enriched parquet 文件的 mtime (秒)。文件不存在返回 None。"""
|
||||
p = _enriched_parquet_path(data_dir, as_of)
|
||||
try:
|
||||
return p.stat().st_mtime
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def read_cache(data_dir: Path) -> dict | None:
|
||||
"""读取策略缓存文件。返回 None 表示无缓存、读取失败或 enriched 数据已更新导致缓存过期。"""
|
||||
path = _cache_path(data_dir)
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if not text.strip():
|
||||
return None
|
||||
cached = json.loads(text)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("读取策略缓存失败: %s", e)
|
||||
return None
|
||||
|
||||
# 校验 enriched mtime: 数据文件变化 → 缓存过期
|
||||
as_of = cached.get("as_of")
|
||||
stored_mtime = cached.get("enriched_mtime")
|
||||
if as_of and stored_mtime:
|
||||
current_mtime = _get_enriched_mtime(data_dir, as_of)
|
||||
if current_mtime is not None and current_mtime != stored_mtime:
|
||||
logger.info("策略缓存过期: enriched 数据已更新 (as_of=%s)", as_of)
|
||||
return None
|
||||
|
||||
return cached
|
||||
|
||||
|
||||
def _rows_to_symbol_map(rows: list[dict]) -> dict[str, dict]:
|
||||
"""将 rows 列表转为 {symbol: row_data} 映射。"""
|
||||
result: dict[str, dict] = {}
|
||||
for row in rows:
|
||||
sym = row.get("symbol")
|
||||
if sym:
|
||||
result[sym] = row
|
||||
return result
|
||||
|
||||
|
||||
def write_cache(
|
||||
data_dir: Path,
|
||||
as_of: str,
|
||||
results: dict[str, Any],
|
||||
) -> None:
|
||||
"""将策略结果写入缓存文件，同时更新今日曾命中集合。
|
||||
|
||||
- 日期变更时重置 today_ever_matched 和 today_ever_rows
|
||||
- 同一天内合并 (并集) 之前曾命中的 symbol，并用最新行数据更新
|
||||
"""
|
||||
path = _cache_path(data_dir)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 读取旧缓存
|
||||
old = read_cache(data_dir)
|
||||
old_as_of = old.get("as_of") if old else None
|
||||
old_ever_rows: dict[str, dict[str, dict]] = old.get("today_ever_rows", {}) if old else {}
|
||||
|
||||
# 当前命中的行数据 → symbol 映射
|
||||
current_row_maps: dict[str, dict[str, dict]] = {}
|
||||
for sid, r in results.items():
|
||||
current_row_maps[sid] = _rows_to_symbol_map(r.get("rows", []))
|
||||
|
||||
if old_as_of and old_as_of == as_of and old_ever_rows:
|
||||
# 同一天: 合并 — 用当前行数据更新旧数据 (保持最新价格等)
|
||||
merged_rows: dict[str, dict[str, dict]] = {}
|
||||
all_keys = set(old_ever_rows.keys()) | set(current_row_maps.keys())
|
||||
for sid in all_keys:
|
||||
old_map = old_ever_rows.get(sid, {})
|
||||
cur_map = current_row_maps.get(sid, {})
|
||||
# 以旧数据为基础，用当前数据覆盖 (当前数据更新鲜)
|
||||
combined = {**old_map, **cur_map}
|
||||
merged_rows[sid] = combined
|
||||
today_ever_rows = merged_rows
|
||||
else:
|
||||
# 新的一天或首次写入
|
||||
today_ever_rows = current_row_maps
|
||||
|
||||
# 从 ever_rows 提取 symbol 列表 (用于快速计数)
|
||||
today_ever_matched = {sid: sorted(maps.keys()) for sid, maps in today_ever_rows.items()}
|
||||
|
||||
# 记录 enriched parquet 文件的 mtime，用于后续校验缓存是否过期
|
||||
enriched_mtime = _get_enriched_mtime(data_dir, as_of)
|
||||
|
||||
payload = {
|
||||
"as_of": as_of,
|
||||
"results": results,
|
||||
"today_ever_matched": today_ever_matched,
|
||||
"today_ever_rows": today_ever_rows,
|
||||
"enriched_mtime": enriched_mtime,
|
||||
"updated_at": int(time.time() * 1000),
|
||||
}
|
||||
try:
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, default=_json_default), encoding="utf-8")
|
||||
total_rows = sum(len(r.get("rows", [])) for r in results.values())
|
||||
total_ever = sum(len(v) for v in today_ever_matched.values())
|
||||
logger.info("策略缓存已写入: %s, %d 策略, %d 命中, %d 曾命中", as_of, len(results), total_rows, total_ever)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("写入策略缓存失败: %s", e)
|
||||
@@ -0,0 +1,130 @@
|
||||
"""自选股服务(§6.1)。
|
||||
|
||||
存储:`data/user_data/watchlist.parquet`,字段 symbol + added_at + note。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.config import settings
|
||||
from app.tickflow.capabilities import Cap, CapabilitySet
|
||||
from app.tickflow.client import get_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _path() -> Path:
|
||||
p = settings.data_dir / "user_data" / "watchlist.parquet"
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def list_symbols() -> list[dict]:
|
||||
p = _path()
|
||||
if not p.exists():
|
||||
return []
|
||||
df = pl.read_parquet(p)
|
||||
if df.is_empty():
|
||||
return []
|
||||
return df.to_dicts()
|
||||
|
||||
|
||||
def add(symbol: str, note: str = "") -> list[dict]:
|
||||
p = _path()
|
||||
if p.exists():
|
||||
df = pl.read_parquet(p)
|
||||
# 已存在则先移除，后面重新插入到最前面
|
||||
if symbol in df["symbol"].to_list():
|
||||
df = df.filter(pl.col("symbol") != symbol)
|
||||
else:
|
||||
df = pl.DataFrame(schema={"symbol": pl.Utf8, "added_at": pl.Utf8, "note": pl.Utf8})
|
||||
|
||||
new_row = pl.DataFrame({
|
||||
"symbol": [symbol],
|
||||
"added_at": [datetime.utcnow().isoformat(timespec="seconds")],
|
||||
"note": [note],
|
||||
})
|
||||
out = pl.concat([new_row, df], how="diagonal_relaxed")
|
||||
out.write_parquet(p)
|
||||
return out.to_dicts()
|
||||
|
||||
|
||||
def remove(symbol: str) -> list[dict]:
|
||||
p = _path()
|
||||
if not p.exists():
|
||||
return []
|
||||
df = pl.read_parquet(p)
|
||||
df = df.filter(pl.col("symbol") != symbol)
|
||||
df.write_parquet(p)
|
||||
return df.to_dicts()
|
||||
|
||||
|
||||
def clear() -> int:
|
||||
"""清空自选列表。返回移除的数量。"""
|
||||
p = _path()
|
||||
if not p.exists():
|
||||
return 0
|
||||
df = pl.read_parquet(p)
|
||||
count = df.height
|
||||
if count > 0:
|
||||
pl.DataFrame(schema={"symbol": pl.Utf8, "added_at": pl.Utf8, "note": pl.Utf8}).write_parquet(p)
|
||||
return count
|
||||
|
||||
|
||||
def fetch_quotes(symbols: list[str], capset: CapabilitySet, timeout_s: float = 8.0) -> list[dict]:
|
||||
"""拉取实时行情。
|
||||
|
||||
优先用 quote.batch;否则降级为 quote.by_symbol 单股请求。
|
||||
timeout_s: 单批次请求超时(秒)，防止 API 卡死阻塞整个请求。
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout
|
||||
|
||||
if not symbols:
|
||||
return []
|
||||
|
||||
tf = get_client()
|
||||
quotes: list[dict] = []
|
||||
|
||||
# 走 batch
|
||||
batch_size = 5
|
||||
if capset.has(Cap.QUOTE_BATCH):
|
||||
lim = capset.limits(Cap.QUOTE_BATCH)
|
||||
batch_size = lim.batch if lim and lim.batch else 50
|
||||
elif capset.has(Cap.QUOTE_BY_SYMBOL):
|
||||
lim = capset.limits(Cap.QUOTE_BY_SYMBOL)
|
||||
batch_size = lim.batch if lim and lim.batch else 5
|
||||
else:
|
||||
# 无任何实时行情能力(none/free 档走 free-api 服务器,不提供实时行情)
|
||||
# 提前返回空,避免发起注定失败的请求
|
||||
return []
|
||||
|
||||
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
|
||||
|
||||
# 用线程池为每个批次加超时保护
|
||||
pool = ThreadPoolExecutor(max_workers=1)
|
||||
for chunk in chunks:
|
||||
try:
|
||||
future = pool.submit(tf.quotes.get, symbols=chunk, as_dataframe=True)
|
||||
raw = future.result(timeout=timeout_s)
|
||||
if raw is None or len(raw) == 0:
|
||||
continue
|
||||
df = pl.from_pandas(raw)
|
||||
rename_map = {
|
||||
"last_price": "price",
|
||||
"ext.change_pct": "pct",
|
||||
"ext.name": "name",
|
||||
}
|
||||
df = df.rename({k: v for k, v in rename_map.items() if k in df.columns})
|
||||
quotes.extend(df.to_dicts())
|
||||
except FuturesTimeout:
|
||||
logger.warning("quote fetch timeout (%.1fs) for %d symbols", timeout_s, len(chunk))
|
||||
break # 超时后不再尝试后续批次
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("quote fetch failed for %d symbols: %s", len(chunk), e)
|
||||
pool.shutdown(wait=False)
|
||||
|
||||
return quotes
|
||||
@@ -0,0 +1,154 @@
|
||||
"""AI 策略生成器 — 读取策略开发文档 + 调用 LLM 生成策略代码。
|
||||
|
||||
职责: 接收用户自然语言描述 → 读取 docs/strategy-guide.md → 调用 LLM → 返回策略代码。
|
||||
不知道: 引擎内部、API、前端、配置持久化、回测。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import logging
|
||||
import re
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 策略开发文档路径
|
||||
GUIDE_PATH = Path(__file__).resolve().parent.parent.parent.parent / "docs" / "strategy-guide.md"
|
||||
|
||||
_SYSTEM_PREFIX = """你是A股量化策略设计专家。根据用户描述的需求，参考下方的《策略开发指南》生成一个完整的策略Python文件。
|
||||
|
||||
核心约束:
|
||||
- 只创建这一个 .py 文件，不要修改任何现有文件，不要跨文件引用
|
||||
- 只 import polars as pl，不 import 其他模块
|
||||
|
||||
要求:
|
||||
1. 用户可能调整的策略阈值通过 META["params"] 暴露；公式常数、固定窗口边界、布尔开关不必强行参数化
|
||||
2. 遵循指南中的文件结构，但优先贴合用户规则，不要为了套模板歪曲策略含义
|
||||
3. ENTRY_SIGNALS/EXIT_SIGNALS 根据策略逻辑自行选择匹配的信号列，不要照搬示例
|
||||
4. scoring 权重根据策略核心逻辑定制，总和 = 1.0
|
||||
5. 优先使用 Polars 表达式、窗口函数、聚合和 with_columns/filter 实现，避免逐行/逐股 Python 循环；只有表达式难以描述的复杂状态机才使用 partition_by/to_dicts
|
||||
6. 直接输出Python代码，不要输出其他内容
|
||||
|
||||
--- 策略开发指南 ---
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class AIStrategyGenerator:
|
||||
"""AI 策略生成器"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._guide_cache: str | None = None
|
||||
|
||||
def _get_guide(self) -> str:
|
||||
if self._guide_cache is None:
|
||||
if GUIDE_PATH.exists():
|
||||
self._guide_cache = GUIDE_PATH.read_text(encoding="utf-8")
|
||||
else:
|
||||
logger.warning("strategy-guide.md not found at %s", GUIDE_PATH)
|
||||
self._guide_cache = ""
|
||||
return self._guide_cache
|
||||
|
||||
async def generate(self, user_prompt: str) -> dict:
|
||||
"""根据用户描述生成策略代码
|
||||
|
||||
Returns: {"code": str, "meta": dict, "valid": bool, "error": str | None}
|
||||
"""
|
||||
guide = self._get_guide()
|
||||
|
||||
# 调用 LLM
|
||||
code = await self._call_llm(user_prompt, guide)
|
||||
|
||||
# 验证
|
||||
try:
|
||||
self._validate_safety(code)
|
||||
except ValueError as e:
|
||||
return {"code": code, "meta": {}, "valid": False, "error": str(e)}
|
||||
|
||||
# 试加载获取 META
|
||||
try:
|
||||
meta = self._extract_meta(code)
|
||||
except Exception as e:
|
||||
return {"code": code, "meta": {}, "valid": False, "error": f"解析META失败: {e}"}
|
||||
|
||||
return {"code": code, "meta": meta, "valid": True, "error": None}
|
||||
|
||||
async def _call_llm(self, user_prompt: str, guide: str) -> str:
|
||||
"""调用 OpenAI 兼容 API（流式，避免 CDN 长连接超时）"""
|
||||
from openai import AsyncOpenAI
|
||||
from app import secrets_store
|
||||
|
||||
ai_key = secrets_store.get_ai_key()
|
||||
if not ai_key:
|
||||
raise RuntimeError("AI API Key 未配置，请在设置页面配置")
|
||||
|
||||
client = AsyncOpenAI(
|
||||
api_key=ai_key,
|
||||
base_url=secrets_store.get_ai_config("ai_base_url", "https://api.alysc.top"),
|
||||
timeout=180.0,
|
||||
max_retries=2,
|
||||
)
|
||||
# 使用流式请求：CDN 收到首个 token 后会持续转发，不会因等待超时
|
||||
stream = await client.chat.completions.create(
|
||||
model=secrets_store.get_ai_config("ai_model", "gpt-5.5"),
|
||||
messages=[
|
||||
{"role": "system", "content": _SYSTEM_PREFIX + guide},
|
||||
{"role": "user", "content": user_prompt},
|
||||
],
|
||||
temperature=0.3,
|
||||
max_tokens=3000,
|
||||
stream=True,
|
||||
)
|
||||
chunks: list[str] = []
|
||||
async for chunk in stream:
|
||||
delta = chunk.choices[0].delta if chunk.choices else None
|
||||
if delta and delta.content:
|
||||
chunks.append(delta.content)
|
||||
content = "".join(chunks).strip()
|
||||
# 提取代码块
|
||||
if "```python" in content:
|
||||
content = content.split("```python", 1)[1].split("```", 1)[0].strip()
|
||||
elif "```" in content:
|
||||
content = content.split("```", 1)[1].split("```", 1)[0].strip()
|
||||
return content
|
||||
|
||||
@staticmethod
|
||||
def _validate_safety(code: str) -> None:
|
||||
"""AST 级安全检查"""
|
||||
tree = ast.parse(code)
|
||||
|
||||
forbidden_modules = {"os", "sys", "subprocess", "socket", "shutil",
|
||||
"pathlib", "http", "urllib", "requests", "httpx"}
|
||||
forbidden_calls = {"open", "exec", "eval", "compile", "__import__",
|
||||
"globals", "locals", "vars", "dir", "getattr",
|
||||
"setattr", "delattr", "type", "input"}
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
if alias.name.split(".")[0] not in ("polars",):
|
||||
if alias.name.split(".")[0] in forbidden_modules:
|
||||
raise ValueError(f"禁止 import {alias.name}")
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
if node.module and node.module.split(".")[0] not in ("polars",):
|
||||
if node.module.split(".")[0] in forbidden_modules:
|
||||
raise ValueError(f"禁止 from {node.module} import")
|
||||
if isinstance(node, ast.Call):
|
||||
if isinstance(node.func, ast.Name) and node.func.id in forbidden_calls:
|
||||
raise ValueError(f"禁止调用 {node.func.id}()")
|
||||
|
||||
@staticmethod
|
||||
def _extract_meta(code: str) -> dict:
|
||||
"""从代码字符串中提取 META 字典（不执行代码）"""
|
||||
tree = ast.parse(code)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Assign):
|
||||
for target in node.targets:
|
||||
if isinstance(target, ast.Name) and target.id == "META":
|
||||
# 找到 META 赋值，用 compile+eval 安全提取
|
||||
# 只允许字面量
|
||||
meta_node = node.value
|
||||
code_obj = compile(ast.Expression(meta_node), "<meta>", "eval")
|
||||
return eval(code_obj, {"__builtins__": {}}) # noqa: S307
|
||||
return {}
|
||||
@@ -0,0 +1,31 @@
|
||||
"""布林突破 — 突破布林上轨 + 放量"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "boll_breakout",
|
||||
"name": "布林突破",
|
||||
"description": "突破布林上轨 + 放量, 强势加速信号",
|
||||
"tags": ["布林", "突破"],
|
||||
"params": [
|
||||
{"id": "vol_ratio_min", "label": "最低量比", "type": "float",
|
||||
"default": 1.5, "min": 0.5, "max": 5.0, "step": 0.1},
|
||||
],
|
||||
"scoring": {"vol_ratio_5d": 0.4, "change_pct": 0.3, "momentum_20d": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_boll_breakout_upper"]
|
||||
EXIT_SIGNALS = ["signal_boll_breakdown_lower"]
|
||||
STOP_LOSS = -0.06
|
||||
MAX_HOLD_DAYS = 15
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
vol_min = params.get("vol_ratio_min", 1.5)
|
||||
return (
|
||||
pl.col("signal_boll_breakout_upper").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= vol_min)
|
||||
)
|
||||
@@ -0,0 +1,35 @@
|
||||
"""断板反包 — 涨停 + 放量 + 涨幅 >3%"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "broken_board_recovery",
|
||||
"name": "断板反包",
|
||||
"description": "连板≥2后断板1-2天, 出现放量反包信号",
|
||||
"tags": ["涨停", "反包"],
|
||||
"params": [
|
||||
{"id": "vol_ratio_min", "label": "最低量比", "type": "float",
|
||||
"default": 1.5, "min": 0.5, "max": 5.0, "step": 0.1},
|
||||
{"id": "change_pct_min", "label": "最低涨幅", "type": "float",
|
||||
"default": 0.03, "min": 0.01, "max": 0.10, "step": 0.01},
|
||||
],
|
||||
"scoring": {"change_pct": 0.4, "vol_ratio_5d": 0.3, "momentum_5d": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_limit_up"]
|
||||
EXIT_SIGNALS = ["signal_ma20_breakdown"]
|
||||
STOP_LOSS = -0.06
|
||||
MAX_HOLD_DAYS = 10
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
vol_min = params.get("vol_ratio_min", 1.5)
|
||||
chg_min = params.get("change_pct_min", 0.03)
|
||||
return (
|
||||
pl.col("signal_limit_up").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= vol_min)
|
||||
& (pl.col("change_pct") > chg_min)
|
||||
)
|
||||
@@ -0,0 +1,29 @@
|
||||
"""均线多头 — MA5>MA10>MA20>MA60 + 短期动量为正"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "bullish_alignment",
|
||||
"name": "均线多头",
|
||||
"description": "MA5>MA10>MA20>MA60多头排列 + 短期动量为正",
|
||||
"tags": ["均线", "多头"],
|
||||
"params": [],
|
||||
"scoring": {"momentum_60d": 0.4, "momentum_20d": 0.3, "turnover_rate": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_ma_golden_5_20", "signal_ma_golden_20_60"]
|
||||
EXIT_SIGNALS = ["signal_ma_dead_5_20", "signal_ma20_breakdown"]
|
||||
STOP_LOSS = -0.06
|
||||
MAX_HOLD_DAYS = 20
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
return (
|
||||
(pl.col("ma5") > pl.col("ma10"))
|
||||
& (pl.col("ma10") > pl.col("ma20"))
|
||||
& (pl.col("ma20") > pl.col("ma60"))
|
||||
& (pl.col("momentum_20d") > 0)
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
"""连板股 — 涨停且连续涨停≥2天"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "consecutive_limit_ups",
|
||||
"name": "连板股",
|
||||
"description": "当日涨停且连续涨停≥2天, 强势追涨",
|
||||
"tags": ["涨停", "连板"],
|
||||
"params": [
|
||||
{"id": "min_boards", "label": "最少连板数", "type": "int",
|
||||
"default": 2, "min": 1, "max": 20, "step": 1},
|
||||
],
|
||||
"scoring": {"consecutive_limit_ups": 0.5, "change_pct": 0.3, "amount": 0.2},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_limit_up"]
|
||||
EXIT_SIGNALS = []
|
||||
STOP_LOSS = -0.05
|
||||
MAX_HOLD_DAYS = 5
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
min_boards = params.get("min_boards", 2)
|
||||
return (
|
||||
pl.col("signal_limit_up").fill_null(False)
|
||||
& (pl.col("consecutive_limit_ups") >= min_boards)
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
"""高换手拉升 — 换手率 > 5% 且涨幅 > 3%, 资金活跃"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "high_turnover_surge",
|
||||
"name": "高换手拉升",
|
||||
"description": "换手率 > 5% 且涨幅 > 3%, 资金活跃",
|
||||
"tags": ["换手率", "放量", "资金"],
|
||||
"params": [
|
||||
{"id": "min_turnover", "label": "最低换手率%", "type": "float",
|
||||
"default": 5.0, "min": 1.0, "max": 20.0, "step": 0.5},
|
||||
{"id": "min_change", "label": "最低涨幅%", "type": "float",
|
||||
"default": 3.0, "min": 1.0, "max": 10.0, "step": 0.5},
|
||||
],
|
||||
"scoring": {"turnover_rate": 0.4, "change_pct": 0.3, "momentum_5d": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 50,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_volume_surge"]
|
||||
EXIT_SIGNALS = ["signal_ma20_breakdown"]
|
||||
STOP_LOSS = -0.05
|
||||
MAX_HOLD_DAYS = 10
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
min_to = params.get("min_turnover", 5.0) / 100.0
|
||||
min_chg = params.get("min_change", 3.0) / 100.0
|
||||
return (
|
||||
(pl.col("turnover_rate") > min_to)
|
||||
& (pl.col("change_pct") > min_chg)
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
"""连板接力 — 近2日涨停且今日涨幅 > 5%, 连板股追踪"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "limit_up_momentum",
|
||||
"name": "连板接力",
|
||||
"description": "连板股 + 今日涨幅 > 5%, 连板接力追踪",
|
||||
"tags": ["涨停", "连板", "接力"],
|
||||
"params": [
|
||||
{"id": "min_change", "label": "最低涨幅%", "type": "float",
|
||||
"default": 5.0, "min": 2.0, "max": 15.0, "step": 0.5},
|
||||
{"id": "min_boards", "label": "最少连板", "type": "int",
|
||||
"default": 1, "min": 1, "max": 10, "step": 1},
|
||||
],
|
||||
"scoring": {"consecutive_limit_ups": 0.4, "change_pct": 0.3, "amount": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 50,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_limit_up"]
|
||||
EXIT_SIGNALS = []
|
||||
STOP_LOSS = -0.05
|
||||
MAX_HOLD_DAYS = 5
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
min_chg = params.get("min_change", 5.0) / 100.0
|
||||
min_boards = params.get("min_boards", 1)
|
||||
return (
|
||||
(pl.col("change_pct") > min_chg)
|
||||
& (pl.col("consecutive_limit_ups") >= min_boards)
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""低波动龙头 — 正动量 + 低波动 + MA20上方"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "low_volatility_leader",
|
||||
"name": "低波动龙头",
|
||||
"description": "20日动量为正 + 年化波动 < 30% + MA20上方",
|
||||
"tags": ["低波动", "龙头"],
|
||||
"params": [
|
||||
{"id": "vol_max", "label": "最大年化波动", "type": "float",
|
||||
"default": 0.30, "min": 0.05, "max": 1.0, "step": 0.01},
|
||||
],
|
||||
"scoring": {"momentum_60d": 0.4, "momentum_20d": 0.3, "turnover_rate": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_ma20_breakout"]
|
||||
EXIT_SIGNALS = ["signal_ma20_breakdown"]
|
||||
STOP_LOSS = -0.05
|
||||
MAX_HOLD_DAYS = 30
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
vol_max = params.get("vol_max", 0.30)
|
||||
return (
|
||||
(pl.col("momentum_20d") > 0)
|
||||
& (pl.col("annual_vol_20d") < vol_max)
|
||||
& (pl.col("close") > pl.col("ma20"))
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""MA金叉 — MA5上穿MA20 + 量能配合 + MA60上方"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "ma_golden_cross",
|
||||
"name": "MA 金叉",
|
||||
"description": "MA5上穿MA20当日触发, 量能配合",
|
||||
"tags": ["均线", "金叉"],
|
||||
"params": [
|
||||
{"id": "vol_ratio_min", "label": "最低量比", "type": "float",
|
||||
"default": 1.2, "min": 0.5, "max": 5.0, "step": 0.1},
|
||||
],
|
||||
"scoring": {"momentum_20d": 0.5, "vol_ratio_5d": 0.3, "change_pct": 0.2},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_ma_golden_5_20"]
|
||||
EXIT_SIGNALS = ["signal_ma_dead_5_20"]
|
||||
STOP_LOSS = -0.06
|
||||
MAX_HOLD_DAYS = 15
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
vol_min = params.get("vol_ratio_min", 1.2)
|
||||
return (
|
||||
pl.col("signal_ma_golden_5_20").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= vol_min)
|
||||
& (pl.col("close") > pl.col("ma60"))
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
"""MACD金叉放量 — MACD金叉当日 + 量能放大"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "macd_golden",
|
||||
"name": "MACD 金叉放量",
|
||||
"description": "MACD金叉当日 + 量能放大",
|
||||
"tags": ["MACD", "金叉", "放量"],
|
||||
"params": [
|
||||
{"id": "vol_ratio_min", "label": "最低量比", "type": "float",
|
||||
"default": 1.5, "min": 0.5, "max": 5.0, "step": 0.1},
|
||||
],
|
||||
"scoring": {"momentum_60d": 0.4, "vol_ratio_5d": 0.3, "change_pct": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_macd_golden"]
|
||||
EXIT_SIGNALS = ["signal_macd_dead"]
|
||||
STOP_LOSS = -0.07
|
||||
MAX_HOLD_DAYS = 20
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
vol_min = params.get("vol_ratio_min", 1.5)
|
||||
return (
|
||||
pl.col("signal_macd_golden").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= vol_min)
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""新低反转 — 60日新低后收阳放量"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "n_day_low_reversal",
|
||||
"name": "新低反转",
|
||||
"description": "触及60日新低后当日收阳放量, 反转信号",
|
||||
"tags": ["反转", "新低"],
|
||||
"params": [
|
||||
{"id": "vol_ratio_min", "label": "最低量比", "type": "float",
|
||||
"default": 1.5, "min": 0.5, "max": 5.0, "step": 0.1},
|
||||
],
|
||||
"scoring": {"change_pct": 0.4, "vol_ratio_5d": 0.3, "momentum_5d": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_n_day_low"]
|
||||
EXIT_SIGNALS = ["signal_ma20_breakdown"]
|
||||
STOP_LOSS = -0.06
|
||||
MAX_HOLD_DAYS = 15
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
vol_min = params.get("vol_ratio_min", 1.5)
|
||||
return (
|
||||
pl.col("signal_n_day_low").fill_null(False)
|
||||
& (pl.col("close") > pl.col("open"))
|
||||
& (pl.col("vol_ratio_5d") >= vol_min)
|
||||
)
|
||||
@@ -0,0 +1,55 @@
|
||||
"""逼近涨停 — 涨幅 > 7% 且距涨停 < 3%, 盘后选股"""
|
||||
import polars as pl
|
||||
|
||||
|
||||
def _limit_pct() -> pl.Expr:
|
||||
"""根据板块和 ST 动态计算涨跌幅限制 (小数)。
|
||||
创业板(300/301)/科创板(688): 20%
|
||||
北交所(.BJ): 30%
|
||||
ST: 5%
|
||||
主板: 10%
|
||||
"""
|
||||
is_st = pl.col("name").str.contains("(?i)ST").fill_null(False)
|
||||
is_cyb = pl.col("symbol").str.starts_with("300") | pl.col("symbol").str.starts_with("301")
|
||||
is_kcb = pl.col("symbol").str.starts_with("688")
|
||||
is_bj = pl.col("symbol").str.contains(r"\.BJ$")
|
||||
return (
|
||||
pl.when(is_st).then(0.05)
|
||||
.when(is_cyb | is_kcb).then(0.20)
|
||||
.when(is_bj).then(0.30)
|
||||
.otherwise(0.10)
|
||||
)
|
||||
|
||||
|
||||
META = {
|
||||
"id": "near_limit_up",
|
||||
"name": "逼近涨停",
|
||||
"description": "涨幅 > 7% 且距涨停 < 3%, 追涨信号",
|
||||
"tags": ["涨停", "追涨"],
|
||||
"params": [
|
||||
{"id": "min_change", "label": "最低涨幅%", "type": "float",
|
||||
"default": 7.0, "min": 3.0, "max": 15.0, "step": 1.0},
|
||||
{"id": "limit_gap", "label": "距涨停空间%", "type": "float",
|
||||
"default": 3.0, "min": 1.0, "max": 10.0, "step": 0.5},
|
||||
],
|
||||
"scoring": {"change_pct": 0.5, "amount": 0.3, "momentum_5d": 0.2},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 50,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = []
|
||||
EXIT_SIGNALS = ["signal_ma20_breakdown"]
|
||||
STOP_LOSS = -0.05
|
||||
MAX_HOLD_DAYS = 5
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
min_chg = params.get("min_change", 7.0) / 100.0
|
||||
gap = params.get("limit_gap", 3.0) / 100.0
|
||||
lp = _limit_pct()
|
||||
return (
|
||||
(pl.col("change_pct") > min_chg)
|
||||
& (pl.col("change_pct") < lp - gap)
|
||||
)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""超跌反弹 — RSI14 < 30 + 收阳 + 放量"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "oversold_bounce",
|
||||
"name": "超跌反弹",
|
||||
"description": "RSI14 < 30超卖区 + 当日收阳 + 放量, 抄底信号",
|
||||
"tags": ["超跌", "反弹", "RSI"],
|
||||
"params": [
|
||||
{"id": "rsi_max", "label": "RSI上限", "type": "float",
|
||||
"default": 30.0, "min": 10.0, "max": 50.0, "step": 1.0},
|
||||
{"id": "vol_ratio_min", "label": "最低量比", "type": "float",
|
||||
"default": 1.2, "min": 0.5, "max": 5.0, "step": 0.1},
|
||||
],
|
||||
"scoring": {"change_pct": 0.3, "vol_ratio_5d": 0.3, "momentum_5d": 0.2, "rsi_14": 0.2},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = []
|
||||
EXIT_SIGNALS = ["signal_ma20_breakdown"]
|
||||
STOP_LOSS = -0.05
|
||||
MAX_HOLD_DAYS = 15
|
||||
ALERTS = [
|
||||
{"field": "rsi_14", "op": "<", "value": 25, "message": "RSI极度超卖"},
|
||||
]
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
rsi_max = params.get("rsi_max", 30.0)
|
||||
vol_min = params.get("vol_ratio_min", 1.2)
|
||||
return (
|
||||
(pl.col("rsi_14") < rsi_max)
|
||||
& (pl.col("close") > pl.col("open"))
|
||||
& (pl.col("vol_ratio_5d") >= vol_min)
|
||||
)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""超跌反弹 — RSI14 < 30 + 涨幅 > 1% + 站上 MA5, 超卖反弹信号"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "oversold_reversal",
|
||||
"name": "超跌反转",
|
||||
"description": "RSI14 < 30超卖 + 涨幅 > 1% + 站上MA5, 超卖反转信号",
|
||||
"tags": ["超跌", "反弹", "RSI"],
|
||||
"params": [
|
||||
{"id": "rsi_max", "label": "RSI上限", "type": "float",
|
||||
"default": 30.0, "min": 10.0, "max": 50.0, "step": 1.0},
|
||||
{"id": "min_change", "label": "最低涨幅%", "type": "float",
|
||||
"default": 1.0, "min": 0.5, "max": 5.0, "step": 0.5},
|
||||
],
|
||||
"scoring": {"change_pct": 0.4, "rsi_14": 0.3, "vol_ratio_5d": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 50,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = []
|
||||
EXIT_SIGNALS = ["signal_ma20_breakdown"]
|
||||
STOP_LOSS = -0.05
|
||||
MAX_HOLD_DAYS = 15
|
||||
ALERTS = [
|
||||
{"field": "rsi_14", "op": "<", "value": 25, "message": "RSI极度超卖"},
|
||||
]
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
rsi_max = params.get("rsi_max", 30.0)
|
||||
min_chg = params.get("min_change", 1.0) / 100.0
|
||||
return (
|
||||
(pl.col("rsi_14") < rsi_max)
|
||||
& (pl.col("change_pct") > min_chg)
|
||||
& (pl.col("close") > pl.col("ma5"))
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
"""均线回踩反弹 — 价格在 MA20 附近(±2%)且 MA 多头排列, 回踩买入"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "pullback_ma20_bounce",
|
||||
"name": "均线回踩反弹",
|
||||
"description": "价格在MA20附近(±2%)且MA5>MA20>MA60多头排列, 回踩买入",
|
||||
"tags": ["回踩", "均线", "反弹"],
|
||||
"params": [
|
||||
{"id": "ma_proximity", "label": "MA偏离度%", "type": "float",
|
||||
"default": 2.0, "min": 0.5, "max": 5.0, "step": 0.5},
|
||||
],
|
||||
"scoring": {"momentum_60d": 0.4, "change_pct": 0.3, "momentum_20d": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 50,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_ma_golden_5_20"]
|
||||
EXIT_SIGNALS = ["signal_ma20_breakdown", "signal_ma_dead_5_20"]
|
||||
STOP_LOSS = -0.05
|
||||
MAX_HOLD_DAYS = 15
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
proximity = params.get("ma_proximity", 2.0) / 100.0
|
||||
return (
|
||||
(pl.col("close") > pl.col("ma20") * (1 - proximity))
|
||||
& (pl.col("close") < pl.col("ma20") * (1 + proximity))
|
||||
& (pl.col("ma5") > pl.col("ma20"))
|
||||
& (pl.col("ma20") > pl.col("ma60"))
|
||||
& (pl.col("change_pct") > 0)
|
||||
)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""缩量回踩 — 回踩MA20附近 + 缩量 + 中期趋势向上"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "pullback_to_support",
|
||||
"name": "缩量回踩",
|
||||
"description": "回踩MA20附近 + 缩量 + 中期趋势向上",
|
||||
"tags": ["回踩", "支撑"],
|
||||
"params": [
|
||||
{"id": "ma_proximity", "label": "均线偏离度", "type": "float",
|
||||
"default": 0.02, "min": 0.01, "max": 0.05, "step": 0.005},
|
||||
{"id": "vol_ratio_max", "label": "最大量比", "type": "float",
|
||||
"default": 0.8, "min": 0.2, "max": 1.5, "step": 0.1},
|
||||
],
|
||||
"scoring": {"momentum_60d": 0.4, "momentum_20d": 0.3, "turnover_rate": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_ma_golden_5_20"]
|
||||
EXIT_SIGNALS = ["signal_ma20_breakdown"]
|
||||
STOP_LOSS = -0.05
|
||||
MAX_HOLD_DAYS = 20
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
proximity = params.get("ma_proximity", 0.02)
|
||||
vol_max = params.get("vol_ratio_max", 0.8)
|
||||
return (
|
||||
(pl.col("close") > pl.col("ma20") * (1 - proximity))
|
||||
& (pl.col("close") < pl.col("ma20") * (1 + proximity))
|
||||
& (pl.col("vol_ratio_5d") < vol_max)
|
||||
& (pl.col("close") > pl.col("ma60"))
|
||||
& (pl.col("momentum_20d") > 0)
|
||||
)
|
||||
@@ -0,0 +1,35 @@
|
||||
"""强势高开 — 高开 > 3% 且保持上涨, 集合竞价强势"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "strong_open",
|
||||
"name": "强势高开",
|
||||
"description": "高开 > 3% 且收盘高于开盘价, 集合竞价强势",
|
||||
"tags": ["高开", "强势"],
|
||||
"params": [
|
||||
{"id": "min_open_gap", "label": "最低高开%", "type": "float",
|
||||
"default": 3.0, "min": 1.0, "max": 10.0, "step": 0.5},
|
||||
{"id": "min_change", "label": "最低涨幅%", "type": "float",
|
||||
"default": 3.0, "min": 1.0, "max": 10.0, "step": 0.5},
|
||||
],
|
||||
"scoring": {"change_pct": 0.4, "amplitude": 0.2, "amount": 0.4},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 50,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = []
|
||||
EXIT_SIGNALS = ["signal_ma20_breakdown"]
|
||||
STOP_LOSS = -0.05
|
||||
MAX_HOLD_DAYS = 10
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
min_gap = params.get("min_open_gap", 3.0) / 100.0
|
||||
min_chg = params.get("min_change", 3.0) / 100.0
|
||||
return (
|
||||
(pl.col("open") > pl.col("prev_close") * (1 + min_gap))
|
||||
& (pl.col("close") > pl.col("open"))
|
||||
& (pl.col("change_pct") > min_chg)
|
||||
)
|
||||
@@ -0,0 +1,42 @@
|
||||
"""趋势突破 — MA60上方 + 60日新高 + 放量"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "trend_breakout",
|
||||
"name": "趋势突破",
|
||||
"description": "MA60上方 + 60日新高 + 量能 ≥ 2倍均量",
|
||||
"tags": ["趋势", "突破", "放量"],
|
||||
"basic_filter": {
|
||||
"price_min": 5,
|
||||
"price_max": 200,
|
||||
"market_cap_min": 20e8,
|
||||
"amount_min": 1e8,
|
||||
"exclude_st": True,
|
||||
"exclude_new_days": 60,
|
||||
},
|
||||
"params": [
|
||||
{"id": "vol_ratio_min", "label": "最低量比", "type": "float",
|
||||
"default": 2.0, "min": 0.5, "max": 10.0, "step": 0.1},
|
||||
],
|
||||
"scoring": {"momentum_60d": 0.4, "vol_ratio_5d": 0.3, "change_pct": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_n_day_high"]
|
||||
EXIT_SIGNALS = ["signal_ma20_breakdown"]
|
||||
STOP_LOSS = -0.08
|
||||
MAX_HOLD_DAYS = 20
|
||||
ALERTS = [
|
||||
{"field": "signal_volume_surge", "message": "放量异动"},
|
||||
]
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
vol_min = params.get("vol_ratio_min", 2.0)
|
||||
return (
|
||||
(pl.col("close") > pl.col("ma60"))
|
||||
& pl.col("signal_n_day_high").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= vol_min)
|
||||
)
|
||||
@@ -0,0 +1,32 @@
|
||||
"""量价齐升 — 突破MA20 + 放量 + 收阳"""
|
||||
import polars as pl
|
||||
|
||||
META = {
|
||||
"id": "volume_price_surge",
|
||||
"name": "量价齐升",
|
||||
"description": "突破MA20 + 放量 + 收阳",
|
||||
"tags": ["量价", "突破"],
|
||||
"params": [
|
||||
{"id": "vol_ratio_min", "label": "最低量比", "type": "float",
|
||||
"default": 2.0, "min": 0.5, "max": 10.0, "step": 0.1},
|
||||
],
|
||||
"scoring": {"vol_ratio_5d": 0.4, "change_pct": 0.3, "momentum_20d": 0.3},
|
||||
"order_by": "score",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
}
|
||||
|
||||
ENTRY_SIGNALS = ["signal_ma20_breakout"]
|
||||
EXIT_SIGNALS = ["signal_ma20_breakdown"]
|
||||
STOP_LOSS = -0.06
|
||||
MAX_HOLD_DAYS = 15
|
||||
ALERTS = []
|
||||
|
||||
|
||||
def filter(df: pl.DataFrame, params: dict) -> pl.Expr:
|
||||
vol_min = params.get("vol_ratio_min", 2.0)
|
||||
return (
|
||||
pl.col("signal_ma20_breakout").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= vol_min)
|
||||
& (pl.col("close") > pl.col("open"))
|
||||
)
|
||||
@@ -0,0 +1,71 @@
|
||||
"""策略配置持久化 — 读写用户覆盖值。
|
||||
|
||||
职责: 将每个策略的用户定制设置（基础参数、策略参数、评分、买卖信号）持久化到 JSON。
|
||||
不知道: 引擎、AI、前端、回测。
|
||||
存储: data/user_data/strategy_overrides/{strategy_id}.json
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _overrides_dir(data_dir: Path) -> Path:
|
||||
d = data_dir / "user_data" / "strategy_overrides"
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def _path(data_dir: Path, strategy_id: str) -> Path:
|
||||
return _overrides_dir(data_dir) / f"{strategy_id}.json"
|
||||
|
||||
|
||||
def load_override(data_dir: Path, strategy_id: str) -> dict:
|
||||
"""读取策略的用户覆盖配置，不存在返回空 dict"""
|
||||
p = _path(data_dir, strategy_id)
|
||||
if not p.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
# 清理 basic_filter 中值为 None/空的键（避免固化无意义的空值）
|
||||
bf = data.get("basic_filter")
|
||||
if isinstance(bf, dict):
|
||||
cleaned = {k: v for k, v in bf.items() if v is not None}
|
||||
if cleaned:
|
||||
data["basic_filter"] = cleaned
|
||||
else:
|
||||
del data["basic_filter"]
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.warning("load override %s failed: %s", strategy_id, e)
|
||||
return {}
|
||||
|
||||
|
||||
def save_override(data_dir: Path, strategy_id: str, overrides: dict) -> None:
|
||||
"""保存策略的用户覆盖配置（全量覆盖写）"""
|
||||
p = _path(data_dir, strategy_id)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(overrides, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def delete_override(data_dir: Path, strategy_id: str) -> None:
|
||||
"""删除策略的用户覆盖配置（重置为默认值）"""
|
||||
p = _path(data_dir, strategy_id)
|
||||
if p.exists():
|
||||
p.unlink()
|
||||
|
||||
|
||||
def list_overrides(data_dir: Path) -> dict[str, dict]:
|
||||
"""返回所有策略的覆盖配置 {strategy_id: overrides}"""
|
||||
d = _overrides_dir(data_dir)
|
||||
result: dict[str, dict] = {}
|
||||
for f in d.glob("*.json"):
|
||||
try:
|
||||
sid = f.stem
|
||||
result[sid] = json.loads(f.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
return result
|
||||
@@ -0,0 +1,207 @@
|
||||
"""自定义信号 — 用户用「字段 + 运算符 + 值」组合出的布尔信号。
|
||||
|
||||
职责:
|
||||
- 从 data/user_data/custom_signals/*.json 加载信号定义
|
||||
- 把每个信号的 conditions 编译成一条 Polars 布尔表达式（AND 组合）
|
||||
- 供 pipeline 在 compute_signals / compute_enriched_today 末尾注入为列
|
||||
|
||||
不知道: 引擎、AI、API、回测、监控。纯函数 + 模块级缓存。
|
||||
|
||||
设计:
|
||||
- 信号列名加前缀 ``csg_`` 避免与内置 ``signal_`` 列冲突。
|
||||
- 回测/选股/监控都按列名找信号，因此注入列后零特殊处理即可三处生效。
|
||||
- 字段白名单 + 固定运算符集，杜绝任意表达式注入。
|
||||
- 第一版只支持 AND（多条件同时满足）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── 常量 ────────────────────────────────────────────────
|
||||
PREFIX = "csg_" # 自定义信号列名前缀
|
||||
ID_RE = re.compile(r"^[a-z0-9_]{1,40}$")
|
||||
OPS = {">", ">=", "<", "<=", "==", "!="}
|
||||
|
||||
# 字段白名单：只允许这些列出现在条件里（防注入）。均为数值型。
|
||||
# 与 ENRICHED_COLUMNS 的数值列保持一致，排除 symbol/date/name 等非数值列。
|
||||
ALLOWED_FIELDS: frozenset[str] = frozenset({
|
||||
# 行情
|
||||
"open", "high", "low", "close", "volume", "amount", "turnover_rate",
|
||||
"consecutive_limit_ups", "consecutive_limit_downs",
|
||||
# 基础
|
||||
"prev_close", "change_pct", "change_amount", "amplitude",
|
||||
# 均线 / 指数均线
|
||||
"ma5", "ma10", "ma20", "ma30", "ma60",
|
||||
"ema5", "ema10", "ema20", "ema30", "ema60",
|
||||
# MACD / BOLL / KDJ / ATR
|
||||
"macd_dif", "macd_dea", "macd_hist",
|
||||
"boll_upper", "boll_lower",
|
||||
"kdj_k", "kdj_d", "kdj_j",
|
||||
"atr_14",
|
||||
# 量价 / 极值 / 动量 / 波动率 / RSI
|
||||
"vol_ma5", "vol_ma10", "vol_ratio_5d",
|
||||
"high_60d", "low_60d",
|
||||
"momentum_5d", "momentum_10d", "momentum_20d", "momentum_30d", "momentum_60d",
|
||||
"annual_vol_20d",
|
||||
"rsi_6", "rsi_14", "rsi_24",
|
||||
})
|
||||
|
||||
# 运算符 → Polars 表达式构造器（输入 col_expr, value）
|
||||
_OP_BUILDERS = {
|
||||
">": lambda c, v: c > v,
|
||||
">=": lambda c, v: c >= v,
|
||||
"<": lambda c, v: c < v,
|
||||
"<=": lambda c, v: c <= v,
|
||||
"==": lambda c, v: c == v,
|
||||
"!=": lambda c, v: c != v,
|
||||
}
|
||||
|
||||
|
||||
# ── 持久化（镜像 strategy/config.py 的写法）──────────────
|
||||
def _dir(data_dir: Path) -> Path:
|
||||
d = data_dir / "user_data" / "custom_signals"
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def _path(data_dir: Path, signal_id: str) -> Path:
|
||||
return _dir(data_dir) / f"{signal_id}.json"
|
||||
|
||||
|
||||
def load_all(data_dir: Path) -> list[dict]:
|
||||
"""读取全部自定义信号定义。损坏的文件被跳过。"""
|
||||
d = _dir(data_dir)
|
||||
out: list[dict] = []
|
||||
for f in sorted(d.glob("*.json")):
|
||||
try:
|
||||
out.append(json.loads(f.read_text(encoding="utf-8")))
|
||||
except Exception as e:
|
||||
logger.warning("custom signal load failed %s: %s", f.name, e)
|
||||
return out
|
||||
|
||||
|
||||
def save_one(data_dir: Path, sig: dict) -> None:
|
||||
p = _path(data_dir, sig["id"])
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(sig, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def delete_one(data_dir: Path, signal_id: str) -> bool:
|
||||
p = _path(data_dir, signal_id)
|
||||
if p.exists():
|
||||
p.unlink()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ── 校验 ────────────────────────────────────────────────
|
||||
def _parse_right(right: str) -> tuple[str, object]:
|
||||
"""解析右值。返回 ('field', colname) 或 ('const', float)。"""
|
||||
if isinstance(right, (int, float)):
|
||||
return ("const", float(right))
|
||||
if not isinstance(right, str):
|
||||
raise ValueError(f"非法右值: {right!r}")
|
||||
if right.startswith("field:"):
|
||||
col = right[len("field:"):]
|
||||
if col not in ALLOWED_FIELDS:
|
||||
raise ValueError(f"右值字段不在白名单: {col}")
|
||||
return ("field", col)
|
||||
# 纯数字
|
||||
try:
|
||||
return ("const", float(right))
|
||||
except ValueError:
|
||||
raise ValueError(f"非法右值（应为 field:xxx 或数字）: {right!r}")
|
||||
|
||||
|
||||
def validate(sig: dict) -> None:
|
||||
"""校验一个信号定义，非法则抛 ValueError（含中文信息）。"""
|
||||
sid = sig.get("id", "")
|
||||
if not isinstance(sid, str) or not ID_RE.match(sid):
|
||||
raise ValueError(f"信号 id 非法（仅小写字母数字下划线，1-40字符）: {sid!r}")
|
||||
if not isinstance(sig.get("name"), str) or not sig["name"].strip():
|
||||
raise ValueError("信号 name 不能为空")
|
||||
if sig.get("kind") not in ("entry", "exit", "both"):
|
||||
raise ValueError("kind 必须是 entry / exit / both")
|
||||
conds = sig.get("conditions")
|
||||
if not isinstance(conds, list) or len(conds) == 0:
|
||||
raise ValueError("conditions 不能为空")
|
||||
if len(conds) > 8:
|
||||
raise ValueError("conditions 最多 8 条")
|
||||
for i, c in enumerate(conds):
|
||||
if not isinstance(c, dict):
|
||||
raise ValueError(f"第 {i+1} 个条件格式错误")
|
||||
left = c.get("left", "")
|
||||
if left not in ALLOWED_FIELDS:
|
||||
raise ValueError(f"第 {i+1} 个条件: 字段 {left!r} 不在白名单")
|
||||
if c.get("op") not in OPS:
|
||||
raise ValueError(f"第 {i+1} 个条件: 运算符 {c.get('op')!r} 非法")
|
||||
_parse_right(c.get("right")) # 会校验右值字段/数字
|
||||
|
||||
|
||||
# ── 编译为 Polars 表达式 ─────────────────────────────────
|
||||
def column_name(signal_id: str) -> str:
|
||||
"""信号 id → DataFrame 列名（加前缀）。"""
|
||||
return f"{PREFIX}{signal_id}"
|
||||
|
||||
|
||||
def build_expressions(signals: list[dict]) -> dict[str, pl.Expr]:
|
||||
"""把多个自定义信号编译成 {column_name: pl.Expr}。
|
||||
|
||||
- 只处理 enabled != False 的信号。
|
||||
- 单个信号内多条件用 ``&`` 串联（AND）。
|
||||
- 编译失败的信号被跳过并告警（不影响其它信号）。
|
||||
"""
|
||||
out: dict[str, pl.Expr] = {}
|
||||
for sig in signals:
|
||||
if sig.get("enabled") is False:
|
||||
continue
|
||||
try:
|
||||
conds = sig["conditions"]
|
||||
col_name = column_name(sig["id"])
|
||||
parts: list[pl.Expr] = []
|
||||
for c in conds:
|
||||
left = c["left"]
|
||||
op = c["op"]
|
||||
kind, val = _parse_right(c["right"])
|
||||
right_expr = pl.col(val) if kind == "field" else val
|
||||
parts.append(_OP_BUILDERS[op](pl.col(left), right_expr))
|
||||
combined = parts[0]
|
||||
for p in parts[1:]:
|
||||
combined = combined & p
|
||||
out[col_name] = combined
|
||||
except Exception as e:
|
||||
logger.warning("custom signal compile failed %s: %s", sig.get("id"), e)
|
||||
return out
|
||||
|
||||
|
||||
def inject(df: pl.DataFrame, exprs: dict[str, pl.Expr]) -> pl.DataFrame:
|
||||
"""把编译好的信号表达式作为列加入 df。仅添加 df 已含其依赖列的信号。"""
|
||||
if df.is_empty() or not exprs:
|
||||
return df
|
||||
cols = set(df.columns)
|
||||
add: dict[str, pl.Expr] = {}
|
||||
for name, expr in exprs.items():
|
||||
# 提取该表达式引用的所有字段列，缺失则跳过（避免运行时报错）
|
||||
needed = _expr_root_columns(expr)
|
||||
if needed.issubset(cols):
|
||||
add[name] = expr
|
||||
if add:
|
||||
df = df.with_columns([e.alias(n) for n, e in add.items()])
|
||||
return df
|
||||
|
||||
|
||||
def _expr_root_columns(expr: pl.Expr) -> set[str]:
|
||||
"""尽力提取表达式里出现的列名。失败则返回空集（保守跳过）。"""
|
||||
try:
|
||||
# Polars 的 meta.root_names() 返回表达式引用的根列名
|
||||
names = expr.meta.root_names()
|
||||
return set(names)
|
||||
except Exception:
|
||||
return set()
|
||||
@@ -0,0 +1,453 @@
|
||||
"""策略引擎 — 加载、执行、评分。
|
||||
|
||||
职责: 从文件系统加载策略 Python 模块，执行两阶段过滤(基础+策略)，
|
||||
通用评分排序。
|
||||
不知道: AI、API、前端、配置持久化、回测。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
import polars as pl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 引擎级默认基础过滤 — 策略未定义 BASIC_FILTER 时兜底
|
||||
DEFAULT_BASIC_FILTER: dict = {
|
||||
"price_min": 3,
|
||||
"price_max": 300,
|
||||
"market_cap_min": 10e8,
|
||||
"float_cap_min": None,
|
||||
"float_cap_max": None,
|
||||
"amount_min": 0.2e8,
|
||||
"amount_max": None,
|
||||
"turnover_min": None,
|
||||
"turnover_max": None,
|
||||
"exclude_st": True,
|
||||
"exclude_new_days": 30,
|
||||
"boards": ["沪主板", "深主板", "创业板", "科创板", "北交所"],
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class StrategyDef:
|
||||
"""加载后的策略定义（只读数据 + filter 函数引用）"""
|
||||
meta: dict
|
||||
basic_filter: dict
|
||||
entry_signals: list[str]
|
||||
exit_signals: list[str]
|
||||
stop_loss: float | None
|
||||
trailing_stop: float | None
|
||||
trailing_take_profit_activate: float | None
|
||||
trailing_take_profit_drawdown: float | None
|
||||
max_hold_days: int | None
|
||||
alerts: list[dict]
|
||||
filter_fn: Callable[[pl.DataFrame, dict], pl.Expr] | None
|
||||
filter_history_fn: Callable[[pl.DataFrame, dict], pl.DataFrame] | None
|
||||
lookback_days: int
|
||||
source: str # "builtin" | "custom" | "ai"
|
||||
file_path: Path | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class StrategyResult:
|
||||
"""策略执行结果"""
|
||||
as_of: date
|
||||
strategy_id: str
|
||||
rows: list[dict] = field(default_factory=list)
|
||||
total: int = 0
|
||||
elapsed_ms: float = 0.0
|
||||
scores: dict[str, float] = field(default_factory=dict)
|
||||
|
||||
|
||||
class StrategyEngine:
|
||||
"""策略引擎 — 策略加载 + 执行 + 评分"""
|
||||
|
||||
def __init__(self, enriched_loader: Callable[[date], pl.DataFrame],
|
||||
enriched_history_loader: Callable[[date, int], pl.DataFrame] | None = None,
|
||||
strategy_dirs: list[Path] | None = None):
|
||||
"""
|
||||
Args:
|
||||
enriched_loader: (date) -> pl.DataFrame, 加载指定日期的 enriched 数据
|
||||
strategy_dirs: 策略文件搜索目录列表
|
||||
"""
|
||||
self._loader = enriched_loader
|
||||
self._history_loader = enriched_history_loader
|
||||
self._strategies: dict[str, StrategyDef] = {}
|
||||
self._strategy_dirs = strategy_dirs or []
|
||||
self._load_all()
|
||||
|
||||
# ================================================================
|
||||
# 加载
|
||||
# ================================================================
|
||||
|
||||
def _load_all(self) -> None:
|
||||
self._strategies.clear()
|
||||
for d in self._strategy_dirs:
|
||||
if not d.exists():
|
||||
continue
|
||||
for f in sorted(d.glob("*.py")):
|
||||
if f.name.startswith("_"):
|
||||
continue
|
||||
try:
|
||||
s = self._load_file(f)
|
||||
self._strategies[s.meta["id"]] = s
|
||||
logger.debug("loaded strategy: %s (%s)", s.meta["id"], s.source)
|
||||
except Exception as e:
|
||||
logger.warning("load strategy %s failed: %s", f.name, e)
|
||||
|
||||
@staticmethod
|
||||
def _load_file(path: Path) -> StrategyDef:
|
||||
"""从 Python 文件加载策略定义"""
|
||||
spec = importlib.util.spec_from_file_location(path.stem, path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ValueError(f"cannot load module from {path}")
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
|
||||
meta = getattr(mod, "META", {})
|
||||
meta.setdefault("id", path.stem)
|
||||
meta.setdefault("name", path.stem)
|
||||
meta.setdefault("description", "")
|
||||
meta.setdefault("tags", [])
|
||||
meta.setdefault("params", [])
|
||||
meta.setdefault("scoring", {})
|
||||
meta.setdefault("order_by", "score")
|
||||
meta.setdefault("descending", True)
|
||||
meta.setdefault("limit", 100)
|
||||
|
||||
# 合并默认基础过滤
|
||||
bf = {**DEFAULT_BASIC_FILTER}
|
||||
strat_bf = getattr(mod, "BASIC_FILTER", None)
|
||||
if strat_bf:
|
||||
bf.update(strat_bf)
|
||||
# meta 里的 basic_filter 也合并（优先级最高）
|
||||
meta_bf = meta.get("basic_filter")
|
||||
if meta_bf:
|
||||
bf.update(meta_bf)
|
||||
|
||||
source = "custom"
|
||||
if "builtin" in str(path).replace("\\", "/"):
|
||||
source = "builtin"
|
||||
elif "/ai/" in str(path).replace("\\", "/") or "\\ai\\" in str(path):
|
||||
source = "ai"
|
||||
|
||||
return StrategyDef(
|
||||
meta=meta,
|
||||
basic_filter=bf,
|
||||
entry_signals=getattr(mod, "ENTRY_SIGNALS", []),
|
||||
exit_signals=getattr(mod, "EXIT_SIGNALS", []),
|
||||
stop_loss=getattr(mod, "STOP_LOSS", None),
|
||||
trailing_stop=getattr(mod, "TRAILING_STOP", None),
|
||||
trailing_take_profit_activate=getattr(mod, "TRAILING_TAKE_PROFIT_ACTIVATE", None),
|
||||
trailing_take_profit_drawdown=getattr(mod, "TRAILING_TAKE_PROFIT_DRAWDOWN", None),
|
||||
max_hold_days=getattr(mod, "MAX_HOLD_DAYS", None),
|
||||
alerts=getattr(mod, "ALERTS", []),
|
||||
filter_fn=getattr(mod, "filter", None),
|
||||
filter_history_fn=getattr(mod, "filter_history", None),
|
||||
lookback_days=int(getattr(mod, "LOOKBACK_DAYS", meta.get("lookback_days", 1)) or 1),
|
||||
source=source,
|
||||
file_path=path,
|
||||
)
|
||||
|
||||
def reload(self) -> None:
|
||||
"""热重载所有策略"""
|
||||
self._load_all()
|
||||
|
||||
# ================================================================
|
||||
# 查询
|
||||
# ================================================================
|
||||
|
||||
def list_strategies(self) -> list[dict]:
|
||||
"""返回所有策略的元信息"""
|
||||
result = []
|
||||
for s in self._strategies.values():
|
||||
result.append({**s.meta, "source": s.source})
|
||||
return result
|
||||
|
||||
def get(self, strategy_id: str) -> StrategyDef:
|
||||
s = self._strategies.get(strategy_id)
|
||||
if not s:
|
||||
raise ValueError(f"unknown strategy: {strategy_id}")
|
||||
return s
|
||||
|
||||
def has(self, strategy_id: str) -> bool:
|
||||
return strategy_id in self._strategies
|
||||
|
||||
# ================================================================
|
||||
# 执行
|
||||
# ================================================================
|
||||
|
||||
def run(
|
||||
self,
|
||||
strategy_id: str,
|
||||
as_of: date,
|
||||
pool: list[str] | None = None,
|
||||
params: dict | None = None,
|
||||
overrides: dict | None = None,
|
||||
precomputed: pl.DataFrame | None = None,
|
||||
precomputed_history: pl.DataFrame | None = None,
|
||||
) -> StrategyResult:
|
||||
"""执行策略: 基础过滤 → 策略过滤 → 评分排序
|
||||
|
||||
Args:
|
||||
strategy_id: 策略 ID
|
||||
as_of: 选股日期
|
||||
pool: 限定股票池
|
||||
params: 策略参数 (用户在设置面板调的值)
|
||||
overrides: 用户覆盖配置 (basic_filter/scoring/stop_loss 等)
|
||||
precomputed: 已加载的 enriched 数据 (run_all 场景复用)
|
||||
precomputed_history: 已加载的历史窗口数据 (run_all 场景复用)
|
||||
"""
|
||||
t0 = time.perf_counter()
|
||||
|
||||
s = self.get(strategy_id)
|
||||
params = params or {}
|
||||
overrides = overrides or {}
|
||||
|
||||
# 加载数据。普通策略只读目标日期；声明 filter_history 的策略读取历史窗口。
|
||||
if s.filter_history_fn:
|
||||
if precomputed_history is not None and not precomputed_history.is_empty():
|
||||
df = precomputed_history
|
||||
elif self._history_loader:
|
||||
df = self._history_loader(as_of, max(1, s.lookback_days))
|
||||
else:
|
||||
logger.warning("strategy %s requires history loader", strategy_id)
|
||||
return StrategyResult(as_of=as_of, strategy_id=strategy_id)
|
||||
if df.is_empty():
|
||||
return StrategyResult(as_of=as_of, strategy_id=strategy_id)
|
||||
df = s.filter_history_fn(df, params)
|
||||
if df.is_empty():
|
||||
return StrategyResult(as_of=as_of, strategy_id=strategy_id)
|
||||
if "date" in df.columns:
|
||||
df = df.filter(pl.col("date") == as_of)
|
||||
elif precomputed is not None and not precomputed.is_empty():
|
||||
df = precomputed
|
||||
else:
|
||||
df = self._loader(as_of)
|
||||
if df.is_empty():
|
||||
return StrategyResult(as_of=as_of, strategy_id=strategy_id)
|
||||
|
||||
# 基础过滤: 策略默认 basic_filter 兜底, 用户 override 优先覆盖。
|
||||
# 这样策略文件里写的 exclude_st/price_min 等默认值即使前端没保存也能生效。
|
||||
bf = dict(s.basic_filter) if s.basic_filter else {}
|
||||
if overrides and overrides.get("basic_filter"):
|
||||
bf.update(overrides["basic_filter"])
|
||||
|
||||
# Stage 1: 基础过滤（enabled 默认开启; 显式 enabled=false 才跳过）
|
||||
if bf and bf.get("enabled", True):
|
||||
df = self._apply_basic_filter(df, bf)
|
||||
|
||||
# Pool 过滤
|
||||
if pool:
|
||||
df = df.filter(pl.col("symbol").is_in(pool))
|
||||
|
||||
# Stage 2: 策略过滤
|
||||
if s.filter_fn:
|
||||
expr = s.filter_fn(df, params)
|
||||
df = df.filter(expr)
|
||||
|
||||
# Stage 3: 评分
|
||||
scoring = s.meta.get("scoring", {})
|
||||
scoring_overrides = overrides.get("scoring")
|
||||
if scoring_overrides:
|
||||
scoring = {**scoring, **scoring_overrides}
|
||||
df = self._apply_scoring(df, scoring)
|
||||
|
||||
# 排序 + 限制
|
||||
limit = s.meta.get("limit", 100)
|
||||
order_desc = s.meta.get("descending", True)
|
||||
if "score" in df.columns:
|
||||
df = df.sort("score", descending=order_desc)
|
||||
elif s.meta.get("order_by") and s.meta["order_by"] != "score":
|
||||
ob = s.meta["order_by"]
|
||||
if ob in df.columns:
|
||||
df = df.sort(ob, descending=order_desc)
|
||||
df = df.head(limit)
|
||||
|
||||
# 输出
|
||||
rows = _sanitize(df.to_dicts())
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
|
||||
scores: dict[str, float] = {}
|
||||
if "score" in df.columns:
|
||||
for r in df.iter_rows(named=True):
|
||||
scores[r["symbol"]] = float(r.get("score") or 0)
|
||||
|
||||
return StrategyResult(
|
||||
as_of=as_of,
|
||||
strategy_id=strategy_id,
|
||||
rows=rows,
|
||||
total=len(rows),
|
||||
elapsed_ms=elapsed,
|
||||
scores=scores,
|
||||
)
|
||||
|
||||
def run_all(self, as_of: date, params_map: dict | None = None,
|
||||
overrides_map: dict | None = None) -> dict[str, StrategyResult]:
|
||||
"""批量执行所有策略 (enriched 只加载一次，基础过滤按策略分组缓存，历史数据共享)"""
|
||||
df = self._loader(as_of)
|
||||
params_map = params_map or {}
|
||||
overrides_map = overrides_map or {}
|
||||
|
||||
# 历史策略: 找最大 lookback，一次加载共享
|
||||
history_strats = [(sid, s) for sid, s in self._strategies.items() if s.filter_history_fn]
|
||||
if history_strats and self._history_loader:
|
||||
max_lookback = max(s.lookback_days for _, s in history_strats)
|
||||
shared_history = self._history_loader(as_of, max(1, max_lookback))
|
||||
else:
|
||||
shared_history = None
|
||||
|
||||
# 按 basic_filter hash 分组，避免重复过滤
|
||||
bf_cache: dict[str, pl.DataFrame] = {}
|
||||
results: dict[str, StrategyResult] = {}
|
||||
|
||||
for sid, strat in self._strategies.items():
|
||||
try:
|
||||
bf_key = _dict_hash(strat.basic_filter)
|
||||
if bf_key not in bf_cache:
|
||||
if strat.basic_filter.get("enabled", True):
|
||||
bf_cache[bf_key] = self._apply_basic_filter(df, strat.basic_filter)
|
||||
else:
|
||||
bf_cache[bf_key] = df
|
||||
base = bf_cache[bf_key]
|
||||
|
||||
# 从已过滤的 base 执行 (filter_history 策略使用共享历史)
|
||||
results[sid] = self.run(
|
||||
sid, as_of,
|
||||
params=params_map.get(sid),
|
||||
overrides=overrides_map.get(sid),
|
||||
precomputed=base,
|
||||
precomputed_history=shared_history,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("run strategy %s failed: %s", sid, e)
|
||||
|
||||
return results
|
||||
|
||||
# ================================================================
|
||||
# 内部: 基础过滤
|
||||
# ================================================================
|
||||
|
||||
@staticmethod
|
||||
def _basic_filter_expr(df: pl.DataFrame, bf: dict) -> pl.Expr | None:
|
||||
"""构建基础过滤表达式。回测可复用为买入候选 mask，不删除行情行。"""
|
||||
exprs: list[pl.Expr] = []
|
||||
if bf.get("price_min") is not None:
|
||||
exprs.append(pl.col("close") >= bf["price_min"])
|
||||
if bf.get("price_max") is not None:
|
||||
exprs.append(pl.col("close") <= bf["price_max"])
|
||||
if bf.get("market_cap_min") is not None and "total_shares" in df.columns:
|
||||
exprs.append(
|
||||
pl.col("close") * pl.col("total_shares") >= bf["market_cap_min"]
|
||||
)
|
||||
if bf.get("market_cap_max") is not None and "total_shares" in df.columns:
|
||||
exprs.append(
|
||||
pl.col("close") * pl.col("total_shares") <= bf["market_cap_max"]
|
||||
)
|
||||
# 流通市值
|
||||
if bf.get("float_cap_min") is not None and "float_shares" in df.columns:
|
||||
exprs.append(
|
||||
pl.col("close") * pl.col("float_shares") >= bf["float_cap_min"]
|
||||
)
|
||||
if bf.get("float_cap_max") is not None and "float_shares" in df.columns:
|
||||
exprs.append(
|
||||
pl.col("close") * pl.col("float_shares") <= bf["float_cap_max"]
|
||||
)
|
||||
if bf.get("amount_min") is not None:
|
||||
exprs.append(pl.col("amount") >= bf["amount_min"])
|
||||
if bf.get("amount_max") is not None:
|
||||
exprs.append(pl.col("amount") <= bf["amount_max"])
|
||||
# 换手率
|
||||
if bf.get("turnover_min") is not None and "turnover_rate" in df.columns:
|
||||
exprs.append(pl.col("turnover_rate") >= bf["turnover_min"])
|
||||
if bf.get("turnover_max") is not None and "turnover_rate" in df.columns:
|
||||
exprs.append(pl.col("turnover_rate") <= bf["turnover_max"])
|
||||
if bf.get("exclude_st") and "name" in df.columns:
|
||||
exprs.append(~pl.col("name").str.contains("(?i)ST|\\*ST|退"))
|
||||
# 板块过滤
|
||||
boards = bf.get("boards")
|
||||
if boards and isinstance(boards, list) and len(boards) > 0:
|
||||
board_exprs: list[pl.Expr] = []
|
||||
for b in boards:
|
||||
if b == "沪主板":
|
||||
board_exprs.append(pl.col("symbol").str.starts_with("60"))
|
||||
elif b == "深主板":
|
||||
board_exprs.append(
|
||||
pl.col("symbol").str.starts_with("00")
|
||||
| pl.col("symbol").str.starts_with("001")
|
||||
)
|
||||
elif b == "创业板":
|
||||
board_exprs.append(
|
||||
pl.col("symbol").str.starts_with("300")
|
||||
| pl.col("symbol").str.starts_with("301")
|
||||
)
|
||||
elif b == "科创板":
|
||||
board_exprs.append(pl.col("symbol").str.starts_with("688"))
|
||||
elif b == "北交所":
|
||||
board_exprs.append(pl.col("symbol").str.contains(r"\.BJ$"))
|
||||
if board_exprs:
|
||||
exprs.append(pl.any_horizontal(board_exprs))
|
||||
if exprs:
|
||||
return pl.all_horizontal(exprs)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _apply_basic_filter(df: pl.DataFrame, bf: dict) -> pl.DataFrame:
|
||||
"""Stage 1: 基础参数过滤"""
|
||||
expr = StrategyEngine._basic_filter_expr(df, bf)
|
||||
if expr is not None:
|
||||
return df.filter(expr)
|
||||
return df
|
||||
|
||||
# ================================================================
|
||||
# 内部: 评分
|
||||
# ================================================================
|
||||
|
||||
@staticmethod
|
||||
def _apply_scoring(df: pl.DataFrame, weights: dict) -> pl.DataFrame:
|
||||
"""通用评分: min-max 归一化 → 加权求和 → 0~100 分"""
|
||||
if not weights:
|
||||
return df
|
||||
total_weight = sum(weights.values())
|
||||
if total_weight <= 0:
|
||||
return df
|
||||
|
||||
score_parts: list[pl.Expr] = []
|
||||
for col, weight in weights.items():
|
||||
if col not in df.columns:
|
||||
continue
|
||||
w = weight / total_weight
|
||||
col_min = pl.col(col).min()
|
||||
col_range = pl.col(col).max() - col_min
|
||||
normalized = pl.when(col_range > 0).then(
|
||||
(pl.col(col) - col_min) / col_range
|
||||
).otherwise(pl.lit(0.5))
|
||||
score_parts.append(normalized * w)
|
||||
|
||||
if not score_parts:
|
||||
return df
|
||||
|
||||
score_expr = score_parts[0]
|
||||
for part in score_parts[1:]:
|
||||
score_expr = score_expr + part
|
||||
return df.with_columns((score_expr * 100).alias("score"))
|
||||
|
||||
|
||||
def _sanitize(rows: list[dict]) -> list[dict]:
|
||||
for r in rows:
|
||||
for k, v in list(r.items()):
|
||||
if isinstance(v, float) and (v != v or abs(v) == float("inf")):
|
||||
r[k] = None
|
||||
return rows
|
||||
|
||||
|
||||
def _dict_hash(d: dict) -> str:
|
||||
"""用于 basic_filter 分组缓存"""
|
||||
return str(sorted(d.items()))
|
||||
@@ -0,0 +1,613 @@
|
||||
"""策略实时监控 — 订阅行情更新，检查策略买卖信号和提醒条件。
|
||||
|
||||
职责: 接收实时行情 DataFrame → 检查监控中策略的信号/提醒 → 推送告警。
|
||||
不知道: 策略加载逻辑、AI、API、配置持久化、回测。
|
||||
依赖: 外部调用 on_quote_update() 传入实时数据。
|
||||
|
||||
本模块含两个评估器:
|
||||
1. StrategyMonitorService — 旧的策略监控 (type=strategy),第二步迁移到 MonitorRuleEngine
|
||||
2. MonitorRuleEngine — 通用规则引擎,覆盖 signal/price/market/strategy 四类,
|
||||
支持 scope (symbols/all/sector) + 多条件 AND/OR + cooldown 去重
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.strategy.custom_signals import _OP_BUILDERS # type: ignore # 复用运算符构造器
|
||||
from app.strategy import config as _strategy_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StrategyAlert:
|
||||
"""策略告警"""
|
||||
type: str # "entry" | "exit" | "alert"
|
||||
strategy_id: str
|
||||
symbol: str
|
||||
name: str | None
|
||||
message: str
|
||||
price: float | None = None
|
||||
change_pct: float | None = None
|
||||
signals: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class StrategyMonitorService:
|
||||
"""策略实时监控服务"""
|
||||
|
||||
def __init__(self, alert_handler: Callable[[StrategyAlert], None] | None = None):
|
||||
"""
|
||||
Args:
|
||||
alert_handler: 告警回调 (如推 SSE)
|
||||
"""
|
||||
self._alert_handler = alert_handler
|
||||
# strategy_id → 监控配置
|
||||
self._watching: dict[str, dict] = {}
|
||||
|
||||
def start(self, strategy_id: str, config: dict) -> None:
|
||||
"""开始监控一个策略
|
||||
|
||||
config: {
|
||||
"entry_signals": ["signal_n_day_high", ...],
|
||||
"exit_signals": ["signal_ma20_breakdown", ...],
|
||||
"alerts": [{"field": "rsi_14", "op": ">", "value": 80, "message": "..."}],
|
||||
}
|
||||
"""
|
||||
self._watching[strategy_id] = config
|
||||
logger.info("strategy monitor started: %s", strategy_id)
|
||||
|
||||
def stop(self, strategy_id: str) -> None:
|
||||
self._watching.pop(strategy_id, None)
|
||||
logger.info("strategy monitor stopped: %s", strategy_id)
|
||||
|
||||
def stop_all(self) -> None:
|
||||
self._watching.clear()
|
||||
|
||||
@property
|
||||
def watching(self) -> dict[str, dict]:
|
||||
return dict(self._watching)
|
||||
|
||||
def on_quote_update(self, df: pl.DataFrame) -> list[StrategyAlert]:
|
||||
"""行情更新后调用。向量化检查所有监控策略。
|
||||
|
||||
Args:
|
||||
df: 实时 enriched 数据 (~5500行)
|
||||
Returns:
|
||||
触发的告警列表
|
||||
"""
|
||||
if not self._watching or df.is_empty():
|
||||
return []
|
||||
|
||||
all_alerts: list[StrategyAlert] = []
|
||||
|
||||
for strategy_id, cfg in self._watching.items():
|
||||
# 买入信号
|
||||
entry_sigs = cfg.get("entry_signals", [])
|
||||
if entry_sigs:
|
||||
for sym, name, price, pct, hit_sigs in self._check_signals(df, entry_sigs):
|
||||
alert = StrategyAlert(
|
||||
type="entry",
|
||||
strategy_id=strategy_id,
|
||||
symbol=sym,
|
||||
name=name,
|
||||
message=f"买入信号触发",
|
||||
price=price,
|
||||
change_pct=pct,
|
||||
signals=hit_sigs,
|
||||
)
|
||||
all_alerts.append(alert)
|
||||
self._emit(alert)
|
||||
|
||||
# 卖出信号
|
||||
exit_sigs = cfg.get("exit_signals", [])
|
||||
if exit_sigs:
|
||||
for sym, name, price, pct, hit_sigs in self._check_signals(df, exit_sigs):
|
||||
alert = StrategyAlert(
|
||||
type="exit",
|
||||
strategy_id=strategy_id,
|
||||
symbol=sym,
|
||||
name=name,
|
||||
message=f"卖出信号触发",
|
||||
price=price,
|
||||
change_pct=pct,
|
||||
signals=hit_sigs,
|
||||
)
|
||||
all_alerts.append(alert)
|
||||
self._emit(alert)
|
||||
|
||||
# 提醒条件
|
||||
for alert_cfg in cfg.get("alerts", []):
|
||||
for sym, name, price, pct in self._check_alert(df, alert_cfg):
|
||||
alert = StrategyAlert(
|
||||
type="alert",
|
||||
strategy_id=strategy_id,
|
||||
symbol=sym,
|
||||
name=name,
|
||||
message=alert_cfg.get("message", "提醒"),
|
||||
price=price,
|
||||
change_pct=pct,
|
||||
)
|
||||
all_alerts.append(alert)
|
||||
self._emit(alert)
|
||||
|
||||
return all_alerts
|
||||
|
||||
def _emit(self, alert: StrategyAlert) -> None:
|
||||
if self._alert_handler:
|
||||
try:
|
||||
self._alert_handler(alert)
|
||||
except Exception as e:
|
||||
logger.warning("alert handler failed: %s", e)
|
||||
|
||||
@staticmethod
|
||||
def _check_signals(
|
||||
df: pl.DataFrame,
|
||||
signals: list[str],
|
||||
) -> list[tuple[str, str | None, float | None, float | None, list[str]]]:
|
||||
"""检查信号列，返回 [(symbol, name, price, change_pct, [hit_signals])]。
|
||||
支持内置 signal_ 与自定义 csg_ 前缀。"""
|
||||
cols = set(df.columns)
|
||||
resolved: list[tuple[str, str]] = [] # (原值, 列名)
|
||||
for s in signals:
|
||||
col = s if (s.startswith("signal_") or s.startswith("csg_")) else f"signal_{s}"
|
||||
if col in cols:
|
||||
resolved.append((s, col))
|
||||
if not resolved:
|
||||
return []
|
||||
|
||||
mask = pl.any_horizontal(pl.col(c).fill_null(False) for _, c in resolved)
|
||||
hit_df = df.filter(mask)
|
||||
|
||||
results = []
|
||||
for row in hit_df.iter_rows(named=True):
|
||||
sym = row.get("symbol", "")
|
||||
name = row.get("name")
|
||||
price = row.get("close")
|
||||
pct = row.get("change_pct")
|
||||
hit_sigs = [orig for orig, col in resolved if row.get(col)]
|
||||
results.append((sym, name, price, pct, hit_sigs))
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def _check_alert(
|
||||
df: pl.DataFrame,
|
||||
alert: dict,
|
||||
) -> list[tuple[str, str | None, float | None, float | None]]:
|
||||
"""检查阈值型提醒条件"""
|
||||
field = alert.get("field", "")
|
||||
if field not in df.columns:
|
||||
return []
|
||||
|
||||
if "op" in alert:
|
||||
# 阈值比较
|
||||
op = alert["op"]
|
||||
value = alert["value"]
|
||||
col = pl.col(field)
|
||||
ops = {
|
||||
">": col > value,
|
||||
">=": col >= value,
|
||||
"<": col < value,
|
||||
"<=": col <= value,
|
||||
}
|
||||
expr = ops.get(op)
|
||||
if expr is None:
|
||||
return []
|
||||
else:
|
||||
# 信号列 (布尔)
|
||||
expr = pl.col(field).fill_null(False)
|
||||
|
||||
hit_df = df.filter(expr)
|
||||
results = []
|
||||
for row in hit_df.iter_rows(named=True):
|
||||
results.append((
|
||||
row.get("symbol", ""),
|
||||
row.get("name"),
|
||||
row.get("close"),
|
||||
row.get("change_pct"),
|
||||
))
|
||||
return results
|
||||
|
||||
|
||||
# ================================================================
|
||||
# 通用监控规则引擎 MonitorRuleEngine
|
||||
# ================================================================
|
||||
|
||||
_SIGNAL_PREFIXES = ("signal_", "csg_")
|
||||
|
||||
|
||||
def _is_signal_field(field: str) -> bool:
|
||||
return any(field.startswith(p) for p in _SIGNAL_PREFIXES)
|
||||
|
||||
|
||||
def _build_condition_mask(df: pl.DataFrame, conditions: list[dict], logic: str) -> pl.DataFrame:
|
||||
"""根据 conditions + logic 构建过滤后的命中 DataFrame。
|
||||
|
||||
conditions: [{"field","op","value"?}] — op=truth 为布尔信号, 否则阈值比较
|
||||
logic: "and" | "or"
|
||||
返回命中行 (含 symbol/name/close/change_pct + 各信号列)
|
||||
"""
|
||||
cols = set(df.columns)
|
||||
parts: list[pl.Expr] = []
|
||||
for c in conditions:
|
||||
field = c["field"]
|
||||
if field not in cols:
|
||||
return df.head(0) # 字段缺失,无法判定 → 空结果
|
||||
op = c["op"]
|
||||
if op == "truth":
|
||||
parts.append(pl.col(field).fill_null(False))
|
||||
elif op in _OP_BUILDERS:
|
||||
parts.append(_OP_BUILDERS[op](pl.col(field), c["value"]))
|
||||
else:
|
||||
return df.head(0)
|
||||
if not parts:
|
||||
return df.head(0)
|
||||
if logic == "or":
|
||||
mask = pl.any_horizontal(parts)
|
||||
else:
|
||||
mask = pl.all_horizontal(parts)
|
||||
return df.filter(mask)
|
||||
|
||||
|
||||
class MonitorRuleEngine:
|
||||
"""通用监控规则引擎 — 接收实时行情 DataFrame,评估所有规则,返回 AlertEvent。
|
||||
|
||||
与 StrategyMonitorService 的区别:
|
||||
- 规则来自 monitor_rules 存储 (用户可配), 而非写死的 strategy config
|
||||
- 支持 scope (symbols/all/sector) 过滤作用域
|
||||
- 支持 conditions + logic (AND/OR) 任意组合
|
||||
- ★ cooldown 去重: 同一 (rule_id, symbol) 在冷却期内不重复触发
|
||||
"""
|
||||
|
||||
def __init__(self, alert_handler: Callable[[dict], None] | None = None):
|
||||
self._alert_handler = alert_handler
|
||||
self._rules: dict[str, dict] = {} # rule_id → rule
|
||||
# (rule_id, symbol) → 上次触发时间戳(秒)。用于 cooldown 去重。
|
||||
self._last_fire: dict[tuple[str, str], float] = {}
|
||||
self._strategy_engine = None # 延迟注入, type=strategy 规则用它跑选股
|
||||
# symbol → 股票名 (enriched DataFrame 已 drop name 列, 触发时从此映射回填)
|
||||
self._name_map: dict[str, str] = {}
|
||||
# 策略选股池状态: strategy_id → 上期选股符号集合 (用于 diff 变更)
|
||||
self._strategy_pools: dict[str, set[str]] = {}
|
||||
# 数据目录 (用于加载策略 overrides)
|
||||
self._data_dir = None
|
||||
|
||||
def set_strategy_engine(self, engine) -> None:
|
||||
"""注入 StrategyEngine, type=strategy 规则据此跑选股。"""
|
||||
self._strategy_engine = engine
|
||||
|
||||
def set_data_dir(self, data_dir) -> None:
|
||||
"""注入数据目录, 用于加载策略的用户覆盖配置。"""
|
||||
self._data_dir = data_dir
|
||||
|
||||
def set_name_map(self, name_map: dict[str, str]) -> None:
|
||||
"""注入 symbol → 股票名 映射, 用于在告警事件里回填 name 字段。
|
||||
|
||||
enriched DataFrame 在 pipeline 计算后不含 name 列 (见 indicators/pipeline.py),
|
||||
触发时从 instruments 表预构建此映射, 保证 AlertEvent.name 有值。
|
||||
"""
|
||||
self._name_map = name_map or {}
|
||||
|
||||
# ── 规则管理 ───────────────────────────────────────
|
||||
def set_rules(self, rules: list[dict]) -> None:
|
||||
"""批量设置规则 (覆盖)。用于启动时 reload。"""
|
||||
self._rules = {}
|
||||
for r in rules:
|
||||
if r.get("enabled") is not False:
|
||||
self._rules[r["id"]] = r
|
||||
logger.info("MonitorRuleEngine: 装载 %d 条规则", len(self._rules))
|
||||
|
||||
def add_rule(self, rule: dict) -> None:
|
||||
if rule.get("enabled") is not False:
|
||||
self._rules[rule["id"]] = rule
|
||||
else:
|
||||
self._rules.pop(rule["id"], None)
|
||||
|
||||
def remove_rule(self, rule_id: str) -> None:
|
||||
self._rules.pop(rule_id, None)
|
||||
# 清理对应的 cooldown 记录
|
||||
self._last_fire = {k: v for k, v in self._last_fire.items() if k[0] != rule_id}
|
||||
|
||||
def clear(self) -> None:
|
||||
self._rules.clear()
|
||||
self._last_fire.clear()
|
||||
|
||||
@property
|
||||
def rules(self) -> dict[str, dict]:
|
||||
return dict(self._rules)
|
||||
|
||||
@property
|
||||
def rule_count(self) -> int:
|
||||
return len(self._rules)
|
||||
|
||||
# ── 评估 ───────────────────────────────────────────
|
||||
def evaluate(self, df: pl.DataFrame) -> list[dict]:
|
||||
"""行情更新后评估所有规则。
|
||||
|
||||
Args:
|
||||
df: 实时 enriched 数据 (~5500行, 含 signal_/csg_/指标列)
|
||||
Returns:
|
||||
触发的 AlertEvent dict 列表 (含 ts/rule_id/source/type/symbol/...)
|
||||
"""
|
||||
if not self._rules or df.is_empty():
|
||||
return []
|
||||
|
||||
now = time.time()
|
||||
events: list[dict] = []
|
||||
|
||||
for rule_id, rule in self._rules.items():
|
||||
try:
|
||||
events.extend(self._evaluate_rule(df, rule, now))
|
||||
except Exception as e:
|
||||
logger.warning("规则评估失败 %s: %s", rule_id, e)
|
||||
|
||||
return events
|
||||
|
||||
def _evaluate_rule(self, df: pl.DataFrame, rule: dict, now: float) -> list[dict]:
|
||||
"""评估单条规则,返回触发的 events。"""
|
||||
# 1. 按 scope 过滤作用域
|
||||
scoped = self._apply_scope(df, rule)
|
||||
if scoped.is_empty():
|
||||
return []
|
||||
|
||||
# 2. 根据 type 构建命中集
|
||||
# 元组格式: (event_type, symbol, name, price, pct, signals)
|
||||
hit_rows: list[tuple[str, str, Any, Any, Any, list[str]]] = []
|
||||
|
||||
rtype = rule.get("type", "signal")
|
||||
if rtype == "strategy":
|
||||
# 策略类型: 跑策略选股 → 对比上期选股池 → 产出 new_entry/dropped 事件
|
||||
hit_rows = self._match_strategy(scoped, rule)
|
||||
else:
|
||||
# signal / price / market: 通用条件匹配
|
||||
for sym, name, price, pct, hit_sigs in self._match_conditions(scoped, rule):
|
||||
hit_rows.append((rtype, sym, name, price, pct, hit_sigs))
|
||||
|
||||
if not hit_rows:
|
||||
return []
|
||||
|
||||
# 3. cooldown 去重 + 生成 events
|
||||
cooldown = rule.get("cooldown_seconds", 3600)
|
||||
severity = rule.get("severity", "info")
|
||||
source = rtype
|
||||
|
||||
events: list[dict] = []
|
||||
for ev_type, sym, name, price, pct, hit_sigs in hit_rows:
|
||||
# cooldown 键: 批量事件用特殊键, 单只事件用 (rule_id, symbol)
|
||||
is_batch = sym == "_batch"
|
||||
if is_batch:
|
||||
key = (rule["id"], f"_{ev_type}_batch")
|
||||
else:
|
||||
key = (rule["id"], sym)
|
||||
last = self._last_fire.get(key)
|
||||
if last is not None and (now - last) < cooldown:
|
||||
continue # 冷却期内, 跳过
|
||||
self._last_fire[key] = now
|
||||
|
||||
# 批量事件: name 存放预构建的消息文本
|
||||
if is_batch:
|
||||
resolved_name = ""
|
||||
message = name # name 字段即批量消息
|
||||
else:
|
||||
resolved_name = name if name else self._name_map.get(sym)
|
||||
message = rule.get("message", "") or self._default_message(rule, ev_type=ev_type, sym=sym, name=resolved_name, pct=pct)
|
||||
|
||||
ev = {
|
||||
"ts": int(now * 1000),
|
||||
"rule_id": rule["id"],
|
||||
"rule_name": rule.get("name", ""),
|
||||
"source": source,
|
||||
"type": ev_type,
|
||||
"symbol": "" if is_batch else sym,
|
||||
"name": resolved_name,
|
||||
"message": message,
|
||||
"price": price,
|
||||
"change_pct": pct,
|
||||
"signals": hit_sigs,
|
||||
"severity": severity,
|
||||
}
|
||||
events.append(ev)
|
||||
if self._alert_handler:
|
||||
try:
|
||||
self._alert_handler(ev)
|
||||
except Exception as e:
|
||||
logger.warning("alert handler failed: %s", e)
|
||||
|
||||
return events
|
||||
|
||||
@staticmethod
|
||||
def _apply_scope(df: pl.DataFrame, rule: dict) -> pl.DataFrame:
|
||||
"""按 scope 过滤 DataFrame。"""
|
||||
scope = rule.get("scope", "symbols")
|
||||
if scope == "all":
|
||||
return df
|
||||
if scope == "symbols":
|
||||
syms = rule.get("symbols", [])
|
||||
if not syms:
|
||||
return df.head(0)
|
||||
return df.filter(pl.col("symbol").is_in(syms))
|
||||
if scope == "sector":
|
||||
# sector 过滤: 需 df 含板块列 (后续接入 ext_data JOIN)
|
||||
# 当前先返回全量, sector 精确过滤第二步完善
|
||||
return df
|
||||
return df
|
||||
|
||||
def _match_strategy(
|
||||
self, df: pl.DataFrame, rule: dict,
|
||||
) -> list[tuple[str, str, Any, Any, Any, list[str]]]:
|
||||
"""策略类型评估: 跑策略选股 → 对比上期选股池 → 产出变更事件。
|
||||
|
||||
返回 [(event_type, symbol, name, price, pct, signals)]
|
||||
event_type: "new_entry" (新入选) | "dropped" (已移出)
|
||||
单只变更逐只返回; 同一策略 >5 只合并为一条批量事件 (symbol="_batch")
|
||||
"""
|
||||
if self._strategy_engine is None:
|
||||
return []
|
||||
sid = rule.get("strategy_id")
|
||||
if not sid:
|
||||
return []
|
||||
try:
|
||||
s = self._strategy_engine.get(sid)
|
||||
except Exception:
|
||||
return []
|
||||
if s is None:
|
||||
return []
|
||||
|
||||
# 需要历史数据的策略跳过 (实时监控不支持 history loader)
|
||||
if s.filter_history_fn:
|
||||
logger.debug("策略 %s 需要历史数据, 跳过实时监控", sid)
|
||||
return []
|
||||
|
||||
# 运行策略选股: 复用当前 enriched DataFrame 跳过数据加载
|
||||
overrides = {}
|
||||
if self._data_dir:
|
||||
try:
|
||||
overrides = _strategy_config.load_override(self._data_dir, sid)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
result = self._strategy_engine.run(
|
||||
sid,
|
||||
as_of=_dt.date.today(),
|
||||
precomputed=df,
|
||||
overrides=overrides,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("策略 %s 选股执行失败: %s", sid, e)
|
||||
return []
|
||||
|
||||
current_pool: set[str] = {r["symbol"] for r in result.rows}
|
||||
prev_pool = self._strategy_pools.get(sid)
|
||||
|
||||
# 首次运行: 仅记录当前选股池, 不产生事件
|
||||
if prev_pool is None:
|
||||
self._strategy_pools[sid] = current_pool
|
||||
return []
|
||||
|
||||
new_entries = current_pool - prev_pool
|
||||
dropped = prev_pool - current_pool
|
||||
|
||||
# 无变更
|
||||
if not new_entries and not dropped:
|
||||
return []
|
||||
|
||||
# 更新存储
|
||||
self._strategy_pools[sid] = current_pool
|
||||
|
||||
sname = s.meta.get("name", "") or s.meta.get("id", sid)
|
||||
|
||||
# 构建查找表 (新入选股票可在 result.rows 中找到; 移出股票需从 df 找)
|
||||
row_map: dict[str, dict] = {r["symbol"]: r for r in result.rows}
|
||||
dropped_map: dict[str, dict] = {}
|
||||
if dropped:
|
||||
try:
|
||||
_dd = df.filter(pl.col("symbol").is_in(list(dropped)))
|
||||
for row in _dd.iter_rows(named=True):
|
||||
dropped_map[row["symbol"]] = row
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
results: list[tuple[str, str, Any, Any, Any, list[str]]] = []
|
||||
|
||||
# ── 新入选 ──
|
||||
new_list = sorted(new_entries)
|
||||
if len(new_list) > 5:
|
||||
names: list[str] = []
|
||||
for sym in new_list:
|
||||
row = row_map.get(sym, {})
|
||||
name = row.get("name") or self._name_map.get(sym, sym)
|
||||
names.append(str(name))
|
||||
message = f"策略「{sname}」进入 {len(new_entries)} 只：{'、'.join(names)}"
|
||||
results.append(("new_entry", "_batch", message, None, None, []))
|
||||
else:
|
||||
for sym in new_list:
|
||||
row = row_map.get(sym, {})
|
||||
name = row.get("name") or self._name_map.get(sym, sym)
|
||||
price = row.get("close")
|
||||
pct = row.get("change_pct")
|
||||
results.append(("new_entry", sym, name, price, pct, []))
|
||||
|
||||
# ── 已移出 ──
|
||||
dropped_list = sorted(dropped)
|
||||
if len(dropped_list) > 5:
|
||||
names = []
|
||||
for sym in dropped_list:
|
||||
row = dropped_map.get(sym, {})
|
||||
name = row.get("name") or self._name_map.get(sym, sym)
|
||||
names.append(str(name))
|
||||
message = f"策略「{sname}」移出 {len(dropped)} 只：{'、'.join(names)}"
|
||||
results.append(("dropped", "_batch", message, None, None, []))
|
||||
else:
|
||||
for sym in dropped_list:
|
||||
row = dropped_map.get(sym, {})
|
||||
name = row.get("name") or self._name_map.get(sym, sym)
|
||||
price = row.get("close")
|
||||
pct = row.get("change_pct")
|
||||
results.append(("dropped", sym, name, price, pct, []))
|
||||
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def _match_conditions(
|
||||
df: pl.DataFrame, rule: dict,
|
||||
) -> list[tuple[str, Any, Any, Any, list[str]]]:
|
||||
"""按 conditions + logic 匹配,返回命中行 [(symbol,name,price,pct,signals)]。"""
|
||||
conditions = rule.get("conditions", [])
|
||||
logic = rule.get("logic", "and")
|
||||
if not conditions:
|
||||
return []
|
||||
hit_df = _build_condition_mask(df, conditions, logic)
|
||||
results = []
|
||||
for row in hit_df.iter_rows(named=True):
|
||||
sym = row.get("symbol", "")
|
||||
name = row.get("name")
|
||||
price = row.get("close")
|
||||
pct = row.get("change_pct")
|
||||
# 收集命中的信号列名 (仅 op=truth 且为真的)
|
||||
hit_sigs = [
|
||||
c["field"] for c in conditions
|
||||
if c.get("op") == "truth" and row.get(c["field"])
|
||||
]
|
||||
results.append((sym, name, price, pct, hit_sigs))
|
||||
return results
|
||||
|
||||
def _default_message(self, rule: dict, ev_type: str = "", sym: str = "",
|
||||
name: str = "", pct: Any = None) -> str:
|
||||
"""生成默认 message。策略类型按变更方向生成。"""
|
||||
rtype = rule.get("type", "signal")
|
||||
if rtype == "strategy":
|
||||
# 从 StrategyEngine 取策略名; 失败则退化为 rule_name 里截取的部分
|
||||
sname = ""
|
||||
sid = rule.get("strategy_id")
|
||||
if sid and self._strategy_engine is not None:
|
||||
try:
|
||||
s = self._strategy_engine.get(sid)
|
||||
sname = s.meta.get("name", "") or s.meta.get("id", "")
|
||||
except Exception: # noqa: BLE001
|
||||
sname = ""
|
||||
if not sname:
|
||||
rn = rule.get("name", "")
|
||||
sname = rn.split(" · ", 1)[1] if " · " in rn else (rn or "策略")
|
||||
|
||||
if ev_type == "new_entry":
|
||||
pct_text = ""
|
||||
if pct is not None:
|
||||
sign = "+" if pct >= 0 else ""
|
||||
pct_text = f" {sign}{pct * 100:.1f}%"
|
||||
return f"策略「{sname}」进入 {name}{pct_text}"
|
||||
elif ev_type == "dropped":
|
||||
pct_text = ""
|
||||
if pct is not None:
|
||||
sign = "+" if pct >= 0 else ""
|
||||
pct_text = f" {sign}{pct * 100:.1f}%"
|
||||
return f"策略「{sname}」移出 {name}{pct_text}"
|
||||
return f"策略「{sname}」变更"
|
||||
|
||||
name_map = {"signal": "信号触发", "price": "价格触发", "market": "市场异动"}
|
||||
return name_map.get(rtype, "监控触发")
|
||||
@@ -0,0 +1,241 @@
|
||||
"""监控规则 — 统一的 MonitorRule 模型,覆盖策略/个股信号/个股价格/市场异动四类。
|
||||
|
||||
职责:
|
||||
- 从 data/user_data/monitor_rules/*.json 加载规则定义
|
||||
- 校验规则字段合法性
|
||||
- 提供 CRUD (load_all / save_one / delete_one)
|
||||
|
||||
不知道: 行情评估引擎、API、告警落盘。纯函数 + 文件存储。
|
||||
|
||||
设计 (镜像 custom_signals.py 的写法):
|
||||
- 一对象一文件 + glob 全扫 + 全量重写
|
||||
- 字段白名单复用 custom_signals.ALLOWED_FIELDS (阈值条件) + 信号列清单 (布尔条件)
|
||||
- id 正则与 custom_signals 一致,保证可纳入同一索引体系
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from app.strategy.custom_signals import ALLOWED_FIELDS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── 常量 ────────────────────────────────────────────────
|
||||
ID_RE = re.compile(r"^[a-z0-9_]{1,40}$")
|
||||
RULE_TYPES = {"strategy", "signal", "price", "market"}
|
||||
SCOPES = {"symbols", "all", "sector"}
|
||||
LOGICS = {"and", "or"}
|
||||
DIRECTIONS = {"entry", "exit", "both"}
|
||||
SEVERITIES = {"info", "warn", "critical"}
|
||||
OPS = {">", ">=", "<", "<=", "==", "!="}
|
||||
|
||||
# 布尔信号列前缀 (op=truth 时 field 取这些)
|
||||
_SIGNAL_PREFIXES = ("signal_", "csg_")
|
||||
|
||||
|
||||
# ── 持久化 (镜像 custom_signals.py) ─────────────────────
|
||||
def _dir(data_dir: Path) -> Path:
|
||||
d = data_dir / "user_data" / "monitor_rules"
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def _path(data_dir: Path, rule_id: str) -> Path:
|
||||
return _dir(data_dir) / f"{rule_id}.json"
|
||||
|
||||
|
||||
def load_all(data_dir: Path) -> list[dict]:
|
||||
"""读取全部监控规则。损坏的文件被跳过。"""
|
||||
d = _dir(data_dir)
|
||||
out: list[dict] = []
|
||||
for f in sorted(d.glob("*.json")):
|
||||
try:
|
||||
out.append(json.loads(f.read_text(encoding="utf-8")))
|
||||
except Exception as e:
|
||||
logger.warning("monitor rule load failed %s: %s", f.name, e)
|
||||
return out
|
||||
|
||||
|
||||
def load_one(data_dir: Path, rule_id: str) -> dict | None:
|
||||
p = _path(data_dir, rule_id)
|
||||
if not p.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
except Exception as e:
|
||||
logger.warning("monitor rule load failed %s: %s", rule_id, e)
|
||||
return None
|
||||
|
||||
|
||||
def save_one(data_dir: Path, rule: dict) -> None:
|
||||
p = _path(data_dir, rule["id"])
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(rule, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def delete_one(data_dir: Path, rule_id: str) -> bool:
|
||||
p = _path(data_dir, rule_id)
|
||||
if p.exists():
|
||||
p.unlink()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ── 校验 ────────────────────────────────────────────────
|
||||
def _is_signal_field(field: str) -> bool:
|
||||
"""判断 field 是否为布尔信号列 (signal_ / csg_ 前缀)。"""
|
||||
return any(field.startswith(p) for p in _SIGNAL_PREFIXES)
|
||||
|
||||
|
||||
def validate(rule: dict) -> None:
|
||||
"""校验一条监控规则,非法则抛 ValueError (含中文信息)。"""
|
||||
rid = rule.get("id", "")
|
||||
if not isinstance(rid, str) or not ID_RE.match(rid):
|
||||
raise ValueError(f"规则 id 非法 (仅小写字母数字下划线, 1-40字符): {rid!r}")
|
||||
if not isinstance(rule.get("name"), str) or not rule["name"].strip():
|
||||
raise ValueError("规则 name 不能为空")
|
||||
if rule.get("type") not in RULE_TYPES:
|
||||
raise ValueError(f"type 必须是 {RULE_TYPES} 之一")
|
||||
|
||||
# 策略类型: 需要 strategy_id + direction,conditions 可空
|
||||
if rule.get("type") == "strategy":
|
||||
if not rule.get("strategy_id"):
|
||||
raise ValueError("策略类型规则必须指定 strategy_id")
|
||||
if rule.get("direction", "entry") not in DIRECTIONS:
|
||||
raise ValueError(f"direction 必须是 {DIRECTIONS} 之一")
|
||||
else:
|
||||
# 信号/价格/市场类型: 需要 conditions
|
||||
conds = rule.get("conditions")
|
||||
if not isinstance(conds, list) or len(conds) == 0:
|
||||
raise ValueError("conditions 不能为空")
|
||||
if len(conds) > 8:
|
||||
raise ValueError("conditions 最多 8 条")
|
||||
if rule.get("logic", "and") not in LOGICS:
|
||||
raise ValueError(f"logic 必须是 {LOGICS} 之一")
|
||||
for i, c in enumerate(conds):
|
||||
if not isinstance(c, dict):
|
||||
raise ValueError(f"第 {i+1} 个条件格式错误")
|
||||
field = c.get("field", "")
|
||||
op = c.get("op", "")
|
||||
if op == "truth":
|
||||
# 布尔信号: field 必须是 signal_/csg_ 前缀
|
||||
if not _is_signal_field(field):
|
||||
raise ValueError(f"第 {i+1} 个条件: op=truth 时 field 必须是信号列 (signal_/csg_ 前缀): {field!r}")
|
||||
elif op in OPS:
|
||||
# 阈值比较: field 必须在白名单, 需要 value
|
||||
if field not in ALLOWED_FIELDS:
|
||||
raise ValueError(f"第 {i+1} 个条件: 阈值字段 {field!r} 不在白名单")
|
||||
if not isinstance(c.get("value"), (int, float)):
|
||||
raise ValueError(f"第 {i+1} 个条件: value 必须是数字")
|
||||
else:
|
||||
raise ValueError(f"第 {i+1} 个条件: op {op!r} 非法 (应为 truth 或 {OPS})")
|
||||
|
||||
# scope 校验
|
||||
if rule.get("scope", "symbols") not in SCOPES:
|
||||
raise ValueError(f"scope 必须是 {SCOPES} 之一")
|
||||
if rule.get("scope") == "symbols":
|
||||
syms = rule.get("symbols")
|
||||
if not isinstance(syms, list) or len(syms) == 0:
|
||||
raise ValueError("scope=symbols 时 symbols 不能为空")
|
||||
|
||||
# 其余枚举
|
||||
if rule.get("severity", "info") not in SEVERITIES:
|
||||
raise ValueError(f"severity 必须是 {SEVERITIES} 之一")
|
||||
cd = rule.get("cooldown_seconds", 3600)
|
||||
if not isinstance(cd, int) or cd < 0:
|
||||
raise ValueError("cooldown_seconds 必须是非负整数")
|
||||
|
||||
|
||||
def normalize(rule: dict) -> dict:
|
||||
"""补全默认字段,返回规范化后的规则 (不校验)。"""
|
||||
r = dict(rule)
|
||||
r.setdefault("enabled", True)
|
||||
r.setdefault("scope", "symbols")
|
||||
r.setdefault("symbols", [])
|
||||
r.setdefault("sector", None)
|
||||
r.setdefault("strategy_id", None)
|
||||
r.setdefault("direction", "entry")
|
||||
r.setdefault("conditions", [])
|
||||
r.setdefault("logic", "and")
|
||||
r.setdefault("cooldown_seconds", 3600)
|
||||
r.setdefault("severity", "info")
|
||||
r.setdefault("message", "")
|
||||
r.setdefault("webhook_url", "")
|
||||
r.setdefault("webhook_enabled", False)
|
||||
r.setdefault("created_at", datetime.now(timezone.utc).isoformat())
|
||||
return r
|
||||
|
||||
|
||||
# 策略监控自动迁移的规则 id 前缀 (固定, 保证幂等)
|
||||
STRATEGY_RULE_PREFIX = "mr_strategy_"
|
||||
|
||||
|
||||
def strategy_rule_id(strategy_id: str) -> str:
|
||||
"""策略监控规则 id = mr_strategy_{strategy_id}。"""
|
||||
return f"{STRATEGY_RULE_PREFIX}{strategy_id}"
|
||||
|
||||
|
||||
def migrate_strategy_monitors(data_dir: Path, strategy_ids: list[str], strategy_names: dict[str, str]) -> list[dict]:
|
||||
"""把 preferences.strategy_monitor_ids 里的策略,同步生成/更新 type=strategy 规则。
|
||||
|
||||
幂等: 已存在的策略规则会被更新 (方向/名称),不会重复创建。
|
||||
已从 strategy_ids 移除的策略, 其规则会被停用 (enabled=False) 而非删除 (保留历史触发记录的关联)。
|
||||
|
||||
Args:
|
||||
data_dir: 数据目录
|
||||
strategy_ids: 当前监控池中的策略 id 列表
|
||||
strategy_names: {strategy_id: 策略名} 用于规则显示名
|
||||
Returns:
|
||||
本次生成/更新的规则列表
|
||||
"""
|
||||
desired = set(strategy_ids)
|
||||
existing = load_all(data_dir)
|
||||
# 已存在的策略规则 {strategy_id: rule}
|
||||
existing_strategy_rules: dict[str, dict] = {}
|
||||
for r in existing:
|
||||
rid = r.get("id", "")
|
||||
if rid.startswith(STRATEGY_RULE_PREFIX):
|
||||
sid = rid[len(STRATEGY_RULE_PREFIX):]
|
||||
if sid:
|
||||
existing_strategy_rules[sid] = r
|
||||
|
||||
touched: list[dict] = []
|
||||
# 1. 为当前监控池的策略 upsert 规则
|
||||
for sid in desired:
|
||||
rule_id = strategy_rule_id(sid)
|
||||
name = strategy_names.get(sid, sid)
|
||||
rule = existing_strategy_rules.get(sid)
|
||||
if rule is None:
|
||||
rule = normalize({
|
||||
"id": rule_id,
|
||||
"name": f"策略监控 · {name}",
|
||||
"type": "strategy",
|
||||
"scope": "all",
|
||||
"strategy_id": sid,
|
||||
"direction": "entry",
|
||||
"conditions": [],
|
||||
"cooldown_seconds": 3600,
|
||||
"enabled": True,
|
||||
})
|
||||
else:
|
||||
rule = dict(rule)
|
||||
rule["enabled"] = True
|
||||
rule["strategy_id"] = sid
|
||||
rule["name"] = f"策略监控 · {name}"
|
||||
rule.setdefault("scope", "all")
|
||||
rule.setdefault("direction", "entry")
|
||||
save_one(data_dir, rule)
|
||||
touched.append(rule)
|
||||
|
||||
# 2. 不在监控池的策略 → 停用其规则 (不删除)
|
||||
for sid, rule in existing_strategy_rules.items():
|
||||
if sid not in desired and rule.get("enabled") is not False:
|
||||
rule = dict(rule)
|
||||
rule["enabled"] = False
|
||||
save_one(data_dir, rule)
|
||||
|
||||
return touched
|
||||
@@ -0,0 +1,65 @@
|
||||
"""策略提示词组装器 — 两步定制流程的提示词生成。
|
||||
|
||||
职责: 加载对应步骤的 Markdown 指南，拼接用户输入，组装 LLM 提示词。
|
||||
不知道: LLM 调用、API、前端、引擎执行。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
_DOCS_DIR = Path(__file__).resolve().parent.parent.parent.parent / "docs"
|
||||
_cache: dict[str, str] = {}
|
||||
|
||||
|
||||
def _load_doc(name: str) -> str:
|
||||
if name not in _cache:
|
||||
path = _DOCS_DIR / name
|
||||
_cache[name] = path.read_text(encoding="utf-8") if path.exists() else ""
|
||||
return _cache[name]
|
||||
|
||||
|
||||
DIRECTION_CN = {"long": "做多", "short": "做空", "monitor": "监控"}
|
||||
|
||||
|
||||
def build_step1(name: str, description: str, direction: str, rules: str, strategy_id: str = "") -> str:
|
||||
"""步骤1：规则 → 完整策略代码（参数 + 信号 + 评分 + 告警）
|
||||
|
||||
注意: strategy-guide.md 已在 ai_generator.py 的 system prompt 中加载，
|
||||
此处不再重复加载以节省 token。
|
||||
"""
|
||||
guide = _load_doc("strategy-builder-step1.md")
|
||||
|
||||
id_line = f"\n策略ID（必须使用此ID）：{strategy_id}" if strategy_id else ""
|
||||
|
||||
return f"""{guide}
|
||||
|
||||
---
|
||||
|
||||
请根据以下用户输入生成完整策略代码：
|
||||
|
||||
策略名称：{name}{id_line}
|
||||
策略描述：{description}
|
||||
选股方向：{DIRECTION_CN.get(direction, direction)}
|
||||
策略规则：
|
||||
{rules}
|
||||
|
||||
只输出 Python 代码。"""
|
||||
|
||||
|
||||
def build_step2(current_code: str, instruction: str) -> str:
|
||||
"""步骤2：修改策略任意部分"""
|
||||
guide = _load_doc("strategy-builder-step2.md")
|
||||
|
||||
return f"""{guide}
|
||||
|
||||
---
|
||||
|
||||
当前策略代码：
|
||||
```python
|
||||
{current_code}
|
||||
```
|
||||
|
||||
用户修改指令：
|
||||
{instruction}
|
||||
|
||||
只输出修改后的完整 Python 代码。"""
|
||||
@@ -0,0 +1 @@
|
||||
"""数据源适配层 — 能力探测 / 调度 / Repository。"""
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Capability 定义(§5.1)。
|
||||
|
||||
业务代码只依赖 CapabilitySet,不读 tiers.yaml,不感知"档位"。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class Cap(StrEnum):
|
||||
"""所有 capability 的命名常量。新增能力时只在这里加一行。"""
|
||||
|
||||
QUOTE_BY_SYMBOL = "quote.by_symbol"
|
||||
QUOTE_BATCH = "quote.batch"
|
||||
QUOTE_POOL = "quote.pool"
|
||||
KLINE_DAILY_BY_SYMBOL = "kline.daily.by_symbol"
|
||||
KLINE_DAILY_BATCH = "kline.daily.batch"
|
||||
KLINE_MINUTE_BY_SYMBOL = "kline.minute.by_symbol"
|
||||
KLINE_MINUTE_BATCH = "kline.minute.batch"
|
||||
INTRADAY = "intraday"
|
||||
INTRADAY_BATCH = "intraday.batch"
|
||||
DEPTH5 = "depth5"
|
||||
DEPTH5_BATCH = "depth5.batch"
|
||||
WEBSOCKET = "websocket"
|
||||
FINANCIAL = "financial"
|
||||
ADJ_FACTOR = "adj_factor"
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class CapabilityLimits:
|
||||
"""单个 capability 的运行时限制。"""
|
||||
rpm: int | None = None # 次/分钟,None 表示未知或不限
|
||||
batch: int | None = None # 标的/次
|
||||
subscribe: int | None = None # WS 订阅上限
|
||||
|
||||
|
||||
class CapabilitySet:
|
||||
"""探测得到的"用户当前可用能力"。业务代码的唯一真理源。"""
|
||||
|
||||
def __init__(self, caps: dict[Cap, CapabilityLimits] | None = None) -> None:
|
||||
self._caps: dict[Cap, CapabilityLimits] = dict(caps or {})
|
||||
|
||||
def has(self, cap: Cap) -> bool:
|
||||
return cap in self._caps
|
||||
|
||||
def limits(self, cap: Cap) -> CapabilityLimits | None:
|
||||
return self._caps.get(cap)
|
||||
|
||||
def require(self, cap: Cap) -> CapabilityLimits:
|
||||
"""断言可用,否则抛 CapabilityDenied。"""
|
||||
if cap not in self._caps:
|
||||
raise CapabilityDenied(cap)
|
||||
return self._caps[cap]
|
||||
|
||||
def all(self) -> dict[Cap, CapabilityLimits]:
|
||||
return dict(self._caps)
|
||||
|
||||
def to_dict(self) -> dict[str, dict]:
|
||||
return {
|
||||
str(cap): {
|
||||
"rpm": lim.rpm,
|
||||
"batch": lim.batch,
|
||||
"subscribe": lim.subscribe,
|
||||
}
|
||||
for cap, lim in self._caps.items()
|
||||
}
|
||||
|
||||
|
||||
class CapabilityDenied(Exception):
|
||||
"""请求的 capability 当前不可用。"""
|
||||
|
||||
def __init__(self, cap: Cap, suggestion: str | None = None) -> None:
|
||||
self.cap = cap
|
||||
self.suggestion = suggestion or f"加购『{cap}』能力可解锁"
|
||||
super().__init__(f"capability not available: {cap}; {self.suggestion}")
|
||||
@@ -0,0 +1,109 @@
|
||||
"""数据源 SDK 封装(§5)。
|
||||
|
||||
进程内单例;Key 来源(优先级):secrets.json > .env。
|
||||
用户改 Key 后需要 `reset_clients()`,然后 `get_client()` 会拿新的。
|
||||
|
||||
5 档体系下服务器归属:
|
||||
- none 档(无 key / 无效 key) → 数据源 SDK .free()(free-api 服务器)
|
||||
- free 档(免费有效 key) → 数据源 SDK .free()(key 被 SDK 忽略,运行时走 free-api)
|
||||
- starter/pro/expert(付费 key) → 数据源 SDK (api_key=key, base_url)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from tickflow import AsyncTickFlow, TickFlow
|
||||
|
||||
from app import secrets_store
|
||||
|
||||
_sync_client: TickFlow | None = None
|
||||
_async_client: AsyncTickFlow | None = None
|
||||
|
||||
|
||||
# ===== 服务器归属判定 =====
|
||||
|
||||
# free-api 服务器默认节点(SDK 默认值),none/free 档运行时走这里。
|
||||
FREE_ENDPOINT = "https://free-api.tickflow.org"
|
||||
# 付费端点默认节点(starter+ 运行时走这里,也是端点切换的默认值)。
|
||||
PAID_ENDPOINT = "https://api.tickflow.org"
|
||||
|
||||
|
||||
def _should_use_free_server() -> bool:
|
||||
"""是否应走 free-api 服务器。
|
||||
|
||||
判定依据:无 key,或当前档位为 none/free。
|
||||
付费档(starter+)走付费端点。
|
||||
"""
|
||||
if not secrets_store.get_tickflow_key():
|
||||
return True
|
||||
# 有 key 时按探测出的档位判定(避免读 capabilities.json 在首次启动前未生成的边界)
|
||||
from app.tickflow.policy import base_tier_name
|
||||
return base_tier_name() in ("none", "free")
|
||||
|
||||
|
||||
def _base_url() -> str | None:
|
||||
"""从 secrets.json 读取用户自定义端点,没有则返回 None(用 SDK 默认)。"""
|
||||
return secrets_store.load().get("tickflow_base_url") or None
|
||||
|
||||
|
||||
def get_client() -> TickFlow:
|
||||
"""同步客户端。能力探测、盘后管道用。"""
|
||||
global _sync_client
|
||||
if _sync_client is None:
|
||||
key = secrets_store.get_tickflow_key()
|
||||
if _should_use_free_server():
|
||||
# none/free 档:走 free-api 服务器(无 key 或免费 key 被 SDK 忽略)
|
||||
_sync_client = TickFlow.free()
|
||||
else:
|
||||
_sync_client = TickFlow(api_key=key, base_url=_base_url())
|
||||
return _sync_client
|
||||
|
||||
|
||||
def get_async_client() -> AsyncTickFlow:
|
||||
"""异步客户端。FastAPI 请求路径上用。"""
|
||||
global _async_client
|
||||
if _async_client is None:
|
||||
key = secrets_store.get_tickflow_key()
|
||||
if _should_use_free_server():
|
||||
_async_client = AsyncTickFlow.free()
|
||||
else:
|
||||
_async_client = AsyncTickFlow(api_key=key, base_url=_base_url())
|
||||
return _async_client
|
||||
|
||||
|
||||
def reset_clients() -> None:
|
||||
"""Key 变化后调用 — 让下一次 get_client() 拿新实例。"""
|
||||
global _sync_client, _async_client
|
||||
_sync_client = None
|
||||
_async_client = None
|
||||
|
||||
|
||||
def current_mode() -> str:
|
||||
"""供 UI 显示当前模式。三态:
|
||||
|
||||
- "none" : 无 key / 无效 key(走 free-api,仅历史日K)
|
||||
- "free" : 免费有效 key(走 free-api,仅历史日K)
|
||||
- "api_key" : 付费 key(starter+,走付费端点,有实时行情)
|
||||
"""
|
||||
if not secrets_store.get_tickflow_key():
|
||||
return "none"
|
||||
from app.tickflow.policy import base_tier_name
|
||||
tier = base_tier_name()
|
||||
if tier in ("none", "free"):
|
||||
return "free" if tier == "free" else "none"
|
||||
return "api_key"
|
||||
|
||||
|
||||
def current_endpoint() -> str:
|
||||
"""返回当前显示用的端点 URL(对应 endpoints.json 列表项)。
|
||||
|
||||
- none/free 档:显示 free-api 服务器节点
|
||||
- 付费档:显示用户自定义端点(测速切换后)或默认付费节点 api.tickflow.org
|
||||
"""
|
||||
if _should_use_free_server():
|
||||
return FREE_ENDPOINT
|
||||
# 自定义端点(付费模式测速切换后):优先返回
|
||||
base = _base_url()
|
||||
if base:
|
||||
return base.rstrip("/")
|
||||
return PAID_ENDPOINT
|
||||
@@ -0,0 +1,550 @@
|
||||
"""能力探测 + CapabilitySet 持久化(§5.3)。
|
||||
|
||||
探测策略:逐 capability 用最小代价请求试探。
|
||||
- 成功 → 记录可用,优先取响应头 X-RateLimit-* 否则用 tiers.yaml 默认
|
||||
- 抛权限错 → 不可用
|
||||
- 抛其他错 → 不可用(谨慎,保留日志)
|
||||
|
||||
Tier Label 算法见 §5.3:基线档 + 补丁能力。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from app.config import settings
|
||||
from app import secrets_store
|
||||
|
||||
from .capabilities import Cap, CapabilityLimits, CapabilitySet
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CAPSET_CACHE_FILE = "capabilities.json"
|
||||
|
||||
# 缓存 schema 版本。capabilities 模型有结构性变更时 bump(如新增/拆分 Cap),
|
||||
# 旧缓存(无此字段或版本更低)会被判定过期,触发重新探测。
|
||||
# v2: 拆分 depth5 → depth5(单只) + depth5.batch(批量)
|
||||
# v3: 探测补全 quote.batch(此前 tiers.yaml 声明了但 _probe_real 漏探测)
|
||||
# v4: 5 档重构 —— 新增 none 档(无key/无效key),free 档重定义(走 free-api 服务器,
|
||||
# 仅历史日K)。判定改为复权因子分水岭:_classify_tier 接管档位判定。
|
||||
_CACHE_SCHEMA_VERSION = 4
|
||||
|
||||
# 探测用最小代价请求:挑流通性最好的 1 只标的试
|
||||
_PROBE_SYMBOL = "600000.SH" # 浦发银行,长期不会退市
|
||||
|
||||
|
||||
def _load_tiers_yaml() -> dict[str, dict[str, dict[str, Any]]]:
|
||||
for path in [settings.tiers_yaml, Path("/app/tiers.yaml"), Path("../tiers.yaml")]:
|
||||
if path.exists():
|
||||
with path.open(encoding="utf-8") as f:
|
||||
return yaml.safe_load(f)
|
||||
raise FileNotFoundError("tiers.yaml not found")
|
||||
|
||||
|
||||
def _tier_to_capset(tier_def: dict[str, dict[str, Any]]) -> CapabilitySet:
|
||||
caps: dict[Cap, CapabilityLimits] = {}
|
||||
for cap_name, limits_dict in tier_def.items():
|
||||
try:
|
||||
cap = Cap(cap_name)
|
||||
except ValueError:
|
||||
logger.warning("unknown cap in tiers.yaml: %s", cap_name)
|
||||
continue
|
||||
caps[cap] = CapabilityLimits(
|
||||
rpm=limits_dict.get("rpm"),
|
||||
batch=limits_dict.get("batch"),
|
||||
subscribe=limits_dict.get("subscribe"),
|
||||
)
|
||||
return CapabilitySet(caps)
|
||||
|
||||
|
||||
def _is_transient(e: Exception) -> bool:
|
||||
"""是否为"可重试的瞬时错误"——网络抖动 / 限流 / 服务端 5xx。
|
||||
|
||||
与权限/参数错误(403/401/400/404)区分:后者重试也无用,不重试。
|
||||
用类名匹配而非 import SDK 异常,避免探测期对 SDK 内部耦合。
|
||||
"""
|
||||
cls = e.__class__.__name__
|
||||
if cls in {
|
||||
"RateLimitError", "InternalServerError", "APIError",
|
||||
"ConnectionError", "TimeoutError", "ConnectError",
|
||||
"ConnectTimeout", "ReadTimeout", "RemoteProtocolError",
|
||||
"httpx.ConnectError", "httpx.TimeoutException",
|
||||
}:
|
||||
return True
|
||||
# APIError 体系下,status_code 5xx/429 视为瞬时
|
||||
status = getattr(e, "status_code", None)
|
||||
if isinstance(status, int) and (status == 429 or status >= 500):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _call_with_retry(fn, attempts: int = 3, backoff: float = 0.6) -> None:
|
||||
"""调用 fn();对瞬时错误退避重试,权限/参数错误立即抛出。
|
||||
|
||||
attempts=总尝试次数(含首次)。返回 None,异常由调用方分类。
|
||||
"""
|
||||
last_exc: Exception | None = None
|
||||
for i in range(attempts):
|
||||
try:
|
||||
fn()
|
||||
return
|
||||
except Exception as e: # noqa: BLE001
|
||||
last_exc = e
|
||||
# 权限/参数类错误:重试无意义,立即抛出交给 try_call 归类
|
||||
if not _is_transient(e):
|
||||
raise
|
||||
# 瞬时错误:最后一轮不再 sleep
|
||||
if i < attempts - 1:
|
||||
time.sleep(backoff * (i + 1))
|
||||
# 重试耗尽,抛出最后一次异常
|
||||
assert last_exc is not None
|
||||
raise last_exc
|
||||
|
||||
|
||||
def _probe_real(tiers: dict) -> tuple[CapabilitySet, list[str]]:
|
||||
"""逐 capability 试探。需要 API key。
|
||||
|
||||
**关键**:探测始终在付费端点上进行,用 key 鉴权验证有效性。
|
||||
绝不能读旧 capabilities 缓存的档位来选服务器 —— 否则首次保存 key 时,
|
||||
旧缓存是 none 档 → get_client() 返回 free 服务器 → free 服务器忽略 key →
|
||||
乱填 key 也能拿到日K → 误判成 free 档(鸡生蛋蛋生鸡的循环依赖 bug)。
|
||||
|
||||
返回 (capset, probe_log)。
|
||||
"""
|
||||
from tickflow import TickFlow
|
||||
from .client import _base_url, PAID_ENDPOINT
|
||||
|
||||
key = secrets_store.get_tickflow_key()
|
||||
# 探测专用客户端:强制走付费端点验证 key。
|
||||
# base_url 用用户自定义端点(若已配置测速切换),否则默认付费端点。
|
||||
probe_base = _base_url() or PAID_ENDPOINT
|
||||
tf = TickFlow(api_key=key, base_url=probe_base)
|
||||
available: dict[Cap, CapabilityLimits] = {}
|
||||
log: list[str] = []
|
||||
|
||||
def try_call(cap: Cap, fn, default_limits: dict[str, Any]) -> None:
|
||||
try:
|
||||
_call_with_retry(fn)
|
||||
available[cap] = CapabilityLimits(
|
||||
rpm=default_limits.get("rpm"),
|
||||
batch=default_limits.get("batch"),
|
||||
subscribe=default_limits.get("subscribe"),
|
||||
)
|
||||
log.append(f"✓ {cap}")
|
||||
except Exception as e: # noqa: BLE001
|
||||
msg = str(e).lower()
|
||||
cls = e.__class__.__name__
|
||||
# PermissionError 类名 / HTTP 403 / 中英文权限关键词都算"明确无权限"
|
||||
is_perm_denied = (
|
||||
cls in {"PermissionError", "AuthorizationError"}
|
||||
or "permission" in msg or "unauthorized" in msg
|
||||
or "403" in msg or "forbidden" in msg
|
||||
or "套餐" in msg or "权限" in msg or "需要" in msg
|
||||
)
|
||||
if is_perm_denied:
|
||||
log.append(f"✗ {cap}(无权限)")
|
||||
else:
|
||||
# 重试耗尽仍失败的瞬时错误 — 标记为疑似,而非直接判定"无此能力"
|
||||
log.append(f"? {cap} ({cls}: {e})")
|
||||
|
||||
# 用各档默认上限作为占位(无 X-RateLimit-* 头时)
|
||||
# 取所有档的并集,逐 cap 试探
|
||||
all_caps_defaults: dict[str, dict[str, Any]] = {}
|
||||
for tier in ("free", "starter", "pro", "expert"):
|
||||
for cap_name, lim in tiers.get(tier, {}).items():
|
||||
all_caps_defaults.setdefault(cap_name, lim)
|
||||
|
||||
def defaults(cap: Cap) -> dict[str, Any]:
|
||||
return all_caps_defaults.get(str(cap), {})
|
||||
|
||||
# 全部用 keyword-only 形式调用,符合 SDK 真实签名
|
||||
# quote.by_symbol
|
||||
try_call(Cap.QUOTE_BY_SYMBOL,
|
||||
lambda: tf.quotes.get(symbols=[_PROBE_SYMBOL], as_dataframe=False),
|
||||
defaults(Cap.QUOTE_BY_SYMBOL))
|
||||
|
||||
# quote.batch — 批量行情(POST /v1/quotes)。用 get_by_symbols 试探。
|
||||
try_call(Cap.QUOTE_BATCH,
|
||||
lambda: tf.quotes.get_by_symbols([_PROBE_SYMBOL], as_dataframe=False),
|
||||
defaults(Cap.QUOTE_BATCH))
|
||||
|
||||
# quote.pool — 用一个真实存在的 universe id 试探。
|
||||
# universes.list() 在 Free 也开放,先拿任意一个 universe id 再用 get_by_universes 试。
|
||||
def _probe_pool():
|
||||
unis = tf.universes.list()
|
||||
if not unis:
|
||||
raise RuntimeError("no universes available")
|
||||
first_id = unis[0]["id"] if isinstance(unis[0], dict) else getattr(unis[0], "id")
|
||||
return tf.quotes.get_by_universes([first_id], as_dataframe=False)
|
||||
|
||||
try_call(Cap.QUOTE_POOL, _probe_pool, defaults(Cap.QUOTE_POOL))
|
||||
|
||||
# kline.daily.by_symbol — Free 也有
|
||||
try_call(Cap.KLINE_DAILY_BY_SYMBOL,
|
||||
lambda: tf.klines.get(_PROBE_SYMBOL, period="1d", count=1, as_dataframe=False),
|
||||
defaults(Cap.KLINE_DAILY_BY_SYMBOL))
|
||||
|
||||
# kline.daily.batch
|
||||
try_call(Cap.KLINE_DAILY_BATCH,
|
||||
lambda: tf.klines.batch([_PROBE_SYMBOL], period="1d", count=1, as_dataframe=False),
|
||||
defaults(Cap.KLINE_DAILY_BATCH))
|
||||
|
||||
# kline.minute.by_symbol
|
||||
try_call(Cap.KLINE_MINUTE_BY_SYMBOL,
|
||||
lambda: tf.klines.get(_PROBE_SYMBOL, period="1m", count=1, as_dataframe=False),
|
||||
defaults(Cap.KLINE_MINUTE_BY_SYMBOL))
|
||||
|
||||
# kline.minute.batch
|
||||
try_call(Cap.KLINE_MINUTE_BATCH,
|
||||
lambda: tf.klines.batch([_PROBE_SYMBOL], period="1m", count=1, as_dataframe=False),
|
||||
defaults(Cap.KLINE_MINUTE_BATCH))
|
||||
|
||||
# intraday
|
||||
try_call(Cap.INTRADAY,
|
||||
lambda: tf.klines.intraday(_PROBE_SYMBOL, count=1, as_dataframe=False),
|
||||
defaults(Cap.INTRADAY))
|
||||
|
||||
# intraday.batch
|
||||
try_call(Cap.INTRADAY_BATCH,
|
||||
lambda: tf.klines.intraday_batch([_PROBE_SYMBOL], count=1, as_dataframe=False),
|
||||
defaults(Cap.INTRADAY_BATCH))
|
||||
|
||||
# depth5 — 按标的查(单只)
|
||||
try_call(Cap.DEPTH5,
|
||||
lambda: tf.depth.get(_PROBE_SYMBOL),
|
||||
defaults(Cap.DEPTH5))
|
||||
|
||||
# depth5.batch — 批量查(SDK 0.1.23+ 提供 depth.batch,对应官方 /v1/depth/batch 端点)
|
||||
try_call(Cap.DEPTH5_BATCH,
|
||||
lambda: tf.depth.batch([_PROBE_SYMBOL]),
|
||||
defaults(Cap.DEPTH5_BATCH))
|
||||
|
||||
# financial — SDK 提供 income / balance_sheet / cash_flow / metrics / shares
|
||||
# 用 metrics 探测(单据最小)
|
||||
try_call(Cap.FINANCIAL,
|
||||
lambda: tf.financials.metrics([_PROBE_SYMBOL], latest=True, as_dataframe=False),
|
||||
defaults(Cap.FINANCIAL))
|
||||
|
||||
# adj_factor — 实际在 klines.ex_factors
|
||||
try_call(Cap.ADJ_FACTOR,
|
||||
lambda: tf.klines.ex_factors([_PROBE_SYMBOL], as_dataframe=False),
|
||||
defaults(Cap.ADJ_FACTOR))
|
||||
|
||||
# websocket 不在探测期试连接(成本太高且阻塞),按档位默认推断
|
||||
# 若 expert 的其他 cap 都通,则推断 websocket 也可用
|
||||
if (Cap.FINANCIAL in available and Cap.INTRADAY_BATCH in available):
|
||||
available[Cap.WEBSOCKET] = CapabilityLimits(
|
||||
subscribe=defaults(Cap.WEBSOCKET).get("subscribe", 100),
|
||||
)
|
||||
log.append("✓ websocket (inferred from expert tier)")
|
||||
|
||||
return CapabilitySet(available), log
|
||||
|
||||
|
||||
def detect_capabilities(force: bool = False) -> CapabilitySet:
|
||||
"""探测当前 API Key 的能力集。"""
|
||||
cache_path = settings.data_dir / _CAPSET_CACHE_FILE
|
||||
if not force and cache_path.exists():
|
||||
with cache_path.open(encoding="utf-8") as f:
|
||||
cached = json.load(f)
|
||||
# schema 版本校验:旧缓存或缺版本号 → 过期,丢弃后重新探测
|
||||
if cached.get("schema_version") == _CACHE_SCHEMA_VERSION:
|
||||
return _capset_from_json(cached)
|
||||
logger.info("capabilities 缓存 schema 版本过期(缓存=%s, 当前=%d), 重新探测",
|
||||
cached.get("schema_version"), _CACHE_SCHEMA_VERSION)
|
||||
|
||||
tiers = _load_tiers_yaml()
|
||||
if settings.use_free_mode:
|
||||
# 无 key —— 归 none 档(走 free-api 服务器,仅历史日K)
|
||||
capset = _tier_to_capset(tiers["none"])
|
||||
_persist(capset, "None", log=["无 API Key(无档 · free-api 服务器)"], missing=[], extras=[])
|
||||
return capset
|
||||
|
||||
# 有 API key — 真实探测
|
||||
try:
|
||||
capset, probe_log = _probe_real(tiers)
|
||||
# 判定档位:无效 key → none,免费 key → free,付费 → starter/pro/expert
|
||||
classified = _classify_tier(capset, tiers)
|
||||
if classified.is_invalid:
|
||||
# 无效 key(连单只日K都拿不到):归 none 档,标记要求清除 key
|
||||
capset = _tier_to_capset(tiers["none"])
|
||||
probe_log.append("⚠ Key 无效(单只日K也无法获取),判定为无档")
|
||||
_persist(capset, "None", log=probe_log, missing=[], extras=[], invalid_key=True)
|
||||
return capset
|
||||
if classified.is_free:
|
||||
# 免费有效 key:能力按 free 档(= none 档能力,走 free-api 服务器)
|
||||
capset = _tier_to_capset(tiers["free"])
|
||||
_persist(capset, "Free", log=probe_log + ["✓ 免费有效 key(运行时走 free-api 服务器)"], missing=[], extras=[])
|
||||
return capset
|
||||
# 付费档(starter+) — 探测出的能力即为真实可用
|
||||
label, missing, extras = _compute_label_and_missing(capset, tiers)
|
||||
capset = _override_limits_with_detected_tier(capset, label, tiers)
|
||||
_persist(capset, label, log=probe_log, missing=missing, extras=extras)
|
||||
return capset
|
||||
except Exception as e:
|
||||
logger.exception("detect_capabilities failed; using none baseline: %s", e)
|
||||
capset = _tier_to_capset(tiers["none"])
|
||||
_persist(capset, "None(探测失败)", log=[f"探测失败:{e}"], missing=[], extras=[])
|
||||
return capset
|
||||
|
||||
|
||||
# ===== Tier 代表性 capability(signature caps)=====
|
||||
# 拥有**任意一个**即认作该档及以上。自上而下匹配。
|
||||
# 这套设计的好处:单个 capability 探测的 transient 失败不会把整体档位"误降"。
|
||||
TIER_SIGNATURES: dict[str, set[Cap]] = {
|
||||
"expert": {Cap.FINANCIAL, Cap.INTRADAY_BATCH, Cap.WEBSOCKET},
|
||||
"pro": {Cap.KLINE_MINUTE_BATCH, Cap.KLINE_MINUTE_BY_SYMBOL,
|
||||
Cap.INTRADAY, Cap.DEPTH5, Cap.DEPTH5_BATCH},
|
||||
"starter": {Cap.QUOTE_BATCH, Cap.KLINE_DAILY_BATCH,
|
||||
Cap.ADJ_FACTOR, Cap.QUOTE_POOL},
|
||||
# free / none 不需 signature — 由 _classify_tier 的分水岭逻辑判定
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class TierClassification:
|
||||
"""档位判定结果。
|
||||
|
||||
判定依据是"复权因子分水岭":
|
||||
- 连单只日K都没有 → 无效 key(is_invalid),归 none 档
|
||||
- 有单只日K、无复权因子 → 免费 key(is_free)
|
||||
- 有复权因子 → 付费档(starter+),具体档位由 signature 决定
|
||||
"""
|
||||
|
||||
tier: str # "none" / "free" / "starter" / "pro" / "expert"
|
||||
is_invalid: bool # 无效 key(连单只日K都拿不到)
|
||||
is_free: bool # 免费有效 key(有日K、无复权因子)
|
||||
|
||||
|
||||
def _classify_tier(capset: CapabilitySet, tiers: dict) -> TierClassification:
|
||||
"""根据探测出的能力集判定档位。
|
||||
|
||||
分水岭是 KLINE_DAILY_BY_SYMBOL(单只日K)与 ADJ_FACTOR(复权因子):
|
||||
- 无单只日K → none(无效 key)
|
||||
- 有日K无复权 → free(免费 key)
|
||||
- 有复权因子 → 走 signature 判定 starter/pro/expert
|
||||
"""
|
||||
held = set(capset.all().keys())
|
||||
|
||||
# 1) 连单只日K都没有 → 无效 key
|
||||
if Cap.KLINE_DAILY_BY_SYMBOL not in held:
|
||||
return TierClassification(tier="none", is_invalid=True, is_free=False)
|
||||
|
||||
# 2) 有日K但无复权因子 → 免费 key
|
||||
if Cap.ADJ_FACTOR not in held:
|
||||
return TierClassification(tier="free", is_invalid=False, is_free=True)
|
||||
|
||||
# 3) 有复权因子 → 付费档,按 signature 自上而下判定
|
||||
if held & TIER_SIGNATURES["expert"]:
|
||||
base = "expert"
|
||||
elif held & TIER_SIGNATURES["pro"]:
|
||||
base = "pro"
|
||||
elif held & TIER_SIGNATURES["starter"]:
|
||||
base = "starter"
|
||||
else:
|
||||
# 有复权因子但无任何代表能力 — 兜底为 starter(复权本身是 starter 特征)
|
||||
base = "starter"
|
||||
return TierClassification(tier=base, is_invalid=False, is_free=False)
|
||||
|
||||
# 补丁友好命名(label 后缀用)
|
||||
_CAP_ALIASES: dict[Cap, str] = {
|
||||
Cap.KLINE_MINUTE_BATCH: "分钟K",
|
||||
Cap.KLINE_MINUTE_BY_SYMBOL: "分钟K",
|
||||
Cap.INTRADAY: "分时",
|
||||
Cap.INTRADAY_BATCH: "批量分时",
|
||||
Cap.DEPTH5: "五档",
|
||||
Cap.DEPTH5_BATCH: "批量五档",
|
||||
Cap.WEBSOCKET: "WS",
|
||||
Cap.FINANCIAL: "财务",
|
||||
Cap.ADJ_FACTOR: "复权",
|
||||
Cap.QUOTE_BATCH: "批量行情",
|
||||
Cap.QUOTE_POOL: "标的池",
|
||||
Cap.KLINE_DAILY_BATCH: "日K批量",
|
||||
}
|
||||
|
||||
|
||||
def _override_limits_with_detected_tier(
|
||||
capset: CapabilitySet, label: str, tiers: dict,
|
||||
) -> CapabilitySet:
|
||||
"""探测完成后,用判档对应的 limits 覆盖每个 cap 的速率/批量。
|
||||
|
||||
判档前每个 cap 用的是"所有档默认值的并集"(为了不漏数据),
|
||||
判档后才知道用户真实档位,limits 用该档的实际值更准。
|
||||
label 可能是 "Pro" / "Pro + 分钟K" / "Pro+" 等组合形式 — 取第一个词当作基线档名。
|
||||
"""
|
||||
base_name = label.split()[0].split("+")[0].strip().lower() # "Pro + 分钟K" → "pro"
|
||||
tier_limits = tiers.get(base_name, {})
|
||||
new_caps: dict[Cap, CapabilityLimits] = {}
|
||||
for cap, _old_lim in capset.all().items():
|
||||
spec = tier_limits.get(cap.value)
|
||||
if spec:
|
||||
new_caps[cap] = CapabilityLimits(
|
||||
rpm=spec.get("rpm"),
|
||||
batch=spec.get("batch"),
|
||||
subscribe=spec.get("subscribe"),
|
||||
)
|
||||
else:
|
||||
# 不在该档定义里(extras),用 expert 档兜底(最宽松)
|
||||
expert_spec = tiers.get("expert", {}).get(cap.value, {})
|
||||
new_caps[cap] = CapabilityLimits(
|
||||
rpm=expert_spec.get("rpm"),
|
||||
batch=expert_spec.get("batch"),
|
||||
subscribe=expert_spec.get("subscribe"),
|
||||
)
|
||||
return CapabilitySet(new_caps)
|
||||
|
||||
|
||||
def _tier_caps_set(tiers: dict, tier_name: str) -> set[Cap]:
|
||||
"""读 tiers.yaml 的某档定义,转为 Cap 集合。"""
|
||||
return {Cap(c) for c in tiers.get(tier_name, {}).keys() if c in {x.value for x in Cap}}
|
||||
|
||||
|
||||
def _compute_label_and_missing(
|
||||
capset: CapabilitySet, tiers: dict,
|
||||
) -> tuple[str, list[str], list[str]]:
|
||||
"""返回 (label, missing_caps, extra_caps)。
|
||||
|
||||
label:档位标签。
|
||||
missing_caps:本档**应有但未探测到**的 capability(用于诊断:可能是探测 bug 或权限丢失)。
|
||||
extra_caps:超出本档的额外 capability(自定义组合)。
|
||||
"""
|
||||
held = set(capset.all().keys())
|
||||
|
||||
# 1) 完全匹配 — 干净命中某档
|
||||
for tier_name in ["free", "starter", "pro", "expert"]:
|
||||
if held == _tier_caps_set(tiers, tier_name):
|
||||
return tier_name.capitalize(), [], []
|
||||
|
||||
# 2) 按 signature 自上而下判档
|
||||
if held & TIER_SIGNATURES["expert"]:
|
||||
base = "expert"
|
||||
elif held & TIER_SIGNATURES["pro"]:
|
||||
base = "pro"
|
||||
elif held & TIER_SIGNATURES["starter"]:
|
||||
base = "starter"
|
||||
else:
|
||||
base = "free"
|
||||
|
||||
base_caps = _tier_caps_set(tiers, base)
|
||||
missing = sorted(c.value for c in (base_caps - held))
|
||||
extras = base_caps and (held - base_caps) or set() # extras 是超出该档的部分
|
||||
|
||||
# 实际超出 = held 中"既不属于本档、也不属于本档下方任何档"的 cap
|
||||
# 简化:extras = held - base_caps
|
||||
extras_set = held - base_caps
|
||||
|
||||
# 3) 拼 label
|
||||
if not extras_set:
|
||||
# 完全在本档内(可能缺一两项 — 由 missing 反映)
|
||||
return base.capitalize(), missing, []
|
||||
|
||||
# 补丁过多 → 用 "≈" 形式
|
||||
if len(extras_set) > 3:
|
||||
return f"{base.capitalize()}+", missing, sorted(c.value for c in extras_set)
|
||||
|
||||
suffix = sorted({_CAP_ALIASES.get(e, str(e)) for e in extras_set})
|
||||
return f"{base.capitalize()} + " + " + ".join(suffix), missing, sorted(c.value for c in extras_set)
|
||||
|
||||
|
||||
def _compute_label(capset: CapabilitySet, tiers: dict) -> str:
|
||||
"""对外简化签名 — 只要 label。"""
|
||||
label, _missing, _extras = _compute_label_and_missing(capset, tiers)
|
||||
return label
|
||||
|
||||
|
||||
def _persist(
|
||||
capset: CapabilitySet,
|
||||
label: str,
|
||||
log: list[str] | None = None,
|
||||
missing: list[str] | None = None,
|
||||
extras: list[str] | None = None,
|
||||
invalid_key: bool = False,
|
||||
) -> None:
|
||||
settings.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
cache_path = settings.data_dir / _CAPSET_CACHE_FILE
|
||||
payload = {
|
||||
"schema_version": _CACHE_SCHEMA_VERSION,
|
||||
"label": label,
|
||||
"capabilities": capset.to_dict(),
|
||||
"probe_log": log or [],
|
||||
"missing_caps": missing or [], # 本档应有但未探测到
|
||||
"extras_caps": extras or [], # 超出本档的额外能力
|
||||
"invalid_key": invalid_key, # 探测出的 key 无效(连单只日K都拿不到)
|
||||
}
|
||||
with cache_path.open("w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def _capset_from_json(data: dict[str, Any]) -> CapabilitySet:
|
||||
caps: dict[Cap, CapabilityLimits] = {}
|
||||
for cap_name, lim in data.get("capabilities", {}).items():
|
||||
try:
|
||||
cap = Cap(cap_name)
|
||||
except ValueError:
|
||||
continue
|
||||
caps[cap] = CapabilityLimits(
|
||||
rpm=lim.get("rpm"),
|
||||
batch=lim.get("batch"),
|
||||
subscribe=lim.get("subscribe"),
|
||||
)
|
||||
return CapabilitySet(caps)
|
||||
|
||||
|
||||
def tier_label() -> str:
|
||||
cache_path = settings.data_dir / _CAPSET_CACHE_FILE
|
||||
if cache_path.exists():
|
||||
with cache_path.open(encoding="utf-8") as f:
|
||||
return json.load(f).get("label", "Unknown")
|
||||
return "Unknown"
|
||||
|
||||
|
||||
def probe_log() -> list[str]:
|
||||
cache_path = settings.data_dir / _CAPSET_CACHE_FILE
|
||||
if cache_path.exists():
|
||||
with cache_path.open(encoding="utf-8") as f:
|
||||
return json.load(f).get("probe_log", [])
|
||||
return []
|
||||
|
||||
|
||||
def missing_caps() -> list[str]:
|
||||
"""本档应有但未探测到的 capability — 通常意味着探测有 bug 或权限边界。"""
|
||||
cache_path = settings.data_dir / _CAPSET_CACHE_FILE
|
||||
if cache_path.exists():
|
||||
with cache_path.open(encoding="utf-8") as f:
|
||||
return json.load(f).get("missing_caps", [])
|
||||
return []
|
||||
|
||||
|
||||
def extras_caps() -> list[str]:
|
||||
cache_path = settings.data_dir / _CAPSET_CACHE_FILE
|
||||
if cache_path.exists():
|
||||
with cache_path.open(encoding="utf-8") as f:
|
||||
return json.load(f).get("extras_caps", [])
|
||||
return []
|
||||
|
||||
|
||||
def is_invalid_key() -> bool:
|
||||
"""最近一次探测是否判定 key 无效(连单只日K都拿不到)。
|
||||
|
||||
settings 层据此清除已存的 key,避免乱填的 key 被持久化。
|
||||
"""
|
||||
cache_path = settings.data_dir / _CAPSET_CACHE_FILE
|
||||
if cache_path.exists():
|
||||
with cache_path.open(encoding="utf-8") as f:
|
||||
return bool(json.load(f).get("invalid_key", False))
|
||||
return False
|
||||
|
||||
|
||||
def base_tier_name() -> str:
|
||||
"""当前档位的基础名(小写): none / free / starter / pro / expert。
|
||||
|
||||
供 client 层判断"是否走 free-api 服务器"(none/free → free 服务器)。
|
||||
"""
|
||||
label = tier_label()
|
||||
return label.split()[0].split("+")[0].strip().lower()
|
||||
@@ -0,0 +1,158 @@
|
||||
"""标的池(Universe)定义(§6.3)。
|
||||
|
||||
Phase 1 实现:
|
||||
- 常用指数成份(沪深 300 / 中证 500 / 上证 50)用数据源 `quote.pool` 端点拉取并缓存
|
||||
- 全 A 通过 instruments.batch 获取
|
||||
- 自选池 = 用户的 watchlist
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.config import settings
|
||||
from app.tickflow.client import get_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PoolId = Literal["CSI300", "CSI500", "SSE50", "CN_Equity_A", "CN_Index", "watchlist"]
|
||||
|
||||
# 数据源 universe id 是内部命名(见 tf.universes.list())。
|
||||
# 没有官方对照表,启动时按名称模糊匹配从 universes.list() 里找。
|
||||
# 常见名:沪深300 / 中证500 / 上证50 / 全 A
|
||||
_POOL_NAME_HINTS = {
|
||||
"CSI300": ["沪深300", "HS300", "CSI300"],
|
||||
"CSI500": ["中证500", "ZZ500", "CSI500"],
|
||||
"SSE50": ["上证50", "SH50", "SSE50"],
|
||||
}
|
||||
|
||||
|
||||
def _find_universe_id(hints: list[str]) -> str | None:
|
||||
"""从 universes.list() 里按 name/id 子串匹配找一个 universe id。"""
|
||||
try:
|
||||
tf = get_client()
|
||||
unis = tf.universes.list()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("universes.list failed: %s", e)
|
||||
return None
|
||||
for u in unis or []:
|
||||
item = u if isinstance(u, dict) else {"id": getattr(u, "id", ""), "name": getattr(u, "name", "")}
|
||||
haystack = (item.get("id", "") + " " + item.get("name", "")).lower()
|
||||
for h in hints:
|
||||
if h.lower() in haystack:
|
||||
return item["id"]
|
||||
return None
|
||||
|
||||
|
||||
def _pool_cache_path(pool_id: str) -> Path:
|
||||
return settings.data_dir / "pools" / f"{pool_id}.parquet"
|
||||
|
||||
|
||||
def get_pool(pool_id: PoolId, refresh: bool = False) -> list[str]:
|
||||
"""返回标的池里的 symbol 列表。"""
|
||||
if pool_id == "watchlist":
|
||||
return _load_watchlist()
|
||||
|
||||
cache = _pool_cache_path(pool_id)
|
||||
if cache.exists() and not refresh:
|
||||
df = pl.read_parquet(cache)
|
||||
return df["symbol"].to_list()
|
||||
|
||||
symbols = _fetch_pool(pool_id)
|
||||
if symbols:
|
||||
cache.parent.mkdir(parents=True, exist_ok=True)
|
||||
pl.DataFrame({"symbol": symbols, "as_of": [date.today()] * len(symbols)}).write_parquet(cache)
|
||||
return symbols
|
||||
|
||||
|
||||
def _fetch_pool(pool_id: PoolId) -> list[str]:
|
||||
"""从数据源拉取池成份。
|
||||
|
||||
实现:先用 universes.list 找到 universe id,再 quotes.get_by_universes 拉成份。
|
||||
"""
|
||||
tf = get_client()
|
||||
|
||||
if pool_id in _POOL_NAME_HINTS:
|
||||
uid = _find_universe_id(_POOL_NAME_HINTS[pool_id])
|
||||
if not uid:
|
||||
logger.warning("无法在数据源 universes 列表里匹配到 %s", pool_id)
|
||||
return []
|
||||
try:
|
||||
df = tf.quotes.get_by_universes([uid], as_dataframe=True)
|
||||
if df is not None and len(df) > 0 and "symbol" in df.columns:
|
||||
return df["symbol"].astype(str).tolist()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("fetch pool %s via universe %s failed: %s", pool_id, uid, e)
|
||||
|
||||
if pool_id == "CN_Equity_A":
|
||||
# 全 A — 优先直接用 CN_Equity_A universe (包含沪深京三市)
|
||||
uid = _find_universe_id(["CN_Equity_A", "沪深京A股", "全A"])
|
||||
if uid:
|
||||
try:
|
||||
df = tf.quotes.get_by_universes([uid], as_dataframe=True)
|
||||
if df is not None and len(df) > 0 and "symbol" in df.columns:
|
||||
return sorted(set(df["symbol"].astype(str).tolist()))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("fetch CN_Equity_A via universe %s failed: %s", uid, e)
|
||||
|
||||
# fallback: 聚合申万一级行业 (覆盖度较低, 缺北交所/新股)
|
||||
try:
|
||||
unis = tf.universes.list()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("universes.list failed: %s", e)
|
||||
unis = []
|
||||
sw1_ids = []
|
||||
for u in unis or []:
|
||||
item = u if isinstance(u, dict) else {"id": getattr(u, "id", "")}
|
||||
uid = item.get("id", "")
|
||||
if "SW1_" in uid:
|
||||
sw1_ids.append(uid)
|
||||
if sw1_ids:
|
||||
try:
|
||||
df = tf.quotes.get_by_universes(sw1_ids, as_dataframe=True)
|
||||
if df is not None and "symbol" in df.columns:
|
||||
return sorted(set(df["symbol"].astype(str).tolist()))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("aggregate SW1 fetch failed: %s", e)
|
||||
|
||||
if pool_id == "CN_Index":
|
||||
uid = _find_universe_id(["CN_Index", "沪深指数", "指数"])
|
||||
ids = [uid] if uid else ["CN_Index"]
|
||||
try:
|
||||
df = tf.quotes.get_by_universes(ids, as_dataframe=True)
|
||||
if df is not None and len(df) > 0 and "symbol" in df.columns:
|
||||
return sorted(set(df["symbol"].astype(str).tolist()))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("fetch CN_Index via universe %s failed: %s", ids, e)
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def _load_watchlist() -> list[str]:
|
||||
"""读取用户自选(由 watchlist service 维护)。"""
|
||||
path = settings.data_dir / "user_data" / "watchlist.parquet"
|
||||
if not path.exists():
|
||||
return []
|
||||
df = pl.read_parquet(path)
|
||||
if df.is_empty() or "symbol" not in df.columns:
|
||||
return []
|
||||
return df["symbol"].to_list()
|
||||
|
||||
|
||||
# 兜底:Free 用户/无 API 时给一个小型可用集合,让 UI 不至于空白
|
||||
DEMO_SYMBOLS = [
|
||||
"600000.SH", # 浦发银行
|
||||
"600036.SH", # 招商银行
|
||||
"600519.SH", # 贵州茅台
|
||||
"601318.SH", # 中国平安
|
||||
"601398.SH", # 工商银行
|
||||
"000001.SZ", # 平安银行
|
||||
"000333.SZ", # 美的集团
|
||||
"000651.SZ", # 格力电器
|
||||
"000858.SZ", # 五粮液
|
||||
"002594.SZ", # 比亚迪
|
||||
]
|
||||
@@ -0,0 +1,988 @@
|
||||
"""Repository 层(§7.4)。
|
||||
|
||||
数据分层:
|
||||
- DuckDB 视图: 冷查询(统计、元数据、用户自定义SQL)
|
||||
- Polars 缓存: 热路径(enriched 最新日 ~5500行 + instruments ~5500行)
|
||||
- Polars scan_parquet: 分钟K/历史日K (predicate pushdown)
|
||||
|
||||
缓存生命周期:
|
||||
- startup 时不加载(数据可能为空)
|
||||
- pipeline 完成后调用 refresh_cache()
|
||||
- 服务层通过 get_enriched_latest() / get_instruments() 获取缓存
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import polars as pl
|
||||
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DataStore:
|
||||
"""唯一的存储入口 — 进程启动时创建。"""
|
||||
|
||||
def __init__(self, data_dir: Path | None = None) -> None:
|
||||
self.data_dir = Path(data_dir or settings.data_dir)
|
||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 关键子目录(§7.2)
|
||||
for sub in (
|
||||
"kline_daily",
|
||||
"kline_daily_enriched",
|
||||
"kline_index_daily",
|
||||
"kline_index_enriched",
|
||||
"kline_minute",
|
||||
"adj_factor",
|
||||
"financials",
|
||||
"instruments",
|
||||
"instruments_index",
|
||||
"instruments_ext",
|
||||
"kline_ext",
|
||||
"pools",
|
||||
"backtest_results",
|
||||
"screener_results",
|
||||
"ai_cache",
|
||||
"user_data",
|
||||
"depth5",
|
||||
):
|
||||
(self.data_dir / sub).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 财务数据子目录
|
||||
for sub in ("metrics", "income", "balance_sheet", "cash_flow"):
|
||||
(self.data_dir / "financials" / sub).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# DuckDB 内存模式 — 不建 .db 文件(§7.1)
|
||||
self.db = duckdb.connect(database=":memory:")
|
||||
self._register_views()
|
||||
|
||||
def _register_views(self) -> None:
|
||||
"""把 Parquet 目录挂载为 DuckDB 视图(§7.3)。"""
|
||||
d = self.data_dir.as_posix()
|
||||
statements = [
|
||||
f"""CREATE OR REPLACE VIEW kline_daily AS
|
||||
SELECT * FROM read_parquet('{d}/kline_daily/**/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW kline_enriched AS
|
||||
SELECT * FROM read_parquet('{d}/kline_daily_enriched/**/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW kline_index_daily AS
|
||||
SELECT * FROM read_parquet('{d}/kline_index_daily/**/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW kline_index_enriched AS
|
||||
SELECT * FROM read_parquet('{d}/kline_index_enriched/**/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW kline_minute AS
|
||||
SELECT * FROM read_parquet('{d}/kline_minute/**/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW adj_factor AS
|
||||
SELECT * FROM read_parquet('{d}/adj_factor/**/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW instruments AS
|
||||
SELECT * FROM read_parquet('{d}/instruments/**/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW instruments_index AS
|
||||
SELECT * FROM read_parquet('{d}/instruments_index/**/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW instruments_ext AS
|
||||
SELECT * FROM read_parquet('{d}/instruments_ext/**/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW kline_ext AS
|
||||
SELECT * FROM read_parquet('{d}/kline_ext/**/*.parquet', union_by_name=true)""",
|
||||
# 财务数据视图
|
||||
f"""CREATE OR REPLACE VIEW financials_metrics AS
|
||||
SELECT * FROM read_parquet('{d}/financials/metrics/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW financials_income AS
|
||||
SELECT * FROM read_parquet('{d}/financials/income/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW financials_balance_sheet AS
|
||||
SELECT * FROM read_parquet('{d}/financials/balance_sheet/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW financials_cash_flow AS
|
||||
SELECT * FROM read_parquet('{d}/financials/cash_flow/*.parquet', union_by_name=true)""",
|
||||
# 五档盘口 sealed 真假涨停(独立旁路存储,不进 enriched)
|
||||
f"""CREATE OR REPLACE VIEW depth5 AS
|
||||
SELECT * FROM read_parquet('{d}/depth5/**/*.parquet', union_by_name=true)""",
|
||||
]
|
||||
for sql in statements:
|
||||
try:
|
||||
self.db.execute(sql)
|
||||
except duckdb.IOException:
|
||||
logger.debug("view registration skipped (no parquet yet): %s", sql[:60])
|
||||
|
||||
|
||||
class KlineRepository:
|
||||
"""日 K / 分钟 K 的读写入口。"""
|
||||
|
||||
def __init__(self, store: DataStore) -> None:
|
||||
self.store = store
|
||||
self.db = store.db
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# ---- Polars 缓存 ----
|
||||
self._enriched_cache: pl.DataFrame | None = None # 最新一天 (~5500行)
|
||||
self._enriched_cache_date: date | None = None
|
||||
self._live_agg_cache: pl.DataFrame | None = None # 预计算聚合表 (~5500行)
|
||||
self._live_agg_cache_date: date | None = None
|
||||
self._instruments_cache: pl.DataFrame | None = None
|
||||
# 完整 enriched 历史 (含所有指标, 供 filter_history 策略使用)
|
||||
self._enriched_history_cache: pl.DataFrame | None = None # ~100万行
|
||||
self._enriched_history_start: date | None = None
|
||||
self._index_instruments_cache: pl.DataFrame | None = None
|
||||
|
||||
# parquet glob 路径
|
||||
self._enriched_glob = str(store.data_dir / "kline_daily_enriched" / "**" / "*.parquet")
|
||||
self._index_enriched_glob = str(store.data_dir / "kline_index_enriched" / "**" / "*.parquet")
|
||||
self._minute_glob = str(store.data_dir / "kline_minute" / "**" / "*.parquet")
|
||||
self._inst_glob = str(store.data_dir / "instruments" / "**" / "*.parquet")
|
||||
self._index_inst_glob = str(store.data_dir / "instruments_index" / "**" / "*.parquet")
|
||||
|
||||
def execute_all(self, sql: str, params: list | None = None) -> list[tuple]:
|
||||
"""线程安全的 SELECT → fetchall。DuckDB 单 connection 非线程安全，所有读路径须走此方法。"""
|
||||
with self._lock:
|
||||
return self.db.execute(sql, params or []).fetchall()
|
||||
|
||||
def execute_one(self, sql: str, params: list | None = None) -> tuple | None:
|
||||
"""线程安全的 SELECT → fetchone。"""
|
||||
with self._lock:
|
||||
return self.db.execute(sql, params or []).fetchone()
|
||||
|
||||
# ================================================================
|
||||
# Polars 缓存管理
|
||||
# ================================================================
|
||||
|
||||
def refresh_cache(self) -> None:
|
||||
"""刷新 Polars 缓存。在 pipeline 完成后、服务启动时调用。"""
|
||||
self._refresh_instruments()
|
||||
self._refresh_index_instruments()
|
||||
self._refresh_enriched()
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
"""清空所有 Polars 内存缓存。
|
||||
|
||||
与 refresh_cache 的区别: refresh_cache 在磁盘无数据时会提前 return,
|
||||
导致内存里的旧缓存残留 (clear 数据后看板仍显示旧数据的根因)。
|
||||
本方法无条件清空, 供清除数据/重置场景调用。
|
||||
"""
|
||||
self._enriched_cache = None
|
||||
self._enriched_cache_date = None
|
||||
self._enriched_history_cache = None
|
||||
self._enriched_history_start = None
|
||||
self._live_agg_cache = None
|
||||
self._live_agg_cache_date = None
|
||||
self._instruments_cache = None
|
||||
self._index_instruments_cache = None
|
||||
|
||||
def _refresh_enriched(self) -> None:
|
||||
"""从 parquet 加载 enriched 最新日到内存 + 构建聚合表。
|
||||
|
||||
enriched parquet 仅存 14 列基础数据。启动时读入历史数据并即时计算完整指标，
|
||||
将结果缓存在内存中供各服务使用。
|
||||
|
||||
优化: 扩大历史读取范围, 同时缓存完整历史 (含指标), 供 filter_history 策略直接复用。
|
||||
"""
|
||||
try:
|
||||
latest = self._latest_enriched_date_duckdb()
|
||||
if not latest:
|
||||
# 磁盘已无数据: 必须清空内存缓存, 否则旧数据会残留
|
||||
# (清数据后看板仍显示旧数据的根因)
|
||||
self.clear_cache()
|
||||
return
|
||||
|
||||
# Step 1: 直接读最新日期的分区文件 (仅 14 列)
|
||||
enriched_dir = self.store.data_dir / "kline_daily_enriched"
|
||||
ds = latest.isoformat() if hasattr(latest, "isoformat") else str(latest)
|
||||
target_parquet = enriched_dir / f"date={ds}" / "part.parquet"
|
||||
|
||||
if not target_parquet.exists():
|
||||
return
|
||||
|
||||
df_latest = pl.read_parquet(target_parquet)
|
||||
if df_latest.is_empty():
|
||||
return
|
||||
|
||||
# Step 2: 读近 300 天 14 列数据 → compute → filter(latest) → 缓存
|
||||
# 300 日历天 ≈ 210 交易日, 覆盖 filter_history 最大 lookback(90) + warmup(60)
|
||||
try:
|
||||
from datetime import timedelta
|
||||
from app.indicators.pipeline import compute_indicators, compute_signals, compute_limit_signals
|
||||
start_full = latest - timedelta(days=300)
|
||||
read_cols = [c for c in ["symbol", "date", "open", "high", "low", "close",
|
||||
"volume", "amount", "raw_close", "raw_high", "raw_low"]
|
||||
if c in df_latest.columns]
|
||||
lf = (
|
||||
pl.scan_parquet(self._enriched_glob)
|
||||
.filter(pl.col("date") >= start_full)
|
||||
.sort(["symbol", "date"])
|
||||
)
|
||||
df_hist = lf.select(read_cols).collect()
|
||||
if not df_hist.is_empty():
|
||||
instruments = self._instruments_cache if self._instruments_cache is not None else pl.DataFrame()
|
||||
df_full = compute_indicators(df_hist)
|
||||
df_full = compute_signals(df_full)
|
||||
if instruments is not None and not instruments.is_empty():
|
||||
df_full = compute_limit_signals(df_full, instruments)
|
||||
|
||||
# JOIN instruments 到完整历史 (filter_history/basic_filter 需要 name/股本等列)
|
||||
if instruments is not None and not instruments.is_empty():
|
||||
inst_cols = [c for c in ["name", "total_shares", "float_shares"]
|
||||
if c in instruments.columns and c not in df_full.columns]
|
||||
if inst_cols:
|
||||
df_full = df_full.join(
|
||||
instruments.select(["symbol", *inst_cols]).unique(subset=["symbol"]),
|
||||
on="symbol",
|
||||
how="left",
|
||||
)
|
||||
|
||||
# 缓存完整历史 (含指标+必要基础信息) 供 filter_history/backtest 直接复用
|
||||
self._enriched_history_cache = df_full
|
||||
self._enriched_history_start = df_full["date"].min()
|
||||
logger.info("enriched 历史缓存: %d rows, %s ~ %s",
|
||||
len(df_full), self._enriched_history_start, latest)
|
||||
|
||||
# 只取最新一天作为 enriched_cache
|
||||
df_today = df_full.filter(pl.col("date") == latest)
|
||||
if not df_today.is_empty():
|
||||
self._enriched_cache = df_today
|
||||
self._enriched_cache_date = latest
|
||||
# 构建盘中递推基准: 若最新分区是今天的实时盘中数据,
|
||||
# 递推状态必须停在上一交易日, 不能把今天作为“昨日”。
|
||||
self._build_live_agg(self._live_agg_baseline_date(latest))
|
||||
logger.info("enriched 缓存已计算: %d 只, 日期 %s (即时计算)", len(df_today), latest)
|
||||
return
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("enriched 即时计算失败, 使用原始 14 列缓存: %s", e)
|
||||
|
||||
# 降级: 直接使用 14 列数据 + 构建 live_agg
|
||||
self._enriched_cache = df_latest
|
||||
self._enriched_cache_date = latest
|
||||
self._build_live_agg(self._live_agg_baseline_date(latest))
|
||||
|
||||
logger.info("enriched 缓存已加载: %d 只, 日期 %s", len(df_latest), latest)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("enriched 缓存刷新失败: %s", e)
|
||||
|
||||
def _build_live_agg(self, latest: date) -> None:
|
||||
"""从 OHLCV 即时计算递推状态 + 窗口聚合, 构建盘中实时聚合表。
|
||||
|
||||
优化: 优先使用 _enriched_history_cache (启动时已计算), 避免重复 compute_indicators。
|
||||
"""
|
||||
from datetime import timedelta
|
||||
from app.indicators.pipeline import _ema_alpha
|
||||
|
||||
start_60d = latest - timedelta(days=90) # 日历90天 ≈ 60个交易日
|
||||
|
||||
# 优先使用已有的历史缓存 (避免重复 scan_parquet + compute_indicators)
|
||||
if self._enriched_history_cache is not None and not self._enriched_history_cache.is_empty():
|
||||
hist_all = self._enriched_history_cache
|
||||
if "date" in hist_all.columns and hist_all["date"].min() <= start_60d:
|
||||
# 从历史缓存中提取所需列 (历史缓存已有指标列)
|
||||
base_cols = ["symbol", "date", "open", "high", "low", "close", "volume",
|
||||
"raw_close", "raw_high", "raw_low"]
|
||||
needed = [c for c in base_cols if c in hist_all.columns]
|
||||
df_hist = hist_all.filter(
|
||||
(pl.col("date") >= start_60d) & (pl.col("date") <= latest)
|
||||
).select(needed).sort(["symbol", "date"])
|
||||
|
||||
# 用历史缓存的指标列提取最新日状态 (无需再次 compute_indicators)
|
||||
state_source = hist_all.filter(pl.col("date") == latest)
|
||||
|
||||
state_cols = [
|
||||
"symbol",
|
||||
"ema5", "ema10", "ema20", "ema30", "ema60",
|
||||
"macd_dea",
|
||||
"kdj_k", "kdj_d",
|
||||
"atr_14",
|
||||
"close", "high", "low",
|
||||
"annual_vol_20d",
|
||||
]
|
||||
existing_state = [c for c in state_cols if c in state_source.columns]
|
||||
agg_a = state_source.select(existing_state)
|
||||
else:
|
||||
df_hist = pl.DataFrame()
|
||||
agg_a = pl.DataFrame()
|
||||
else:
|
||||
# 降级: 读 parquet + compute_indicators
|
||||
df_hist, agg_a = self._build_live_agg_from_parquet(latest, start_60d)
|
||||
|
||||
if df_hist.is_empty():
|
||||
self._live_agg_cache = pl.DataFrame()
|
||||
self._live_agg_cache_date = None
|
||||
return
|
||||
|
||||
if agg_a.is_empty():
|
||||
self._live_agg_cache = pl.DataFrame()
|
||||
self._live_agg_cache_date = None
|
||||
return
|
||||
|
||||
# 单独计算 _ema12 / _ema26 (compute_indicators 内部会 drop 掉)
|
||||
df_ema = df_hist.sort(["symbol", "date"]).with_columns([
|
||||
pl.col("close").ewm_mean(alpha=_ema_alpha(12), adjust=False).over("symbol").alias("_ema12"),
|
||||
pl.col("close").ewm_mean(alpha=_ema_alpha(26), adjust=False).over("symbol").alias("_ema26"),
|
||||
]).filter(pl.col("date") == latest).select("symbol", "_ema12", "_ema26")
|
||||
|
||||
agg_a = agg_a.join(df_ema, on="symbol", how="inner")
|
||||
|
||||
# 单独计算 RSI 状态列 (compute_indicators 内部会 drop 掉)
|
||||
df_rsi_base = df_hist.sort(["symbol", "date"]).with_columns(
|
||||
pl.col("close").diff().over("symbol").alias("_daily_delta")
|
||||
)
|
||||
gain = pl.when(pl.col("_daily_delta") > 0).then(pl.col("_daily_delta")).otherwise(0.0)
|
||||
loss = pl.when(pl.col("_daily_delta") < 0).then(-pl.col("_daily_delta")).otherwise(0.0)
|
||||
rsi_exprs = []
|
||||
for n in (6, 14, 24):
|
||||
a = 1.0 / n
|
||||
rsi_exprs.append(gain.ewm_mean(alpha=a, adjust=False).over("symbol").alias(f"_rsi_avg_gain_{n}"))
|
||||
rsi_exprs.append(loss.ewm_mean(alpha=a, adjust=False).over("symbol").alias(f"_rsi_avg_loss_{n}"))
|
||||
df_rsi = (
|
||||
df_rsi_base
|
||||
.with_columns(rsi_exprs)
|
||||
.filter(pl.col("date") == latest)
|
||||
.select("symbol", *[f"_rsi_avg_gain_{n}" for n in (6, 14, 24)],
|
||||
*[f"_rsi_avg_loss_{n}" for n in (6, 14, 24)])
|
||||
)
|
||||
agg_a = agg_a.join(df_rsi, on="symbol", how="inner")
|
||||
|
||||
# 前复权因子: adj_factor = close(复权) / raw_close(原始)
|
||||
if "raw_close" in df_hist.columns:
|
||||
adj_factor_df = (
|
||||
df_hist.filter(pl.col("date") == latest)
|
||||
.select("symbol", (pl.col("close") / pl.col("raw_close")).alias("_adj_factor"))
|
||||
)
|
||||
agg_a = agg_a.join(adj_factor_df, on="symbol", how="left")
|
||||
if "_adj_factor" in agg_a.columns:
|
||||
agg_a = agg_a.with_columns(pl.col("_adj_factor").fill_null(1.0))
|
||||
|
||||
# annual_vol_20d 递推状态: 最近 19 天日收益率的部分和 / 平方和
|
||||
df_daily_pct = (
|
||||
df_hist.sort(["symbol", "date"])
|
||||
.with_columns(
|
||||
pl.col("close").pct_change().over("symbol").alias("_daily_pct")
|
||||
)
|
||||
)
|
||||
df_vol = df_daily_pct.group_by("symbol").agg([
|
||||
pl.col("_daily_pct").tail(19).sum().alias("_vol_19d_pct_sum"),
|
||||
(pl.col("_daily_pct") ** 2).tail(19).sum().alias("_vol_19d_pct_sq_sum"),
|
||||
])
|
||||
agg_a = agg_a.join(df_vol, on="symbol", how="left")
|
||||
|
||||
# 昨日连板数: 从 enriched parquet 取 (用于增量计算同向 +1)
|
||||
lf = pl.scan_parquet(self._enriched_glob).filter(pl.col("date") == latest)
|
||||
consec_cols = [c for c in ["symbol", "consecutive_limit_ups", "consecutive_limit_downs"]
|
||||
if c in lf.collect_schema().names()]
|
||||
if len(consec_cols) == 3:
|
||||
consec_df = lf.select(consec_cols).collect()
|
||||
if not consec_df.is_empty():
|
||||
consec = consec_df.select(
|
||||
"symbol",
|
||||
pl.col("consecutive_limit_ups").alias("_prev_consec_up"),
|
||||
pl.col("consecutive_limit_downs").alias("_prev_consec_down"),
|
||||
)
|
||||
agg_a = agg_a.join(consec, on="symbol", how="left")
|
||||
|
||||
# B类: 按 symbol 分组聚合 — 窗口统计
|
||||
agg_b = (
|
||||
df_hist.sort(["symbol", "date"])
|
||||
.group_by("symbol")
|
||||
.agg([
|
||||
pl.col("close").tail(4).sum().alias("_ma5_partial_sum"),
|
||||
pl.col("close").tail(9).sum().alias("_ma10_partial_sum"),
|
||||
pl.col("close").tail(19).sum().alias("_ma20_partial_sum"),
|
||||
pl.col("close").tail(29).sum().alias("_ma30_partial_sum"),
|
||||
pl.col("close").tail(59).sum().alias("_ma60_partial_sum"),
|
||||
|
||||
pl.col("close").tail(19).sum().alias("_boll_partial_sum"),
|
||||
(pl.col("close").tail(19) ** 2).sum().alias("_boll_partial_sq_sum"),
|
||||
|
||||
pl.col("high").tail(59).max().alias("_high_59d"),
|
||||
pl.col("low").tail(59).min().alias("_low_59d"),
|
||||
|
||||
pl.col("close").tail(5).first().alias("_close_5d_ago"),
|
||||
pl.col("close").tail(10).first().alias("_close_10d_ago"),
|
||||
pl.col("close").tail(20).first().alias("_close_20d_ago"),
|
||||
pl.col("close").tail(30).first().alias("_close_30d_ago"),
|
||||
pl.col("close").tail(60).first().alias("_close_60d_ago"),
|
||||
|
||||
pl.col("volume").tail(4).sum().alias("_vol_ma5_partial_sum"),
|
||||
pl.col("volume").tail(9).sum().alias("_vol_ma10_partial_sum"),
|
||||
|
||||
pl.col("low").tail(8).min().alias("_kdj_8d_low"),
|
||||
pl.col("high").tail(8).max().alias("_kdj_8d_high"),
|
||||
|
||||
pl.col("close").tail(59).len().alias("_window_len"),
|
||||
])
|
||||
)
|
||||
|
||||
self._live_agg_cache = agg_a.join(agg_b, on="symbol", how="inner")
|
||||
self._live_agg_cache_date = latest
|
||||
|
||||
def _live_agg_baseline_date(self, latest: date) -> date:
|
||||
"""盘中递推基准日期。当天实时分区存在时使用上一可用交易日。"""
|
||||
if latest != date.today():
|
||||
return latest
|
||||
try:
|
||||
row = self.execute_one(
|
||||
"SELECT max(date) FROM kline_enriched WHERE date < ?",
|
||||
[latest],
|
||||
)
|
||||
if row and row[0]:
|
||||
d = row[0]
|
||||
return d if isinstance(d, date) else date.fromisoformat(str(d))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return latest
|
||||
|
||||
def _build_live_agg_from_parquet(self, latest: date, start_60d: date) -> tuple[pl.DataFrame, pl.DataFrame]:
|
||||
"""降级路径: 从 parquet 读取数据并计算指标 (当 _enriched_history_cache 不可用时)。"""
|
||||
from app.indicators.pipeline import compute_indicators
|
||||
|
||||
lf = (
|
||||
pl.scan_parquet(self._enriched_glob)
|
||||
.filter(pl.col("date") >= start_60d)
|
||||
.filter(pl.col("date") <= latest)
|
||||
.sort(["symbol", "date"])
|
||||
)
|
||||
|
||||
read_cols = [c for c in ["symbol", "date", "open", "high", "low", "close", "volume",
|
||||
"raw_close", "raw_high", "raw_low"]
|
||||
if c in lf.collect_schema().names()]
|
||||
df_hist = lf.select(read_cols).collect()
|
||||
|
||||
if df_hist.is_empty():
|
||||
return df_hist, pl.DataFrame()
|
||||
|
||||
df_with_indicators = compute_indicators(df_hist)
|
||||
|
||||
state_cols = [
|
||||
"symbol",
|
||||
"ema5", "ema10", "ema20", "ema30", "ema60",
|
||||
"macd_dea",
|
||||
"kdj_k", "kdj_d",
|
||||
"atr_14",
|
||||
"close", "high", "low",
|
||||
"annual_vol_20d",
|
||||
]
|
||||
existing_state = [c for c in state_cols if c in df_with_indicators.columns]
|
||||
agg_a = df_with_indicators.filter(pl.col("date") == latest).select(existing_state)
|
||||
|
||||
return df_hist, agg_a
|
||||
|
||||
def _refresh_instruments(self) -> None:
|
||||
"""加载 instruments 到内存。"""
|
||||
try:
|
||||
df = pl.scan_parquet(self._inst_glob).collect()
|
||||
if not df.is_empty():
|
||||
self._instruments_cache = df
|
||||
logger.info("instruments 缓存已加载: %d 只", len(df))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("instruments 缓存刷新失败: %s", e)
|
||||
|
||||
def _refresh_index_instruments(self) -> None:
|
||||
"""加载指数 instruments 到内存。"""
|
||||
try:
|
||||
df = pl.scan_parquet(self._index_inst_glob).collect()
|
||||
if not df.is_empty():
|
||||
self._index_instruments_cache = df
|
||||
logger.info("index instruments 缓存已加载: %d 只", len(df))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("index instruments 缓存刷新跳过: %s", e)
|
||||
|
||||
def get_enriched_latest(self) -> tuple[pl.DataFrame, date | None]:
|
||||
"""返回缓存的 enriched 最新日 DataFrame + 日期。如无缓存则懒加载。"""
|
||||
if self._enriched_cache is None:
|
||||
self._refresh_enriched()
|
||||
if self._enriched_cache is None:
|
||||
return pl.DataFrame(), self._enriched_cache_date
|
||||
return self._enriched_cache, self._enriched_cache_date
|
||||
|
||||
def get_enriched_history(self, target_date: date, lookback_days: int) -> pl.DataFrame | None:
|
||||
"""返回预计算的 enriched 历史数据 (仅 lookback 范围, 不含 warmup)。
|
||||
|
||||
warmup 部分在 _refresh_enriched 计算指标时已使用, 策略只需要最终的 lookback 窗口。
|
||||
返回 ~33万行 (90日历天) 而非 ~107万行, filter_history 策略的 group_by 快 20x+。
|
||||
"""
|
||||
cache = self._enriched_history_cache
|
||||
if cache is None or cache.is_empty():
|
||||
return None
|
||||
if "date" not in cache.columns:
|
||||
return None
|
||||
cache_max = cache["date"].max()
|
||||
cache_min = cache["date"].min()
|
||||
from datetime import timedelta
|
||||
# 验证缓存覆盖完整范围 (含 warmup)
|
||||
warmup_start = target_date - timedelta(days=(lookback_days + 60) * 2)
|
||||
if cache_min > warmup_start or cache_max < target_date:
|
||||
return None
|
||||
# 只返回 lookback 范围 (日历天数 ≈ 2/3 交易日, 足够覆盖)
|
||||
lookback_start = target_date - timedelta(days=lookback_days)
|
||||
return cache.filter((pl.col("date") >= lookback_start) & (pl.col("date") <= target_date))
|
||||
|
||||
def get_enriched_range(
|
||||
self,
|
||||
start: date,
|
||||
end: date,
|
||||
symbols: list[str] | None = None,
|
||||
columns: list[str] | None = None,
|
||||
) -> pl.DataFrame | None:
|
||||
"""从预计算 enriched 历史缓存返回完整区间；缓存不覆盖时返回 None。"""
|
||||
if self._enriched_history_cache is None:
|
||||
self._refresh_enriched()
|
||||
cache = self._enriched_history_cache
|
||||
if cache is None or cache.is_empty() or "date" not in cache.columns:
|
||||
return None
|
||||
|
||||
cache_min = cache["date"].min()
|
||||
cache_max = cache["date"].max()
|
||||
if cache_min > start or cache_max < end:
|
||||
return None
|
||||
|
||||
df = cache.filter((pl.col("date") >= start) & (pl.col("date") <= end))
|
||||
if symbols is not None:
|
||||
df = df.filter(pl.col("symbol").is_in(symbols))
|
||||
if columns and not df.is_empty():
|
||||
existing = [c for c in columns if c in df.columns]
|
||||
if "symbol" not in existing and "symbol" in df.columns:
|
||||
existing.insert(0, "symbol")
|
||||
if "date" not in existing and "date" in df.columns:
|
||||
existing.insert(1, "date")
|
||||
df = df.select(existing)
|
||||
return df.sort(["symbol", "date"])
|
||||
|
||||
def get_live_agg(self) -> pl.DataFrame:
|
||||
"""返回盘中实时指标预计算聚合表。如无缓存则懒加载。"""
|
||||
if self._live_agg_cache is None:
|
||||
self._refresh_enriched()
|
||||
if self._live_agg_cache is None:
|
||||
return pl.DataFrame()
|
||||
return self._live_agg_cache
|
||||
|
||||
def get_instruments(self) -> pl.DataFrame:
|
||||
"""返回缓存的 instruments DataFrame。如无缓存则懒加载。"""
|
||||
if self._instruments_cache is None:
|
||||
self._refresh_instruments()
|
||||
if self._instruments_cache is None:
|
||||
return pl.DataFrame()
|
||||
return self._instruments_cache
|
||||
|
||||
def get_index_instruments(self) -> pl.DataFrame:
|
||||
"""返回缓存的指数 instruments DataFrame。如无缓存则懒加载。"""
|
||||
if self._index_instruments_cache is None:
|
||||
self._refresh_index_instruments()
|
||||
if self._index_instruments_cache is None:
|
||||
return pl.DataFrame()
|
||||
return self._index_instruments_cache
|
||||
|
||||
def get_index_symbol_set(self) -> set[str]:
|
||||
"""返回已缓存指数 symbol 集合。"""
|
||||
df = self.get_index_instruments()
|
||||
if df.is_empty() or "symbol" not in df.columns:
|
||||
return set()
|
||||
return set(df["symbol"].cast(pl.Utf8).to_list())
|
||||
|
||||
def enriched_latest_date(self) -> date | None:
|
||||
"""返回缓存中的 enriched 最新日期。"""
|
||||
return self._enriched_cache_date
|
||||
|
||||
# ================================================================
|
||||
# 热路径: Polars 查询 (Chart / Screener / Signals / Intraday)
|
||||
# ================================================================
|
||||
|
||||
def get_daily(
|
||||
self,
|
||||
symbol: str,
|
||||
start: date,
|
||||
end: date,
|
||||
columns: list[str] | None = None,
|
||||
) -> pl.DataFrame:
|
||||
"""单股日K查询 — 从14列parquet读取后即时计算指标。"""
|
||||
from datetime import timedelta
|
||||
|
||||
# 扩展范围用于指标预热 (MA60 需要 ~60 交易日 ≈ 120 日历日)
|
||||
warmup_start = start - timedelta(days=150)
|
||||
|
||||
# 扫描14列 parquet
|
||||
df = self._scan_daily_symbol(symbol, warmup_start, end, None)
|
||||
if not df.is_empty():
|
||||
df = self._compute_enriched_range(df)
|
||||
|
||||
# 尝试用缓存数据覆盖最新日 (盘中更准确)
|
||||
cached, cache_date = self.get_enriched_latest()
|
||||
if not df.is_empty() and cached is not None and not cached.is_empty() and cache_date:
|
||||
if start <= cache_date <= end:
|
||||
cached_part = self._filter_cached(cached, symbol, None)
|
||||
if not cached_part.is_empty():
|
||||
df = df.filter(pl.col("date") != cache_date)
|
||||
common_cols = [c for c in df.columns if c in cached_part.columns]
|
||||
df = pl.concat([df.select(common_cols), cached_part.select(common_cols)])
|
||||
|
||||
# 裁剪到请求范围
|
||||
if not df.is_empty():
|
||||
df = df.filter((pl.col("date") >= start) & (pl.col("date") <= end))
|
||||
|
||||
if columns and not df.is_empty():
|
||||
existing = [c for c in columns if c in df.columns]
|
||||
df = df.select(existing)
|
||||
|
||||
return df
|
||||
|
||||
def get_daily_batch(
|
||||
self,
|
||||
symbols: list[str],
|
||||
start: date,
|
||||
end: date,
|
||||
columns: list[str] | None = None,
|
||||
) -> pl.DataFrame:
|
||||
"""批量日K查询。"""
|
||||
cached, cache_date = self.get_enriched_latest()
|
||||
if cached is not None and not cached.is_empty() and cache_date:
|
||||
if start >= cache_date:
|
||||
return self._filter_cached_batch(cached, symbols, columns)
|
||||
|
||||
# 回退 scan_parquet
|
||||
return self._scan_daily_batch(symbols, start, end, columns)
|
||||
|
||||
def get_index_daily(
|
||||
self,
|
||||
symbol: str,
|
||||
start: date,
|
||||
end: date,
|
||||
columns: list[str] | None = None,
|
||||
) -> pl.DataFrame:
|
||||
"""指数日K查询 — 从独立指数 enriched parquet 读取后即时计算通用指标。"""
|
||||
from datetime import timedelta
|
||||
|
||||
warmup_start = start - timedelta(days=150)
|
||||
df = self._scan_index_daily_symbol(symbol, warmup_start, end, None)
|
||||
if not df.is_empty():
|
||||
df = self._compute_index_enriched_range(df)
|
||||
df = df.filter((pl.col("date") >= start) & (pl.col("date") <= end))
|
||||
if columns and not df.is_empty():
|
||||
existing = [c for c in columns if c in df.columns]
|
||||
df = df.select(existing)
|
||||
return df
|
||||
|
||||
def get_minute(
|
||||
self,
|
||||
symbol: str,
|
||||
trade_date: date,
|
||||
) -> pl.DataFrame:
|
||||
"""分钟K查询 — Polars scan_parquet + predicate pushdown。"""
|
||||
try:
|
||||
return pl.scan_parquet(self._minute_glob).filter(
|
||||
(pl.col("symbol") == symbol)
|
||||
& (pl.col("datetime").dt.date() == trade_date)
|
||||
).sort("datetime").collect()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("分钟K查询失败: %s", e)
|
||||
return pl.DataFrame()
|
||||
|
||||
# ================================================================
|
||||
# Polars 查询内部方法
|
||||
# ================================================================
|
||||
|
||||
def _compute_enriched_range(self, df: pl.DataFrame) -> pl.DataFrame:
|
||||
"""对14列enriched数据即时计算完整指标+信号。输入应含足够预热行数。"""
|
||||
from app.indicators.pipeline import compute_indicators, compute_signals, compute_limit_signals, filter_halt_days
|
||||
if df.is_empty() or df.height < 2:
|
||||
return df
|
||||
# 兜底过滤历史脏数据中的停牌日 (close 可能被填充为前收盘价)
|
||||
df = filter_halt_days(df)
|
||||
if df.is_empty() or df.height < 2:
|
||||
return df
|
||||
try:
|
||||
df = compute_indicators(df)
|
||||
df = compute_signals(df)
|
||||
instruments = self.get_instruments()
|
||||
df = compute_limit_signals(df, instruments)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("on-demand compute failed: %s", e)
|
||||
return df
|
||||
|
||||
def _compute_index_enriched_range(self, df: pl.DataFrame) -> pl.DataFrame:
|
||||
"""指数只计算通用技术指标和通用信号，跳过涨跌停/股本/市值逻辑。"""
|
||||
from app.indicators.pipeline import compute_indicators, compute_signals
|
||||
if df.is_empty() or df.height < 2:
|
||||
return df
|
||||
try:
|
||||
df = compute_indicators(df)
|
||||
df = compute_signals(df)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("index on-demand compute failed: %s", e)
|
||||
return df
|
||||
|
||||
def _filter_cached(self, cached: pl.DataFrame, symbol: str, columns: list[str] | None) -> pl.DataFrame:
|
||||
df = cached.filter(pl.col("symbol") == symbol)
|
||||
if columns and not df.is_empty():
|
||||
existing = [c for c in columns if c in df.columns]
|
||||
df = df.select(existing)
|
||||
return df
|
||||
|
||||
def _filter_cached_batch(self, cached: pl.DataFrame, symbols: list[str], columns: list[str] | None) -> pl.DataFrame:
|
||||
df = cached.filter(pl.col("symbol").is_in(symbols))
|
||||
if columns and not df.is_empty():
|
||||
existing = [c for c in columns if c in df.columns]
|
||||
df = df.select(existing)
|
||||
return df.sort(["symbol", "date"])
|
||||
|
||||
def _scan_daily_symbol(self, symbol: str, start: date, end: date, columns: list[str] | None) -> pl.DataFrame:
|
||||
try:
|
||||
lf = pl.scan_parquet(self._enriched_glob,
|
||||
cast_options=pl.ScanCastOptions(integer_cast="allow-float")).filter(
|
||||
(pl.col("symbol") == symbol)
|
||||
& (pl.col("date") >= start)
|
||||
& (pl.col("date") <= end)
|
||||
).sort("date")
|
||||
if columns:
|
||||
schema_names = lf.collect_schema().names()
|
||||
existing = [c for c in columns if c in schema_names]
|
||||
lf = lf.select(existing)
|
||||
return lf.collect()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("日K查询失败: %s", e)
|
||||
return pl.DataFrame()
|
||||
|
||||
def _scan_daily_batch(self, symbols: list[str], start: date, end: date, columns: list[str] | None) -> pl.DataFrame:
|
||||
try:
|
||||
lf = pl.scan_parquet(self._enriched_glob,
|
||||
cast_options=pl.ScanCastOptions(integer_cast="allow-float")).filter(
|
||||
(pl.col("symbol").is_in(symbols))
|
||||
& (pl.col("date") >= start)
|
||||
& (pl.col("date") <= end)
|
||||
).sort(["symbol", "date"])
|
||||
if columns:
|
||||
schema_names = lf.collect_schema().names()
|
||||
existing = [c for c in columns if c in schema_names]
|
||||
lf = lf.select(existing)
|
||||
return lf.collect()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("日K批量查询失败: %s", e)
|
||||
return pl.DataFrame()
|
||||
|
||||
def _scan_index_daily_symbol(self, symbol: str, start: date, end: date, columns: list[str] | None) -> pl.DataFrame:
|
||||
try:
|
||||
lf = pl.scan_parquet(self._index_enriched_glob,
|
||||
cast_options=pl.ScanCastOptions(integer_cast="allow-float")).filter(
|
||||
(pl.col("symbol") == symbol)
|
||||
& (pl.col("date") >= start)
|
||||
& (pl.col("date") <= end)
|
||||
).sort("date")
|
||||
if columns:
|
||||
schema_names = lf.collect_schema().names()
|
||||
existing = [c for c in columns if c in schema_names]
|
||||
lf = lf.select(existing)
|
||||
return lf.collect()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("指数日K查询失败: %s", e)
|
||||
return pl.DataFrame()
|
||||
|
||||
def _merge_cached_and_scan(
|
||||
self,
|
||||
cached: pl.DataFrame,
|
||||
cache_date: date,
|
||||
symbol: str,
|
||||
start: date,
|
||||
end: date,
|
||||
columns: list[str] | None,
|
||||
) -> pl.DataFrame:
|
||||
"""合并缓存部分 + scan 历史部分。
|
||||
|
||||
历史部分用 strict < cache_date, 避免与缓存重复。
|
||||
两部分 schema 可能不一致 (增量 vs 全量), concat 前对齐列。
|
||||
"""
|
||||
hist = self._scan_daily_symbol(symbol, start, cache_date, columns)
|
||||
cached_part = self._filter_cached(cached, symbol, columns)
|
||||
if hist.is_empty():
|
||||
return cached_part
|
||||
if cached_part.is_empty():
|
||||
return hist
|
||||
# 去重: 历史部分可能包含 cache_date, 去掉后再合并
|
||||
hist = hist.filter(pl.col("date") < cache_date)
|
||||
# 对齐列: 取交集, 统一类型
|
||||
common_cols = [c for c in hist.columns if c in cached_part.columns]
|
||||
hist = hist.select(common_cols)
|
||||
cached_part = cached_part.select(common_cols)
|
||||
# 统一类型: 历史可能是 Float64, 缓存可能是 Int64, 统一为 cast
|
||||
for c in common_cols:
|
||||
if hist[c].dtype != cached_part[c].dtype:
|
||||
# 统一到更宽的类型
|
||||
target = hist[c].dtype if hist.height > cached_part.height else cached_part[c].dtype
|
||||
hist = hist.with_columns(pl.col(c).cast(target))
|
||||
cached_part = cached_part.with_columns(pl.col(c).cast(target))
|
||||
return pl.concat([hist, cached_part])
|
||||
|
||||
# ================================================================
|
||||
# DuckDB 查询 (冷路径: 统计/元数据/自定义SQL)
|
||||
# ================================================================
|
||||
|
||||
def latest_minute_date(self, symbol: str) -> date | None:
|
||||
try:
|
||||
with self._lock:
|
||||
row = self.db.execute(
|
||||
"SELECT max(CAST(datetime AS DATE)) FROM kline_minute WHERE symbol = ?",
|
||||
[symbol],
|
||||
).fetchone()
|
||||
if row and row[0]:
|
||||
return row[0] if isinstance(row[0], date) else date.fromisoformat(str(row[0]))
|
||||
except duckdb.CatalogException:
|
||||
pass
|
||||
return None
|
||||
|
||||
def earliest_daily_date(self) -> date | None:
|
||||
"""本地日K数据的最早日期。"""
|
||||
try:
|
||||
with self._lock:
|
||||
res = self.db.execute(
|
||||
"SELECT min(date) FROM kline_daily",
|
||||
).fetchone()
|
||||
if res and res[0]:
|
||||
d = res[0]
|
||||
return d if isinstance(d, date) else date.fromisoformat(str(d))
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
def earliest_minute_date(self) -> date | None:
|
||||
"""本地分钟K数据的最早日期。"""
|
||||
try:
|
||||
with self._lock:
|
||||
res = self.db.execute(
|
||||
"SELECT min(CAST(datetime AS DATE)) FROM kline_minute",
|
||||
).fetchone()
|
||||
if res and res[0]:
|
||||
d = res[0]
|
||||
return d if isinstance(d, date) else date.fromisoformat(str(d))
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
def latest_daily_date(self) -> date | None:
|
||||
"""本地日K数据的最新日期。"""
|
||||
try:
|
||||
with self._lock:
|
||||
res = self.db.execute(
|
||||
"SELECT max(date) FROM kline_daily",
|
||||
).fetchone()
|
||||
if res and res[0]:
|
||||
d = res[0]
|
||||
return d if isinstance(d, date) else date.fromisoformat(str(d))
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
def _latest_enriched_date_duckdb(self) -> date | None:
|
||||
try:
|
||||
with self._lock:
|
||||
res = self.db.execute(
|
||||
"SELECT max(date) FROM kline_enriched",
|
||||
).fetchone()
|
||||
if res and res[0]:
|
||||
d = res[0]
|
||||
return d if isinstance(d, date) else date.fromisoformat(str(d))
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
return None
|
||||
|
||||
# ================================================================
|
||||
# 写入 (Pipeline / Sync)
|
||||
# ================================================================
|
||||
|
||||
def append_daily(self, df: pl.DataFrame) -> None:
|
||||
"""按日分区写入日K数据 (merge-upsert)。"""
|
||||
if df.is_empty():
|
||||
return
|
||||
self._write_daily_partition(df, "kline_daily")
|
||||
|
||||
def append_enriched(self, df: pl.DataFrame) -> None:
|
||||
"""按日分区写入 enriched 数据 (merge-upsert)。磁盘仅写入 14 列存储列。"""
|
||||
if df.is_empty():
|
||||
return
|
||||
from app.indicators.pipeline import ENRICHED_STORAGE_COLS
|
||||
storage_cols = [c for c in ENRICHED_STORAGE_COLS if c in df.columns]
|
||||
df_storage = df.select(storage_cols)
|
||||
self._write_daily_partition(df_storage, "kline_daily_enriched")
|
||||
|
||||
def append_index_daily(self, df: pl.DataFrame) -> None:
|
||||
"""按日分区写入指数日K数据 (merge-upsert)。"""
|
||||
if df.is_empty():
|
||||
return
|
||||
self._write_daily_partition(df, "kline_index_daily")
|
||||
|
||||
def append_index_enriched(self, df: pl.DataFrame) -> None:
|
||||
"""按日分区写入指数 enriched 数据。磁盘仅写入通用基础行情窄表。"""
|
||||
if df.is_empty():
|
||||
return
|
||||
from app.indicators.pipeline import ENRICHED_STORAGE_COLS
|
||||
storage_cols = [c for c in ENRICHED_STORAGE_COLS if c in df.columns]
|
||||
df_storage = df.select(storage_cols)
|
||||
self._write_daily_partition(df_storage, "kline_index_enriched")
|
||||
|
||||
def save_index_instruments(self, df: pl.DataFrame) -> None:
|
||||
"""保存指数标的维表。"""
|
||||
if df.is_empty() or "symbol" not in df.columns:
|
||||
return
|
||||
out = self.store.data_dir / "instruments_index" / "instruments_index.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.unique(subset=["symbol"], keep="last").sort("symbol").write_parquet(out)
|
||||
self._index_instruments_cache = None
|
||||
self._refresh_index_instruments()
|
||||
|
||||
def refresh_index_views(self) -> None:
|
||||
"""刷新指数相关 DuckDB 视图。"""
|
||||
d = self.store.data_dir.as_posix()
|
||||
statements = [
|
||||
f"""CREATE OR REPLACE VIEW kline_index_daily AS
|
||||
SELECT * FROM read_parquet('{d}/kline_index_daily/**/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW kline_index_enriched AS
|
||||
SELECT * FROM read_parquet('{d}/kline_index_enriched/**/*.parquet', union_by_name=true)""",
|
||||
f"""CREATE OR REPLACE VIEW instruments_index AS
|
||||
SELECT * FROM read_parquet('{d}/instruments_index/**/*.parquet', union_by_name=true)""",
|
||||
]
|
||||
for sql in statements:
|
||||
try:
|
||||
with self._lock:
|
||||
self.db.execute(sql)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("index view refresh skipped: %s", e)
|
||||
|
||||
def _write_daily_partition(self, df: pl.DataFrame, table: str) -> None:
|
||||
"""按 date 分区写入 parquet，每个日期一个文件，支持 merge-upsert。"""
|
||||
base = self.store.data_dir / table
|
||||
for date_df in df.partition_by("date"):
|
||||
dt = date_df["date"][0]
|
||||
ds = dt.isoformat() if hasattr(dt, "isoformat") else str(dt)
|
||||
out = base / f"date={ds}" / "part.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
if out.exists():
|
||||
existing = pl.read_parquet(out)
|
||||
date_df = pl.concat([existing, date_df], how="diagonal_relaxed").unique(
|
||||
subset=["symbol", "date"], keep="last"
|
||||
)
|
||||
date_df = date_df.sort(["symbol", "date"])
|
||||
date_df.write_parquet(out)
|
||||
|
||||
def flush_live_daily(self, df: pl.DataFrame) -> None:
|
||||
"""覆写当天 kline_daily 分区 (实时行情落盘, 非merge)。"""
|
||||
if df.is_empty() or "date" not in df.columns:
|
||||
return
|
||||
base = self.store.data_dir / "kline_daily"
|
||||
dt = df["date"][0]
|
||||
ds = dt.isoformat() if hasattr(dt, "isoformat") else str(dt)
|
||||
out = base / f"date={ds}" / "part.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.sort(["symbol", "date"]).write_parquet(out)
|
||||
|
||||
def flush_live_enriched(self, df: pl.DataFrame) -> None:
|
||||
"""覆写当天 kline_daily_enriched 分区 (实时 enriched 落盘, 非merge)。
|
||||
|
||||
内存缓存保留完整指标列供各服务使用，磁盘仅写入 14 列存储列。
|
||||
"""
|
||||
if df.is_empty() or "date" not in df.columns:
|
||||
return
|
||||
# 内存缓存: 保留完整 66 列
|
||||
self._enriched_cache = df.sort(["symbol"])
|
||||
dt = df["date"][0]
|
||||
self._enriched_cache_date = dt
|
||||
# 磁盘写入: 仅 14 列存储列
|
||||
from app.indicators.pipeline import ENRICHED_STORAGE_COLS
|
||||
storage_cols = [c for c in ENRICHED_STORAGE_COLS if c in df.columns]
|
||||
df_storage = df.select(storage_cols).sort(["symbol"])
|
||||
base = self.store.data_dir / "kline_daily_enriched"
|
||||
ds = dt.isoformat() if hasattr(dt, "isoformat") else str(dt)
|
||||
out = base / f"date={ds}" / "part.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
df_storage.write_parquet(out)
|
||||
@@ -0,0 +1,66 @@
|
||||
"""请求调度器(§5.6)。
|
||||
|
||||
按 capability 分别维护令牌桶。Phase 0:基础实现;Phase 1 接入批量合并、优先级队列。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .capabilities import Cap, CapabilitySet
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Bucket:
|
||||
capacity: int # 每分钟令牌数
|
||||
tokens: float
|
||||
last_refill: float # 单位:秒
|
||||
|
||||
def consume(self, n: int = 1) -> float:
|
||||
"""尝试消费 n 个令牌,返回需要等待的秒数(0 表示无需等待)。"""
|
||||
now = time.monotonic()
|
||||
elapsed = now - self.last_refill
|
||||
# 60s 内补满 capacity,匀速补
|
||||
refill = elapsed * (self.capacity / 60.0)
|
||||
self.tokens = min(self.capacity, self.tokens + refill)
|
||||
self.last_refill = now
|
||||
|
||||
if self.tokens >= n:
|
||||
self.tokens -= n
|
||||
return 0.0
|
||||
deficit = n - self.tokens
|
||||
# 还需多少秒才能补齐
|
||||
wait = deficit / (self.capacity / 60.0)
|
||||
# 不预扣,留给下一次再竞争(避免饿死优先级高的请求)
|
||||
return wait
|
||||
|
||||
|
||||
class Scheduler:
|
||||
"""每个 capability 一个桶。"""
|
||||
|
||||
def __init__(self, capset: CapabilitySet) -> None:
|
||||
self._capset = capset
|
||||
self._buckets: dict[Cap, _Bucket] = {}
|
||||
self._locks: dict[Cap, asyncio.Lock] = {}
|
||||
for cap, lim in capset.all().items():
|
||||
if lim.rpm:
|
||||
self._buckets[cap] = _Bucket(
|
||||
capacity=lim.rpm,
|
||||
tokens=lim.rpm,
|
||||
last_refill=time.monotonic(),
|
||||
)
|
||||
self._locks[cap] = asyncio.Lock()
|
||||
|
||||
async def acquire(self, cap: Cap, n: int = 1) -> None:
|
||||
"""阻塞直到拿到 n 个令牌。无桶 = 不限流(由调用方保证)。"""
|
||||
bucket = self._buckets.get(cap)
|
||||
if bucket is None:
|
||||
return
|
||||
lock = self._locks[cap]
|
||||
async with lock:
|
||||
while True:
|
||||
wait = bucket.consume(n)
|
||||
if wait == 0:
|
||||
return
|
||||
await asyncio.sleep(wait)
|
||||
@@ -0,0 +1,98 @@
|
||||
"""管理员创建的访问 UUID 持久化存储。
|
||||
|
||||
位置:`data/user_data/access_uuids.json`,权限 0600。
|
||||
与 secrets.json 分离,避免凭据文件过大且便于独立管理。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _path() -> Path:
|
||||
from app.config import settings
|
||||
p = settings.data_dir / "user_data" / "access_uuids.json"
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def _load() -> list[dict]:
|
||||
p = _path()
|
||||
if p.exists():
|
||||
try:
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("access_uuids.json malformed: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
def _save(records: list[dict]) -> list[dict]:
|
||||
p = _path()
|
||||
p.write_text(json.dumps(records, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
try:
|
||||
os.chmod(p, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
return records
|
||||
|
||||
|
||||
def list_uuids() -> list[dict]:
|
||||
"""返回所有 UUID 记录(按创建时间倒序)。"""
|
||||
records = _load()
|
||||
records.sort(key=lambda r: r.get("created_at", 0), reverse=True)
|
||||
return records
|
||||
|
||||
|
||||
def exists(uuid: str) -> bool:
|
||||
"""检查 UUID 是否存在且启用。"""
|
||||
normalized = uuid.strip()
|
||||
return any(r.get("uuid") == normalized and r.get("enabled", True) for r in _load())
|
||||
|
||||
|
||||
def create(label: str = "") -> dict:
|
||||
"""创建一个新的访问 UUID。"""
|
||||
records = _load()
|
||||
new_uuid = str(uuid.uuid4())
|
||||
record = {
|
||||
"uuid": new_uuid,
|
||||
"label": (label or "").strip(),
|
||||
"enabled": True,
|
||||
"created_at": int(time.time()),
|
||||
}
|
||||
records.append(record)
|
||||
_save(records)
|
||||
return record
|
||||
|
||||
|
||||
def delete(uuid: str) -> bool:
|
||||
"""删除指定 UUID。"""
|
||||
records = _load()
|
||||
original_len = len(records)
|
||||
records = [r for r in records if r.get("uuid") != uuid.strip()]
|
||||
if len(records) == original_len:
|
||||
return False
|
||||
_save(records)
|
||||
return True
|
||||
|
||||
|
||||
def toggle(uuid: str, enabled: bool) -> bool:
|
||||
"""启用/禁用指定 UUID。"""
|
||||
records = _load()
|
||||
found = False
|
||||
for r in records:
|
||||
if r.get("uuid") == uuid.strip():
|
||||
r["enabled"] = enabled
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
return False
|
||||
_save(records)
|
||||
return True
|
||||
Reference in New Issue
Block a user