初始化工程

This commit is contained in:
2026-07-01 22:07:55 +08:00
parent 9bb6d7a654
commit fc2b0944d2
248 changed files with 65020 additions and 316 deletions
+1
View File
@@ -0,0 +1 @@
"""业务服务层。"""
+209
View File
@@ -0,0 +1,209 @@
"""告警触发记录存储 — JSONL 追加写 + 滚动清理。
职责:
- 把每次触发的 AlertEvent 追加写入 data/user_data/alerts.jsonl
- 提供查询 (按来源/类型过滤、时间倒序、限量)
- 滚动清理: 保留近 N 天 + 上限 M 条 (取交集)
设计:
- JSONL 每行一个 JSON 对象,便于增量追加和流式读取
- 清理策略: 追加后按需 prune (按 ts 删旧),避免文件无限膨胀
- 读时全量加载到内存过滤 (记录量受上限约束, 5000 条量级无压力)
"""
from __future__ import annotations
import json
import logging
import threading
from pathlib import Path
logger = logging.getLogger(__name__)
# 保留策略
MAX_DAYS = 7
MAX_RECORDS = 5000
# 每隔多少次写入触发一次清理 (避免每次写都 prune)
PRUNE_EVERY = 20
_lock = threading.Lock()
_write_count = 0
def _path(data_dir: Path) -> Path:
p = data_dir / "user_data" / "alerts.jsonl"
p.parent.mkdir(parents=True, exist_ok=True)
return p
def append(data_dir: Path, event: dict) -> None:
"""追加一条触发记录。event 应含 ts(毫秒)、rule_id、source 等字段。"""
line = json.dumps(event, ensure_ascii=False)
with _lock:
p = _path(data_dir)
with p.open("a", encoding="utf-8") as f:
f.write(line + "\n")
global _write_count
_write_count += 1
if _write_count >= PRUNE_EVERY:
_write_count = 0
_prune_locked(p)
def append_many(data_dir: Path, events: list[dict]) -> None:
"""批量追加。"""
if not events:
return
with _lock:
p = _path(data_dir)
with p.open("a", encoding="utf-8") as f:
for ev in events:
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
global _write_count
_write_count += len(events)
if _write_count >= PRUNE_EVERY:
_write_count = 0
_prune_locked(p)
def list_recent(
data_dir: Path,
days: int = MAX_DAYS,
limit: int = MAX_RECORDS,
source: str | None = None,
type: str | None = None,
) -> list[dict]:
"""读取近 N 天记录,按时间倒序,支持按 source/type 过滤。"""
import time
cutoff = (time.time() - days * 86400) * 1000 # 毫秒
out: list[dict] = []
p = _path(data_dir)
if not p.exists():
return []
try:
with p.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
ev = json.loads(line)
except Exception:
continue
if ev.get("ts", 0) < cutoff:
continue
if source and ev.get("source") != source:
continue
if type and ev.get("type") != type:
continue
out.append(ev)
except Exception as e:
logger.warning("alert_store read failed: %s", e)
return []
# 时间倒序 + 截断
out.sort(key=lambda x: x.get("ts", 0), reverse=True)
return out[:limit]
def clear(data_dir: Path) -> int:
"""清空全部记录,返回清除的条数。"""
with _lock:
p = _path(data_dir)
if not p.exists():
return 0
count = 0
try:
with p.open("r", encoding="utf-8") as f:
count = sum(1 for line in f if line.strip())
except Exception:
pass
p.write_text("", encoding="utf-8")
return count
def delete_one(data_dir: Path, ts: int) -> bool:
"""删除指定 ts 的单条记录,返回是否删除成功。
JSONL 无主键, 用 ts(毫秒时间戳) 作为标识。
若存在多条同 ts, 只删第一条。
"""
with _lock:
p = _path(data_dir)
if not p.exists():
return False
kept: list[dict] = []
deleted = False
try:
with p.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
ev = json.loads(line)
except Exception:
continue
if not deleted and ev.get("ts") == ts:
deleted = True
continue
kept.append(ev)
except Exception as e:
logger.warning("alert_store delete_one read failed: %s", e)
return False
if not deleted:
return False
try:
with p.open("w", encoding="utf-8") as f:
for ev in kept:
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
except Exception as e:
logger.warning("alert_store delete_one write failed: %s", e)
return False
return True
return count
def count(data_dir: Path) -> int:
"""返回当前记录总数。"""
p = _path(data_dir)
if not p.exists():
return 0
try:
with p.open("r", encoding="utf-8") as f:
return sum(1 for line in f if line.strip())
except Exception:
return 0
def _prune_locked(p: Path) -> None:
"""(调用方需持锁) 保留近 MAX_DAYS 天 + 上限 MAX_RECORDS 条。"""
import time
cutoff = (time.time() - MAX_DAYS * 86400) * 1000
kept: list[dict] = []
try:
with p.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
ev = json.loads(line)
except Exception:
continue
if ev.get("ts", 0) >= cutoff:
kept.append(ev)
except FileNotFoundError:
return
except Exception as e:
logger.warning("alert_store prune read failed: %s", e)
return
# 上限截断 (保留最新的)
if len(kept) > MAX_RECORDS:
kept.sort(key=lambda x: x.get("ts", 0))
kept = kept[-MAX_RECORDS:]
# 重写文件
try:
with p.open("w", encoding="utf-8") as f:
for ev in kept:
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
except Exception as e:
logger.warning("alert_store prune write failed: %s", e)
+392
View File
@@ -0,0 +1,392 @@
"""回测服务(§6.7)。
包 vectorbt — 全项目唯一一处出现 pandas。
"""
from __future__ import annotations
import logging
import uuid
from dataclasses import dataclass, field
from datetime import date
from typing import Literal
import numpy as np
import pandas as pd
import polars as pl
from app.config import settings
from app.tickflow.repository import KlineRepository
logger = logging.getLogger(__name__)
# vectorbt 是 optional extras(见 pyproject.toml).未装时只有 backtest 不可用,其他功能正常.
_vbt = None
_vbt_unavailable_reason: str | None = None
class VectorbtUnavailable(RuntimeError):
"""vectorbt 未安装 — 提示用户 `uv sync --extra backtest`."""
def _get_vbt():
global _vbt, _vbt_unavailable_reason
if _vbt is not None:
return _vbt
if _vbt_unavailable_reason is not None:
raise VectorbtUnavailable(_vbt_unavailable_reason)
try:
import vectorbt as vbt
_vbt = vbt
return _vbt
except ImportError as e:
_vbt_unavailable_reason = (
"vectorbt 未安装 — 它是回测的可选依赖.macOS Intel 用户先 `brew install cmake` "
"然后 `uv sync --extra backtest`"
)
logger.warning("vectorbt unavailable: %s", e)
raise VectorbtUnavailable(_vbt_unavailable_reason) from e
def is_available() -> bool:
"""供 API 层快速检测."""
try:
_get_vbt()
return True
except VectorbtUnavailable:
return False
SignalKind = Literal[
"macd_golden", "macd_dead",
"ma_golden_5_20", "ma_dead_5_20",
"ma_golden_20_60",
"ma20_breakout", "ma20_breakdown",
"n_day_high", "n_day_low",
"boll_breakout_upper", "boll_breakdown_lower",
"volume_surge",
"rsi_oversold", "rsi_overbought",
"stop_loss", "trailing_stop", "max_hold",
]
@dataclass
class BacktestConfig:
symbols: list[str]
start: date
end: date
# 买入信号(任一触发即买)
entries: list[str] = field(default_factory=list)
# 卖出信号(任一触发即卖)
exits: list[str] = field(default_factory=list)
# 其他参数
stop_loss_pct: float | None = None # 例 -0.05 = -5%
max_hold_days: int | None = None
fees_pct: float = 0.0002 # 万二佣金
slippage_bps: float = 5 # 5 bps
# 撮合
matching: Literal["close_t", "open_t+1"] = "close_t"
rsi_oversold_threshold: float = 30
rsi_overbought_threshold: float = 70
@dataclass
class BacktestResult:
run_id: str
config: dict
stats: dict
equity_curve: list[dict] # [{date, value}]
trades: list[dict] # [{symbol, entry_date, exit_date, pnl_pct, ...}]
per_symbol_stats: list[dict] # 每只股票的统计
# enriched 表里的信号列名映射
_SIGNAL_COLS: dict[SignalKind, str] = {
"macd_golden": "signal_macd_golden",
"macd_dead": "signal_macd_dead",
"ma_golden_5_20": "signal_ma_golden_5_20",
"ma_dead_5_20": "signal_ma_dead_5_20",
"ma_golden_20_60": "signal_ma_golden_20_60",
"ma20_breakout": "signal_ma20_breakout",
"ma20_breakdown": "signal_ma20_breakdown",
"n_day_high": "signal_n_day_high",
"n_day_low": "signal_n_day_low",
"boll_breakout_upper": "signal_boll_breakout_upper",
"boll_breakdown_lower": "signal_boll_breakdown_lower",
"volume_surge": "signal_volume_surge",
}
class BacktestService:
def __init__(self, repo: KlineRepository) -> None:
self.repo = repo
def _load_panel(
self,
symbols: list[str],
start: date,
end: date,
) -> pd.DataFrame:
"""加载 [date × symbol] 价格面板 — Polars scan_parquet + 即时计算指标。
**全项目唯一从 Polars 转 pandas 的边界**(§7.4 / ADR-19)。
"""
try:
enriched_glob = str(self.repo.store.data_dir / "kline_daily_enriched" / "**" / "*.parquet")
df = (
pl.scan_parquet(enriched_glob)
.filter(
(pl.col("symbol").is_in(symbols))
& (pl.col("date") >= start)
& (pl.col("date") <= end)
)
.sort(["date", "symbol"])
.collect()
)
except Exception as e: # noqa: BLE001
logger.warning("backtest load failed: %s", e)
return pd.DataFrame()
if df.is_empty():
return pd.DataFrame()
# 即时计算指标 + 信号
from app.indicators.pipeline import compute_all
df = compute_all(df)
# 选择需要的列
needed_cols = [
"date", "symbol", "open", "high", "low", "close", "volume",
"rsi_14", "signal_macd_golden", "signal_macd_dead",
"signal_ma_golden_5_20", "signal_ma_dead_5_20",
"signal_ma_golden_20_60",
"signal_ma20_breakout", "signal_ma20_breakdown",
"signal_n_day_high", "signal_n_day_low",
"signal_boll_breakout_upper", "signal_boll_breakdown_lower",
"signal_volume_surge",
]
existing = [c for c in needed_cols if c in df.columns]
df = df.select(existing)
# to_pandas 边界
return df.to_pandas(use_pyarrow_extension_array=False)
def _build_signal_matrix(
self,
panel: pd.DataFrame,
kinds: list[str],
config: BacktestConfig,
) -> pd.DataFrame:
"""从面板构造 [date × symbol] 的布尔信号矩阵。"""
if not kinds or panel.empty:
return pd.DataFrame()
# pivot 成 [date × symbol] 形式
result = None
for kind in kinds:
mat = None
if kind in _SIGNAL_COLS:
col = _SIGNAL_COLS[kind]
mat = panel.pivot(index="date", columns="symbol", values=col).fillna(False).astype(bool)
elif kind == "rsi_oversold":
mat = (panel.pivot(index="date", columns="symbol", values="rsi_14")
< config.rsi_oversold_threshold)
elif kind == "rsi_overbought":
mat = (panel.pivot(index="date", columns="symbol", values="rsi_14")
> config.rsi_overbought_threshold)
# stop_loss / trailing / max_hold 通过 vectorbt 参数处理,不参与信号矩阵
if mat is not None:
result = mat if result is None else (result | mat)
return result if result is not None else pd.DataFrame()
def run(self, config: BacktestConfig) -> BacktestResult:
vbt = _get_vbt()
run_id = uuid.uuid4().hex[:10]
panel = self._load_panel(config.symbols, config.start, config.end)
if panel.empty:
return BacktestResult(
run_id=run_id,
config=_config_to_dict(config),
stats={"error": "no data"},
equity_curve=[],
trades=[],
per_symbol_stats=[],
)
# 价格面板
close = panel.pivot(index="date", columns="symbol", values="close")
# 信号矩阵
entries = self._build_signal_matrix(panel, config.entries, config)
exits = self._build_signal_matrix(panel, config.exits, config)
# 对齐 index/columns
if not entries.empty:
entries = entries.reindex_like(close).fillna(False).astype(bool)
else:
entries = pd.DataFrame(False, index=close.index, columns=close.columns)
if not exits.empty:
exits = exits.reindex_like(close).fillna(False).astype(bool)
else:
exits = pd.DataFrame(False, index=close.index, columns=close.columns)
if not entries.any().any():
return BacktestResult(
run_id=run_id,
config=_config_to_dict(config),
stats={"error": "no buy signals"},
equity_curve=[],
trades=[],
per_symbol_stats=[],
)
# T+1 适配:vectorbt 默认信号当根 K 撮合
# close_t 撮合:维持默认
# open_t+1 撮合:shift 信号 1 根 + 用 open 作为价
if config.matching == "open_t+1":
entries = entries.shift(1).fillna(False).astype(bool)
exits = exits.shift(1).fillna(False).astype(bool)
price = panel.pivot(index="date", columns="symbol", values="open")
else:
price = close
# 跑回测
try:
pf_kwargs = dict(
close=close,
entries=entries,
exits=exits,
price=price,
fees=config.fees_pct,
slippage=config.slippage_bps / 10000.0,
freq="1D",
)
if config.stop_loss_pct is not None:
pf_kwargs["sl_stop"] = abs(config.stop_loss_pct)
if config.max_hold_days is not None:
# vectorbt 没有内置 max-hold;用时间退出近似:
# 在 max_hold_days 后强制 exit
exits_idx = entries.copy()
for col in entries.columns:
entry_rows = np.where(entries[col].values)[0]
for i in entry_rows:
end_i = min(i + config.max_hold_days, len(entries) - 1)
if end_i > i:
exits_idx.iloc[end_i][col] = True
pf_kwargs["exits"] = (exits | exits_idx).astype(bool)
pf = vbt.Portfolio.from_signals(**pf_kwargs)
except Exception as e: # noqa: BLE001
logger.exception("vectorbt backtest failed")
return BacktestResult(
run_id=run_id,
config=_config_to_dict(config),
stats={"error": str(e)},
equity_curve=[],
trades=[],
per_symbol_stats=[],
)
# 提取结果
try:
stats_series = pf.stats(silence_warnings=True)
if isinstance(stats_series, pd.DataFrame):
# 多列时取 agg
stats_dict = stats_series.mean(numeric_only=True).to_dict()
else:
stats_dict = stats_series.to_dict()
except Exception: # noqa: BLE001
stats_dict = {}
# 净值曲线(组合平均)
equity = pf.value().mean(axis=1) if isinstance(pf.value(), pd.DataFrame) else pf.value()
equity_curve = [
{"date": str(idx.date() if hasattr(idx, "date") else idx), "value": float(v)}
for idx, v in equity.items() if pd.notna(v)
]
# 交易记录
try:
trades_df = pf.trades.records_readable
trades = trades_df.to_dict(orient="records") if not trades_df.empty else []
# 字段名美化
trades = [
{
"symbol": t.get("Column", t.get("Symbol", "")),
"entry_date": str(t.get("Entry Timestamp", t.get("Entry Date", ""))),
"exit_date": str(t.get("Exit Timestamp", t.get("Exit Date", ""))),
"entry_price": float(t.get("Avg Entry Price", t.get("Avg. Entry Price", 0))),
"exit_price": float(t.get("Avg Exit Price", t.get("Avg. Exit Price", 0))),
"pnl_pct": float(t.get("Return", t.get("PnL %", 0))),
"duration": str(t.get("Duration", "")),
}
for t in trades
]
except Exception: # noqa: BLE001
trades = []
# 每标的统计
per_symbol = []
try:
total_ret = pf.total_return()
if isinstance(total_ret, pd.Series):
for sym, ret in total_ret.items():
if pd.notna(ret):
per_symbol.append({"symbol": sym, "total_return": float(ret)})
except Exception: # noqa: BLE001
pass
result = BacktestResult(
run_id=run_id,
config=_config_to_dict(config),
stats={k: _json_safe(v) for k, v in stats_dict.items()},
equity_curve=equity_curve,
trades=trades,
per_symbol_stats=per_symbol,
)
# 落盘
self._persist(result)
return result
def _persist(self, result: BacktestResult) -> None:
out_dir = settings.data_dir / "backtest_results"
out_dir.mkdir(parents=True, exist_ok=True)
# 用 polars 写一份汇总
summary = pl.DataFrame({
"run_id": [result.run_id],
"stats_json": [str(result.stats)],
"n_trades": [len(result.trades)],
})
summary.write_parquet(out_dir / f"run_id={result.run_id}.parquet")
def get_result(self, run_id: str) -> BacktestResult | None:
# Phase 1:只保留近似落盘,完整结果保存在内存的近期 cache 中
# 简化:重新 run 比缓存复杂结果代价小,暂不实现 get_result
return None
def _config_to_dict(c: BacktestConfig) -> dict:
return {
"symbols": c.symbols,
"start": str(c.start),
"end": str(c.end),
"entries": c.entries,
"exits": c.exits,
"stop_loss_pct": c.stop_loss_pct,
"max_hold_days": c.max_hold_days,
"fees_pct": c.fees_pct,
"slippage_bps": c.slippage_bps,
"matching": c.matching,
}
def _json_safe(v):
if isinstance(v, (int, float, str, bool)) or v is None:
return v
if isinstance(v, (np.floating, np.integer)):
return float(v) if not np.isnan(float(v)) else None
if hasattr(v, "isoformat"):
return v.isoformat()
return str(v)
+575
View File
@@ -0,0 +1,575 @@
"""五档盘口 sealed(真假涨停/跌停) 服务 — 独立旁路线。
架构(完全解耦):
- 只读 enriched(拿涨跌停名单), 不写回 enriched(14列不动)
- sealed 存独立 parquet(data/depth5/date=xxx/part.parquet)
- limit_ladder API 查询时 LEFT JOIN(同 ext_columns 机制)
- signal_limit_up 永远是"价格涨停", sealed 是叠加的真假判定层
数据流:
盘中轮询线程(交易时段, 独立 sleep, 不绑行情轮询):
读 enriched 内存缓存(线程安全) → 涨跌停名单 → tf.depth.batch
→ 算 sealed → 更新内存缓存(不落盘) → sealed_ready=True
盘后定版 job(可配置时间, 默认15:02):
最后拉一次 → 落盘 depth5 parquet(定版)
三层防护节流("设过大设上限, 设过小设最小值"):
① 套餐范围 clamp: Pro 10~120s, Expert 3~300s
② 限速安全 clamp: safe = 60/((rpm*0.8)/batches), 涨跌停多就自动放慢
③ 系统接管通知: 用户设置会超限时, 推 toast 告知已自动调整
"""
from __future__ import annotations
import logging
import math
import threading
import time
from datetime import date, datetime, time as dt_time
from pathlib import Path
import polars as pl
logger = logging.getLogger(__name__)
# 套餐 → (轮询间隔下限s, 上限s)
TIER_INTERVAL_RANGE: dict[str, tuple[float, float]] = {
"pro": (10.0, 120.0),
"expert": (3.0, 300.0),
}
# 兜底: 其他有 DEPTH5_BATCH 的套餐按 pro 范围
DEFAULT_RANGE = (10.0, 120.0)
# 限速余量: 只用 rpm 的 80%, 给系统其他 depth 调用留空间
RPM_MARGIN = 0.8
# 间隔硬下限/上限(任何套餐)
INTERVAL_HARD_MIN = 10.0
INTERVAL_HARD_MAX = 300.0
class DepthService:
"""五档盘口 sealed 服务 — 单例。"""
def __init__(self) -> None:
self._lock = threading.Lock()
self._running = False
self._thread: threading.Thread | None = None
self._repo = None # 延迟注入(KlineRepository)
self._app_state = None # 延迟注入(FastAPI app.state)
# 内存缓存: {symbol: SealedEntry}
# SealedEntry = {sealed_up, sealed_down, ask1_vol, bid1_vol, status, fetched_ts}
self._sealed_cache: dict[str, dict] = {}
self._sealed_ready = False
self._sealed_date: date | None = None # sealed 数据对应的交易日(可能是昨天,如休市)
self._sealed_fetched_ts: float = 0.0 # 上次拉取的 perf_counter
self._sealed_fetched_at: float = 0.0 # 上次拉取的 wall-clock 时间戳
self._persisted_date: date | None = None # 已落盘的日期
# 系统接管状态(防通知刷屏)
self._last_taken_over: bool | None = None
self._last_user_interval: float | None = None
# ================================================================
# 注入
# ================================================================
def set_repo(self, repo) -> None:
self._repo = repo
def set_app_state(self, app_state) -> None:
self._app_state = app_state
# ================================================================
# 生命周期
# ================================================================
def boot_check(self) -> None:
"""启动补跑: 当天 depth5 文件不存在则 finalize 一次; 已存在则恢复内存缓存。"""
if not self._has_capability():
logger.info("depth sealed: 无 DEPTH5_BATCH 能力, 跳过启动补跑")
return
today = date.today()
if self._persisted_for_date(today):
# parquet 已存在: 恢复内存缓存(避免重启后每次查询都读 parquet)
self._restore_from_parquet(today)
return
logger.info("depth sealed: 启动补跑今天定版")
try:
self.finalize()
except Exception as e: # noqa: BLE001
logger.warning("depth sealed 启动补跑失败: %s", e)
def _restore_from_parquet(self, d: date) -> None:
"""从 parquet 恢复内存缓存(服务重启后)。"""
if not self._repo:
return
out = self._repo.store.data_dir / "depth5" / f"date={d.isoformat()}" / "part.parquet"
if not out.exists():
return
try:
df = pl.read_parquet(out)
cache: dict[str, dict] = {}
for row in df.to_dicts():
sym = row.get("symbol")
if not sym:
continue
cache[sym] = {
"sealed_up": row.get("sealed_up"),
"sealed_down": row.get("sealed_down"),
"ask1_vol": row.get("ask1_vol"),
"bid1_vol": row.get("bid1_vol"),
"status": row.get("status"),
"fetched_ts": row.get("fetched_at"),
}
with self._lock:
self._sealed_cache = cache
self._sealed_ready = True
self._sealed_date = d
self._persisted_date = d
logger.info("depth sealed: 从 parquet 恢复 %d 只 (日期=%s)", len(cache), d)
except Exception as e: # noqa: BLE001
logger.warning("depth sealed 从 parquet 恢复失败: %s", e)
def start_polling(self) -> None:
"""启动盘中轮询线程(连板梯队监控开启 + 有能力 + 交易时段)。"""
if self._running:
return
if not self._has_capability():
return
from app.services import preferences
if not preferences.get_limit_ladder_monitor_enabled():
return
self._running = True
self._thread = threading.Thread(target=self._poll_loop, daemon=True)
self._thread.start()
logger.info("depth sealed 盘中轮询已启动")
def stop_polling(self) -> None:
"""停止盘中轮询线程。"""
self._running = False
if self._thread:
self._thread.join(timeout=10)
self._thread = None
logger.info("depth sealed 盘中轮询已停止")
def apply_monitor_toggle(self, enabled: bool) -> None:
"""连板梯队监控开关切换时调用: 开启→启动轮询, 关闭→停止轮询。"""
if enabled:
self.start_polling()
else:
self.stop_polling()
def run_once(self) -> dict:
"""手动触发一次修正(立即拉取 depth + 更新内存缓存)。
不受监控开关限制 — 用户可随时手动修正一次。
返回 {"ok": bool, "count": int, "msg": str}
"""
if not self._has_capability():
return {"ok": False, "count": 0, "msg": "无五档盘口能力(需 Pro+)"}
try:
self._fetch_and_seal(persist=True) # 落盘, 刷新页面不丢
with self._lock:
count = len(self._sealed_cache)
return {"ok": True, "count": count, "msg": f"已修正 {count}"}
except Exception as e: # noqa: BLE001
logger.warning("depth run_once 失败: %s", e)
return {"ok": False, "count": 0, "msg": f"修正失败: {e}"}
# ================================================================
# 核心拉取
# ================================================================
def _fetch_and_seal(self, persist: bool = False) -> None:
"""拉一次 depth.batch, 算 sealed, 更新内存缓存(可选落盘)。
persist=True: 盘后定版, 写 depth5 parquet
persist=False: 盘中轮询, 只更新内存缓存
"""
if not self._repo:
return
# 只读 enriched 内存缓存(线程安全, 避免和 quote_service 写盘竞态)
enriched, enriched_date = self._repo.get_enriched_latest()
if enriched.is_empty():
return
# 筛涨跌停名单(用 fill_null 防止列缺失)
syms_up: list[str] = []
syms_down: list[str] = []
if "signal_limit_up" in enriched.columns:
syms_up = enriched.filter(
pl.col("signal_limit_up").fill_null(False)
)["symbol"].to_list()
if "signal_limit_down" in enriched.columns:
syms_down = enriched.filter(
pl.col("signal_limit_down").fill_null(False)
)["symbol"].to_list()
all_syms = list(dict.fromkeys(syms_up + syms_down)) # 去重保序
if not all_syms:
logger.debug("depth sealed: 当日无涨跌停股, 跳过")
return
# 拉 depth(涨跌停一次拉, 按 capset batch 切片)
depth_data = self._call_depth_batch(all_syms)
if not depth_data:
logger.warning("depth sealed: depth.batch 返回空")
return
up_set = set(syms_up)
down_set = set(syms_down)
now_perf = time.perf_counter()
now_wall = time.time()
new_cache: dict[str, dict] = {}
for sym, d in depth_data.items():
ask_vols = d.get("ask_volumes") or []
bid_vols = d.get("bid_volumes") or []
ask1 = ask_vols[0] if ask_vols else None
bid1 = bid_vols[0] if bid_vols else None
# depth 返回的 timestamp(毫秒 epoch), 回退到当前 wall-clock
depth_ts = d.get("timestamp")
fetched = (depth_ts / 1000.0) if isinstance(depth_ts, (int, float)) and depth_ts else now_wall
entry = {
# 涨停真封: 涨停价上卖一(主动卖压)为 0
"sealed_up": (ask1 == 0) if sym in up_set and ask1 is not None else None,
# 跌停真封: 跌停价上买一为 0
"sealed_down": (bid1 == 0) if sym in down_set and bid1 is not None else None,
"ask1_vol": ask1,
"bid1_vol": bid1,
"status": "limit_down" if sym in down_set and sym not in up_set else "limit_up",
"fetched_ts": fetched,
}
new_cache[sym] = entry
with self._lock:
self._sealed_cache = new_cache
self._sealed_ready = True
self._sealed_date = enriched_date # 记录数据对应的交易日(可能是昨天,如休市)
self._sealed_fetched_ts = now_perf
self._sealed_fetched_at = now_wall
logger.info("depth sealed: 拉取 %d 只 (涨停%d/跌停%d) 日期=%s%s",
len(new_cache), len(syms_up), len(syms_down),
enriched_date, " → 落盘" if persist else "")
# 缓存已更新: 通知 SSE 推 depth_updated, 触发连板梯队刷新封单数据。
self._notify_depth_updated(len(new_cache))
if persist and enriched_date:
self._persist(enriched_date)
def _call_depth_batch(self, symbols: list[str]) -> dict:
"""调 tf.depth.batch, 按 capset 的 batch 切片 + 节流。返回 {symbol: MarketDepth}。"""
from app.tickflow.client import get_client
tf = get_client()
capset = self._get_capset()
lim = capset.limits(__import__("app.tickflow.capabilities", fromlist=["Cap"]).Cap.DEPTH5_BATCH)
batch_size = (lim.batch if lim and lim.batch else 100)
rpm = (lim.rpm if lim and lim.rpm else 30)
# 批间隔 = 60/rpm(匀速)
inter_batch = 60.0 / rpm if rpm > 0 else 2.0
result: dict = {}
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
for i, chunk in enumerate(chunks):
if i > 0:
time.sleep(inter_batch)
try:
# SDK 的 batch 内部已按 batch_size 切, 这里再切一层防单请求过大
data = tf.depth.batch(chunk)
if isinstance(data, dict):
result.update(data)
except Exception as e: # noqa: BLE001
logger.warning("depth.batch 第 %d 批失败(%d 只): %s", i + 1, len(chunk), e)
# 单批失败不影响其他批
return result
def finalize(self) -> None:
"""盘后定版: 拉一次 + 落盘。"""
if not self._has_capability():
return
self._fetch_and_seal(persist=True)
# ================================================================
# 落盘
# ================================================================
def _persist(self, today: date) -> None:
"""把内存缓存写 depth5/date=今天/part.parquet。"""
with self._lock:
cache = dict(self._sealed_cache)
if not cache:
return
rows = []
for sym, e in cache.items():
rows.append({
"symbol": sym,
"sealed_up": e.get("sealed_up"),
"sealed_down": e.get("sealed_down"),
"ask1_vol": e.get("ask1_vol"),
"bid1_vol": e.get("bid1_vol"),
"status": e.get("status"),
"fetched_at": e.get("fetched_ts"),
})
df = pl.DataFrame(rows)
ds = today.isoformat()
out = self._repo.store.data_dir / "depth5" / f"date={ds}" / "part.parquet"
out.parent.mkdir(parents=True, exist_ok=True)
df.write_parquet(out)
self._persisted_date = today
logger.info("depth sealed 落盘: %d 行 → %s", df.height, out)
def _persisted_for_date(self, d: date) -> bool:
"""检查某日 depth5 文件是否已存在。"""
if not self._repo:
return False
out = self._repo.store.data_dir / "depth5" / f"date={d.isoformat()}" / "part.parquet"
return out.exists()
# ================================================================
# 查询(供 limit_ladder API 用)
# ================================================================
def get_sealed_map(self, target_date: date, is_down: bool) -> dict:
"""返回 {symbol: {sealed, vol, ready, age}} 供 JOIN。
优先内存缓存(盘中), 回退 parquet(历史/盘后)。
sealed: bool | None (None=待确认或降级)
vol: 封单量(int) | None
ready: sealed 数据是否就绪(False→降级标识)
age: 距上次拉取秒数(盘后定版为 None)
"""
# 内存缓存(sealed 数据对应的交易日 = target_date 时才用)
if self._sealed_date and target_date == self._sealed_date and self._sealed_ready and self._sealed_cache:
return self._read_from_memory(is_down)
# parquet(历史或盘后定版)
return self._read_from_parquet(target_date, is_down)
def _read_from_memory(self, is_down: bool) -> dict:
sealed_key = "sealed_down" if is_down else "sealed_up"
# 封单量: 涨停=买一量(涨停价买单堆积), 跌停=卖一量(跌停价卖单堆积)
vol_key = "ask1_vol" if is_down else "bid1_vol"
now = time.perf_counter()
with self._lock:
cache = dict(self._sealed_cache)
fetched_ts = self._sealed_fetched_ts
age = (now - fetched_ts) if fetched_ts else 0.0
result = {}
for sym, e in cache.items():
result[sym] = {
"sealed": e.get(sealed_key),
"vol": e.get(vol_key),
"ready": True,
"age": age,
}
return result
def _read_from_parquet(self, target_date: date, is_down: bool) -> dict:
if not self._repo:
return {}
out = self._repo.store.data_dir / "depth5" / f"date={target_date.isoformat()}" / "part.parquet"
if not out.exists():
return {}
try:
df = pl.read_parquet(out)
except Exception as e: # noqa: BLE001
logger.warning("depth5 parquet 读取失败: %s", e)
return {}
sealed_key = "sealed_down" if is_down else "sealed_up"
# 封单量: 涨停=买一量, 跌停=卖一量
vol_key = "ask1_vol" if is_down else "bid1_vol"
result = {}
for row in df.to_dicts():
sym = row.get("symbol")
if not sym:
continue
result[sym] = {
"sealed": row.get(sealed_key),
"vol": row.get(vol_key),
"ready": True,
"age": None, # 盘后定版, 无 age
}
return result
def is_sealed_ready(self, target_date: date) -> bool:
"""sealed 数据是否就绪(供前端降级判定)。"""
# 内存缓存对应的数据日 == 查询日 → 看内存就绪状态
if self._sealed_date and target_date == self._sealed_date:
return self._sealed_ready
# 其他日期: 有 parquet 就 ready
return self._persisted_for_date(target_date)
def get_sealed_age(self, target_date: date) -> float | None:
"""返回 sealed 数据 age(秒), 盘后定版为 None。"""
if self._sealed_date and target_date == self._sealed_date and self._sealed_ready and self._sealed_fetched_ts:
return time.perf_counter() - self._sealed_fetched_ts
return None
# ================================================================
# 盘中轮询线程
# ================================================================
def _poll_loop(self) -> None:
"""盘中轮询: 按 capset 自适应间隔拉 depth, 更新内存缓存。"""
while self._running:
try:
if self._is_trading_hours():
self._poll_once()
else:
logger.debug("depth sealed: 非交易时段, 跳过")
except Exception as e: # noqa: BLE001
logger.warning("depth sealed 轮询异常: %s", e)
# 等待下一轮(用 _running 检查保证能及时退出)
interval = self._current_sleep_interval()
waited = 0.0
while self._running and waited < interval:
time.sleep(0.5)
waited += 0.5
def _poll_once(self) -> None:
"""单次轮询: 算间隔(三层防护) → 拉取 → 检测系统接管通知。"""
# 数当前涨跌停股
n = self._count_limit_stocks()
if n == 0:
return
interval, taken_over, user_interval = self._compute_interval(n)
# 系统接管通知(状态切换时才推, 防刷屏)
if taken_over and (self._last_taken_over is False or self._last_user_interval != user_interval):
self._notify_takeover(n, user_interval, interval)
self._last_taken_over = taken_over
self._last_user_interval = user_interval
self._fetch_and_seal(persist=False)
def _current_sleep_interval(self) -> float:
"""计算当前 sleep 间隔(供 _poll_loop 等待用)。"""
n = self._count_limit_stocks()
if n == 0:
return 30.0 # 无涨跌停, 慢轮询
interval, _, _ = self._compute_interval(n)
return interval
# ================================================================
# 三层防护节流
# ================================================================
def _compute_interval(self, n_symbols: int) -> tuple[float, bool, float]:
"""三层防护计算实际轮询间隔。
返回 (actual_interval, taken_over, user_interval)
- actual_interval: 实际使用的间隔(秒)
- taken_over: 是否被系统接管(用户设置会超限)
- user_interval: 用户设置(经套餐 clamp 后)的间隔
"""
from app.services import preferences
from app.tickflow.policy import tier_label
capset = self._get_capset()
lim = capset.limits(__import__("app.tickflow.capabilities", fromlist=["Cap"]).Cap.DEPTH5_BATCH)
batch_size = (lim.batch if lim and lim.batch else 100)
rpm = (lim.rpm if lim and lim.rpm else 30)
# ① 套餐范围 clamp
tier = tier_label().split()[0].split("+")[0].strip().lower()
lo, hi = TIER_INTERVAL_RANGE.get(tier, DEFAULT_RANGE)
raw_user = preferences.get_depth_polling_interval()
user_interval = max(lo, min(hi, raw_user))
# ② 限速安全 clamp
batches = max(1, math.ceil(n_symbols / batch_size))
usable_rpm = rpm * RPM_MARGIN
calls_per_min = usable_rpm / batches if batches > 0 else usable_rpm
safe_interval = 60.0 / calls_per_min if calls_per_min > 0 else INTERVAL_HARD_MAX
# 实际: 取用户设置和安全的较大值
actual = max(user_interval, safe_interval)
# 硬上下限
actual = max(INTERVAL_HARD_MIN, min(actual, INTERVAL_HARD_MAX))
taken_over = safe_interval > user_interval
return actual, taken_over, user_interval
def _count_limit_stocks(self) -> int:
"""数当前涨跌停股总数(供节流计算)。"""
if not self._repo:
return 0
enriched, _ = self._repo.get_enriched_latest()
if enriched.is_empty():
return 0
n = 0
if "signal_limit_up" in enriched.columns:
n += enriched.filter(pl.col("signal_limit_up").fill_null(False)).height
if "signal_limit_down" in enriched.columns:
n += enriched.filter(pl.col("signal_limit_down").fill_null(False)).height
return n
# ================================================================
# 通知
# ================================================================
def _notify_takeover(self, n_stocks: int, user_interval: float, actual_interval: float) -> None:
"""系统接管通知: 复用 quote_service 的 _pending_alerts 通道。"""
if not self._app_state:
return
qs = getattr(self._app_state, "quote_service", None)
if not qs:
return
msg = (f"五档轮询: 当前涨跌停 {n_stocks} 只, 您设置的 {user_interval:.0f} 秒间隔会超限, "
f"系统已自动调整为 {actual_interval:.0f}")
alert = {
"source": "depth",
"type": "takeover",
"message": msg,
}
try:
with qs._lock:
qs._pending_alerts.append(alert)
qs._alert_event.set()
except Exception as e: # noqa: BLE001
logger.debug("depth 接管通知推送失败: %s", e)
def _notify_depth_updated(self, count: int) -> None:
"""修正完成通知: set quote_service._depth_update_event, SSE 推 depth_updated 刷新连板梯队。"""
if not self._app_state:
return
qs = getattr(self._app_state, "quote_service", None)
if not qs:
return
try:
qs.notify_depth_updated()
except Exception as e: # noqa: BLE001
logger.debug("depth 更新通知推送失败: %s", e)
# ================================================================
# 工具
# ================================================================
def _has_capability(self) -> bool:
capset = self._get_capset()
from app.tickflow.capabilities import Cap
return capset.has(Cap.DEPTH5_BATCH)
def _get_capset(self):
"""获取当前 capset(优先 app.state, 回退 detect)。"""
if self._app_state:
cs = getattr(self._app_state, "capabilities", None)
if cs:
return cs
from app.tickflow.policy import detect_capabilities
return detect_capabilities()
@staticmethod
def _is_trading_hours() -> bool:
now = datetime.now()
t = now.time()
morning = dt_time(9, 25) <= t <= dt_time(11, 35)
afternoon = dt_time(12, 55) <= t <= dt_time(15, 5)
return now.weekday() < 5 and (morning or afternoon)
+514
View File
@@ -0,0 +1,514 @@
"""扩展数据服务 — 配置管理 + 文件解析 + Parquet 存储。"""
from __future__ import annotations
import json
import logging
from datetime import date, datetime
from pathlib import Path
from typing import Literal
import polars as pl
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# 配置模型
# ---------------------------------------------------------------------------
class ExtField:
"""扩展字段定义。"""
__slots__ = ("name", "dtype", "label")
def __init__(self, name: str, dtype: str = "string", label: str = "") -> None:
self.name = name
self.dtype = dtype # string | int | float | bool
self.label = label or name
def to_dict(self) -> dict:
return {"name": self.name, "dtype": self.dtype, "label": self.label}
@classmethod
def from_dict(cls, d: dict) -> ExtField:
return cls(d["name"], d.get("dtype", "string"), d.get("label", ""))
class PullConfig:
"""定时拉取配置。"""
__slots__ = (
"url", "method", "headers", "body", "response_path",
"field_map", "schedule_minutes", "enabled",
"last_run", "last_status", "last_message", "last_rows",
)
def __init__(
self,
url: str = "",
method: str = "GET",
headers: dict[str, str] | None = None,
body: str | None = None,
response_path: str = "",
field_map: dict[str, str] | None = None,
schedule_minutes: int = 1440,
enabled: bool = False,
last_run: str | None = None,
last_status: str | None = None,
last_message: str | None = None,
last_rows: int | None = None,
) -> None:
self.url = url
self.method = method # GET | POST
self.headers = headers or {}
self.body = body # JSON string (POST body template)
self.response_path = response_path # dot-path to rows array, e.g. "data.list"
self.field_map = field_map or {} # external_name → config_field_name
self.schedule_minutes = schedule_minutes
self.enabled = enabled
self.last_run = last_run
self.last_status = last_status # "success" | "error"
self.last_message = last_message
self.last_rows = last_rows
def to_dict(self) -> dict:
return {
"url": self.url,
"method": self.method,
"headers": self.headers,
"body": self.body,
"response_path": self.response_path,
"field_map": self.field_map,
"schedule_minutes": self.schedule_minutes,
"enabled": self.enabled,
"last_run": self.last_run,
"last_status": self.last_status,
"last_message": self.last_message,
"last_rows": self.last_rows,
}
@classmethod
def from_dict(cls, d: dict) -> PullConfig:
if not d:
return cls()
return cls(
url=d.get("url", ""),
method=d.get("method", "GET"),
headers=d.get("headers"),
body=d.get("body"),
response_path=d.get("response_path", ""),
field_map=d.get("field_map"),
schedule_minutes=d.get("schedule_minutes", 1440),
enabled=d.get("enabled", False),
last_run=d.get("last_run"),
last_status=d.get("last_status"),
last_message=d.get("last_message"),
last_rows=d.get("last_rows"),
)
class ExtConfig:
"""一个扩展数据源的完整配置。"""
__slots__ = (
"id", "label", "mode", "fields", "description",
"symbol_map", "code_map",
"created_at", "updated_at", "pull",
)
def __init__(
self,
id: str,
label: str,
mode: Literal["snapshot", "timeseries"],
fields: list[ExtField],
description: str = "",
symbol_map: dict | None = None,
code_map: dict | None = None,
created_at: str | None = None,
updated_at: str | None = None,
pull: PullConfig | None = None,
) -> None:
self.id = id
self.label = label
self.mode = mode
self.fields = fields
self.description = description
# 映射关系: {"type": "mapped", "col": "原始列名"} 或 {"type": "computed", "from": "symbol|code", "method": "strip_exchange|append_exchange"}
self.symbol_map = symbol_map or {}
self.code_map = code_map or {}
self.created_at = created_at or datetime.now().isoformat()
self.updated_at = updated_at or datetime.now().isoformat()
self.pull = pull
def to_dict(self) -> dict:
d = {
"id": self.id,
"label": self.label,
"mode": self.mode,
"fields": [f.to_dict() for f in self.fields],
"description": self.description,
"symbol_map": self.symbol_map,
"code_map": self.code_map,
"created_at": self.created_at,
"updated_at": self.updated_at,
}
if self.pull:
d["pull"] = self.pull.to_dict()
return d
@classmethod
def from_dict(cls, d: dict) -> ExtConfig:
return cls(
id=d["id"],
label=d["label"],
mode=d["mode"],
fields=[ExtField.from_dict(f) for f in d.get("fields", [])],
description=d.get("description", ""),
symbol_map=d.get("symbol_map"),
code_map=d.get("code_map"),
created_at=d.get("created_at"),
updated_at=d.get("updated_at"),
pull=PullConfig.from_dict(d["pull"]) if d.get("pull") else None,
)
# ---------------------------------------------------------------------------
# 配置持久化
# ---------------------------------------------------------------------------
class ExtConfigStore:
"""扩展数据配置文件读写 — 每个表独立目录 data/ext/{config_id}/config.json。"""
def __init__(self, data_dir: Path) -> None:
self._base = data_dir / "ext_data"
def _config_path(self, config_id: str) -> Path:
return self._base / config_id / "config.json"
def load_all(self) -> list[ExtConfig]:
# 兼容旧版: 如果目录为空且旧配置文件存在则迁移
if not self._base.exists() or not any(self._base.iterdir()):
old = self._base.parent / "ext_configs.json"
if not old.exists():
old = self._base.parent / "ext_configs.json.bak"
if old.exists():
self._migrate_legacy(old)
if not self._base.exists():
return []
configs = []
for d in sorted(self._base.iterdir()):
cp = d / "config.json"
if d.is_dir() and cp.exists():
try:
raw = json.loads(cp.read_text(encoding="utf-8"))
configs.append(ExtConfig.from_dict(raw))
except Exception as e:
logger.warning("扩展表配置解析失败 %s: %s", cp, e)
return configs
def get(self, config_id: str) -> ExtConfig | None:
cp = self._config_path(config_id)
if not cp.exists():
return None
try:
raw = json.loads(cp.read_text(encoding="utf-8"))
return ExtConfig.from_dict(raw)
except Exception:
return None
def upsert(self, config: ExtConfig) -> None:
config.updated_at = datetime.now().isoformat()
cp = self._config_path(config.id)
cp.parent.mkdir(parents=True, exist_ok=True)
cp.write_text(
json.dumps(config.to_dict(), ensure_ascii=False, indent=2),
encoding="utf-8",
)
def delete(self, config_id: str) -> bool:
import shutil
cp = self._config_path(config_id)
if not cp.exists():
return False
shutil.rmtree(cp.parent, ignore_errors=True)
return True
def _migrate_legacy(self, old_path: Path) -> None:
"""一次性迁移旧版 ext_configs.json 到独立目录结构。"""
try:
raw = json.loads(old_path.read_text(encoding="utf-8"))
configs = [ExtConfig.from_dict(d) for d in raw]
for c in configs:
cp = self._config_path(c.id)
cp.parent.mkdir(parents=True, exist_ok=True)
cp.write_text(
json.dumps(c.to_dict(), ensure_ascii=False, indent=2),
encoding="utf-8",
)
# 迁移完成后重命名旧文件作为备份
backup = old_path.with_suffix(".json.bak")
old_path.rename(backup)
logger.info("ext_configs.json 已迁移至 ext/ (备份: %s)", backup.name)
except Exception as e:
logger.warning("ext_configs 迁移失败: %s", e)
# ---------------------------------------------------------------------------
# CSV / Excel 解析 → Parquet 写入
# ---------------------------------------------------------------------------
_POLARS_DTYPE_MAP = {
"string": pl.Utf8,
"int": pl.Int64,
"float": pl.Float64,
"bool": pl.Boolean,
}
def build_code_lookup(data_dir: Path) -> dict[str, str]:
"""从 instruments 维表构建 code → symbol 映射。"""
path = data_dir / "instruments" / "instruments.parquet"
if not path.exists():
return {}
try:
df = pl.read_parquet(path, columns=["code", "symbol"])
return dict(zip(df["code"].to_list(), df["symbol"].to_list()))
except Exception:
return {}
def normalize_symbol(series: pl.Series, lookup: dict[str, str] | None = None) -> pl.Series:
"""将 symbol 列标准化为 代码.交易所 格式。
优先使用 instruments 维表查找 code → symbol,确保 100% 准确。
查不到时按规则兜底:6开头 → .SH,其余 → .SZ。
"""
_lookup = lookup or {}
def _fix_one(val: str) -> str:
if not val:
return val
val = val.strip()
# 已经是标准格式(含 .),直接返回
if "." in val:
return val
# 纯6位数字代码 → 优先查维表
if len(val) == 6 and val.isdigit():
mapped = _lookup.get(val)
if mapped:
return mapped
# 兜底规则
if val.startswith(("6",)):
return f"{val}.SH"
else:
return f"{val}.SZ"
return val
return series.map_elements(_fix_one, return_dtype=pl.Utf8)
def ensure_utf8_csv(file_path: Path) -> Path:
"""确保 CSV 文件以 UTF-8 编码可读,非 UTF-8(如 GBK/GB18030)则转换。
国内行情软件(同花顺/东财/通达信)和 Windows 中文 Excel 导出的 CSV 多为
GBK 系编码,Polars 的 read_csv 默认按 UTF-8 解析会抛 "invalid utf-8 sequence"
这里在交给 Polars 前做一次编码规范化。
返回值:若已是 UTF-8 则返回原路径;否则在同目录写一个 *.utf8 文件并返回它
(调用方用临时目录,随目录一起清理)。
"""
raw = file_path.read_bytes()
# BOM 处理:UTF-8-SIG 等带 BOM 文件直接交给 Polars(它认识 BOM)
try:
raw.decode("utf-8")
return file_path # 已是合法 UTF-8
except UnicodeDecodeError:
pass
# 依次尝试常见中文编码,第一个能完整解码的即为命中
for enc in ("gb18030", "gbk", "gb2312", "big5"):
try:
text = raw.decode(enc)
except UnicodeDecodeError:
continue
out_path = file_path.with_suffix(file_path.suffix + ".utf8")
out_path.write_text(text, encoding="utf-8")
logger.info("CSV 编码转换 %s%s (%s)", file_path.name, out_path.name, enc)
return out_path
# 都无法解码:返回原路径,让 Polars 抛出更精确的原始错误
return file_path
def parse_upload_file(file_path: Path, symbol_col: str = "symbol", data_dir: Path | None = None) -> pl.DataFrame:
"""解析上传的 CSV / Excel 文件为 Polars DataFrame。"""
suffix = file_path.suffix.lower()
if suffix == ".csv":
df = pl.read_csv(ensure_utf8_csv(file_path), infer_schema_length=10000)
elif suffix in (".xlsx", ".xls"):
df = pl.read_excel(file_path)
else:
raise ValueError(f"不支持的文件格式: {suffix}")
if symbol_col not in df.columns:
# 尝试模糊匹配
candidates = [c for c in df.columns if c.lower() in ("symbol", "code", "代码", "标的")]
if candidates:
df = df.rename({candidates[0]: symbol_col})
else:
raise ValueError(f"未找到标的代码列 (symbol),可选列: {df.columns}")
# 确保 symbol 列为字符串并标准化
lookup = build_code_lookup(data_dir) if data_dir else None
df = df.with_columns(normalize_symbol(df[symbol_col].cast(pl.Utf8), lookup))
return df
def cast_df_to_schema(df: pl.DataFrame, fields: list[ExtField]) -> pl.DataFrame:
"""按配置的字段类型转换 DataFrame 列类型。"""
for f in fields:
if f.name in df.columns:
target = _POLARS_DTYPE_MAP.get(f.dtype, pl.Utf8)
df = df.with_columns(pl.col(f.name).cast(target))
return df
def _config_dir(config_id: str, data_dir: Path) -> Path:
"""返回扩展配置的根目录 data/ext_data/{config_id}/。"""
return data_dir / "ext_data" / config_id
def write_ext_parquet(
df: pl.DataFrame,
config: ExtConfig,
data_dir: Path,
snapshot_date: date | None = None,
) -> int:
"""将 DataFrame 写入扩展数据 Parquet。
目录结构:
- snapshot: data/ext_data/{id}/part.parquet(与 config.json 同级,覆盖写)
- timeseries: data/ext_data/{id}/timeseries/date=xxx/part.parquet(按日分区)
Returns:
写入行数。
"""
snap = snapshot_date or date.today()
cfg_dir = _config_dir(config.id, data_dir)
# 标准化 symbol 列: 用维表查找 → 准确匹配交易所
if "symbol" in df.columns:
lookup = build_code_lookup(data_dir)
df = df.with_columns(normalize_symbol(df["symbol"], lookup))
if config.mode == "snapshot":
# 快照: 与 config.json 同级,直接覆盖
cfg_dir.mkdir(parents=True, exist_ok=True)
out_path = cfg_dir / "part.parquet"
# 如果已有文件,合并去重后覆盖
if out_path.exists():
try:
existing = pl.read_parquet(out_path)
key = "symbol" if "symbol" in df.columns else df.columns[0]
df = pl.concat([existing, df]).unique(subset=[key], keep="last")
except Exception:
pass
else:
# 时序: timeseries/ 下按日期分区
out_dir = cfg_dir / "timeseries" / f"date={snap}"
out_dir.mkdir(parents=True, exist_ok=True)
out_path = out_dir / "part.parquet"
# 如果已有文件,合并去重
if out_path.exists():
try:
existing = pl.read_parquet(out_path)
key = "symbol" if "symbol" in df.columns else df.columns[0]
df = pl.concat([existing, df]).unique(subset=[key], keep="last")
except Exception:
pass
df = cast_df_to_schema(df, config.fields)
df.write_parquet(out_path)
logger.info("扩展表写入: %s%s (%d 行)", config.id, out_path, len(df))
return len(df)
def delete_ext_parquet(config_id: str, data_dir: Path) -> None:
"""删除扩展数据源关联的所有 Parquet 数据(保留 config.json)。
- snapshot: 删除 ext_data/{id}/part.parquet
- timeseries: 删除 ext_data/{id}/timeseries/ 目录
"""
cfg_dir = _config_dir(config_id, data_dir)
# 删除快照文件
snap = cfg_dir / "part.parquet"
if snap.exists():
snap.unlink()
# 删除时序目录
ts_dir = cfg_dir / "timeseries"
if ts_dir.exists():
import shutil
shutil.rmtree(ts_dir, ignore_errors=True)
def fix_symbol_format(config: ExtConfig, data_dir: Path) -> int:
"""扫描该扩展配置的所有 Parquet 文件,将 symbol 列标准化为 代码.交易所 格式。
- snapshot: 扫描 ext_data/{id}/part.parquet
- timeseries: 扫描 ext_data/{id}/timeseries/date=xxx/part.parquet
Returns:
修复的文件数。
"""
cfg_dir = _config_dir(config.id, data_dir)
if not cfg_dir.exists():
return 0
# 收集需要扫描的 parquet 文件列表
parquet_files: list[Path] = []
if config.mode == "snapshot":
p = cfg_dir / "part.parquet"
if p.exists():
parquet_files.append(p)
else:
ts_dir = cfg_dir / "timeseries"
if ts_dir.exists():
for part_dir in sorted(ts_dir.iterdir()):
if not part_dir.is_dir() or not part_dir.name.startswith("date="):
continue
p = part_dir / "part.parquet"
if p.exists():
parquet_files.append(p)
fixed = 0
lookup = build_code_lookup(data_dir)
for parquet_path in parquet_files:
try:
df = pl.read_parquet(parquet_path)
if "symbol" not in df.columns:
continue
old = df["symbol"].to_list()
df = df.with_columns(normalize_symbol(df["symbol"], lookup))
new = df["symbol"].to_list()
if old != new:
df.write_parquet(parquet_path)
fixed += 1
logger.info("代码格式修复: %s/%s (%d 行)", config.id, parquet_path.parent.name, len(df))
except Exception as e:
logger.warning("代码格式修复跳过 %s: %s", parquet_path, e)
return fixed
def rows_to_parquet(
rows: list[dict],
config: ExtConfig,
data_dir: Path,
snapshot_date: date | None = None,
) -> int:
"""将 JSON 行列表转为 DataFrame 写入 Parquet,复用 write_ext_parquet 的存储逻辑。
Returns:
写入行数。
"""
df = pl.DataFrame(rows)
if "symbol" in df.columns:
df = df.with_columns(pl.col("symbol").cast(pl.Utf8))
return write_ext_parquet(df, config, data_dir, snapshot_date=snapshot_date)
+216
View File
@@ -0,0 +1,216 @@
"""扩展数据定时拉取引擎 — 从外部 API 拉取数据写入 Parquet。"""
from __future__ import annotations
import asyncio
import json
import logging
import threading
from datetime import date, datetime, timezone
from functools import reduce
from typing import Any
import httpx
from app.services.ext_data import (
ExtConfig,
ExtConfigStore,
PullConfig,
rows_to_parquet,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# 响应解析
# ---------------------------------------------------------------------------
def _extract_rows(data: Any, path: str) -> list[dict]:
"""按 dot-path 从 JSON 响应中提取行数组。
例: path="data.list" → response["data"]["list"]
如果 path 为空,直接将 data 视为数组。
"""
if not path:
if isinstance(data, list):
return data
raise ValueError("response_path 为空但响应不是数组")
keys = path.split(".")
current = data
for key in keys:
if isinstance(current, dict):
if key not in current:
raise ValueError(f"响应中不存在路径 '{path}',缺失键 '{key}'")
current = current[key]
elif isinstance(current, list):
try:
current = current[int(key)]
except (ValueError, IndexError) as e:
raise ValueError(f"响应路径 '{path}' 解析失败: {e}") from e
else:
raise ValueError(f"响应路径 '{path}' 中间值不是 dict/list: {type(current)}")
if not isinstance(current, list):
raise ValueError(f"路径 '{path}' 指向的不是数组,而是 {type(current)}")
return current
def _apply_field_map(rows: list[dict], field_map: dict[str, str]) -> list[dict]:
"""将外部字段名映射为内部配置字段名。field_map: {外部名: 内部名}。"""
if not field_map:
return rows
mapped = []
for row in rows:
new_row: dict = {}
for k, v in row.items():
mapped_key = field_map.get(k, k)
new_row[mapped_key] = v
mapped.append(new_row)
return mapped
# ---------------------------------------------------------------------------
# 拉取执行
# ---------------------------------------------------------------------------
async def fetch_and_ingest(
config: ExtConfig,
data_dir,
) -> tuple[int, str]:
"""执行一次拉取: 请求外部 API → 解析响应 → 写入 Parquet。
Returns:
(rows_written, date_str)
"""
pull = config.pull
if not pull or not pull.url:
raise ValueError("拉取未配置或 URL 为空")
async with httpx.AsyncClient(timeout=30) as client:
headers = pull.headers or {}
kwargs: dict[str, Any] = {"headers": headers}
if pull.method.upper() == "POST" and pull.body:
kwargs["content"] = pull.body
if "content-type" not in {k.lower() for k in headers}:
kwargs["headers"]["Content-Type"] = "application/json"
resp = await client.request(pull.method.upper(), pull.url, **kwargs)
resp.raise_for_status()
# 解析 JSON
try:
data = resp.json()
except Exception as e:
raise ValueError(f"响应不是有效 JSON: {e}") from e
# 提取行
rows = _extract_rows(data, pull.response_path)
if not rows:
raise ValueError("提取到的行数为 0")
# 字段映射
rows = _apply_field_map(rows, pull.field_map)
# 校验 symbol 列
if rows and "symbol" not in rows[0]:
raise ValueError("数据行中缺少 symbol 字段,请配置 field_map 映射")
# 写入
snap = date.today()
n = rows_to_parquet(rows, config, data_dir, snapshot_date=snap)
return n, snap.isoformat()
# ---------------------------------------------------------------------------
# 调度器
# ---------------------------------------------------------------------------
class PullScheduler:
"""后台调度器:为每个启用了 pull 的 ExtConfig 维护定时任务。"""
def __init__(self) -> None:
self._tasks: dict[str, asyncio.Task] = {}
self._running = False
self._lock = threading.Lock()
def start(self, data_dir) -> None:
"""启动调度(在 lifespan startup 调用)。"""
self._running = True
self._data_dir = data_dir
logger.info("PullScheduler started")
def stop(self) -> None:
"""停止所有任务。"""
self._running = False
for task in self._tasks.values():
task.cancel()
self._tasks.clear()
logger.info("PullScheduler stopped")
def refresh(self, data_dir) -> None:
"""重新加载配置,更新调度任务(增/删/改)。"""
self._data_dir = data_dir
store = ExtConfigStore(data_dir)
configs = store.load_all()
active_ids: set[str] = set()
for config in configs:
if not config.pull or not config.pull.enabled or not config.pull.url:
continue
active_ids.add(config.id)
if config.id not in self._tasks:
# 新增调度
task = asyncio.create_task(self._run_loop(config))
self._tasks[config.id] = task
logger.info("PullScheduler: scheduled %s (every %d min)", config.id, config.pull.schedule_minutes)
# 移除不再活跃的
for cid in list(self._tasks):
if cid not in active_ids:
self._tasks[cid].cancel()
del self._tasks[cid]
logger.info("PullScheduler: removed %s", cid)
async def _run_loop(self, config: ExtConfig) -> None:
"""单个配置的定时拉取循环。"""
try:
while self._running:
pull = config.pull
if not pull:
break
interval = max(pull.schedule_minutes * 60, 60) # 至少 60s
await asyncio.sleep(interval)
if not self._running:
break
try:
# 重新加载最新配置(用户可能中途修改)
store = ExtConfigStore(self._data_dir)
fresh = store.get(config.id)
if not fresh or not fresh.pull or not fresh.pull.enabled:
break
n, d = await fetch_and_ingest(fresh, self._data_dir)
fresh.pull.last_run = datetime.now(timezone.utc).isoformat()
fresh.pull.last_status = "success"
fresh.pull.last_message = f"{n} rows @ {d}"
fresh.pull.last_rows = n
store.upsert(fresh)
logger.info("PullScheduler: %s success, %d rows", config.id, n)
except Exception as e:
store = ExtConfigStore(self._data_dir)
fresh = store.get(config.id)
if fresh and fresh.pull:
fresh.pull.last_run = datetime.now(timezone.utc).isoformat()
fresh.pull.last_status = "error"
fresh.pull.last_message = str(e)[:200]
store.upsert(fresh)
logger.warning("PullScheduler: %s error: %s", config.id, e)
except asyncio.CancelledError:
pass
# 全局单例
pull_scheduler = PullScheduler()
+224
View File
@@ -0,0 +1,224 @@
"""向前扩展历史数据 — 完全独立于 daily_pipeline 的盘后管道。
用户从日 K 卡片手动触发,指定往前补的时长 (x 天/月/年)。
流程:
1. 获取当前最早日期
2. 向前拉日 K batch (start = 最早日期 - offset, end = 最早日期)
3. 向前拉除权因子 (同范围)
4. 全量重算 enriched
5. 刷新视图 + 缓存
⚠️ 本模块不导入 daily_pipeline 的任何函数,只复用基础设施:
- kline_sync.sync_and_persist_daily_batch / sync_adj_factor
- indicators.pipeline.run_pipeline
- pipeline_jobs.JobStore
- tickflow.repository.KlineRepository
"""
from __future__ import annotations
import logging
from collections.abc import Callable
from datetime import date, datetime, timedelta
from app.services import kline_sync
from app.services.pipeline_jobs import job_store
from app.tickflow.capabilities import Cap, CapabilitySet
from app.tickflow.repository import KlineRepository
logger = logging.getLogger(__name__)
def _noop(stage: str, pct: int, msg: str, **kwargs) -> None: # noqa: ARG001
pass
def _invalidate(table: str | None = None) -> None:
from app.api.data import invalidate_data_cache
invalidate_data_cache(table)
def _resolve_universe(capset: CapabilitySet) -> list[str]:
"""解析标的池 — 与 daily_pipeline 独立的副本。"""
if capset.has(Cap.KLINE_DAILY_BATCH):
try:
from app.tickflow.pools import get_pool
all_a = get_pool("CN_Equity_A", refresh=True)
if all_a:
return sorted(all_a)
except Exception as e:
logger.warning("CN_Equity_A pool unavailable: %s", e)
from app.tickflow.pools import DEMO_SYMBOLS, get_pool as _get_pool
from app.config import settings
from pathlib import Path
import polars as pl
base: set[str] = set(DEMO_SYMBOLS)
base.update(_get_pool("watchlist"))
d = Path(settings.data_dir)
inst_path = d / "instruments" / "instruments.parquet"
if inst_path.exists():
try:
inst = pl.read_parquet(inst_path, columns=["symbol"])
base.update(inst["symbol"].to_list())
except Exception as e:
logger.warning("instruments supplement failed: %s", e)
return sorted(base)
def _refresh_single_view(repo: KlineRepository, name: str) -> None:
"""刷新单个 DuckDB 视图。"""
d = repo.store.data_dir.as_posix()
paths = {
"kline_daily": f"{d}/kline_daily/**/*.parquet",
"kline_enriched": f"{d}/kline_daily_enriched/**/*.parquet",
"kline_minute": f"{d}/kline_minute/**/*.parquet",
"adj_factor": f"{d}/adj_factor/**/*.parquet",
"instruments": f"{d}/instruments/**/*.parquet",
}
path = paths.get(name)
if not path:
return
try:
repo.db.execute(
f"CREATE OR REPLACE VIEW {name} AS "
f"SELECT * FROM read_parquet('{path}', union_by_name=true)"
)
except Exception as e:
logger.warning("refresh view %s failed: %s", name, e)
def compute_offset(value: int, unit: str) -> timedelta:
"""将用户输入的 value + unit 转成 timedelta。"""
if unit == "day":
return timedelta(days=value)
elif unit == "month":
return timedelta(days=value * 30)
elif unit == "year":
return timedelta(days=value * 365)
else:
raise ValueError(f"不支持的单位: {unit}")
def run_extend_history(
repo: KlineRepository,
capset: CapabilitySet,
value: int,
unit: str,
on_progress: Callable | None = None,
) -> dict:
"""向前扩展历史数据的主函数。
完全独立于 daily_pipeline.run_now(),不调用其任何逻辑。
返回结果 dict 供 job_store 记录。
"""
emit = on_progress or _noop
# 0. 计算时间偏移
offset = compute_offset(value, unit)
today = date.today()
# 1. 获取当前最早日期
emit("extend_history", 2, "检查当前数据范围…")
earliest = repo.earliest_daily_date()
if not earliest:
return {"error": "本地无日K数据,请先执行一次完整同步"}
new_start = earliest - offset
# 不能超过今天
if new_start >= earliest:
return {"error": "扩展范围无效,请增大时间跨度"}
# 2. 解析标的池
emit("extend_history", 5, "解析标的池…")
universe = _resolve_universe(capset)
if not universe:
return {"error": "标的池为空"}
emit("extend_history", 8, f"标的池: {len(universe)}")
start_str = new_start.strftime("%Y-%m-%d")
end_str = earliest.strftime("%Y-%m-%d")
# 3. 拉日 K
emit("extend_history", 10, f"获取日K [{start_str} ~ {end_str}]…")
logger.info("extend_history: daily K [%s ~ %s], %d symbols", start_str, end_str, len(universe))
def _daily_chunk(cur: int, tot: int) -> None:
emit("extend_history", 10 + int(35 * cur / tot),
f"日K 批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
written_daily = kline_sync.sync_and_persist_daily_batch(
universe, repo, capset,
start_date=datetime.combine(new_start, datetime.min.time()),
end_date=datetime.combine(earliest, datetime.min.time()),
on_chunk_done=_daily_chunk,
)
emit("extend_history", 45, f"日K 完成,写入 {written_daily}")
logger.info("extend_history: daily K done, %d rows", written_daily)
_refresh_single_view(repo, "kline_daily")
_invalidate("daily")
# 4. 拉除权因子 (新范围)
written_adj = 0
adj_start = datetime.combine(new_start, datetime.min.time())
adj_end = datetime.combine(today, datetime.min.time())
adj_start_str = new_start.strftime("%Y-%m-%d")
adj_end_str = today.strftime("%Y-%m-%d")
if capset.has(Cap.ADJ_FACTOR):
emit("extend_history", 48, f"获取除权因子 [{adj_start_str} ~ {adj_end_str}]…")
logger.info("extend_history: adj_factor [%s ~ %s]", adj_start_str, adj_end_str)
def _adj_chunk(cur: int, tot: int) -> None:
emit("extend_history", 48 + int(10 * cur / tot),
f"除权因子批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
written_adj, _affected = kline_sync.sync_adj_factor(
universe, repo, capset,
start_time=adj_start, end_time=adj_end,
on_chunk_done=_adj_chunk,
)
emit("extend_history", 60, f"除权因子完成,{written_adj}")
logger.info("extend_history: adj_factor done, %d rows", written_adj)
_refresh_single_view(repo, "adj_factor")
_invalidate("adj_factor")
else:
emit("extend_history", 60, "除权因子跳过(无权限)")
logger.info("extend_history: adj_factor skipped, no ADJ_FACTOR capability")
# 5. 全量重算 enriched
emit("extend_history", 65, "全量计算 enriched…")
logger.info("extend_history: full enriched rebuild start")
from app.indicators.pipeline import run_pipeline
written_enriched = run_pipeline()
enriched_dir = repo.store.data_dir / "kline_daily_enriched"
enriched_days = len(list(enriched_dir.glob("date=*"))) if enriched_dir.exists() else 0
emit("extend_history", 92, f"enriched 完成,覆盖 {enriched_days}")
logger.info("extend_history: enriched done, %d days", enriched_days)
_refresh_single_view(repo, "kline_enriched")
_invalidate("enriched")
# 6. 刷新视图
emit("extend_history", 95, "刷新视图…")
_refresh_single_view(repo, "kline_daily")
_refresh_single_view(repo, "kline_enriched")
_refresh_single_view(repo, "adj_factor")
_invalidate(None)
# 7. 统计结果
daily_dir = repo.store.data_dir / "kline_daily"
daily_days = len(list(daily_dir.glob("date=*"))) if daily_dir.exists() else 0
emit("extend_history", 100, f"完成,已扩展至 {new_start}")
return {
"earliest_before": earliest.isoformat(),
"earliest_after": new_start.isoformat(),
"daily_rows": written_daily,
"daily_days": daily_days,
"adj_factor_rows": written_adj,
"enriched_days": enriched_days,
"universe_size": len(universe),
}
+314
View File
@@ -0,0 +1,314 @@
"""财务数据独立同步服务。
解耦于 K-line 管道, 自有调度 + 自有存储。
能力门控: Cap.FINANCIAL (Expert 套餐)
"""
from __future__ import annotations
import asyncio
import logging
import threading
from datetime import date, datetime, timezone
from pathlib import Path
from typing import Any
import polars as pl
from app.tickflow.capabilities import Cap, CapabilitySet
logger = logging.getLogger(__name__)
# 每个 API 请求最多 100 个标的
_BATCH_SIZE = 100
# 4 张财务表
FINANCIAL_TABLES = ("metrics", "income", "balance_sheet", "cash_flow")
# ================================================================
# 同步函数
# ================================================================
def _get_symbols(data_dir: Path) -> list[str]:
"""从 instruments 表获取标的列表。"""
inst_path = data_dir / "instruments" / "instruments.parquet"
if not inst_path.exists():
return []
try:
df = pl.read_parquet(inst_path, columns=["symbol"])
return df["symbol"].to_list()
except Exception as e:
logger.warning("读取 instruments 失败: %s", e)
return []
def _sync_table(
table: str,
symbols: list[str],
data_dir: Path,
capset: CapabilitySet,
latest_only: bool = True,
) -> int:
"""同步单张财务表。返回写入的行数。"""
if not capset.has(Cap.FINANCIAL):
logger.info("sync_%s skipped: no FINANCIAL capability", table)
return 0
if not symbols:
logger.warning("sync_%s skipped: no symbols", table)
return 0
from app.tickflow.client import get_client
tf = get_client()
# 分批拉取
api_method = {
"metrics": tf.financials.metrics,
"income": tf.financials.income,
"balance_sheet": tf.financials.balance_sheet,
"cash_flow": tf.financials.cash_flow,
}[table]
all_records: list[dict] = []
total_batches = (len(symbols) + _BATCH_SIZE - 1) // _BATCH_SIZE
for i in range(0, len(symbols), _BATCH_SIZE):
chunk = symbols[i : i + _BATCH_SIZE]
batch_num = i // _BATCH_SIZE + 1
try:
data = api_method(chunk, latest=latest_only)
# data 格式: { "600519.SH": [record, ...], ... }
if isinstance(data, dict):
for sym, records in data.items():
if isinstance(records, list):
for rec in records:
if isinstance(rec, dict):
rec["symbol"] = sym
all_records.append(rec)
logger.debug("sync_%s batch %d/%d: %d records", table, batch_num, total_batches, len(data) if isinstance(data, dict) else 0)
except Exception as e:
logger.warning("sync_%s batch %d/%d failed: %s", table, batch_num, total_batches, e)
if not all_records:
return 0
df = pl.DataFrame(all_records)
if df.is_empty():
return 0
# 确保 symbol 列存在
if "symbol" not in df.columns:
return 0
# 写入 Parquet (全量覆盖)
out_dir = data_dir / "financials" / table
out_dir.mkdir(parents=True, exist_ok=True)
out_file = out_dir / "part.parquet"
df.write_parquet(out_file)
logger.info("sync_%s done: %d records written", table, len(df))
return len(df)
def sync_metrics(data_dir: Path, capset: CapabilitySet) -> int:
"""同步核心财务指标 (metrics)。"""
symbols = _get_symbols(data_dir)
return _sync_table("metrics", symbols, data_dir, capset, latest_only=True)
def sync_income(data_dir: Path, capset: CapabilitySet) -> int:
"""同步利润表。"""
symbols = _get_symbols(data_dir)
return _sync_table("income", symbols, data_dir, capset, latest_only=True)
def sync_balance_sheet(data_dir: Path, capset: CapabilitySet) -> int:
"""同步资产负债表。"""
symbols = _get_symbols(data_dir)
return _sync_table("balance_sheet", symbols, data_dir, capset, latest_only=True)
def sync_cash_flow(data_dir: Path, capset: CapabilitySet) -> int:
"""同步现金流量表。"""
symbols = _get_symbols(data_dir)
return _sync_table("cash_flow", symbols, data_dir, capset, latest_only=True)
def sync_all(data_dir: Path, capset: CapabilitySet) -> dict[str, int]:
"""同步所有财务表。返回 {table: rows}。"""
if not capset.has(Cap.FINANCIAL):
logger.info("sync_all financials skipped: no FINANCIAL capability")
return {}
symbols = _get_symbols(data_dir)
results: dict[str, int] = {}
for table in FINANCIAL_TABLES:
results[table] = _sync_table(table, symbols, data_dir, capset, latest_only=True)
# 同步完成后注册 DuckDB 视图
_refresh_financials_views(data_dir)
return results
# ================================================================
# DuckDB 视图
# ================================================================
def _refresh_financials_views(data_dir: Path) -> None:
"""刷新财务表 DuckDB 视图 (在 DataStore.db 上注册)。"""
d = data_dir.as_posix()
views = {
"financials_metrics": f"{d}/financials/metrics/*.parquet",
"financials_income": f"{d}/financials/income/*.parquet",
"financials_balance_sheet": f"{d}/financials/balance_sheet/*.parquet",
"financials_cash_flow": f"{d}/financials/cash_flow/*.parquet",
}
for name, path in views.items():
out = data_dir / "financials" / name.replace("financials_", "") / "part.parquet"
if not out.exists():
continue
# 视图注册需要由 DataStore 完成,这里只做日志
logger.debug("financial parquet ready: %s (%d rows)", name, out.stat().st_size)
def get_financial_df(data_dir: Path, table: str) -> pl.DataFrame:
"""读取本地财务 Parquet。"""
path = data_dir / "financials" / table / "part.parquet"
if not path.exists():
return pl.DataFrame()
try:
return pl.read_parquet(path)
except Exception as e:
logger.warning("读取 financials/%s 失败: %s", table, e)
return pl.DataFrame()
# ================================================================
# 调度器
# ================================================================
class FinancialScheduler:
"""独立调度器: 每周同步 metrics, 每季度同步三张报表。"""
def __init__(self) -> None:
self._task: asyncio.Task | None = None
self._running = False
self._data_dir: Path | None = None
self._capset: CapabilitySet | None = None
self._lock = threading.Lock()
self._last_sync: dict[str, str] = {} # {table: iso_timestamp}
# 手动同步(run_now)是否正在进行。前端据此显示"同步中"并防重复点击。
self._is_syncing = False
def start(self, data_dir: Path, capset: CapabilitySet) -> None:
if not capset.has(Cap.FINANCIAL):
logger.info("FinancialScheduler skipped: no FINANCIAL capability")
return
self._data_dir = data_dir
self._capset = capset
self._running = True
self._task = asyncio.create_task(self._run_loop())
logger.info("FinancialScheduler started")
def stop(self) -> None:
self._running = False
if self._task:
self._task.cancel()
self._task = None
logger.info("FinancialScheduler stopped")
def update(self, data_dir: Path, capset: CapabilitySet) -> None:
"""运行时更新数据目录和能力集。
用户在设置页更换/清除 Key 后,能力集可能变化,无需重启服务即可让
财务调度器生效或失效。
"""
had_financial = self._capset is not None and self._capset.has(Cap.FINANCIAL)
has_financial = capset.has(Cap.FINANCIAL)
self._data_dir = data_dir
self._capset = capset
if has_financial and not self._running:
self.start(data_dir, capset)
elif had_financial and not has_financial and self._running:
self.stop()
async def _run_loop(self) -> None:
"""每周执行一次 metrics 同步。"""
try:
while self._running:
# 首次启动等 60s, 之后每 7 天执行一次
await asyncio.sleep(60)
if not self._running:
break
# 每周: 只同步 metrics
try:
rows = sync_metrics(self._data_dir, self._capset)
self._last_sync["metrics"] = datetime.now(timezone.utc).isoformat()
logger.info("FinancialScheduler: metrics synced, %d rows", rows)
except Exception as e:
logger.warning("FinancialScheduler: metrics sync failed: %s", e)
# 等待下一次 (7天)
for _ in range(7 * 24 * 60): # 每分钟检查一次 _running
if not self._running:
break
await asyncio.sleep(60)
except asyncio.CancelledError:
pass
def run_now(self, table: str | None = None) -> dict[str, int]:
"""手动触发同步。table=None 同步全部。
用 _is_syncing 标志防并发:若已有同步在进行,本次直接跳过,
避免重复请求拖慢服务端 / 触发上游限流。
"""
if not self._capset or not self._capset.has(Cap.FINANCIAL):
return {}
with self._lock:
if self._is_syncing:
logger.info("financial sync skipped: already running")
return {"_skipped": 1}
self._is_syncing = True
try:
if table:
fn = {
"metrics": sync_metrics,
"income": sync_income,
"balance_sheet": sync_balance_sheet,
"cash_flow": sync_cash_flow,
}.get(table)
if not fn:
return {}
rows = fn(self._data_dir, self._capset)
self._last_sync[table] = datetime.now(timezone.utc).isoformat()
return {table: rows}
else:
# 全部同步: 逐表执行, 每张完成立即更新 last_sync,
# 让前端轮询 /status 能看到进度递增 (而非等全部完成才一次性更新)。
symbols = _get_symbols(self._data_dir)
result: dict[str, int] = {}
for t in FINANCIAL_TABLES:
result[t] = _sync_table(t, symbols, self._data_dir, self._capset, latest_only=True)
self._last_sync[t] = datetime.now(timezone.utc).isoformat()
_refresh_financials_views(self._data_dir)
return result
finally:
with self._lock:
self._is_syncing = False
@property
def is_syncing(self) -> bool:
"""手动同步是否正在进行(供 /status 返回,前端据此显示"同步中")。"""
with self._lock:
return self._is_syncing
@property
def last_sync(self) -> dict[str, str]:
return dict(self._last_sync)
# 全局单例
financial_scheduler = FinancialScheduler()
+146
View File
@@ -0,0 +1,146 @@
"""指数数据同步服务。"""
from __future__ import annotations
import logging
import gc
from datetime import datetime, timedelta
import polars as pl
from app.indicators.pipeline import compute_enriched
from app.services import kline_sync, preferences
from app.tickflow.capabilities import Cap, CapabilitySet
from app.tickflow.client import get_client
from app.tickflow.repository import KlineRepository
logger = logging.getLogger(__name__)
def _quotes_to_index_instruments(resp) -> pl.DataFrame:
"""将数据源 quotes 响应规范为指数 instruments。"""
if resp is None:
return pl.DataFrame()
if isinstance(resp, pl.DataFrame):
df = resp
elif hasattr(resp, "columns"):
df = pl.from_pandas(resp.reset_index() if hasattr(resp, "reset_index") else resp)
else:
rows: list[dict] = []
for q in resp or []:
item = q if isinstance(q, dict) else {}
ext = item.get("ext") or {}
symbol = item.get("symbol")
if not symbol:
continue
rows.append({
"symbol": str(symbol),
"name": ext.get("name") or item.get("name") or str(symbol),
})
df = pl.DataFrame(rows)
if df.is_empty() or "symbol" not in df.columns:
return pl.DataFrame()
rename = {"ts_code": "symbol"}
df = df.rename({k: v for k, v in rename.items() if k in df.columns})
if "name" not in df.columns:
if "ext" in df.columns:
df = df.with_columns(pl.col("symbol").cast(pl.Utf8).alias("name"))
else:
df = df.with_columns(pl.col("symbol").cast(pl.Utf8).alias("name"))
result = df.select([
pl.col("symbol").cast(pl.Utf8),
pl.col("name").cast(pl.Utf8),
]).with_columns([
pl.col("symbol").str.split(".").list.first().alias("code"),
pl.lit("index").alias("asset_type"),
])
return result.unique(subset=["symbol"], keep="last").sort("symbol")
def sync_index_instruments(repo: KlineRepository) -> int:
"""同步 CN_Index 指数标的维表,返回指数数量。"""
tf = get_client()
resp = None
errors: list[str] = []
for kwargs in (
{"universes": ["CN_Index"]},
{"universes": ["CN_Index"], "as_dataframe": False},
):
try:
resp = tf.quotes.get_by_universes(**kwargs)
if resp is not None and len(resp) > 0:
break
except Exception as e: # noqa: BLE001
errors.append(str(e))
resp = None
if resp is None or len(resp) == 0:
logger.warning("CN_Index universe returned empty: %s", "; ".join(errors))
return 0
instruments = _quotes_to_index_instruments(resp)
if instruments.is_empty():
return 0
repo.save_index_instruments(instruments)
repo.refresh_index_views()
return instruments.height
def sync_and_persist_index_daily(
repo: KlineRepository,
capset: CapabilitySet,
count: int | None = None,
start_date: datetime | None = None,
end_date: datetime | None = None,
) -> int:
"""同步指数日K到独立 parquet,并计算指数 enriched。"""
if not capset.has(Cap.KLINE_DAILY_BATCH):
return 0
instruments = repo.get_index_instruments()
if instruments.is_empty():
sync_index_instruments(repo)
instruments = repo.get_index_instruments()
if instruments.is_empty() or "symbol" not in instruments.columns:
return 0
symbols = sorted(set(instruments["symbol"].to_list()))
lim = capset.limits(Cap.KLINE_DAILY_BATCH)
batch_size = preferences.get_index_daily_batch_size()
if lim and lim.batch:
batch_size = min(batch_size, lim.batch)
rpm = lim.rpm if lim else None
end_time = end_date or datetime.now()
start_time = start_date or (end_time - timedelta(days=365))
total_rows = 0
interval = (60.0 / rpm) if rpm else 0
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
for i, chunk in enumerate(chunks):
if i > 0 and interval > 0 and len(chunks) > rpm:
import time
time.sleep(interval)
raw = kline_sync.sync_daily_batch(
chunk,
count=count,
batch_size=None,
start_time=start_time,
end_time=end_time,
)
if raw.is_empty():
continue
repo.append_index_daily(raw)
enriched = compute_enriched(raw, factors=None, instruments=None)
repo.append_index_enriched(enriched)
total_rows += raw.height
logger.info("index daily synced: %d/%d chunks, +%d rows", i + 1, len(chunks), raw.height)
del raw, enriched
gc.collect()
repo.refresh_index_views()
return total_rows
+122
View File
@@ -0,0 +1,122 @@
"""标的维表同步服务。
盘前 9:10 调用 tf.exchanges.get_instruments("SH"/"SZ"/"BJ", type="stock")
获取全量标的元数据,flatten ext 字段,写入 instruments.parquet。
Starter+ 盘后可用 quotes.get(universes) 顺便补充 name。
"""
from __future__ import annotations
import logging
from datetime import date
from pathlib import Path
import polars as pl
from app.tickflow.client import get_client
logger = logging.getLogger(__name__)
_EXCHANGES = ["SH", "SZ", "BJ"]
def _flatten_instruments(items: list[dict]) -> list[dict]:
"""把 SDK 返回的 Instrument 列表 flatten 成扁平行。"""
rows = []
for item in items:
row = {
"symbol": item.get("symbol"),
"name": item.get("name"),
"code": item.get("code"),
"exchange": item.get("exchange"),
"region": item.get("region"),
"type": item.get("type"),
}
ext = item.get("ext") or {}
row["listing_date"] = ext.get("listing_date")
row["total_shares"] = ext.get("total_shares")
row["float_shares"] = ext.get("float_shares")
row["tick_size"] = ext.get("tick_size")
row["limit_up"] = ext.get("limit_up")
row["limit_down"] = ext.get("limit_down")
rows.append(row)
return rows
def sync_instruments(data_dir: Path) -> int:
"""全量同步标的维表 → data/instruments/instruments.parquet。
返回写入的行数。
"""
tf = get_client()
all_rows: list[dict] = []
for ex in _EXCHANGES:
try:
items = tf.exchanges.get_instruments(ex, instrument_type="stock")
if items:
all_rows.extend(_flatten_instruments(items))
logger.info("instruments %s: %d stocks", ex, len(items))
except Exception as e:
logger.warning("get_instruments(%s) failed: %s", ex, e)
if not all_rows:
return 0
df = pl.DataFrame(all_rows)
df = df.with_columns(pl.lit(date.today()).alias("as_of"))
out = data_dir / "instruments" / "instruments.parquet"
out.parent.mkdir(parents=True, exist_ok=True)
df.write_parquet(out)
logger.info("instruments synced: %d rows → %s", df.height, out)
return df.height
def enrich_names_from_quotes(
data_dir: Path,
quotes_data: list[dict],
) -> int:
"""从 quotes 响应中提取 name,更新 instruments 维表(兜底补充)。
盘后 quotes.get(universes) 返回的数据中包含 ext.name
用来补充 instruments 中可能缺失的 name。
"""
if not quotes_data:
return 0
# 构建 symbol → name 映射
name_map: dict[str, str] = {}
for q in quotes_data:
symbol = q.get("symbol", "")
ext = q.get("ext") or {}
name = ext.get("name") or q.get("name", "")
if symbol and name:
name_map[symbol] = name
if not name_map:
return 0
inst_path = data_dir / "instruments" / "instruments.parquet"
if not inst_path.exists():
return 0
df = pl.read_parquet(inst_path)
# 只更新空 name 的行
updates = pl.DataFrame({
"symbol": list(name_map.keys()),
"_new_name": list(name_map.values()),
})
df = df.join(updates, on="symbol", how="left")
df = df.with_columns(
pl.when(pl.col("name").is_null() | (pl.col("name") == ""))
.then(pl.col("_new_name"))
.otherwise(pl.col("name"))
.alias("name"),
).drop("_new_name")
df.write_parquet(inst_path)
logger.info("instruments name enriched from quotes: %d names", len(name_map))
return len(name_map)
+617
View File
@@ -0,0 +1,617 @@
"""日 K 同步服务(§7.7 Step 1)。
调度器在 capability 允许下,把符号集合的日 K 批量同步到本地 Parquet。
策略:
- 日 K 仅使用 `kline.daily.batch`
- 除权因子仅使用 `adj_factor`
"""
from __future__ import annotations
import logging
import time
from collections.abc import Callable
from datetime import datetime, timedelta
import polars as pl
from app.indicators.pipeline import filter_halt_days
from app.tickflow.capabilities import Cap, CapabilitySet
from app.tickflow.client import get_client
from app.tickflow.repository import KlineRepository
logger = logging.getLogger(__name__)
# 标准列(无论 SDK 返回什么形状,我们把它规范成这套)
CANONICAL_DAILY_COLS = [
"symbol", "date", "open", "high", "low", "close", "volume", "amount",
]
def _normalize_daily(df_in, default_symbol: str | None = None) -> pl.DataFrame:
"""把 SDK 返回的 pandas/任意 DataFrame 规范成 canonical 列。"""
if df_in is None or len(df_in) == 0:
return pl.DataFrame()
if not isinstance(df_in, pl.DataFrame):
df = pl.from_pandas(df_in.reset_index() if hasattr(df_in, "reset_index") else df_in)
else:
df = df_in
# 兼容字段名差异
rename_map = {
"ts_code": "symbol",
"trade_date": "date",
"vol": "volume",
"amt": "amount",
"datetime": "date",
}
df = df.rename({k: v for k, v in rename_map.items() if k in df.columns})
if "symbol" not in df.columns and default_symbol is not None:
df = df.with_columns(pl.lit(default_symbol).alias("symbol"))
# 类型规范
if "date" in df.columns and df.schema["date"] != pl.Date:
df = df.with_columns(pl.col("date").cast(pl.Date, strict=False))
for col in ("open", "high", "low", "close"):
if col in df.columns:
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
for col in ("volume", "amount"):
if col in df.columns:
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
# 过滤停牌日 (open/high 为 0; close 可能被填充为前收盘价, 不能用全零判断)
df = filter_halt_days(df)
# 只保留 canonical 列
keep = [c for c in CANONICAL_DAILY_COLS if c in df.columns]
return df.select(keep)
def sync_daily_batch(symbols: list[str],
count: int | None = None,
batch_size: int | None = None,
rpm: int | None = None,
start_time: datetime | None = None,
end_time: datetime | None = None,
on_chunk_done: Callable[[int, int], None] | None = None) -> pl.DataFrame:
"""批量拉取多股日 K。
优先使用 start_time / end_time 区间 + count=10000,确保覆盖完整时间段。
仅传 count 时按条数回溯。
"""
tf = get_client()
out: list[pl.DataFrame] = []
interval = (60.0 / rpm) if rpm else 0
if batch_size is None:
chunks = [symbols]
else:
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
for i, chunk in enumerate(chunks):
if i > 0 and interval > 0 and len(chunks) > rpm:
time.sleep(interval)
try:
if start_time and end_time:
raw = tf.klines.batch(
chunk, period="1d", adjust="none",
start_time=_datetime_to_ms(start_time),
end_time=_datetime_to_ms(end_time),
count=10000,
as_dataframe=True, show_progress=False,
)
else:
raw = tf.klines.batch(chunk, period="1d", count=count or 250, adjust="none",
as_dataframe=True, show_progress=False)
except Exception as e: # noqa: BLE001
logger.warning("batch fetch failed for %d symbols: %s", len(chunk), e)
continue
# 兼容两种形态:dict[sym → df] 和扁平 df
if isinstance(raw, dict):
for sym, sub in raw.items():
if sub is None or len(sub) == 0:
continue
out.append(_normalize_daily(sub, default_symbol=sym))
elif raw is not None and len(raw) > 0:
out.append(_normalize_daily(raw))
if on_chunk_done:
on_chunk_done(i + 1, len(chunks))
if not out:
return pl.DataFrame()
return pl.concat(out, how="diagonal_relaxed")
def sync_and_persist_daily_batch(
symbols: list[str],
repo: KlineRepository,
capset: CapabilitySet,
count: int | None = None,
start_date: datetime | None = None,
end_date: datetime | None = None,
on_chunk_done: Callable[[int, int], None] | None = None,
) -> int:
"""批量同步日 K 并落到 Parquet。返回写入的行数。
start_date/end_date: 外部传入的时间范围(由 pipeline 根据已有数据计算)。
未传入时默认拉最近 1 年。
"""
if not symbols or not capset.has(Cap.KLINE_DAILY_BATCH):
return 0
lim = capset.limits(Cap.KLINE_DAILY_BATCH)
batch_size = lim.batch if lim and lim.batch else 100
rpm = lim.rpm if lim else None
end_time = end_date or datetime.now()
start_time = start_date or (end_time - timedelta(days=365))
df = sync_daily_batch(
symbols, count=count, batch_size=batch_size, rpm=rpm,
start_time=start_time, end_time=end_time,
on_chunk_done=on_chunk_done,
)
if df.is_empty():
return 0
repo.append_daily(df)
try:
d = repo.store.data_dir.as_posix()
repo.db.execute(
f"""CREATE OR REPLACE VIEW kline_daily AS
SELECT * FROM read_parquet('{d}/kline_daily/**/*.parquet', union_by_name=true)"""
)
except Exception as e: # noqa: BLE001
logger.warning("refresh view failed: %s", e)
return df.height
def sync_daily_by_quotes(repo: KlineRepository) -> int:
"""用实时行情接口拉全市场当日数据,覆写 kline_daily 今天分区。
一个请求覆盖 ~5500 只股票,比 batch K-line 快几个数量级。
返回写入的行数。
"""
from datetime import date as _date
from app.tickflow.client import get_client
tf = get_client()
try:
resp = tf.quotes.get_by_universes(universes=["CN_Equity_A"])
except Exception as e:
logger.warning("get_by_universes failed: %s", e)
return 0
if not resp:
logger.warning("get_by_universes returned empty")
return 0
records = []
for q in resp:
ext = q.get("ext") or {}
records.append({
"symbol": q.get("symbol"),
"open": q.get("open"),
"high": q.get("high"),
"low": q.get("low"),
"close": q.get("last_price"),
"volume": q.get("volume"),
"amount": q.get("amount"),
})
df = pl.DataFrame(records)
if df.is_empty():
return 0
today = _date.today()
daily_df = df.with_columns(pl.lit(today).cast(pl.Date).alias("date"))
# 过滤停牌 (open/high 为 0; close 可能被填充为前收盘价, 不能用全零判断)
daily_df = filter_halt_days(daily_df)
repo.flush_live_daily(daily_df)
logger.info("sync_daily_by_quotes: %d symbols flushed for %s", daily_df.height, today)
return daily_df.height
def sync_adj_factor(symbols: list[str], repo: KlineRepository,
capset: CapabilitySet,
start_time: datetime | None = None,
end_time: datetime | None = None,
on_chunk_done: Callable[[int, int], None] | None = None) -> tuple[int, list[str]]:
"""同步除权因子(Starter+)。SDK 接口:`tf.klines.ex_factors(symbols=...)`。
支持增量: 传 start_time/end_time 只拉取该时间范围内的新除权事件。
返回 (写入行数, 受影响的 symbol 列表) — 供 enriched 局部重算使用。
"""
if not capset.has(Cap.ADJ_FACTOR) or not symbols:
return 0, []
tf = get_client()
lim = capset.limits(Cap.ADJ_FACTOR)
batch_size = lim.batch if lim and lim.batch else 50
rpm = lim.rpm if lim else 30
interval = 60.0 / rpm if rpm else 0
# 构建 SDK 参数
sdk_kwargs: dict = {"as_dataframe": True, "batch_size": batch_size, "show_progress": False}
if start_time:
sdk_kwargs["start_time"] = _datetime_to_ms(start_time)
if end_time:
sdk_kwargs["end_time"] = _datetime_to_ms(end_time)
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
all_dfs: list[pl.DataFrame] = []
for i, chunk in enumerate(chunks):
if i > 0 and interval > 0 and len(chunks) > rpm:
time.sleep(interval)
try:
raw = tf.klines.ex_factors(chunk, **sdk_kwargs)
if raw is not None and len(raw) > 0:
all_dfs.append(pl.from_pandas(
raw.reset_index() if hasattr(raw, "reset_index") else raw
))
logger.debug("adj_factor chunk %d/%d: %d symbols", i + 1, len(chunks), len(chunk))
except Exception as e: # noqa: BLE001
logger.warning("adj_factor chunk %d failed: %s", i + 1, e)
if on_chunk_done:
on_chunk_done(i + 1, len(chunks))
if not all_dfs:
return 0, []
new_data = pl.concat(all_dfs, how="diagonal_relaxed") if len(all_dfs) > 1 else all_dfs[0]
# 提取受影响的 symbol 列表(合并前)
affected = new_data["symbol"].unique().to_list()
out = repo.store.data_dir / "adj_factor" / "all.parquet"
out.parent.mkdir(parents=True, exist_ok=True)
if out.exists():
existing = pl.read_parquet(out)
before = existing.height
merged = pl.concat([existing, new_data]).unique(
subset=["symbol", "trade_date"], keep="last",
).sort(["symbol", "trade_date"])
merged.write_parquet(out)
added = merged.height - before
logger.info("adj_factor merged: %d total (+%d new), %d/%d symbols",
merged.height, added, new_data.height, len(symbols))
return added, affected
else:
new_data.sort(["symbol", "trade_date"]).write_parquet(out)
logger.info("adj_factor synced: %d rows (%d symbols)", new_data.height, len(symbols))
return new_data.height, affected
# ===== 分钟 K 同步 =====
CANONICAL_MINUTE_COLS = [
"symbol", "datetime", "open", "high", "low", "close", "volume", "amount",
]
def _normalize_minute(df_in, default_symbol: str | None = None) -> pl.DataFrame:
"""把 SDK 返回的分钟 K 数据规范成 canonical 列。"""
if df_in is None or len(df_in) == 0:
return pl.DataFrame()
if not isinstance(df_in, pl.DataFrame):
df = pl.from_pandas(df_in.reset_index() if hasattr(df_in, "reset_index") else df_in)
else:
df = df_in
rename_map = {
"ts_code": "symbol",
"vol": "volume",
"amt": "amount",
}
df = df.rename({k: v for k, v in rename_map.items() if k in df.columns})
# datetime 列:优先用 timestamp(毫秒精度),其次 trade_time
if "timestamp" in df.columns:
df = df.with_columns(
pl.from_epoch("timestamp", time_unit="ms").alias("datetime"),
).drop("timestamp")
for drop_col in ("trade_time", "trade_date"):
if drop_col in df.columns:
df = df.drop(drop_col)
elif "trade_time" in df.columns:
df = df.rename({"trade_time": "datetime"})
if "trade_date" in df.columns:
df = df.drop("trade_date")
elif "trade_date" in df.columns:
df = df.rename({"trade_date": "datetime"})
if "symbol" not in df.columns and default_symbol is not None:
df = df.with_columns(pl.lit(default_symbol).alias("symbol"))
# 类型规范:统一转 Datetime('us')
if "datetime" in df.columns:
dt_type = df.schema["datetime"]
if not isinstance(dt_type, pl.Datetime) or dt_type.time_unit != "us":
df = df.with_columns(pl.col("datetime").cast(pl.Datetime("us"), strict=False))
for col in ("open", "high", "low", "close"):
if col in df.columns:
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
for col in ("volume", "amount"):
if col in df.columns:
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
keep = [c for c in CANONICAL_MINUTE_COLS if c in df.columns]
return df.select(keep)
def _datetime_to_ms(dt: datetime) -> int:
"""datetime → 毫秒时间戳 (供 SDK start_time / end_time 使用)。"""
return int(dt.timestamp() * 1000)
def sync_minute_batch(
symbols: list[str],
start_time: datetime | None = None,
end_time: datetime | None = None,
count: int | None = None,
batch_size: int | None = None,
rpm: int | None = None,
on_chunk_done: Callable[[int, int], None] | None = None,
) -> pl.DataFrame:
"""批量拉取多股分钟 K。
优先使用 start_time / end_time 区间, 确保所有标的覆盖同一时间段。
count 仅作为 fallback 保留。
on_chunk_done(current, total) 每个 chunk 完成后回调。
"""
tf = get_client()
out: list[pl.DataFrame] = []
interval = (60.0 / rpm) if rpm else 0
if batch_size is None:
chunks = [symbols]
else:
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
for i, chunk in enumerate(chunks):
if i > 0 and interval > 0 and len(chunks) > rpm:
time.sleep(interval)
try:
if start_time and end_time:
raw = tf.klines.batch(
chunk, period="1m",
start_time=_datetime_to_ms(start_time),
end_time=_datetime_to_ms(end_time),
count=10000,
as_dataframe=True, show_progress=False,
)
else:
raw = tf.klines.batch(chunk, period="1m", count=count or 1200,
as_dataframe=True, show_progress=False)
except Exception as e: # noqa: BLE001
logger.warning("minute batch fetch failed for %d symbols: %s", len(chunk), e)
continue
if isinstance(raw, dict):
for sym, sub in raw.items():
if sub is None or len(sub) == 0:
continue
out.append(_normalize_minute(sub, default_symbol=sym))
elif raw is not None and len(raw) > 0:
out.append(_normalize_minute(raw))
if on_chunk_done:
on_chunk_done(i + 1, len(chunks))
if not out:
return pl.DataFrame()
return pl.concat(out, how="diagonal_relaxed")
def fetch_minute_single(symbol: str, trade_date: date) -> pl.DataFrame:
"""从数据源实时拉取单股单日分钟 K(不写入本地)。"""
from datetime import datetime
start_time = datetime(trade_date.year, trade_date.month, trade_date.day, 9, 25, 0)
end_time = datetime(trade_date.year, trade_date.month, trade_date.day, 15, 5, 0)
tf = get_client()
try:
raw = tf.klines.batch(
[symbol], period="1m",
start_time=_datetime_to_ms(start_time),
end_time=_datetime_to_ms(end_time),
count=10000,
as_dataframe=True, show_progress=False,
)
except Exception as e:
logger.warning("fetch_minute_single(%s, %s) failed: %s", symbol, trade_date, e)
return pl.DataFrame()
if isinstance(raw, dict):
sub = raw.get(symbol)
return _normalize_minute(sub) if sub is not None and len(sub) > 0 else pl.DataFrame()
if raw is not None and len(raw) > 0:
return _normalize_minute(raw)
return pl.DataFrame()
def _latest_minute_datetime(repo: KlineRepository) -> datetime | None:
"""本地分钟 K 数据的最新时间。"""
try:
res = repo.execute_one("SELECT max(datetime) FROM kline_minute")
if res and res[0]:
d = res[0]
if isinstance(d, datetime):
return d
return datetime.fromisoformat(str(d))
except Exception: # noqa: BLE001
pass
return None
def _cleanup_null_datetime_minute(repo: KlineRepository) -> None:
"""检测并清除 datetime 全为 null 的旧版分钟 K 数据(迁移用)。"""
minute_dir = repo.store.data_dir / "kline_minute"
if not minute_dir.exists():
return
try:
row = repo.execute_one(
"SELECT count(*) AS total, count(datetime) AS non_null FROM kline_minute"
)
if row and row[0] > 0 and (row[1] is None or row[1] == 0):
# 全部 datetime 为 null — 清除所有分钟 K parquet
n = 0
for f in minute_dir.rglob("*.parquet"):
f.unlink()
n += 1
logger.info("cleaned %d corrupted minute-K parquet files (null datetime)", n)
except Exception as e: # noqa: BLE001
logger.debug("minute cleanup check failed: %s", e)
def _migrate_symbol_to_date_partition(repo: KlineRepository) -> None:
"""将旧版 symbol= 分区迁移为 date= 分区。迁移完成后删除旧目录。"""
minute_dir = repo.store.data_dir / "kline_minute"
if not minute_dir.exists():
return
old_dirs = [d for d in minute_dir.iterdir() if d.is_dir() and d.name.startswith("symbol=")]
if not old_dirs:
return
logger.info("migrating %d symbol-partitioned minute-K dirs to date partition…", len(old_dirs))
all_frames: list[pl.DataFrame] = []
for sym_dir in old_dirs:
for pq in sym_dir.glob("*.parquet"):
try:
df = pl.read_parquet(pq)
if "datetime" in df.columns:
df = df.filter(pl.col("datetime").is_not_null())
if not df.is_empty():
all_frames.append(df)
except Exception: # noqa: BLE001
pass
if not all_frames:
# 数据全部不可用,直接删旧目录
for d in old_dirs:
d.mkdir(parents=True, exist_ok=True)
for f in d.rglob("*"):
if f.is_file():
f.unlink()
d.rmdir()
return
combined = pl.concat(all_frames, how="diagonal_relaxed")
combined = combined.unique(subset=["symbol", "datetime"], keep="last")
# 按日期写新分区
combined = combined.with_columns(pl.col("datetime").dt.date().alias("_trade_date"))
for day_df in combined.partition_by("_trade_date"):
trade_date = day_df["_trade_date"][0]
out = minute_dir / f"date={trade_date}" / "part.parquet"
out.parent.mkdir(parents=True, exist_ok=True)
day_df = day_df.drop("_trade_date").sort("symbol", "datetime")
day_df.write_parquet(out)
# 删旧目录
for d in old_dirs:
for f in d.rglob("*"):
if f.is_file():
f.unlink()
# 移除空目录
try:
d.rmdir()
except OSError:
pass
logger.info("minute-K migration done: %d rows migrated", combined.height)
def sync_and_persist_minute(
symbols: list[str],
repo: KlineRepository,
capset: CapabilitySet,
days: int = 5,
on_chunk_done: Callable[[int, int], None] | None = None,
) -> int:
"""同步分钟 K 并存到 Parquet(仅 raw,不前复权)。返回写入行数。
使用 start_time / end_time 区间拉取, 确保所有标的覆盖同一时间段。
on_chunk_done(current, total) 每个 chunk 完成后回调。
"""
if not symbols or not capset.has(Cap.KLINE_MINUTE_BATCH):
return 0
# 迁移:旧版 _normalize_minute 未转换 timestamp→datetime,导致全部 datetime 为 null
# 检测到后直接清除(这些数据无法使用)
_cleanup_null_datetime_minute(repo)
# 迁移:旧版按 symbol= 分区转为 date= 分区
_migrate_symbol_to_date_partition(repo)
now = datetime.now()
# 计算时间区间: 首次拉取回溯 N 天, 增量从最后数据时间开始
last_dt = _latest_minute_datetime(repo)
if last_dt:
start_time = last_dt
else:
start_time = now - timedelta(days=days)
end_time = now
lim = capset.limits(Cap.KLINE_MINUTE_BATCH)
batch_size = lim.batch if lim and lim.batch else 100
rpm = lim.rpm if lim else 30
df = sync_minute_batch(symbols, start_time=start_time, end_time=end_time,
batch_size=batch_size, rpm=rpm,
on_chunk_done=on_chunk_done)
if df.is_empty():
return 0
# 按日期分区写: data/kline_minute/date={YYYY-MM-DD}/part.parquet
df = df.with_columns(
pl.col("datetime").dt.date().alias("_trade_date")
)
written = 0
for day_df in df.partition_by("_trade_date"):
trade_date = day_df["_trade_date"][0]
out = repo.store.data_dir / "kline_minute" / f"date={trade_date}" / "part.parquet"
out.parent.mkdir(parents=True, exist_ok=True)
if out.exists():
existing = pl.read_parquet(out)
if "datetime" in existing.columns:
existing = existing.filter(pl.col("datetime").is_not_null())
day_df = pl.concat([existing, day_df.drop("_trade_date")]).unique(
subset=["symbol", "datetime"], keep="last",
)
else:
day_df = day_df.drop("_trade_date")
day_df = day_df.sort("symbol", "datetime")
day_df.write_parquet(out)
written += day_df.height
# 刷新视图
try:
d = repo.store.data_dir.as_posix()
repo.db.execute(
f"""CREATE OR REPLACE VIEW kline_minute AS
SELECT * FROM read_parquet('{d}/kline_minute/**/*.parquet', union_by_name=true)"""
)
except Exception as e: # noqa: BLE001
logger.warning("refresh kline_minute view failed: %s", e)
logger.info("minute K synced: %d rows (%d symbols)", written, len(symbols))
return written
+112
View File
@@ -0,0 +1,112 @@
"""系统通知适配器 — 调用操作系统原生通知命令。
职责: 把后端产生的告警事件推送到操作系统通知中心。
窗口最小化 / 被遮挡 / 后台运行时都能弹通知。
平台实现:
- macOS: osascript (系统已内置)
- Linux: notify-send (系统已内置)
- Windows: 暂不支持原生通知中心 (无额外依赖实现)
设计: 失败静默降级, 绝不因通知失败阻断告警主流程 (落盘 / SSE 推送)。
通知去重不在本层做, 复用 MonitorRuleEngine 的 cooldown 逻辑。
"""
from __future__ import annotations
import logging
import subprocess
import sys
from pathlib import Path
logger = logging.getLogger(__name__)
# 单次通知最长字符 (避免某些平台截断报错)
_MAX_LEN = 200
# 避免重复探测平台能力, 缓存一次
_backend_cache: str | None = None
def _detect_backend() -> str | None:
"""探测当前平台可用的通知后端。"""
global _backend_cache
if _backend_cache is not None:
return _backend_cache if _backend_cache != "none" else None
backend = None
if sys.platform == "darwin":
backend = "osascript"
elif sys.platform.startswith("linux"):
backend = "notify-send"
_backend_cache = backend or "none"
return backend
def _truncate(text: str) -> str:
"""截断超长文本, 避免平台通知上限报错。"""
text = (text or "").strip()
return text[:_MAX_LEN] + ("" if len(text) > _MAX_LEN else "")
def notify(title: str, message: str, icon: Path | None = None) -> bool:
"""推送一条系统通知。
Args:
title: 通知标题
message: 通知正文
icon: 可选图标路径 (部分平台支持)
Returns:
True=成功送达, False=失败或无可用后端。
失败静默, 不抛异常 (通知是辅助通道, 不能阻断告警主流程)。
"""
title = _truncate(title)
message = _truncate(message)
if not title:
return False
backend = _detect_backend()
if backend is None:
return False
try:
if backend == "osascript":
return _notify_osascript(title, message)
if backend == "notify-send":
return _notify_notify_send(title, message, icon)
except Exception as e: # noqa: BLE001
logger.debug("系统通知失败 (%s): %s", backend, e)
return False
return False
def _notify_osascript(title: str, message: str) -> bool:
"""macOS 通知 (osascript) — 调用系统 AppleScript。"""
# 转义双引号, 避免 AppleScript 注入
safe_title = title.replace('"', '\\"')
safe_msg = message.replace('"', '\\"')
script = (
f'display notification "{safe_msg}" with title "{safe_title}"'
)
result = subprocess.run( # noqa: S603, S607
["osascript", "-e", script],
capture_output=True,
timeout=5,
)
return result.returncode == 0
def _notify_notify_send(title: str, message: str, icon: Path | None) -> bool:
"""Linux 通知 (notify-send) — freedesktop.org 标准。"""
args = ["notify-send", title]
if icon and Path(icon).exists():
args.extend(["--icon", str(icon)])
args.append(message)
result = subprocess.run( # noqa: S603
args,
capture_output=True,
timeout=5,
)
return result.returncode == 0
+250
View File
@@ -0,0 +1,250 @@
"""异步盘后管道任务注册表 — 每个 job 独立 JSON 文件。
设计:
- job_store/ 文件夹,每个 job 一个 {id}.json,最多保留 max_jobs 个文件
- running/pending 状态的 job 仅存内存(高频读写)
- succeeded/failed 后写入独立文件并从内存释放
- 列表查询 = 内存中的活跃 job + 磁盘文件扫描,按时间排序
- 单个查询 = 内存优先,没有则读磁盘
- 创建新 job 前检查文件数量,>= max_jobs 时删除最老的文件
"""
from __future__ import annotations
import json
import logging
import os
import threading
import uuid
from datetime import datetime
from pathlib import Path
from typing import Any, Literal
logger = logging.getLogger(__name__)
JobStatus = Literal["pending", "running", "succeeded", "failed"]
def _default_store_dir() -> Path:
from app.config import settings
return settings.data_dir / "job_store"
_STORE_DIR = _default_store_dir()
class JobStore:
def __init__(self, max_jobs: int = 50, store_dir: Path = _STORE_DIR) -> None:
self._max_jobs = max_jobs
self._store_dir = store_dir
self._active_jobs: dict[str, dict[str, Any]] = {} # running/pending
self._active_id: str | None = None
self._lock = threading.Lock()
self._store_dir.mkdir(parents=True, exist_ok=True)
# ===== persistence =====
def _write_file(self, job: dict[str, Any]) -> None:
"""将终态 job 写入独立 JSON 文件。"""
path = self._store_dir / f"{job['id']}.json"
try:
path.write_text(
json.dumps(job, ensure_ascii=False, indent=None),
encoding="utf-8",
)
except Exception:
logger.warning("failed to write job file %s", path)
def _read_file(self, job_id: str) -> dict[str, Any] | None:
"""从磁盘读取单个 job 文件。"""
path = self._store_dir / f"{job_id}.json"
if not path.exists():
return None
try:
return json.loads(path.read_text("utf-8"))
except Exception:
logger.warning("failed to read job file %s", path)
return None
def _delete_oldest(self) -> None:
"""删除最老的 job 文件,保持文件数量 < max_jobs。"""
try:
files = sorted(self._store_dir.glob("*.json"), key=lambda f: f.stat().st_mtime)
except Exception:
return
while len(files) >= self._max_jobs:
oldest = files.pop(0)
try:
oldest.unlink()
except Exception:
logger.warning("failed to delete old job file %s", oldest)
def _job_files_sorted(self) -> list[dict[str, Any]]:
"""扫描磁盘上所有 job 文件,按 started_at 从新到旧排序。"""
jobs: list[dict[str, Any]] = []
for f in self._store_dir.glob("*.json"):
try:
jobs.append(json.loads(f.read_text("utf-8")))
except Exception:
continue
jobs.sort(key=lambda j: j.get("started_at") or "", reverse=True)
return jobs
# ===== lifecycle =====
def create(self) -> str:
with self._lock:
if self._active_id and self._active_jobs.get(self._active_id, {}).get("status") == "running":
return self._active_id
job_id = uuid.uuid4().hex[:10]
self._active_jobs[job_id] = {
"id": job_id,
"status": "pending",
"stage": "init",
"progress": 0,
"stage_pct": 0,
"log": [],
"started_at": None,
"finished_at": None,
"duration_s": None,
"result": None,
"error": None,
}
self._active_id = job_id
return job_id
def start(self, job_id: str) -> None:
with self._lock:
j = self._active_jobs.get(job_id)
if not j:
return
j["status"] = "running"
j["started_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z"
def succeed(self, job_id: str, result: Any) -> None:
with self._lock:
j = self._active_jobs.pop(job_id, None)
if not j:
return
j["status"] = "succeeded"
j["finished_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z"
j["progress"] = 100
j["result"] = result
j["duration_s"] = _duration_s(j)
if self._active_id == job_id:
self._active_id = None
self._delete_oldest()
self._write_file(j)
def fail(self, job_id: str, error: str) -> None:
with self._lock:
j = self._active_jobs.pop(job_id, None)
if not j:
return
j["status"] = "failed"
j["finished_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z"
j["error"] = error
j["duration_s"] = _duration_s(j)
if self._active_id == job_id:
self._active_id = None
self._delete_oldest()
self._write_file(j)
# ===== progress =====
def progress(self, job_id: str, stage: str, pct: int, msg: str,
stage_pct: int | None = None, skip_log: bool = False) -> None:
with self._lock:
j = self._active_jobs.get(job_id)
if not j:
return
j["stage"] = stage
j["progress"] = max(0, min(100, int(pct)))
if stage_pct is not None:
j["stage_pct"] = max(0, min(100, int(stage_pct)))
elif j["stage"] != stage:
j["stage_pct"] = 0
entry = {
"ts": datetime.utcnow().isoformat(timespec="seconds") + "Z",
"stage": stage,
"msg": msg,
}
if skip_log:
entry["_skip"] = True
if skip_log and j["log"] and j["log"][-1].get("stage") == stage and j["log"][-1].get("_skip"):
j["log"][-1] = entry
else:
j["log"].append(entry)
if len(j["log"]) > 200:
j["log"] = j["log"][-200:]
# ===== query =====
def get(self, job_id: str) -> dict[str, Any] | None:
# 内存中的活跃 job 优先
j = self._active_jobs.get(job_id)
if j:
return j
# 否则从磁盘读
return self._read_file(job_id)
def list_recent(self, limit: int = 20) -> list[dict[str, Any]]:
# 合并: 内存中的活跃 job + 磁盘文件
all_jobs: list[dict[str, Any]] = list(self._active_jobs.values())
all_jobs.extend(self._job_files_sorted())
# 按 started_at 从新到旧排序,去重(理论上不会有重复)
seen: set[str] = set()
result: list[dict[str, Any]] = []
for j in sorted(all_jobs, key=lambda x: x.get("started_at") or "", reverse=True):
jid = j["id"]
if jid in seen:
continue
seen.add(jid)
result.append(_summary(j))
if len(result) >= limit:
break
return result
def active_id(self) -> str | None:
return self._active_id
def clear(self) -> None:
"""清空所有任务(内存 + 磁盘文件)。"""
with self._lock:
self._active_jobs.clear()
self._active_id = None
for f in self._store_dir.glob("*.json"):
try:
f.unlink()
except Exception:
pass
def _summary(j: dict[str, Any]) -> dict[str, Any]:
return {
"id": j["id"],
"status": j["status"],
"stage": j["stage"],
"progress": j["progress"],
"stage_pct": j.get("stage_pct", 0),
"started_at": j["started_at"],
"finished_at": j["finished_at"],
"duration_s": j["duration_s"],
"result": j["result"],
"error": j["error"],
}
def _duration_s(j: dict[str, Any]) -> float | None:
if not j.get("started_at") or not j.get("finished_at"):
return None
try:
s = datetime.fromisoformat(j["started_at"])
e = datetime.fromisoformat(j["finished_at"])
return round((e - s).total_seconds(), 2)
except Exception: # noqa: BLE001
return None
# 进程内单例
job_store = JobStore()
+313
View File
@@ -0,0 +1,313 @@
"""用户偏好设置持久化。
存储位置: data/user_data/preferences.json
沿用 secrets_store 的 merge-write 模式,但不做 chmod 0600 (非敏感数据)。
"""
from __future__ import annotations
import json
import logging
from pathlib import Path
logger = logging.getLogger(__name__)
def _path() -> Path:
from app.config import settings
p = settings.data_dir / "user_data" / "preferences.json"
p.parent.mkdir(parents=True, exist_ok=True)
return p
def load() -> dict:
p = _path()
if p.exists():
try:
return json.loads(p.read_text(encoding="utf-8"))
except Exception as e: # noqa: BLE001
logger.warning("preferences.json malformed: %s", e)
return {}
def save(updates: dict) -> dict:
"""合并写入。返回新内容。"""
current = load()
current.update(updates)
_path().write_text(
json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8",
)
return current
def get_realtime_quotes_enabled() -> bool:
return load().get("realtime_quotes_enabled", False)
def get_indices_nav_pinned() -> bool:
"""侧栏指数报价卡片是否固定显示。默认 True(常驻)。
关闭后,卡片跟随实时行情开关(仅实时开时显示)。"""
return load().get("indices_nav_pinned", True)
def get_realtime_quote_interval() -> float:
return load().get("realtime_quote_interval", 10.0)
def set_realtime_quote_interval(interval: float) -> float:
"""保存行情轮询间隔(不在此做 min/max 校验,由调用方按档位限制)。"""
current = load()
current["realtime_quote_interval"] = interval
_path().write_text(
json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8",
)
return interval
def get_minute_sync_enabled() -> bool:
return load().get("minute_sync_enabled", False)
def get_minute_sync_days() -> int:
return max(1, min(30, load().get("minute_sync_days", 5)))
def get_pipeline_schedule() -> dict:
"""返回盘后管道调度时间 {"hour": 15, "minute": 30}。"""
d = load().get("pipeline_schedule", {"hour": 15, "minute": 30})
return {"hour": d.get("hour", 15), "minute": d.get("minute", 30)}
def set_pipeline_schedule(hour: int, minute: int) -> dict:
h = max(0, min(23, hour))
m = max(0, min(59, minute))
# 盘后不早于 15:00
if h * 60 + m < 15 * 60:
h, m = 15, 0
save({"pipeline_schedule": {"hour": h, "minute": m}})
return {"hour": h, "minute": m}
def get_instruments_schedule() -> dict:
"""返回盘前标的维表调度时间 {"hour": 9, "minute": 10}。"""
d = load().get("instruments_schedule", {"hour": 9, "minute": 10})
return {"hour": d.get("hour", 9), "minute": d.get("minute", 10)}
def set_instruments_schedule(hour: int, minute: int) -> dict:
h = max(0, min(23, hour))
m = max(0, min(59, minute))
# 盘前不晚于 09:15
if h * 60 + m > 9 * 60 + 15:
h, m = 9, 15
save({"instruments_schedule": {"hour": h, "minute": m}})
return {"hour": h, "minute": m}
def get_enriched_batch_size() -> int:
"""返回 enriched 全量计算每批 symbol 数量。"""
return max(1, min(10000, load().get("enriched_batch_size", 1000)))
def set_enriched_batch_size(size: int) -> int:
"""保存 enriched 全量计算批次大小。"""
size = max(10, min(6000, size))
save({"enriched_batch_size": size})
return size
def get_index_daily_batch_size() -> int:
"""返回指数日 K 同步每批 symbol 数量。"""
return max(1, min(10000, load().get("index_daily_batch_size", 100)))
def set_index_daily_batch_size(size: int) -> int:
"""保存指数日 K 同步批次大小。"""
size = max(1, min(10000, size))
save({"index_daily_batch_size": size})
return size
# ── 五档盘口 sealed(真假涨停) 配置 ──────────────────────
def get_limit_ladder_monitor_enabled() -> bool:
"""连板梯队 5 档监控开关。关闭时 depth 不轮询(连板梯队降级显示)。"""
return load().get("limit_ladder_monitor_enabled", False)
def get_depth_polling_interval() -> float:
"""depth 盘中轮询间隔(秒)。默认 20(Pro/Expert 都适用)。"""
return float(load().get("depth_polling_interval", 20.0))
def set_depth_polling_interval(interval: float) -> float:
"""保存 depth 轮询间隔。套餐范围 clamp 由 depth_service 按档位做。"""
interval = max(1.0, min(600.0, float(interval)))
save({"depth_polling_interval": interval})
return interval
def get_depth_finalize_time() -> dict:
"""盘后 sealed 定版时间 {"hour": 15, "minute": 2}。范围 15:01~18:00。"""
d = load().get("depth_finalize_time", {"hour": 15, "minute": 2})
return {"hour": d.get("hour", 15), "minute": d.get("minute", 2)}
def set_depth_finalize_time(hour: int, minute: int) -> dict:
"""保存盘后 sealed 定版时间,强制范围 15:01~18:00。"""
h = max(0, min(23, hour))
m = max(0, min(59, minute))
# 下限 15:01, 上限 18:00
if h * 60 + m < 15 * 60 + 1:
h, m = 15, 1
if h * 60 + m > 18 * 60:
h, m = 18, 0
save({"depth_finalize_time": {"hour": h, "minute": m}})
return {"hour": h, "minute": m}
# ===== 实时监控 =====
# 页面 SSE 刷新配置: { "watchlist": true, "monitor": true, ... }
# 可刷新的页面列表及其默认值
SSE_REFRESH_PAGES_DEFAULT = {
"watchlist": True,
"limit-ladder": False,
}
SIDEBAR_INDEX_SYMBOLS_DEFAULT = ["000001.SH", "399001.SZ", "399006.SZ", "000680.SH"]
def get_sse_refresh_pages() -> dict[str, bool]:
"""返回每个页面的 SSE 刷新开关。"""
stored = load().get("sse_refresh_pages", {})
# 合并默认值 (新增页面自动出现)
result = dict(SSE_REFRESH_PAGES_DEFAULT)
result.update(stored)
return result
def set_sse_refresh_pages(pages: dict[str, bool]) -> dict[str, bool]:
"""保存页面 SSE 刷新配置。"""
save({"sse_refresh_pages": pages})
return get_sse_refresh_pages()
def get_sidebar_index_symbols() -> list[str]:
"""返回左侧菜单显示的指数代码。"""
stored = load().get("sidebar_index_symbols", SIDEBAR_INDEX_SYMBOLS_DEFAULT)
allowed = set(SIDEBAR_INDEX_SYMBOLS_DEFAULT)
return [s for s in stored if s in allowed]
def get_strategy_monitor_enabled() -> bool:
"""策略告警评估总开关。"""
return load().get("strategy_monitor_enabled", False)
def get_system_notify_enabled() -> bool:
"""系统通知开关 — 开启后监控告警同时推送到操作系统通知中心。"""
return load().get("system_notify_enabled", False)
def set_system_notify_enabled(enabled: bool) -> bool:
"""保存系统通知开关。"""
save({"system_notify_enabled": bool(enabled)})
return bool(enabled)
def get_screener_auto_run() -> bool:
"""选股页进入时是否自动运行所有策略 (获取命中数)。默认开。"""
return load().get("screener_auto_run", True)
def get_strategy_monitor_ids() -> list[str]:
"""返回监控池中的策略 ID。"""
return load().get("strategy_monitor_ids", [])
def set_realtime_monitor_config(cfg: dict) -> dict:
"""批量更新实时监控配置。"""
updates = {}
if "sse_refresh_pages" in cfg:
updates["sse_refresh_pages"] = cfg["sse_refresh_pages"]
if "strategy_monitor_enabled" in cfg:
updates["strategy_monitor_enabled"] = cfg["strategy_monitor_enabled"]
if "strategy_monitor_ids" in cfg:
updates["strategy_monitor_ids"] = cfg["strategy_monitor_ids"]
if "sidebar_index_symbols" in cfg:
allowed = set(SIDEBAR_INDEX_SYMBOLS_DEFAULT)
updates["sidebar_index_symbols"] = [s for s in cfg["sidebar_index_symbols"] if s in allowed]
if "screener_auto_run" in cfg:
updates["screener_auto_run"] = bool(cfg["screener_auto_run"])
if updates:
save(updates)
return get_realtime_monitor_config()
def get_realtime_monitor_config() -> dict:
"""返回完整的实时监控配置。"""
return {
"sse_refresh_pages": get_sse_refresh_pages(),
"strategy_monitor_enabled": get_strategy_monitor_enabled(),
"strategy_monitor_ids": get_strategy_monitor_ids(),
"sidebar_index_symbols": get_sidebar_index_symbols(),
"screener_auto_run": get_screener_auto_run(),
}
def get_nav_order() -> list[str]:
"""返回左侧菜单的自定义排序(内置页面 path + 扩展分析菜单 id)。"""
return load().get("nav_order", [])
def set_nav_order(order: list[str]) -> list[str]:
"""保存左侧菜单排序。"""
save({"nav_order": order})
return get_nav_order()
def get_nav_hidden() -> list[str]:
"""返回左侧菜单中隐藏的项 id 列表。"""
return load().get("nav_hidden", [])
def set_nav_hidden(hidden: list[str]) -> list[str]:
"""保存左侧菜单隐藏项。"""
save({"nav_hidden": hidden})
return get_nav_hidden()
def get_watchlist_columns() -> list[dict] | None:
"""返回自选列表列配置。"""
return load().get("watchlist_columns")
def set_watchlist_columns(columns: list[dict]) -> list[dict]:
"""保存自选列表列配置。"""
save({"watchlist_columns": columns})
return columns
def get_screener_result_columns() -> list[dict] | None:
"""返回策略结果列表列配置。"""
return load().get("screener_result_columns")
def set_screener_result_columns(columns: list[dict]) -> list[dict]:
"""保存策略结果列表列配置。"""
save({"screener_result_columns": columns})
return columns
# ===== 首次使用引导 =====
def get_onboarding_completed() -> bool:
"""是否已完成首次使用向导。默认 False(新用户)。"""
return bool(load().get("onboarding_completed", False))
def set_onboarding_completed(done: bool = True) -> bool:
"""标记首次使用向导完成状态。"""
save({"onboarding_completed": bool(done)})
return bool(done)
+779
View File
@@ -0,0 +1,779 @@
"""全局实时行情服务。
集中管理全市场行情拉取 + enriched 缓存,供盘中选股、自选股等所有模块复用。
架构:
- 后台线程轮询数据源 get_by_universes(["CN_Equity_A", "CN_Index"])
- 拉取行情 → 写 kline_daily (不复权) + 增量计算 enriched → 写盘 + 更新缓存
- _enriched_cache 是唯一的盘中数据源 (OHLCV + 全套技术指标)
- _live_agg_cache 是递推状态 (只加载一次, 盘中不变)
数据流 (每轮 ~15s):
1. API 拉取 → raw_records (临时变量)
2. raw_records → 写 kline_daily (不复权原始价格)
3. raw_records → 更新 _enriched_cache 的 OHLCV
4. 增量计算 enriched 指标 (~50ms)
5. 写 kline_daily_enriched + 替换 _enriched_cache
6. 通知 SSE
生命周期:
- 服务启动时读取 preferences,若 enabled 则自动启动线程
- 运行中可通过 API 切换开关
- 关闭时停止线程
"""
from __future__ import annotations
import logging
import threading
import time
from datetime import date, datetime, time as dt_time
import polars as pl
logger = logging.getLogger(__name__)
class QuoteService:
"""全局实时行情服务 — 单例。"""
CORE_INDEX_SYMBOLS = ("000001.SH", "399001.SZ", "399006.SZ", "000680.SH")
# 档位 → 最小轮询间隔 (秒)
TIER_MIN_INTERVAL = {
"expert": 1.0,
"pro": 2.0,
"starter": 3.0,
}
DEFAULT_INTERVAL = 10.0
MAX_INTERVAL = 60.0
def __init__(self) -> None:
self._lock = threading.Lock()
self._running = False
self._enabled = False # 全局开关 (持久化到 preferences)
self._interval = self.DEFAULT_INTERVAL
self._thread: threading.Thread | None = None
self._repo = None # 延迟注入, 避免循环导入
self._update_event = threading.Event() # SSE 通知: 行情更新后 set
self._alert_event = threading.Event() # SSE 通知: 有告警时 set
self._depth_update_event = threading.Event() # SSE 通知: depth 五档修正后 set (刷新连板梯队)
self._pending_alerts: list[dict] = [] # 待推送的告警
self._max_pending_alerts: int = 1000 # 背压上限: 超出丢弃最旧
self._strategy_monitor = None # 延迟注入
self._app_state = None # 延迟注入 (FastAPI app.state)
# 拉取元信息 (给 SSE / status 用)
self._fetch_time: float = 0.0 # perf_counter (用于计算 quote_age_ms)
self._fetch_ms: float = 0.0 # 拉取耗时 (毫秒)
self._fetched_at: float = 0.0 # 拉取完成的 Unix 时间戳 (毫秒)
self._symbol_count: int = 0
self._index_symbol_count: int = 0
self._index_quotes_cache: pl.DataFrame | None = None
# ================================================================
# 生命周期
# ================================================================
def start(self, interval: float = 0.0) -> None:
"""启动后台行情轮询线程。"""
if self._running:
return
if interval <= 0:
from app.services import preferences
interval = preferences.get_realtime_quote_interval()
self._interval = self._clamp_interval(interval)
self._running = True
self._enabled = True
self._thread = threading.Thread(target=self._poll_loop, daemon=True)
self._thread.start()
self._save_enabled(True)
logger.info("行情服务已启动, 轮询间隔 %.1fs", self._interval)
def stop(self) -> None:
"""停止后台行情轮询线程。"""
self._running = False
self._enabled = False
if self._thread:
self._thread.join(timeout=10)
self._thread = None
self._save_enabled(False)
logger.info("行情服务已停止")
def enable(self) -> bool:
"""开启自动行情 (不立即启动线程,等下一个交易时段)。
none/free 档无实时行情权限,拒绝开启并返回 False;
starter+ 正常启动。返回值表示是否真正开启。
"""
if not self.is_realtime_allowed():
logger.warning("实时行情开启被拒:当前档位(none/free)无实时行情权限")
return False
self._enabled = True
self._save_enabled(True)
if not self._running:
from app.services import preferences
self._interval = self._clamp_interval(preferences.get_realtime_quote_interval())
self._running = True
self._thread = threading.Thread(target=self._poll_loop, daemon=True)
self._thread.start()
logger.info("行情服务已启用, 轮询间隔 %.1fs", self._interval)
def disable(self) -> None:
"""关闭自动行情。"""
self.stop()
logger.info("行情服务已关闭")
def boot_check(self) -> None:
"""启动时检查 preferences,若 enabled 则自动启动。
none/free 档无实时行情权限:即使 preferences 标记为 enabled,
也不启动,并同步 preferences 为关闭(避免 UI 误显示已开启)。
"""
from app.services import preferences
if not self.is_realtime_allowed():
if preferences.get_realtime_quotes_enabled():
self._save_enabled(False)
logger.info("实时行情未启动:当前档位(none/free)无实时行情权限")
return
if preferences.get_realtime_quotes_enabled():
self.start()
def set_repo(self, repo) -> None:
"""注入 KlineRepository, 用于实时落盘。"""
self._repo = repo
def set_app_state(self, app_state) -> None:
"""注入 FastAPI app.state, 用于获取 strategy_monitor 等单例。"""
self._app_state = app_state
def set_interval(self, interval: float) -> float:
"""运行时更新轮询间隔(立即生效)。"""
clamped = self._clamp_interval(interval)
self._interval = clamped
from app.services import preferences
preferences.set_realtime_quote_interval(clamped)
logger.info("轮询间隔已更新为 %.1fs", clamped)
return clamped
def get_min_interval(self) -> float:
"""返回当前档位允许的最小间隔。"""
return self._tier_min_interval()
def wait_for_update(self, timeout: float = 30.0) -> bool:
"""阻塞等待下一次行情更新 (供 SSE 线程使用)。"""
self._update_event.clear()
return self._update_event.wait(timeout=timeout)
def wait_for_alert(self, timeout: float = 30.0) -> bool:
"""阻塞等待告警 (供 SSE 线程使用)。"""
self._alert_event.clear()
return self._alert_event.wait(timeout=timeout)
def notify_depth_updated(self) -> None:
"""五档盘口修正完成后调用: 通知 SSE 推送 depth_updated, 触发连板梯队刷新。
与行情/告警通道独立 — 只刷新连板梯队, 不连带刷新 watchlist 等。
"""
self._depth_update_event.set()
def wait_for_depth_update(self, timeout: float = 30.0) -> bool:
"""阻塞等待 depth 修正 (供 SSE 线程使用)。"""
self._depth_update_event.clear()
return self._depth_update_event.wait(timeout=timeout)
def pop_alerts(self) -> list[dict]:
"""取走所有待推送的告警 (线程安全)。"""
with self._lock:
alerts = self._pending_alerts
self._pending_alerts = []
return alerts
# ================================================================
# 档位感知间隔限制
# ================================================================
@staticmethod
def _current_tier() -> str:
"""获取当前档位名(小写)。"""
from app.tickflow.policy import tier_label
return tier_label().split()[0].split("+")[0].strip().lower()
@classmethod
def is_realtime_allowed(cls) -> bool:
"""当前档位是否允许使用实时行情。
none/free 档走 free-api 服务器,无实时行情权限 → 不允许;
starter+ 付费档走付费端点,有实时行情 → 允许。
"""
return cls._current_tier() not in ("none", "free")
@classmethod
def _tier_min_interval(cls) -> float:
tier = cls._current_tier()
return cls.TIER_MIN_INTERVAL.get(tier, cls.DEFAULT_INTERVAL)
def _clamp_interval(self, interval: float) -> float:
return max(self._tier_min_interval(), min(self.MAX_INTERVAL, interval))
# ================================================================
# 行情数据访问
# ================================================================
def get_enriched_today(self) -> tuple[pl.DataFrame, date | None]:
"""返回今天 enriched 数据 + 日期 (线程安全)。
所有页面统一通过此方法获取实时行情 + 技术指标。
"""
if not self._repo:
return pl.DataFrame(), None
return self._repo.get_enriched_latest()
def get_quotes_compat(self) -> pl.DataFrame:
"""兼容接口: 返回行情 DataFrame (用于盘中选股等需要 last_price/prev_close 的场景)。
从 _enriched_cache 取 today 的数据, 只选行情基础列, 补上 last_price 别名。
不返回指标列, 避免 JOIN live_agg 时列名冲突。
"""
df, _ = self.get_enriched_today()
if df.is_empty():
return df
# 只取盘中选股需要的行情基础列
keep = [c for c in [
"symbol", "close", "open", "high", "low", "volume", "amount",
"prev_close", "change_pct", "change_amount", "amplitude", "turnover_rate",
] if c in df.columns]
df = df.select(keep)
# enriched 的 close 等价于 last_price
if "close" in df.columns and "last_price" not in df.columns:
df = df.with_columns(pl.col("close").alias("last_price"))
return df
def get_index_quotes(self, symbols: list[str] | None = None) -> pl.DataFrame:
"""返回实时指数行情缓存。不会触发数据源请求。"""
with self._lock:
df = self._index_quotes_cache.clone() if self._index_quotes_cache is not None else pl.DataFrame()
if df.is_empty():
return df
if symbols:
return df.filter(pl.col("symbol").is_in(symbols))
return df
def status(self) -> dict:
"""返回行情服务状态。"""
age = (time.perf_counter() - self._fetch_time) * 1000 if self._fetch_time else -1
return {
"enabled": self._enabled,
"running": self._running,
"interval_s": self._interval,
"symbol_count": self._symbol_count,
"index_symbol_count": self._index_symbol_count,
"quote_age_ms": round(age, 0) if age >= 0 else None,
"is_trading_hours": self._is_trading_hours(),
"last_fetch_ms": round(self._fetched_at, 0) if self._fetched_at else None,
}
def refresh(self) -> dict:
"""手动触发一次行情拉取。"""
self._fetch_quotes()
return self.status()
# ================================================================
# 后台轮询
# ================================================================
def _poll_loop(self) -> None:
while self._running and self._enabled:
try:
if self._is_trading_hours():
self._fetch_quotes()
else:
logger.debug("非交易时段, 跳过行情轮询")
except Exception as e: # noqa: BLE001
logger.warning("行情轮询异常: %s", e)
waited = 0.0
while self._running and self._enabled and waited < self._interval:
time.sleep(0.5)
waited += 0.5
def _fetch_quotes(self) -> None:
"""拉取全市场行情 → 写 daily + 计算 enriched + 更新缓存。"""
from app.tickflow.client import get_client
tf = get_client()
t0 = time.perf_counter()
now_ts = time.perf_counter()
try:
all_index_symbols = set(self._repo.get_index_symbol_set()) if self._repo else set()
all_index_symbols.update(self.CORE_INDEX_SYMBOLS)
resp = tf.quotes.get_by_universes(universes=["CN_Equity_A", "CN_Index"])
except Exception as e: # noqa: BLE001
logger.warning("行情拉取失败: %s", e)
return
if not resp:
logger.warning("行情数据为空")
return
# ---- 解析 API 响应 (临时变量, 用完丢弃) ----
records = []
for q in resp:
ext = q.get("ext") or {}
last_price = q.get("last_price")
prev_close = q.get("prev_close")
change_amount = ext.get("change_amount")
change_pct = ext.get("change_pct")
if change_amount is None and last_price is not None and prev_close is not None:
change_amount = float(last_price) - float(prev_close)
if change_pct is None and change_amount is not None and prev_close not in (None, 0):
change_pct = float(change_amount) / float(prev_close) * 100
records.append({
"symbol": q.get("symbol"),
"name": q.get("name") or ext.get("name"),
"last_price": last_price,
"prev_close": prev_close,
"open": q.get("open"),
"high": q.get("high"),
"low": q.get("low"),
"volume": q.get("volume"),
"amount": q.get("amount"),
"change_pct": change_pct,
"change_amount": change_amount,
"amplitude": ext.get("amplitude"),
"turnover_rate": ext.get("turnover_rate"),
"timestamp": q.get("timestamp"),
"session": q.get("session"),
})
index_records = [r for r in records if r.get("symbol") in all_index_symbols]
stock_records = [r for r in records if r.get("symbol") not in all_index_symbols]
fetch_ms = (time.perf_counter() - t0) * 1000
fetched_at = time.time() * 1000
# ---- 更新元信息 ----
with self._lock:
self._fetch_time = now_ts
self._fetch_ms = fetch_ms
self._fetched_at = fetched_at
self._symbol_count = len(stock_records)
self._index_symbol_count = len(index_records)
self._index_quotes_cache = self._build_index_quotes(index_records)
logger.info("行情刷新: %d 只股票, %d 只指数, 耗时 %.0fms", len(stock_records), len(index_records), fetch_ms)
# ---- 写 kline_daily (不复权原始价格, 只有 OHLCV) ----
daily_df = self._build_daily(stock_records)
if not daily_df.is_empty() and self._repo:
try:
self._repo.flush_live_daily(daily_df)
except Exception as e: # noqa: BLE001
logger.warning("日K写盘失败: %s", e)
# ---- 构建 API 直接值的补充表 (不写 daily, 只用于 enriched 计算) ----
quote_extra = self._build_quote_extra(stock_records)
# ---- 增量计算 enriched + 写盘 + 更新缓存 ----
if not daily_df.is_empty() and self._repo:
self._flush_live_enriched(daily_df, quote_extra)
# ---- 通知 SSE ----
self._update_event.set()
# ---- 策略监控 + 告警评估 ----
self._evaluate_monitors(daily_df, quote_extra)
# ================================================================
# 工具
# ================================================================
@staticmethod
def _build_daily(records: list[dict]) -> pl.DataFrame:
"""将 API records 转为日K格式 DataFrame (只有 OHLCV, 写 kline_daily 用)。"""
if not records:
return pl.DataFrame()
df = pl.DataFrame(records)
cols_map = {
"symbol": "symbol",
"last_price": "close",
"open": "open",
"high": "high",
"low": "low",
"volume": "volume",
"amount": "amount",
}
select_exprs = []
for src, dst in cols_map.items():
if src in df.columns:
select_exprs.append(pl.col(src).alias(dst))
if not select_exprs:
return pl.DataFrame()
result = df.select(select_exprs).with_columns(
pl.lit(date.today()).cast(pl.Date).alias("date"),
)
# 修复: API 在非交易时段可能返回 open/high/low=0 或 null,
# 导致蜡烛从 0 开始。用 close 填充这些异常值。
for col in ("open", "high", "low"):
if col in result.columns:
result = result.with_columns(
pl.when((pl.col(col) == 0) | pl.col(col).is_null())
.then(pl.col("close"))
.otherwise(pl.col(col))
.alias(col)
)
return result
@staticmethod
def _build_quote_extra(records: list[dict]) -> pl.DataFrame:
"""构建 API 直接提供的补充字段 (不写 daily, 只传给 enriched 计算)。
包含: prev_close, change_pct, change_amount, amplitude, turnover_rate。
"""
if not records:
return pl.DataFrame()
df = pl.DataFrame(records)
keep = [c for c in [
"symbol", "prev_close", "change_pct", "change_amount",
"amplitude", "turnover_rate",
] if c in df.columns]
if not keep or "symbol" not in keep:
return pl.DataFrame()
return df.select(keep)
@staticmethod
def _build_index_quotes(records: list[dict]) -> pl.DataFrame:
"""构建指数实时行情缓存,不落股票 parquet。
注意: API 返回的 change_pct/amplitude 是小数 (0.0366 = 3.66%),
统一转成百分比输出, 与 _fallback_index_quotes_from_daily 口径一致
(前端指数侧不×100, 直接 toFixed(2)% 展示)。
"""
if not records:
return pl.DataFrame()
df = pl.DataFrame(records)
keep = [c for c in [
"symbol", "name", "last_price", "prev_close", "open", "high", "low",
"volume", "amount", "change_pct", "change_amount", "amplitude", "timestamp", "session",
] if c in df.columns]
if not keep or "symbol" not in keep:
return pl.DataFrame()
df = df.select(keep)
# change_pct / amplitude: 小数 → 百分比 (统一指数展示口径)
for col in ("change_pct", "amplitude"):
if col in df.columns:
df = df.with_columns((pl.col(col).cast(pl.Float64) * 100).alias(col))
if "last_price" in df.columns and "close" not in df.columns:
df = df.with_columns(pl.col("last_price").alias("close"))
return df
@staticmethod
def _is_trading_hours() -> bool:
now = datetime.now()
t = now.time()
morning = dt_time(9, 15) <= t <= dt_time(11, 35)
afternoon = dt_time(12, 55) <= t <= dt_time(15, 5)
return now.weekday() < 5 and (morning or afternoon)
@staticmethod
def _save_enabled(enabled: bool) -> None:
from app.services import preferences
preferences.save({"realtime_quotes_enabled": enabled})
# ================================================================
# 策略监控
# ================================================================
def _evaluate_monitors(self, daily_df: pl.DataFrame, quote_extra: pl.DataFrame | None) -> None:
"""行情更新后评估统一监控规则引擎,并刷新策略结果缓存。"""
try:
# 获取 enriched 数据 (刚算好的)
enriched_today, enriched_date = self.get_enriched_today()
if enriched_today.is_empty():
return
all_alerts: list[dict] = []
# 通用监控规则评估 (统一引擎: signal/price/market/strategy)
if self._app_state:
engine = getattr(self._app_state, "monitor_engine", None)
if engine and engine.rule_count > 0:
# 预构建 symbol → name 映射 (enriched 已 drop name 列, 引擎触发时回填用)
try:
inst_df = self._app_state.repo.get_instruments()
if not inst_df.is_empty() and "symbol" in inst_df.columns and "name" in inst_df.columns:
engine.set_name_map({
row["symbol"]: row["name"]
for row in inst_df.select(["symbol", "name"]).iter_rows(named=True)
if row.get("name")
})
except Exception as e: # noqa: BLE001
logger.debug("name_map 构建失败 (不影响监控): %s", e)
rule_events = engine.evaluate(enriched_today)
if rule_events:
# 落盘到 alerts.jsonl
try:
from app.services import alert_store
alert_store.append_many(
self._app_state.repo.store.data_dir, rule_events,
)
except Exception as e: # noqa: BLE001
logger.warning("告警落盘失败: %s", e)
# 转为 SSE 推送格式 (兼容旧 alert schema)
for ev in rule_events:
all_alerts.append({
"source": ev["source"],
"type": ev["type"],
"rule_id": ev.get("rule_id"),
"strategy_id": ev.get("rule_id") if ev["source"] == "strategy" else None,
"symbol": ev["symbol"],
"name": ev["name"],
"message": ev["message"],
"price": ev["price"],
"change_pct": ev["change_pct"],
"signals": ev["signals"],
"severity": ev.get("severity", "info"),
})
# 刷新策略结果缓存 (实时行情开启时,每轮行情更新后自动重算)
if self._enabled and self._app_state:
self._refresh_strategy_cache(enriched_today, enriched_date)
# 推入待推送队列 + 通知 SSE (含背压保护)
if all_alerts:
with self._lock:
self._pending_alerts.extend(all_alerts)
# 背压: 超出上限丢弃最旧
if len(self._pending_alerts) > self._max_pending_alerts:
overflow = len(self._pending_alerts) - self._max_pending_alerts
self._pending_alerts = self._pending_alerts[overflow:]
self._alert_event.set()
logger.info("监控评估完成: %d 条通知", len(all_alerts))
# 系统通知 (可选通道, 由 preferences 开关控制)。
# cooldown 去重已在 MonitorRuleEngine 做过, 这里只负责转发。
self._maybe_send_system_notifications(all_alerts)
except Exception as e: # noqa: BLE001
logger.warning("监控评估失败: %s", e)
def _maybe_send_system_notifications(self, all_alerts: list[dict]) -> None:
"""把告警转发到操作系统通知中心 (由 preferences 开关控制)。
- 开关关闭: 直接返回
- 开关开启: 逐条发系统通知; 失败静默, 不阻断主流程
- 去重: 复用 MonitorRuleEngine 的 cooldown, 此处不重复去重
- 批量策略事件 (symbol="") 聚合为一条通知, 避免刷屏
"""
try:
from app.services import preferences
from app.services import notify_adapter
if not preferences.get_system_notify_enabled():
return
for ev in all_alerts:
# 通知标题: 用 source 分类 (策略/信号/价格/异动)
source = ev.get("source", "")
source_label = {
"strategy": "策略", "signal": "信号",
"price": "价格", "market": "异动",
}.get(source, source or "通知")
name = ev.get("name") or ""
symbol = ev.get("symbol") or ""
message = ev.get("message") or ""
# 正文: 优先用现成 message, 拼上 symbol/name 让用户一眼定位
if symbol:
body = f"{symbol} {name} {message}".strip()
else:
body = message or name
title = f"Stock Panel · {source_label}"
notify_adapter.notify(title, body)
except Exception as e: # noqa: BLE001
logger.debug("系统通知发送异常 (不影响告警主流程): %s", e)
def _refresh_strategy_cache(self, enriched_today: pl.DataFrame, enriched_date: date | None) -> None:
"""利用已计算好的 enriched 数据,运行策略池并写入缓存。"""
import math
from dataclasses import asdict
from app.services import strategy_cache
from app.services.screener import PRESET_STRATEGIES, ScreenerService
from app.strategy import config as strategy_config
try:
if enriched_date is None:
return
as_of = enriched_date
data_dir = self._repo.store.data_dir
svc = ScreenerService(self._repo)
engine = getattr(self._app_state, "strategy_engine", None)
# 确定要运行的策略: 策略监控池中的策略
monitor_ids = self._get_monitor_pool_ids()
if not monitor_ids:
return
# 一次加载所有 override
all_overrides = strategy_config.list_overrides(data_dir)
# 历史策略: 只在需要时加载
shared_history = None
history_strats = []
if engine:
id_set = set(monitor_ids)
history_strats = [
(sid, s) for sid, s in engine._strategies.items()
if s.filter_history_fn and sid in id_set
]
if history_strats:
max_lb = max(s.lookback_days for _, s in history_strats)
shared_history = svc._load_enriched_history(as_of, max(1, max_lb))
results: dict[str, dict] = {}
for sid in monitor_ids:
try:
overrides = all_overrides.get(sid, {})
bf = overrides.get("basic_filter") if overrides else None
dl = overrides.get("display_limit") if overrides else None
if dl is None and overrides and "display_limit" in overrides:
dl = 0
if sid in PRESET_STRATEGIES:
r = svc.run_preset(sid, as_of=as_of, precomputed=enriched_today, basic_filter=bf, display_limit=dl)
elif engine:
r = engine.run(
sid, as_of, overrides=overrides or None,
precomputed=enriched_today, precomputed_history=shared_history,
)
if dl is not None and dl > 0:
r.rows = r.rows[:dl]
r.total = min(r.total, dl)
else:
continue
# sanitize NaN/Inf
rows = []
for row_dict in asdict(r).get("rows", []):
for k, v in list(row_dict.items()):
if isinstance(v, float) and not math.isfinite(v):
row_dict[k] = None
rows.append(row_dict)
results[sid] = {"total": r.total, "as_of": str(as_of), "rows": rows}
except Exception: # noqa: BLE001
continue
if results:
strategy_cache.write_cache(data_dir, str(as_of), results)
except Exception as e: # noqa: BLE001
logger.warning("策略缓存刷新失败: %s", e)
def _get_monitor_pool_ids(self) -> list[str]:
"""获取策略监控池中的策略 ID 列表。"""
from app.services import preferences
ids = preferences.get_strategy_monitor_ids()
if not ids:
return []
return [sid for sid in ids if sid]
@staticmethod
def _get_strategy_monitor():
"""获取 StrategyMonitorService — 不再使用, 改用 _app_state 注入。"""
return None
# ================================================================
# enriched 增量计算
# ================================================================
def _flush_live_enriched(self, daily_df: pl.DataFrame, quote_extra: pl.DataFrame = None) -> None:
"""增量计算今天的 enriched: 用昨天的递推状态 + 今天 OHLCV → 只算今天 5500 行。
quote_extra: API 直接提供的补充字段 (prev_close, change_pct 等),
不写 daily, 直接传给 compute_enriched_today 避免重复计算。
"""
try:
today = date.today()
t0 = time.perf_counter()
# ---- 尝试增量路径 ----
live_agg = self._repo.get_live_agg()
prev_enriched, prev_date = self._repo.get_enriched_latest()
use_incremental = (
not live_agg.is_empty()
and not prev_enriched.is_empty()
and prev_date is not None
)
if use_incremental:
from app.indicators.pipeline import compute_enriched_today
instruments = self._repo.get_instruments()
# 将 API 直接提供的补充字段 JOIN 到 daily_df
today_ohlcv = daily_df
if quote_extra is not None and not quote_extra.is_empty():
today_ohlcv = daily_df.join(quote_extra, on="symbol", how="left")
enriched_today = compute_enriched_today(
live_agg=live_agg,
prev_enriched=prev_enriched,
today_ohlcv=today_ohlcv,
instruments=instruments,
)
if enriched_today.is_empty():
logger.warning("增量计算结果为空, 回退到全量计算")
use_incremental = False
# ---- 全量回退路径 ----
if not use_incremental:
from datetime import timedelta
from app.indicators.pipeline import compute_enriched
logger.info("enriched 全量计算 (live_agg=%s, 上次日期=%s)",
"ok" if not live_agg.is_empty() else "", prev_date)
cutoff = today - timedelta(days=90)
daily_glob = str(self._repo.store.data_dir / "kline_daily" / "**" / "*.parquet")
ohlcv_cols = ["symbol", "date", "open", "high", "low", "close", "volume", "amount"]
hist_df = (
pl.scan_parquet(daily_glob)
.filter(pl.col("date") >= cutoff)
.sort(["symbol", "date"])
.collect()
)
if hist_df.is_empty():
return
hist_cols = [c for c in ohlcv_cols if c in hist_df.columns]
hist_df = hist_df.select(hist_cols).filter(pl.col("date") != today)
daily_ohlcv = daily_df.select([c for c in ohlcv_cols if c in daily_df.columns])
full_df = pl.concat([hist_df, daily_ohlcv], how="diagonal_relaxed")
full_df = full_df.sort(["symbol", "date"])
factor_path = self._repo.store.data_dir / "adj_factor" / "all.parquet"
factors = pl.DataFrame()
if factor_path.exists():
try:
factors = pl.read_parquet(factor_path)
except Exception:
pass
instruments = self._repo.get_instruments()
enriched_full = compute_enriched(full_df, factors=factors, instruments=instruments)
enriched_today = enriched_full.filter(pl.col("date") == today)
if enriched_today.is_empty():
return
# ---- 写盘 + 更新缓存 ----
self._repo.flush_live_enriched(enriched_today)
elapsed = time.perf_counter() - t0
mode_label = "增量" if use_incremental else "全量"
logger.info("enriched %s: %d 只, %s, 耗时 %.0fms",
mode_label, len(enriched_today), today, elapsed * 1000)
except Exception as e: # noqa: BLE001
logger.warning("enriched 计算失败: %s", e)
+593
View File
@@ -0,0 +1,593 @@
"""Screener 服务(§6.3)。
性能优化:
- enriched parquet 仅存 14 列基础数据, 指标和信号即时计算
- preset 策略: 从内存缓存或即时计算获取完整指标, ~10-50ms
- custom SQL: DuckDB (用户传 SQL WHERE 字符串), ~10-50ms
"""
from __future__ import annotations
import logging
import time
from dataclasses import dataclass, field
from datetime import date, timedelta
import polars as pl
from app.tickflow.repository import KlineRepository
logger = logging.getLogger(__name__)
# ── 进程级历史数据缓存 (避免 run_all 每次重新扫描 parquet + 计算指标) ──
_history_cache: dict[tuple[date, int], tuple[float, pl.DataFrame]] = {}
_HISTORY_CACHE_TTL = 120.0 # 秒
# 内置预设策略 — Polars 表达式方式
PRESET_STRATEGIES: dict[str, dict] = {
"trend_breakout": {
"name": "趋势突破",
"description": "MA60 上方 + 60 日新高 + 量能 ≥ 2 倍均量",
"filter": (
(pl.col("close") > pl.col("ma60"))
& pl.col("signal_n_day_high").fill_null(False)
& (pl.col("vol_ratio_5d") >= 2.0)
),
"order_by": "momentum_60d",
"descending": True,
"limit": 100,
},
"ma_golden_cross": {
"name": "MA 金叉",
"description": "MA5 上穿 MA20 当日触发,量能配合",
"filter": (
pl.col("signal_ma_golden_5_20").fill_null(False)
& (pl.col("vol_ratio_5d") >= 1.2)
& (pl.col("close") > pl.col("ma60"))
),
"order_by": "momentum_20d",
"descending": True,
"limit": 100,
},
"macd_golden": {
"name": "MACD 金叉放量",
"description": "MACD 金叉当日 + 量能放大",
"filter": (
pl.col("signal_macd_golden").fill_null(False)
& (pl.col("vol_ratio_5d") >= 1.5)
),
"order_by": "momentum_60d",
"descending": True,
"limit": 100,
},
"volume_price_surge": {
"name": "量价齐升",
"description": "突破 MA20 + 放量 + 收阳",
"filter": (
pl.col("signal_ma20_breakout").fill_null(False)
& (pl.col("vol_ratio_5d") >= 2.0)
& (pl.col("close") > pl.col("open"))
),
"order_by": "vol_ratio_5d",
"descending": True,
"limit": 100,
},
"low_volatility_leader": {
"name": "低波动龙头",
"description": "20 日动量为正 + 年化波动 < 30% + MA20 上方",
"filter": (
(pl.col("momentum_20d") > 0)
& (pl.col("annual_vol_20d") < 0.30)
& (pl.col("close") > pl.col("ma20"))
),
"order_by": "momentum_60d",
"descending": True,
"limit": 100,
},
"broken_board_recovery": {
"name": "断板反包",
"description": "连板 ≥2 后断板 1-2 天,出现放量反包信号",
"filter": (
pl.col("signal_limit_up").fill_null(False)
& (pl.col("vol_ratio_5d") >= 1.5)
& (pl.col("change_pct") > 0.03)
),
"order_by": "change_pct",
"descending": True,
"limit": 100,
},
"oversold_bounce": {
"name": "超跌反弹",
"description": "RSI14 < 30 超卖区 + 当日收阳 + 放量,抄底信号",
"filter": (
(pl.col("rsi_14") < 30)
& (pl.col("close") > pl.col("open"))
& (pl.col("vol_ratio_5d") >= 1.2)
),
"order_by": "rsi_14",
"descending": False,
"limit": 100,
},
"boll_breakout": {
"name": "布林突破",
"description": "突破布林上轨 + 放量,强势加速信号",
"filter": (
pl.col("signal_boll_breakout_upper").fill_null(False)
& (pl.col("vol_ratio_5d") >= 1.5)
),
"order_by": "vol_ratio_5d",
"descending": True,
"limit": 100,
},
"bullish_alignment": {
"name": "均线多头",
"description": "MA5 > MA10 > MA20 > MA60 多头排列 + 短期动量为正",
"filter": (
(pl.col("ma5") > pl.col("ma10"))
& (pl.col("ma10") > pl.col("ma20"))
& (pl.col("ma20") > pl.col("ma60"))
& (pl.col("momentum_20d") > 0)
),
"order_by": "momentum_60d",
"descending": True,
"limit": 100,
},
"consecutive_limit_ups": {
"name": "连板股",
"description": "当日涨停且连续涨停 ≥ 2 天,强势追涨",
"filter": (
pl.col("signal_limit_up").fill_null(False)
& (pl.col("consecutive_limit_ups") >= 2)
),
"order_by": "consecutive_limit_ups",
"descending": True,
"limit": 100,
},
"pullback_to_support": {
"name": "缩量回踩",
"description": "回踩 MA20 附近 + 缩量 + 中期趋势向上",
"filter": (
(pl.col("close") > pl.col("ma20") * 0.98)
& (pl.col("close") < pl.col("ma20") * 1.02)
& (pl.col("vol_ratio_5d") < 0.8)
& (pl.col("close") > pl.col("ma60"))
& (pl.col("momentum_20d") > 0)
),
"order_by": "momentum_60d",
"descending": True,
"limit": 100,
},
"n_day_low_reversal": {
"name": "新低反转",
"description": "触及 60 日新低后当日收阳放量,反转信号",
"filter": (
pl.col("signal_n_day_low").fill_null(False)
& (pl.col("close") > pl.col("open"))
& (pl.col("vol_ratio_5d") >= 1.5)
),
"order_by": "change_pct",
"descending": True,
"limit": 100,
},
}
@dataclass
class ScreenerResult:
as_of: date
strategy: str | None
rows: list[dict] = field(default_factory=list)
total: int = 0
elapsed_ms: float = 0.0
class ScreenerService:
def __init__(self, repo: KlineRepository) -> None:
self.repo = repo
@staticmethod
def clear_history_cache() -> None:
"""清空进程级 _history_cache (TTL 缓存)。
清除数据后调用, 避免内存里的旧历史窗口残留导致策略/看板仍命中旧数据。
"""
_history_cache.clear()
def _load_enriched_for_date(self, target_date: date) -> pl.DataFrame:
"""从 enriched parquet 读取指定日期的基础数据并即时计算完整指标+信号。
enriched parquet 仅存 14 列。读取后需要即时计算 ma/ema/macd/kdj/rsi/boll/momentum/signal 等列。
对于最新日, 优先使用内存缓存 (已包含完整指标)。
"""
# 优先使用 repo 最新日缓存
cache, cache_date = self.repo.get_enriched_latest()
if cache is not None and not cache.is_empty() and cache_date == target_date:
df = cache
# JOIN instruments
df_i = self.repo.get_instruments()
if not df_i.is_empty():
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"] if c in df_i.columns]
if "name" not in df.columns:
df = df.join(df_i.select(inst_cols), on="symbol", how="left")
return df
# 尝试从 repo 级预计算历史缓存中提取目标日期
cached_hist = self.repo.get_enriched_history(target_date, 1)
if cached_hist is not None and not cached_hist.is_empty() and "date" in cached_hist.columns:
df = cached_hist.filter(pl.col("date") == target_date)
if not df.is_empty():
logger.debug("_load_enriched_for_date: repo history cache for %s", target_date)
# JOIN instruments
df_i = self.repo.get_instruments()
if not df_i.is_empty():
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"] if c in df_i.columns]
if "name" not in df.columns:
df = df.join(df_i.select(inst_cols), on="symbol", how="left")
return df
# 历史日期: 从 parquet 读取 14 列, 即时计算指标 (慢路径)
enriched_dir = self.repo.store.data_dir / "kline_daily_enriched"
ds = target_date.isoformat()
target_parquet = enriched_dir / f"date={ds}" / "part.parquet"
if not target_parquet.exists():
return pl.DataFrame()
try:
df = pl.read_parquet(target_parquet)
except Exception as e: # noqa: BLE001
logger.warning("load_enriched_for_date failed: %s", e)
return pl.DataFrame()
if df.is_empty():
return df
# 即时计算指标: 需要加载历史窗口作 warmup
df_full = self._compute_enriched_full(df, target_date)
return df_full
def _compute_enriched_full(self, df_target: pl.DataFrame, target_date: date) -> pl.DataFrame:
"""从 14 列基础数据即时计算完整 enriched (含全部指标和信号)。
读取历史数据作为指标计算的 warmup, 计算完成后只返回目标日期的行。
"""
from app.indicators.pipeline import compute_indicators, compute_signals, compute_limit_signals
# 加载 warmup 历史 (目标日期前 ~120 天)
enriched_dir = self.repo.store.data_dir / "kline_daily_enriched"
start = target_date - timedelta(days=150)
read_cols = ["symbol", "date", "open", "high", "low", "close", "volume",
"amount", "raw_close", "raw_high", "raw_low"]
try:
lf = (
pl.scan_parquet(str(enriched_dir / "**" / "*.parquet"))
.filter(
(pl.col("date") >= start)
& (pl.col("date") <= target_date)
)
.sort(["symbol", "date"])
)
available = [c for c in read_cols if c in lf.schema]
df_hist = lf.select(available).collect()
except Exception as e: # noqa: BLE001
logger.warning("warmup history load failed: %s", e)
df_hist = df_target
if df_hist.is_empty():
df_hist = df_target
# 计算指标
df_full = compute_indicators(df_hist)
df_full = compute_signals(df_full)
# 计算涨跌停信号 (需要 instruments)
instruments = self.repo.get_instruments()
if instruments is not None and not instruments.is_empty():
df_full = compute_limit_signals(df_full, instruments)
# 只保留目标日期
df_result = df_full.filter(pl.col("date") == target_date)
# JOIN instruments (name, total_shares, float_shares)
if not instruments.is_empty():
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"] if c in instruments.columns]
if "name" not in df_result.columns:
df_result = df_result.join(instruments.select(inst_cols), on="symbol", how="left")
return df_result
def _load_enriched_history(self, target_date: date, lookback_days: int) -> pl.DataFrame:
"""读取目标日期之前的基础行情数据, 供历史窗口策略使用。
优先从 repo 内存缓存获取 (启动时已预计算), 命中时 0ms。
缓存 miss 时走 scan_parquet + compute_indicators 慢路径。
"""
# 优先级 1: repo 级预计算缓存 (启动时 _refresh_enriched 已计算完整历史)
t0 = time.perf_counter()
cached = self.repo.get_enriched_history(target_date, lookback_days)
if cached is not None and not cached.is_empty():
# JOIN instruments (repo 缓存不含 name 等列)
instruments = self.repo.get_instruments()
if instruments is not None and not instruments.is_empty() and "name" not in cached.columns:
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"]
if c in instruments.columns]
cached = cached.join(instruments.select(inst_cols), on="symbol", how="left")
elapsed = (time.perf_counter() - t0) * 1000
logger.info("_load_enriched_history(%s, %d): repo cache hit, %.1fms, %d rows",
target_date, lookback_days, elapsed, len(cached))
return cached
# 优先级 2: 进程级 history_cache (之前的 TTL 缓存)
cache_key = (target_date, lookback_days)
now = time.monotonic()
ttl_cached = _history_cache.get(cache_key)
if ttl_cached is not None:
ts, cached_df = ttl_cached
if now - ts < _HISTORY_CACHE_TTL:
logger.debug("history TTL cache hit: %s lookback=%d", target_date, lookback_days)
return cached_df
del _history_cache[cache_key]
# 优先级 3: scan_parquet + compute_indicators (慢路径, ~5s)
logger.warning("_load_enriched_history cache miss, computing indicators (%s, %d)...",
target_date, lookback_days)
from app.indicators.pipeline import compute_indicators, compute_signals, compute_limit_signals
warmup = 60
start = target_date - timedelta(days=min((lookback_days + warmup) * 2, 180))
enriched_dir = self.repo.store.data_dir / "kline_daily_enriched"
read_cols = ["symbol", "date", "open", "high", "low", "close", "volume",
"amount", "raw_close", "raw_high", "raw_low"]
try:
lf = (
pl.scan_parquet(str(enriched_dir / "**" / "*.parquet"))
.filter((pl.col("date") >= start) & (pl.col("date") <= target_date))
.sort(["symbol", "date"])
)
available = [c for c in read_cols if c in lf.collect_schema().names()]
df_hist = lf.select(available).collect()
except Exception as e: # noqa: BLE001
logger.warning("load_enriched_history failed: %s", e)
return pl.DataFrame()
if df_hist.is_empty():
return pl.DataFrame()
df_full = compute_indicators(df_hist)
df_full = compute_signals(df_full)
instruments = self.repo.get_instruments()
if instruments is not None and not instruments.is_empty():
df_full = compute_limit_signals(df_full, instruments)
if instruments is not None and not instruments.is_empty():
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"] if c in instruments.columns]
if "name" not in df_full.columns:
df_full = df_full.join(instruments.select(inst_cols), on="symbol", how="left")
# 裁剪掉 warmup 部分, 只保留 lookback 范围 (减少 group_by 开销)
lookback_start = target_date - timedelta(days=lookback_days)
if "date" in df_full.columns:
df_full = df_full.filter(pl.col("date") >= lookback_start)
df_full = df_full.sort(["symbol", "date"])
elapsed = (time.perf_counter() - t0) * 1000
logger.info("_load_enriched_history(%s, %d): computed in %.1fms, %d rows",
target_date, lookback_days, elapsed, len(df_full))
_history_cache[cache_key] = (now, df_full)
if len(_history_cache) > 10:
expired = [k for k, (ts, _) in _history_cache.items() if now - ts > _HISTORY_CACHE_TTL]
for k in expired:
del _history_cache[k]
return df_full
def run(
self,
as_of: date,
conditions: list[str],
order_by: str | None = None,
limit: int = 30,
pool: list[str] | None = None,
) -> ScreenerResult:
"""自定义 SQL 条件选股。
先通过 Polars 即时计算完整指标, 再用 DuckDB 做 SQL WHERE 过滤。
kline_enriched DuckDB 视图只有 14 列, 不能直接用于指标过滤。
"""
t0 = time.perf_counter()
if not conditions:
return ScreenerResult(as_of=as_of, strategy=None)
# 从即时计算获取完整 enriched 数据
df = self._load_enriched_for_date(as_of)
if df.is_empty():
return ScreenerResult(as_of=as_of, strategy=None)
# Pool 过滤
if pool:
df = df.filter(pl.col("symbol").is_in(pool))
# 用 DuckDB 做 SQL 过滤 (注册临时视图)
try:
import duckdb
con = duckdb.connect(database=":memory:")
con.register("enriched", df.to_arrow())
where = " AND ".join(f"({c})" for c in conditions)
sql = f"SELECT * FROM enriched WHERE {where}"
if order_by:
sql += f" ORDER BY {order_by}"
if limit:
sql += f" LIMIT {limit}"
df_result = con.execute(sql).pl()
con.close()
except Exception as e: # noqa: BLE001
logger.warning("screener SQL query failed: %s", e)
df_result = pl.DataFrame()
rows = df_result.to_dicts() if not df_result.is_empty() else []
elapsed = (time.perf_counter() - t0) * 1000
return ScreenerResult(
as_of=as_of,
strategy=None,
rows=rows,
total=len(rows),
elapsed_ms=elapsed,
)
def run_preset(
self,
strategy_id: str,
as_of: date,
pool: list[str] | None = None,
precomputed: pl.DataFrame | None = None,
basic_filter: dict | None = None,
display_limit: int | None = None,
) -> ScreenerResult:
"""预设策略选股 — 从 enriched 读取预计算好的指标列后过滤。
- precomputed 不为空: 直接复用(run_all 场景)
- precomputed 为空: 从 enriched 读目标日期
- basic_filter: 用户保存的基础参数过滤(boards、价格等)
"""
t0 = time.perf_counter()
strat = PRESET_STRATEGIES.get(strategy_id)
if not strat:
raise ValueError(f"unknown strategy: {strategy_id}")
if precomputed is not None and not precomputed.is_empty():
df = precomputed
else:
df = self._load_enriched_for_date(as_of)
if df.is_empty():
return ScreenerResult(as_of=as_of, strategy=strategy_id)
# 应用用户基础参数过滤(boards、价格区间等)
if basic_filter and basic_filter.get("enabled", True):
df = self._apply_basic_filter(df, basic_filter)
# 应用策略过滤
df = df.filter(strat["filter"])
# 应用 pool
if pool:
df = df.filter(pl.col("symbol").is_in(pool))
# 排序 + 限制
order_col = strat["order_by"]
if order_col in df.columns:
df = df.sort(order_col, descending=strat.get("descending", True))
# display_limit: None=不限制, 0=全部, N=前N个
if display_limit == 0:
limit = None # 不限制
elif display_limit is not None:
limit = display_limit
else:
limit = None # 未配置时默认不限制
if limit is not None and limit > 0:
df = df.head(limit)
# 基于排序列生成 0-100 评分 (与 StrategyEngine 统一)
if order_col in df.columns and not df.is_empty():
col_vals = df[order_col].cast(pl.Float64)
col_min = col_vals.min()
col_max = col_vals.max()
col_range = col_max - col_min
if col_range and col_range > 0:
normalized = (col_vals - col_min) / col_range
else:
normalized = pl.Series("norm", [0.5] * len(df))
if not strat.get("descending", True):
normalized = 1.0 - normalized
df = df.with_columns((normalized * 100).alias("score"))
rows = df.to_dicts()
elapsed = (time.perf_counter() - t0) * 1000
# sanitize
for r in rows:
for k, v in list(r.items()):
if isinstance(v, float) and (v != v or abs(v) == float("inf")):
r[k] = None
return ScreenerResult(
as_of=as_of,
strategy=strategy_id,
rows=rows,
total=len(rows),
elapsed_ms=elapsed,
)
@staticmethod
def _apply_basic_filter(df: pl.DataFrame, bf: dict) -> pl.DataFrame:
"""应用用户基础参数过滤(boards、价格区间、市值等)"""
exprs: list[pl.Expr] = []
if bf.get("price_min") is not None:
exprs.append(pl.col("close") >= bf["price_min"])
if bf.get("price_max") is not None:
exprs.append(pl.col("close") <= bf["price_max"])
if bf.get("float_cap_min") is not None and "float_shares" in df.columns:
exprs.append(pl.col("close") * pl.col("float_shares") >= bf["float_cap_min"])
if bf.get("float_cap_max") is not None and "float_shares" in df.columns:
exprs.append(pl.col("close") * pl.col("float_shares") <= bf["float_cap_max"])
if bf.get("amount_min") is not None:
exprs.append(pl.col("amount") >= bf["amount_min"])
if bf.get("amount_max") is not None:
exprs.append(pl.col("amount") <= bf["amount_max"])
if bf.get("turnover_min") is not None and "turnover_rate" in df.columns:
exprs.append(pl.col("turnover_rate") >= bf["turnover_min"])
if bf.get("turnover_max") is not None and "turnover_rate" in df.columns:
exprs.append(pl.col("turnover_rate") <= bf["turnover_max"])
if bf.get("exclude_st") and "name" in df.columns:
exprs.append(~pl.col("name").str.contains("(?i)ST|\\*ST|退"))
# 板块过滤
boards = bf.get("boards")
if boards and isinstance(boards, list) and len(boards) > 0:
board_exprs: list[pl.Expr] = []
for b in boards:
if b == "沪主板":
board_exprs.append(pl.col("symbol").str.starts_with("60"))
elif b == "深主板":
board_exprs.append(
pl.col("symbol").str.starts_with("00")
| pl.col("symbol").str.starts_with("001")
)
elif b == "创业板":
board_exprs.append(
pl.col("symbol").str.starts_with("300")
| pl.col("symbol").str.starts_with("301")
)
elif b == "科创板":
board_exprs.append(pl.col("symbol").str.starts_with("688"))
elif b == "北交所":
board_exprs.append(pl.col("symbol").str.contains(r"\.BJ$"))
if board_exprs:
exprs.append(pl.any_horizontal(board_exprs))
if exprs:
return df.filter(pl.all_horizontal(exprs))
return df
def latest_date(self) -> date | None:
d = self.repo.enriched_latest_date()
if d:
return d
# 回退 DuckDB
try:
res = self.repo.execute_one(
"SELECT max(date) FROM kline_enriched",
)
if res and res[0]:
d = res[0]
return d if isinstance(d, date) else date.fromisoformat(str(d))
except Exception: # noqa: BLE001
return None
return None
+150
View File
@@ -0,0 +1,150 @@
"""策略结果缓存 — 写入本地文件,供策略页面秒加载。
缓存结构:
{
"as_of": "2024-01-15",
"results": { strategy_id: { total, as_of, rows } },
"today_ever_matched": { strategy_id: [symbol, ...] }, // 今日曾命中 symbol 并集
"today_ever_rows": { strategy_id: { symbol: row_data } },// 今日曾命中的完整行数据
"updated_at": 1705324800000 # Unix ms
}
文件路径: data/user_data/strategy_cache.json
"""
from __future__ import annotations
import json
import logging
import time
from datetime import date, datetime
from pathlib import Path
from typing import Any
def _json_default(obj: Any) -> Any:
"""处理 date/datetime 等 JSON 不认识的类型。"""
if isinstance(obj, date):
return obj.isoformat()
if isinstance(obj, datetime):
return obj.isoformat()
raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable")
logger = logging.getLogger(__name__)
_CACHE_FILENAME = "strategy_cache.json"
def _cache_path(data_dir: Path) -> Path:
return data_dir / "user_data" / _CACHE_FILENAME
def _enriched_parquet_path(data_dir: Path, as_of: str) -> Path:
"""返回 enriched parquet 文件路径。"""
return data_dir / "kline_daily_enriched" / f"date={as_of}" / "part.parquet"
def _get_enriched_mtime(data_dir: Path, as_of: str) -> float | None:
"""返回 enriched parquet 文件的 mtime (秒)。文件不存在返回 None。"""
p = _enriched_parquet_path(data_dir, as_of)
try:
return p.stat().st_mtime
except FileNotFoundError:
return None
def read_cache(data_dir: Path) -> dict | None:
"""读取策略缓存文件。返回 None 表示无缓存、读取失败或 enriched 数据已更新导致缓存过期。"""
path = _cache_path(data_dir)
if not path.exists():
return None
try:
text = path.read_text(encoding="utf-8")
if not text.strip():
return None
cached = json.loads(text)
except Exception as e: # noqa: BLE001
logger.warning("读取策略缓存失败: %s", e)
return None
# 校验 enriched mtime: 数据文件变化 → 缓存过期
as_of = cached.get("as_of")
stored_mtime = cached.get("enriched_mtime")
if as_of and stored_mtime:
current_mtime = _get_enriched_mtime(data_dir, as_of)
if current_mtime is not None and current_mtime != stored_mtime:
logger.info("策略缓存过期: enriched 数据已更新 (as_of=%s)", as_of)
return None
return cached
def _rows_to_symbol_map(rows: list[dict]) -> dict[str, dict]:
"""将 rows 列表转为 {symbol: row_data} 映射。"""
result: dict[str, dict] = {}
for row in rows:
sym = row.get("symbol")
if sym:
result[sym] = row
return result
def write_cache(
data_dir: Path,
as_of: str,
results: dict[str, Any],
) -> None:
"""将策略结果写入缓存文件,同时更新今日曾命中集合。
- 日期变更时重置 today_ever_matched 和 today_ever_rows
- 同一天内合并 (并集) 之前曾命中的 symbol,并用最新行数据更新
"""
path = _cache_path(data_dir)
path.parent.mkdir(parents=True, exist_ok=True)
# 读取旧缓存
old = read_cache(data_dir)
old_as_of = old.get("as_of") if old else None
old_ever_rows: dict[str, dict[str, dict]] = old.get("today_ever_rows", {}) if old else {}
# 当前命中的行数据 → symbol 映射
current_row_maps: dict[str, dict[str, dict]] = {}
for sid, r in results.items():
current_row_maps[sid] = _rows_to_symbol_map(r.get("rows", []))
if old_as_of and old_as_of == as_of and old_ever_rows:
# 同一天: 合并 — 用当前行数据更新旧数据 (保持最新价格等)
merged_rows: dict[str, dict[str, dict]] = {}
all_keys = set(old_ever_rows.keys()) | set(current_row_maps.keys())
for sid in all_keys:
old_map = old_ever_rows.get(sid, {})
cur_map = current_row_maps.get(sid, {})
# 以旧数据为基础,用当前数据覆盖 (当前数据更新鲜)
combined = {**old_map, **cur_map}
merged_rows[sid] = combined
today_ever_rows = merged_rows
else:
# 新的一天或首次写入
today_ever_rows = current_row_maps
# 从 ever_rows 提取 symbol 列表 (用于快速计数)
today_ever_matched = {sid: sorted(maps.keys()) for sid, maps in today_ever_rows.items()}
# 记录 enriched parquet 文件的 mtime,用于后续校验缓存是否过期
enriched_mtime = _get_enriched_mtime(data_dir, as_of)
payload = {
"as_of": as_of,
"results": results,
"today_ever_matched": today_ever_matched,
"today_ever_rows": today_ever_rows,
"enriched_mtime": enriched_mtime,
"updated_at": int(time.time() * 1000),
}
try:
path.write_text(json.dumps(payload, ensure_ascii=False, default=_json_default), encoding="utf-8")
total_rows = sum(len(r.get("rows", [])) for r in results.values())
total_ever = sum(len(v) for v in today_ever_matched.values())
logger.info("策略缓存已写入: %s, %d 策略, %d 命中, %d 曾命中", as_of, len(results), total_rows, total_ever)
except Exception as e: # noqa: BLE001
logger.warning("写入策略缓存失败: %s", e)
+130
View File
@@ -0,0 +1,130 @@
"""自选股服务(§6.1)。
存储:`data/user_data/watchlist.parquet`,字段 symbol + added_at + note。
"""
from __future__ import annotations
import logging
from datetime import datetime
from pathlib import Path
import polars as pl
from app.config import settings
from app.tickflow.capabilities import Cap, CapabilitySet
from app.tickflow.client import get_client
logger = logging.getLogger(__name__)
def _path() -> Path:
p = settings.data_dir / "user_data" / "watchlist.parquet"
p.parent.mkdir(parents=True, exist_ok=True)
return p
def list_symbols() -> list[dict]:
p = _path()
if not p.exists():
return []
df = pl.read_parquet(p)
if df.is_empty():
return []
return df.to_dicts()
def add(symbol: str, note: str = "") -> list[dict]:
p = _path()
if p.exists():
df = pl.read_parquet(p)
# 已存在则先移除,后面重新插入到最前面
if symbol in df["symbol"].to_list():
df = df.filter(pl.col("symbol") != symbol)
else:
df = pl.DataFrame(schema={"symbol": pl.Utf8, "added_at": pl.Utf8, "note": pl.Utf8})
new_row = pl.DataFrame({
"symbol": [symbol],
"added_at": [datetime.utcnow().isoformat(timespec="seconds")],
"note": [note],
})
out = pl.concat([new_row, df], how="diagonal_relaxed")
out.write_parquet(p)
return out.to_dicts()
def remove(symbol: str) -> list[dict]:
p = _path()
if not p.exists():
return []
df = pl.read_parquet(p)
df = df.filter(pl.col("symbol") != symbol)
df.write_parquet(p)
return df.to_dicts()
def clear() -> int:
"""清空自选列表。返回移除的数量。"""
p = _path()
if not p.exists():
return 0
df = pl.read_parquet(p)
count = df.height
if count > 0:
pl.DataFrame(schema={"symbol": pl.Utf8, "added_at": pl.Utf8, "note": pl.Utf8}).write_parquet(p)
return count
def fetch_quotes(symbols: list[str], capset: CapabilitySet, timeout_s: float = 8.0) -> list[dict]:
"""拉取实时行情。
优先用 quote.batch;否则降级为 quote.by_symbol 单股请求。
timeout_s: 单批次请求超时(秒),防止 API 卡死阻塞整个请求。
"""
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout
if not symbols:
return []
tf = get_client()
quotes: list[dict] = []
# 走 batch
batch_size = 5
if capset.has(Cap.QUOTE_BATCH):
lim = capset.limits(Cap.QUOTE_BATCH)
batch_size = lim.batch if lim and lim.batch else 50
elif capset.has(Cap.QUOTE_BY_SYMBOL):
lim = capset.limits(Cap.QUOTE_BY_SYMBOL)
batch_size = lim.batch if lim and lim.batch else 5
else:
# 无任何实时行情能力(none/free 档走 free-api 服务器,不提供实时行情)
# 提前返回空,避免发起注定失败的请求
return []
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
# 用线程池为每个批次加超时保护
pool = ThreadPoolExecutor(max_workers=1)
for chunk in chunks:
try:
future = pool.submit(tf.quotes.get, symbols=chunk, as_dataframe=True)
raw = future.result(timeout=timeout_s)
if raw is None or len(raw) == 0:
continue
df = pl.from_pandas(raw)
rename_map = {
"last_price": "price",
"ext.change_pct": "pct",
"ext.name": "name",
}
df = df.rename({k: v for k, v in rename_map.items() if k in df.columns})
quotes.extend(df.to_dicts())
except FuturesTimeout:
logger.warning("quote fetch timeout (%.1fs) for %d symbols", timeout_s, len(chunk))
break # 超时后不再尝试后续批次
except Exception as e: # noqa: BLE001
logger.warning("quote fetch failed for %d symbols: %s", len(chunk), e)
pool.shutdown(wait=False)
return quotes