将项目文件整理到 refer 目录
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""业务服务层。"""
|
||||
@@ -0,0 +1,209 @@
|
||||
"""告警触发记录存储 — JSONL 追加写 + 滚动清理。
|
||||
|
||||
职责:
|
||||
- 把每次触发的 AlertEvent 追加写入 data/user_data/alerts.jsonl
|
||||
- 提供查询 (按来源/类型过滤、时间倒序、限量)
|
||||
- 滚动清理: 保留近 N 天 + 上限 M 条 (取交集)
|
||||
|
||||
设计:
|
||||
- JSONL 每行一个 JSON 对象,便于增量追加和流式读取
|
||||
- 清理策略: 追加后按需 prune (按 ts 删旧),避免文件无限膨胀
|
||||
- 读时全量加载到内存过滤 (记录量受上限约束, 5000 条量级无压力)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 保留策略
|
||||
MAX_DAYS = 7
|
||||
MAX_RECORDS = 5000
|
||||
# 每隔多少次写入触发一次清理 (避免每次写都 prune)
|
||||
PRUNE_EVERY = 20
|
||||
|
||||
_lock = threading.Lock()
|
||||
_write_count = 0
|
||||
|
||||
|
||||
def _path(data_dir: Path) -> Path:
|
||||
p = data_dir / "user_data" / "alerts.jsonl"
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def append(data_dir: Path, event: dict) -> None:
|
||||
"""追加一条触发记录。event 应含 ts(毫秒)、rule_id、source 等字段。"""
|
||||
line = json.dumps(event, ensure_ascii=False)
|
||||
with _lock:
|
||||
p = _path(data_dir)
|
||||
with p.open("a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
global _write_count
|
||||
_write_count += 1
|
||||
if _write_count >= PRUNE_EVERY:
|
||||
_write_count = 0
|
||||
_prune_locked(p)
|
||||
|
||||
|
||||
def append_many(data_dir: Path, events: list[dict]) -> None:
|
||||
"""批量追加。"""
|
||||
if not events:
|
||||
return
|
||||
with _lock:
|
||||
p = _path(data_dir)
|
||||
with p.open("a", encoding="utf-8") as f:
|
||||
for ev in events:
|
||||
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
|
||||
global _write_count
|
||||
_write_count += len(events)
|
||||
if _write_count >= PRUNE_EVERY:
|
||||
_write_count = 0
|
||||
_prune_locked(p)
|
||||
|
||||
|
||||
def list_recent(
|
||||
data_dir: Path,
|
||||
days: int = MAX_DAYS,
|
||||
limit: int = MAX_RECORDS,
|
||||
source: str | None = None,
|
||||
type: str | None = None,
|
||||
) -> list[dict]:
|
||||
"""读取近 N 天记录,按时间倒序,支持按 source/type 过滤。"""
|
||||
import time
|
||||
cutoff = (time.time() - days * 86400) * 1000 # 毫秒
|
||||
out: list[dict] = []
|
||||
p = _path(data_dir)
|
||||
if not p.exists():
|
||||
return []
|
||||
try:
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if ev.get("ts", 0) < cutoff:
|
||||
continue
|
||||
if source and ev.get("source") != source:
|
||||
continue
|
||||
if type and ev.get("type") != type:
|
||||
continue
|
||||
out.append(ev)
|
||||
except Exception as e:
|
||||
logger.warning("alert_store read failed: %s", e)
|
||||
return []
|
||||
# 时间倒序 + 截断
|
||||
out.sort(key=lambda x: x.get("ts", 0), reverse=True)
|
||||
return out[:limit]
|
||||
|
||||
|
||||
def clear(data_dir: Path) -> int:
|
||||
"""清空全部记录,返回清除的条数。"""
|
||||
with _lock:
|
||||
p = _path(data_dir)
|
||||
if not p.exists():
|
||||
return 0
|
||||
count = 0
|
||||
try:
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
count = sum(1 for line in f if line.strip())
|
||||
except Exception:
|
||||
pass
|
||||
p.write_text("", encoding="utf-8")
|
||||
return count
|
||||
|
||||
|
||||
def delete_one(data_dir: Path, ts: int) -> bool:
|
||||
"""删除指定 ts 的单条记录,返回是否删除成功。
|
||||
|
||||
JSONL 无主键, 用 ts(毫秒时间戳) 作为标识。
|
||||
若存在多条同 ts, 只删第一条。
|
||||
"""
|
||||
with _lock:
|
||||
p = _path(data_dir)
|
||||
if not p.exists():
|
||||
return False
|
||||
kept: list[dict] = []
|
||||
deleted = False
|
||||
try:
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if not deleted and ev.get("ts") == ts:
|
||||
deleted = True
|
||||
continue
|
||||
kept.append(ev)
|
||||
except Exception as e:
|
||||
logger.warning("alert_store delete_one read failed: %s", e)
|
||||
return False
|
||||
if not deleted:
|
||||
return False
|
||||
try:
|
||||
with p.open("w", encoding="utf-8") as f:
|
||||
for ev in kept:
|
||||
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
|
||||
except Exception as e:
|
||||
logger.warning("alert_store delete_one write failed: %s", e)
|
||||
return False
|
||||
return True
|
||||
return count
|
||||
|
||||
|
||||
def count(data_dir: Path) -> int:
|
||||
"""返回当前记录总数。"""
|
||||
p = _path(data_dir)
|
||||
if not p.exists():
|
||||
return 0
|
||||
try:
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
return sum(1 for line in f if line.strip())
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def _prune_locked(p: Path) -> None:
|
||||
"""(调用方需持锁) 保留近 MAX_DAYS 天 + 上限 MAX_RECORDS 条。"""
|
||||
import time
|
||||
cutoff = (time.time() - MAX_DAYS * 86400) * 1000
|
||||
kept: list[dict] = []
|
||||
try:
|
||||
with p.open("r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
ev = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if ev.get("ts", 0) >= cutoff:
|
||||
kept.append(ev)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
except Exception as e:
|
||||
logger.warning("alert_store prune read failed: %s", e)
|
||||
return
|
||||
# 上限截断 (保留最新的)
|
||||
if len(kept) > MAX_RECORDS:
|
||||
kept.sort(key=lambda x: x.get("ts", 0))
|
||||
kept = kept[-MAX_RECORDS:]
|
||||
# 重写文件
|
||||
try:
|
||||
with p.open("w", encoding="utf-8") as f:
|
||||
for ev in kept:
|
||||
f.write(json.dumps(ev, ensure_ascii=False) + "\n")
|
||||
except Exception as e:
|
||||
logger.warning("alert_store prune write failed: %s", e)
|
||||
@@ -0,0 +1,392 @@
|
||||
"""回测服务(§6.7)。
|
||||
|
||||
包 vectorbt — 全项目唯一一处出现 pandas。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from typing import Literal
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import polars as pl
|
||||
|
||||
from app.config import settings
|
||||
from app.tickflow.repository import KlineRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# vectorbt 是 optional extras(见 pyproject.toml).未装时只有 backtest 不可用,其他功能正常.
|
||||
_vbt = None
|
||||
_vbt_unavailable_reason: str | None = None
|
||||
|
||||
|
||||
class VectorbtUnavailable(RuntimeError):
|
||||
"""vectorbt 未安装 — 提示用户 `uv sync --extra backtest`."""
|
||||
|
||||
|
||||
def _get_vbt():
|
||||
global _vbt, _vbt_unavailable_reason
|
||||
if _vbt is not None:
|
||||
return _vbt
|
||||
if _vbt_unavailable_reason is not None:
|
||||
raise VectorbtUnavailable(_vbt_unavailable_reason)
|
||||
try:
|
||||
import vectorbt as vbt
|
||||
_vbt = vbt
|
||||
return _vbt
|
||||
except ImportError as e:
|
||||
_vbt_unavailable_reason = (
|
||||
"vectorbt 未安装 — 它是回测的可选依赖.macOS Intel 用户先 `brew install cmake` "
|
||||
"然后 `uv sync --extra backtest`"
|
||||
)
|
||||
logger.warning("vectorbt unavailable: %s", e)
|
||||
raise VectorbtUnavailable(_vbt_unavailable_reason) from e
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
"""供 API 层快速检测."""
|
||||
try:
|
||||
_get_vbt()
|
||||
return True
|
||||
except VectorbtUnavailable:
|
||||
return False
|
||||
|
||||
|
||||
SignalKind = Literal[
|
||||
"macd_golden", "macd_dead",
|
||||
"ma_golden_5_20", "ma_dead_5_20",
|
||||
"ma_golden_20_60",
|
||||
"ma20_breakout", "ma20_breakdown",
|
||||
"n_day_high", "n_day_low",
|
||||
"boll_breakout_upper", "boll_breakdown_lower",
|
||||
"volume_surge",
|
||||
"rsi_oversold", "rsi_overbought",
|
||||
"stop_loss", "trailing_stop", "max_hold",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class BacktestConfig:
|
||||
symbols: list[str]
|
||||
start: date
|
||||
end: date
|
||||
# 买入信号(任一触发即买)
|
||||
entries: list[str] = field(default_factory=list)
|
||||
# 卖出信号(任一触发即卖)
|
||||
exits: list[str] = field(default_factory=list)
|
||||
# 其他参数
|
||||
stop_loss_pct: float | None = None # 例 -0.05 = -5%
|
||||
max_hold_days: int | None = None
|
||||
fees_pct: float = 0.0002 # 万二佣金
|
||||
slippage_bps: float = 5 # 5 bps
|
||||
# 撮合
|
||||
matching: Literal["close_t", "open_t+1"] = "close_t"
|
||||
rsi_oversold_threshold: float = 30
|
||||
rsi_overbought_threshold: float = 70
|
||||
|
||||
|
||||
@dataclass
|
||||
class BacktestResult:
|
||||
run_id: str
|
||||
config: dict
|
||||
stats: dict
|
||||
equity_curve: list[dict] # [{date, value}]
|
||||
trades: list[dict] # [{symbol, entry_date, exit_date, pnl_pct, ...}]
|
||||
per_symbol_stats: list[dict] # 每只股票的统计
|
||||
|
||||
|
||||
# enriched 表里的信号列名映射
|
||||
_SIGNAL_COLS: dict[SignalKind, str] = {
|
||||
"macd_golden": "signal_macd_golden",
|
||||
"macd_dead": "signal_macd_dead",
|
||||
"ma_golden_5_20": "signal_ma_golden_5_20",
|
||||
"ma_dead_5_20": "signal_ma_dead_5_20",
|
||||
"ma_golden_20_60": "signal_ma_golden_20_60",
|
||||
"ma20_breakout": "signal_ma20_breakout",
|
||||
"ma20_breakdown": "signal_ma20_breakdown",
|
||||
"n_day_high": "signal_n_day_high",
|
||||
"n_day_low": "signal_n_day_low",
|
||||
"boll_breakout_upper": "signal_boll_breakout_upper",
|
||||
"boll_breakdown_lower": "signal_boll_breakdown_lower",
|
||||
"volume_surge": "signal_volume_surge",
|
||||
}
|
||||
|
||||
|
||||
class BacktestService:
|
||||
def __init__(self, repo: KlineRepository) -> None:
|
||||
self.repo = repo
|
||||
|
||||
def _load_panel(
|
||||
self,
|
||||
symbols: list[str],
|
||||
start: date,
|
||||
end: date,
|
||||
) -> pd.DataFrame:
|
||||
"""加载 [date × symbol] 价格面板 — Polars scan_parquet + 即时计算指标。
|
||||
|
||||
**全项目唯一从 Polars 转 pandas 的边界**(§7.4 / ADR-19)。
|
||||
"""
|
||||
try:
|
||||
enriched_glob = str(self.repo.store.data_dir / "kline_daily_enriched" / "**" / "*.parquet")
|
||||
df = (
|
||||
pl.scan_parquet(enriched_glob)
|
||||
.filter(
|
||||
(pl.col("symbol").is_in(symbols))
|
||||
& (pl.col("date") >= start)
|
||||
& (pl.col("date") <= end)
|
||||
)
|
||||
.sort(["date", "symbol"])
|
||||
.collect()
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("backtest load failed: %s", e)
|
||||
return pd.DataFrame()
|
||||
|
||||
if df.is_empty():
|
||||
return pd.DataFrame()
|
||||
|
||||
# 即时计算指标 + 信号
|
||||
from app.indicators.pipeline import compute_all
|
||||
df = compute_all(df)
|
||||
|
||||
# 选择需要的列
|
||||
needed_cols = [
|
||||
"date", "symbol", "open", "high", "low", "close", "volume",
|
||||
"rsi_14", "signal_macd_golden", "signal_macd_dead",
|
||||
"signal_ma_golden_5_20", "signal_ma_dead_5_20",
|
||||
"signal_ma_golden_20_60",
|
||||
"signal_ma20_breakout", "signal_ma20_breakdown",
|
||||
"signal_n_day_high", "signal_n_day_low",
|
||||
"signal_boll_breakout_upper", "signal_boll_breakdown_lower",
|
||||
"signal_volume_surge",
|
||||
]
|
||||
existing = [c for c in needed_cols if c in df.columns]
|
||||
df = df.select(existing)
|
||||
|
||||
# to_pandas 边界
|
||||
return df.to_pandas(use_pyarrow_extension_array=False)
|
||||
|
||||
def _build_signal_matrix(
|
||||
self,
|
||||
panel: pd.DataFrame,
|
||||
kinds: list[str],
|
||||
config: BacktestConfig,
|
||||
) -> pd.DataFrame:
|
||||
"""从面板构造 [date × symbol] 的布尔信号矩阵。"""
|
||||
if not kinds or panel.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
# pivot 成 [date × symbol] 形式
|
||||
result = None
|
||||
for kind in kinds:
|
||||
mat = None
|
||||
if kind in _SIGNAL_COLS:
|
||||
col = _SIGNAL_COLS[kind]
|
||||
mat = panel.pivot(index="date", columns="symbol", values=col).fillna(False).astype(bool)
|
||||
elif kind == "rsi_oversold":
|
||||
mat = (panel.pivot(index="date", columns="symbol", values="rsi_14")
|
||||
< config.rsi_oversold_threshold)
|
||||
elif kind == "rsi_overbought":
|
||||
mat = (panel.pivot(index="date", columns="symbol", values="rsi_14")
|
||||
> config.rsi_overbought_threshold)
|
||||
# stop_loss / trailing / max_hold 通过 vectorbt 参数处理,不参与信号矩阵
|
||||
|
||||
if mat is not None:
|
||||
result = mat if result is None else (result | mat)
|
||||
return result if result is not None else pd.DataFrame()
|
||||
|
||||
def run(self, config: BacktestConfig) -> BacktestResult:
|
||||
vbt = _get_vbt()
|
||||
run_id = uuid.uuid4().hex[:10]
|
||||
|
||||
panel = self._load_panel(config.symbols, config.start, config.end)
|
||||
if panel.empty:
|
||||
return BacktestResult(
|
||||
run_id=run_id,
|
||||
config=_config_to_dict(config),
|
||||
stats={"error": "no data"},
|
||||
equity_curve=[],
|
||||
trades=[],
|
||||
per_symbol_stats=[],
|
||||
)
|
||||
|
||||
# 价格面板
|
||||
close = panel.pivot(index="date", columns="symbol", values="close")
|
||||
|
||||
# 信号矩阵
|
||||
entries = self._build_signal_matrix(panel, config.entries, config)
|
||||
exits = self._build_signal_matrix(panel, config.exits, config)
|
||||
|
||||
# 对齐 index/columns
|
||||
if not entries.empty:
|
||||
entries = entries.reindex_like(close).fillna(False).astype(bool)
|
||||
else:
|
||||
entries = pd.DataFrame(False, index=close.index, columns=close.columns)
|
||||
if not exits.empty:
|
||||
exits = exits.reindex_like(close).fillna(False).astype(bool)
|
||||
else:
|
||||
exits = pd.DataFrame(False, index=close.index, columns=close.columns)
|
||||
|
||||
if not entries.any().any():
|
||||
return BacktestResult(
|
||||
run_id=run_id,
|
||||
config=_config_to_dict(config),
|
||||
stats={"error": "no buy signals"},
|
||||
equity_curve=[],
|
||||
trades=[],
|
||||
per_symbol_stats=[],
|
||||
)
|
||||
|
||||
# T+1 适配:vectorbt 默认信号当根 K 撮合
|
||||
# close_t 撮合:维持默认
|
||||
# open_t+1 撮合:shift 信号 1 根 + 用 open 作为价
|
||||
if config.matching == "open_t+1":
|
||||
entries = entries.shift(1).fillna(False).astype(bool)
|
||||
exits = exits.shift(1).fillna(False).astype(bool)
|
||||
price = panel.pivot(index="date", columns="symbol", values="open")
|
||||
else:
|
||||
price = close
|
||||
|
||||
# 跑回测
|
||||
try:
|
||||
pf_kwargs = dict(
|
||||
close=close,
|
||||
entries=entries,
|
||||
exits=exits,
|
||||
price=price,
|
||||
fees=config.fees_pct,
|
||||
slippage=config.slippage_bps / 10000.0,
|
||||
freq="1D",
|
||||
)
|
||||
if config.stop_loss_pct is not None:
|
||||
pf_kwargs["sl_stop"] = abs(config.stop_loss_pct)
|
||||
if config.max_hold_days is not None:
|
||||
# vectorbt 没有内置 max-hold;用时间退出近似:
|
||||
# 在 max_hold_days 后强制 exit
|
||||
exits_idx = entries.copy()
|
||||
for col in entries.columns:
|
||||
entry_rows = np.where(entries[col].values)[0]
|
||||
for i in entry_rows:
|
||||
end_i = min(i + config.max_hold_days, len(entries) - 1)
|
||||
if end_i > i:
|
||||
exits_idx.iloc[end_i][col] = True
|
||||
pf_kwargs["exits"] = (exits | exits_idx).astype(bool)
|
||||
|
||||
pf = vbt.Portfolio.from_signals(**pf_kwargs)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.exception("vectorbt backtest failed")
|
||||
return BacktestResult(
|
||||
run_id=run_id,
|
||||
config=_config_to_dict(config),
|
||||
stats={"error": str(e)},
|
||||
equity_curve=[],
|
||||
trades=[],
|
||||
per_symbol_stats=[],
|
||||
)
|
||||
|
||||
# 提取结果
|
||||
try:
|
||||
stats_series = pf.stats(silence_warnings=True)
|
||||
if isinstance(stats_series, pd.DataFrame):
|
||||
# 多列时取 agg
|
||||
stats_dict = stats_series.mean(numeric_only=True).to_dict()
|
||||
else:
|
||||
stats_dict = stats_series.to_dict()
|
||||
except Exception: # noqa: BLE001
|
||||
stats_dict = {}
|
||||
|
||||
# 净值曲线(组合平均)
|
||||
equity = pf.value().mean(axis=1) if isinstance(pf.value(), pd.DataFrame) else pf.value()
|
||||
equity_curve = [
|
||||
{"date": str(idx.date() if hasattr(idx, "date") else idx), "value": float(v)}
|
||||
for idx, v in equity.items() if pd.notna(v)
|
||||
]
|
||||
|
||||
# 交易记录
|
||||
try:
|
||||
trades_df = pf.trades.records_readable
|
||||
trades = trades_df.to_dict(orient="records") if not trades_df.empty else []
|
||||
# 字段名美化
|
||||
trades = [
|
||||
{
|
||||
"symbol": t.get("Column", t.get("Symbol", "")),
|
||||
"entry_date": str(t.get("Entry Timestamp", t.get("Entry Date", ""))),
|
||||
"exit_date": str(t.get("Exit Timestamp", t.get("Exit Date", ""))),
|
||||
"entry_price": float(t.get("Avg Entry Price", t.get("Avg. Entry Price", 0))),
|
||||
"exit_price": float(t.get("Avg Exit Price", t.get("Avg. Exit Price", 0))),
|
||||
"pnl_pct": float(t.get("Return", t.get("PnL %", 0))),
|
||||
"duration": str(t.get("Duration", "")),
|
||||
}
|
||||
for t in trades
|
||||
]
|
||||
except Exception: # noqa: BLE001
|
||||
trades = []
|
||||
|
||||
# 每标的统计
|
||||
per_symbol = []
|
||||
try:
|
||||
total_ret = pf.total_return()
|
||||
if isinstance(total_ret, pd.Series):
|
||||
for sym, ret in total_ret.items():
|
||||
if pd.notna(ret):
|
||||
per_symbol.append({"symbol": sym, "total_return": float(ret)})
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
result = BacktestResult(
|
||||
run_id=run_id,
|
||||
config=_config_to_dict(config),
|
||||
stats={k: _json_safe(v) for k, v in stats_dict.items()},
|
||||
equity_curve=equity_curve,
|
||||
trades=trades,
|
||||
per_symbol_stats=per_symbol,
|
||||
)
|
||||
|
||||
# 落盘
|
||||
self._persist(result)
|
||||
return result
|
||||
|
||||
def _persist(self, result: BacktestResult) -> None:
|
||||
out_dir = settings.data_dir / "backtest_results"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
# 用 polars 写一份汇总
|
||||
summary = pl.DataFrame({
|
||||
"run_id": [result.run_id],
|
||||
"stats_json": [str(result.stats)],
|
||||
"n_trades": [len(result.trades)],
|
||||
})
|
||||
summary.write_parquet(out_dir / f"run_id={result.run_id}.parquet")
|
||||
|
||||
def get_result(self, run_id: str) -> BacktestResult | None:
|
||||
# Phase 1:只保留近似落盘,完整结果保存在内存的近期 cache 中
|
||||
# 简化:重新 run 比缓存复杂结果代价小,暂不实现 get_result
|
||||
return None
|
||||
|
||||
|
||||
def _config_to_dict(c: BacktestConfig) -> dict:
|
||||
return {
|
||||
"symbols": c.symbols,
|
||||
"start": str(c.start),
|
||||
"end": str(c.end),
|
||||
"entries": c.entries,
|
||||
"exits": c.exits,
|
||||
"stop_loss_pct": c.stop_loss_pct,
|
||||
"max_hold_days": c.max_hold_days,
|
||||
"fees_pct": c.fees_pct,
|
||||
"slippage_bps": c.slippage_bps,
|
||||
"matching": c.matching,
|
||||
}
|
||||
|
||||
|
||||
def _json_safe(v):
|
||||
if isinstance(v, (int, float, str, bool)) or v is None:
|
||||
return v
|
||||
if isinstance(v, (np.floating, np.integer)):
|
||||
return float(v) if not np.isnan(float(v)) else None
|
||||
if hasattr(v, "isoformat"):
|
||||
return v.isoformat()
|
||||
return str(v)
|
||||
@@ -0,0 +1,575 @@
|
||||
"""五档盘口 sealed(真假涨停/跌停) 服务 — 独立旁路线。
|
||||
|
||||
架构(完全解耦):
|
||||
- 只读 enriched(拿涨跌停名单), 不写回 enriched(14列不动)
|
||||
- sealed 存独立 parquet(data/depth5/date=xxx/part.parquet)
|
||||
- limit_ladder API 查询时 LEFT JOIN(同 ext_columns 机制)
|
||||
- signal_limit_up 永远是"价格涨停", sealed 是叠加的真假判定层
|
||||
|
||||
数据流:
|
||||
盘中轮询线程(交易时段, 独立 sleep, 不绑行情轮询):
|
||||
读 enriched 内存缓存(线程安全) → 涨跌停名单 → tf.depth.batch
|
||||
→ 算 sealed → 更新内存缓存(不落盘) → sealed_ready=True
|
||||
盘后定版 job(可配置时间, 默认15:02):
|
||||
最后拉一次 → 落盘 depth5 parquet(定版)
|
||||
|
||||
三层防护节流("设过大设上限, 设过小设最小值"):
|
||||
① 套餐范围 clamp: Pro 10~120s, Expert 3~300s
|
||||
② 限速安全 clamp: safe = 60/((rpm*0.8)/batches), 涨跌停多就自动放慢
|
||||
③ 系统接管通知: 用户设置会超限时, 推 toast 告知已自动调整
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
from datetime import date, datetime, time as dt_time
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# 套餐 → (轮询间隔下限s, 上限s)
|
||||
TIER_INTERVAL_RANGE: dict[str, tuple[float, float]] = {
|
||||
"pro": (10.0, 120.0),
|
||||
"expert": (3.0, 300.0),
|
||||
}
|
||||
# 兜底: 其他有 DEPTH5_BATCH 的套餐按 pro 范围
|
||||
DEFAULT_RANGE = (10.0, 120.0)
|
||||
|
||||
# 限速余量: 只用 rpm 的 80%, 给系统其他 depth 调用留空间
|
||||
RPM_MARGIN = 0.8
|
||||
# 间隔硬下限/上限(任何套餐)
|
||||
INTERVAL_HARD_MIN = 10.0
|
||||
INTERVAL_HARD_MAX = 300.0
|
||||
|
||||
|
||||
class DepthService:
|
||||
"""五档盘口 sealed 服务 — 单例。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._running = False
|
||||
self._thread: threading.Thread | None = None
|
||||
self._repo = None # 延迟注入(KlineRepository)
|
||||
self._app_state = None # 延迟注入(FastAPI app.state)
|
||||
|
||||
# 内存缓存: {symbol: SealedEntry}
|
||||
# SealedEntry = {sealed_up, sealed_down, ask1_vol, bid1_vol, status, fetched_ts}
|
||||
self._sealed_cache: dict[str, dict] = {}
|
||||
self._sealed_ready = False
|
||||
self._sealed_date: date | None = None # sealed 数据对应的交易日(可能是昨天,如休市)
|
||||
self._sealed_fetched_ts: float = 0.0 # 上次拉取的 perf_counter
|
||||
self._sealed_fetched_at: float = 0.0 # 上次拉取的 wall-clock 时间戳
|
||||
self._persisted_date: date | None = None # 已落盘的日期
|
||||
|
||||
# 系统接管状态(防通知刷屏)
|
||||
self._last_taken_over: bool | None = None
|
||||
self._last_user_interval: float | None = None
|
||||
|
||||
# ================================================================
|
||||
# 注入
|
||||
# ================================================================
|
||||
|
||||
def set_repo(self, repo) -> None:
|
||||
self._repo = repo
|
||||
|
||||
def set_app_state(self, app_state) -> None:
|
||||
self._app_state = app_state
|
||||
|
||||
# ================================================================
|
||||
# 生命周期
|
||||
# ================================================================
|
||||
|
||||
def boot_check(self) -> None:
|
||||
"""启动补跑: 当天 depth5 文件不存在则 finalize 一次; 已存在则恢复内存缓存。"""
|
||||
if not self._has_capability():
|
||||
logger.info("depth sealed: 无 DEPTH5_BATCH 能力, 跳过启动补跑")
|
||||
return
|
||||
today = date.today()
|
||||
if self._persisted_for_date(today):
|
||||
# parquet 已存在: 恢复内存缓存(避免重启后每次查询都读 parquet)
|
||||
self._restore_from_parquet(today)
|
||||
return
|
||||
logger.info("depth sealed: 启动补跑今天定版")
|
||||
try:
|
||||
self.finalize()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth sealed 启动补跑失败: %s", e)
|
||||
|
||||
def _restore_from_parquet(self, d: date) -> None:
|
||||
"""从 parquet 恢复内存缓存(服务重启后)。"""
|
||||
if not self._repo:
|
||||
return
|
||||
out = self._repo.store.data_dir / "depth5" / f"date={d.isoformat()}" / "part.parquet"
|
||||
if not out.exists():
|
||||
return
|
||||
try:
|
||||
df = pl.read_parquet(out)
|
||||
cache: dict[str, dict] = {}
|
||||
for row in df.to_dicts():
|
||||
sym = row.get("symbol")
|
||||
if not sym:
|
||||
continue
|
||||
cache[sym] = {
|
||||
"sealed_up": row.get("sealed_up"),
|
||||
"sealed_down": row.get("sealed_down"),
|
||||
"ask1_vol": row.get("ask1_vol"),
|
||||
"bid1_vol": row.get("bid1_vol"),
|
||||
"status": row.get("status"),
|
||||
"fetched_ts": row.get("fetched_at"),
|
||||
}
|
||||
with self._lock:
|
||||
self._sealed_cache = cache
|
||||
self._sealed_ready = True
|
||||
self._sealed_date = d
|
||||
self._persisted_date = d
|
||||
logger.info("depth sealed: 从 parquet 恢复 %d 只 (日期=%s)", len(cache), d)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth sealed 从 parquet 恢复失败: %s", e)
|
||||
|
||||
def start_polling(self) -> None:
|
||||
"""启动盘中轮询线程(连板梯队监控开启 + 有能力 + 交易时段)。"""
|
||||
if self._running:
|
||||
return
|
||||
if not self._has_capability():
|
||||
return
|
||||
from app.services import preferences
|
||||
if not preferences.get_limit_ladder_monitor_enabled():
|
||||
return
|
||||
self._running = True
|
||||
self._thread = threading.Thread(target=self._poll_loop, daemon=True)
|
||||
self._thread.start()
|
||||
logger.info("depth sealed 盘中轮询已启动")
|
||||
|
||||
def stop_polling(self) -> None:
|
||||
"""停止盘中轮询线程。"""
|
||||
self._running = False
|
||||
if self._thread:
|
||||
self._thread.join(timeout=10)
|
||||
self._thread = None
|
||||
logger.info("depth sealed 盘中轮询已停止")
|
||||
|
||||
def apply_monitor_toggle(self, enabled: bool) -> None:
|
||||
"""连板梯队监控开关切换时调用: 开启→启动轮询, 关闭→停止轮询。"""
|
||||
if enabled:
|
||||
self.start_polling()
|
||||
else:
|
||||
self.stop_polling()
|
||||
|
||||
def run_once(self) -> dict:
|
||||
"""手动触发一次修正(立即拉取 depth + 更新内存缓存)。
|
||||
|
||||
不受监控开关限制 — 用户可随时手动修正一次。
|
||||
返回 {"ok": bool, "count": int, "msg": str}
|
||||
"""
|
||||
if not self._has_capability():
|
||||
return {"ok": False, "count": 0, "msg": "无五档盘口能力(需 Pro+)"}
|
||||
try:
|
||||
self._fetch_and_seal(persist=True) # 落盘, 刷新页面不丢
|
||||
with self._lock:
|
||||
count = len(self._sealed_cache)
|
||||
return {"ok": True, "count": count, "msg": f"已修正 {count} 只"}
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth run_once 失败: %s", e)
|
||||
return {"ok": False, "count": 0, "msg": f"修正失败: {e}"}
|
||||
|
||||
# ================================================================
|
||||
# 核心拉取
|
||||
# ================================================================
|
||||
|
||||
def _fetch_and_seal(self, persist: bool = False) -> None:
|
||||
"""拉一次 depth.batch, 算 sealed, 更新内存缓存(可选落盘)。
|
||||
|
||||
persist=True: 盘后定版, 写 depth5 parquet
|
||||
persist=False: 盘中轮询, 只更新内存缓存
|
||||
"""
|
||||
if not self._repo:
|
||||
return
|
||||
|
||||
# 只读 enriched 内存缓存(线程安全, 避免和 quote_service 写盘竞态)
|
||||
enriched, enriched_date = self._repo.get_enriched_latest()
|
||||
if enriched.is_empty():
|
||||
return
|
||||
|
||||
# 筛涨跌停名单(用 fill_null 防止列缺失)
|
||||
syms_up: list[str] = []
|
||||
syms_down: list[str] = []
|
||||
if "signal_limit_up" in enriched.columns:
|
||||
syms_up = enriched.filter(
|
||||
pl.col("signal_limit_up").fill_null(False)
|
||||
)["symbol"].to_list()
|
||||
if "signal_limit_down" in enriched.columns:
|
||||
syms_down = enriched.filter(
|
||||
pl.col("signal_limit_down").fill_null(False)
|
||||
)["symbol"].to_list()
|
||||
|
||||
all_syms = list(dict.fromkeys(syms_up + syms_down)) # 去重保序
|
||||
if not all_syms:
|
||||
logger.debug("depth sealed: 当日无涨跌停股, 跳过")
|
||||
return
|
||||
|
||||
# 拉 depth(涨跌停一次拉, 按 capset batch 切片)
|
||||
depth_data = self._call_depth_batch(all_syms)
|
||||
if not depth_data:
|
||||
logger.warning("depth sealed: depth.batch 返回空")
|
||||
return
|
||||
|
||||
up_set = set(syms_up)
|
||||
down_set = set(syms_down)
|
||||
now_perf = time.perf_counter()
|
||||
now_wall = time.time()
|
||||
|
||||
new_cache: dict[str, dict] = {}
|
||||
for sym, d in depth_data.items():
|
||||
ask_vols = d.get("ask_volumes") or []
|
||||
bid_vols = d.get("bid_volumes") or []
|
||||
ask1 = ask_vols[0] if ask_vols else None
|
||||
bid1 = bid_vols[0] if bid_vols else None
|
||||
# depth 返回的 timestamp(毫秒 epoch), 回退到当前 wall-clock
|
||||
depth_ts = d.get("timestamp")
|
||||
fetched = (depth_ts / 1000.0) if isinstance(depth_ts, (int, float)) and depth_ts else now_wall
|
||||
entry = {
|
||||
# 涨停真封: 涨停价上卖一(主动卖压)为 0
|
||||
"sealed_up": (ask1 == 0) if sym in up_set and ask1 is not None else None,
|
||||
# 跌停真封: 跌停价上买一为 0
|
||||
"sealed_down": (bid1 == 0) if sym in down_set and bid1 is not None else None,
|
||||
"ask1_vol": ask1,
|
||||
"bid1_vol": bid1,
|
||||
"status": "limit_down" if sym in down_set and sym not in up_set else "limit_up",
|
||||
"fetched_ts": fetched,
|
||||
}
|
||||
new_cache[sym] = entry
|
||||
|
||||
with self._lock:
|
||||
self._sealed_cache = new_cache
|
||||
self._sealed_ready = True
|
||||
self._sealed_date = enriched_date # 记录数据对应的交易日(可能是昨天,如休市)
|
||||
self._sealed_fetched_ts = now_perf
|
||||
self._sealed_fetched_at = now_wall
|
||||
|
||||
logger.info("depth sealed: 拉取 %d 只 (涨停%d/跌停%d) 日期=%s%s",
|
||||
len(new_cache), len(syms_up), len(syms_down),
|
||||
enriched_date, " → 落盘" if persist else "")
|
||||
|
||||
# 缓存已更新: 通知 SSE 推 depth_updated, 触发连板梯队刷新封单数据。
|
||||
self._notify_depth_updated(len(new_cache))
|
||||
|
||||
if persist and enriched_date:
|
||||
self._persist(enriched_date)
|
||||
|
||||
def _call_depth_batch(self, symbols: list[str]) -> dict:
|
||||
"""调 tf.depth.batch, 按 capset 的 batch 切片 + 节流。返回 {symbol: MarketDepth}。"""
|
||||
from app.tickflow.client import get_client
|
||||
tf = get_client()
|
||||
|
||||
capset = self._get_capset()
|
||||
lim = capset.limits(__import__("app.tickflow.capabilities", fromlist=["Cap"]).Cap.DEPTH5_BATCH)
|
||||
batch_size = (lim.batch if lim and lim.batch else 100)
|
||||
rpm = (lim.rpm if lim and lim.rpm else 30)
|
||||
# 批间隔 = 60/rpm(匀速)
|
||||
inter_batch = 60.0 / rpm if rpm > 0 else 2.0
|
||||
|
||||
result: dict = {}
|
||||
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
|
||||
for i, chunk in enumerate(chunks):
|
||||
if i > 0:
|
||||
time.sleep(inter_batch)
|
||||
try:
|
||||
# SDK 的 batch 内部已按 batch_size 切, 这里再切一层防单请求过大
|
||||
data = tf.depth.batch(chunk)
|
||||
if isinstance(data, dict):
|
||||
result.update(data)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth.batch 第 %d 批失败(%d 只): %s", i + 1, len(chunk), e)
|
||||
# 单批失败不影响其他批
|
||||
return result
|
||||
|
||||
def finalize(self) -> None:
|
||||
"""盘后定版: 拉一次 + 落盘。"""
|
||||
if not self._has_capability():
|
||||
return
|
||||
self._fetch_and_seal(persist=True)
|
||||
|
||||
# ================================================================
|
||||
# 落盘
|
||||
# ================================================================
|
||||
|
||||
def _persist(self, today: date) -> None:
|
||||
"""把内存缓存写 depth5/date=今天/part.parquet。"""
|
||||
with self._lock:
|
||||
cache = dict(self._sealed_cache)
|
||||
if not cache:
|
||||
return
|
||||
|
||||
rows = []
|
||||
for sym, e in cache.items():
|
||||
rows.append({
|
||||
"symbol": sym,
|
||||
"sealed_up": e.get("sealed_up"),
|
||||
"sealed_down": e.get("sealed_down"),
|
||||
"ask1_vol": e.get("ask1_vol"),
|
||||
"bid1_vol": e.get("bid1_vol"),
|
||||
"status": e.get("status"),
|
||||
"fetched_at": e.get("fetched_ts"),
|
||||
})
|
||||
df = pl.DataFrame(rows)
|
||||
ds = today.isoformat()
|
||||
out = self._repo.store.data_dir / "depth5" / f"date={ds}" / "part.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.write_parquet(out)
|
||||
self._persisted_date = today
|
||||
logger.info("depth sealed 落盘: %d 行 → %s", df.height, out)
|
||||
|
||||
def _persisted_for_date(self, d: date) -> bool:
|
||||
"""检查某日 depth5 文件是否已存在。"""
|
||||
if not self._repo:
|
||||
return False
|
||||
out = self._repo.store.data_dir / "depth5" / f"date={d.isoformat()}" / "part.parquet"
|
||||
return out.exists()
|
||||
|
||||
# ================================================================
|
||||
# 查询(供 limit_ladder API 用)
|
||||
# ================================================================
|
||||
|
||||
def get_sealed_map(self, target_date: date, is_down: bool) -> dict:
|
||||
"""返回 {symbol: {sealed, vol, ready, age}} 供 JOIN。
|
||||
|
||||
优先内存缓存(盘中), 回退 parquet(历史/盘后)。
|
||||
sealed: bool | None (None=待确认或降级)
|
||||
vol: 封单量(int) | None
|
||||
ready: sealed 数据是否就绪(False→降级标识)
|
||||
age: 距上次拉取秒数(盘后定版为 None)
|
||||
"""
|
||||
# 内存缓存(sealed 数据对应的交易日 = target_date 时才用)
|
||||
if self._sealed_date and target_date == self._sealed_date and self._sealed_ready and self._sealed_cache:
|
||||
return self._read_from_memory(is_down)
|
||||
# parquet(历史或盘后定版)
|
||||
return self._read_from_parquet(target_date, is_down)
|
||||
|
||||
def _read_from_memory(self, is_down: bool) -> dict:
|
||||
sealed_key = "sealed_down" if is_down else "sealed_up"
|
||||
# 封单量: 涨停=买一量(涨停价买单堆积), 跌停=卖一量(跌停价卖单堆积)
|
||||
vol_key = "ask1_vol" if is_down else "bid1_vol"
|
||||
now = time.perf_counter()
|
||||
with self._lock:
|
||||
cache = dict(self._sealed_cache)
|
||||
fetched_ts = self._sealed_fetched_ts
|
||||
age = (now - fetched_ts) if fetched_ts else 0.0
|
||||
result = {}
|
||||
for sym, e in cache.items():
|
||||
result[sym] = {
|
||||
"sealed": e.get(sealed_key),
|
||||
"vol": e.get(vol_key),
|
||||
"ready": True,
|
||||
"age": age,
|
||||
}
|
||||
return result
|
||||
|
||||
def _read_from_parquet(self, target_date: date, is_down: bool) -> dict:
|
||||
if not self._repo:
|
||||
return {}
|
||||
out = self._repo.store.data_dir / "depth5" / f"date={target_date.isoformat()}" / "part.parquet"
|
||||
if not out.exists():
|
||||
return {}
|
||||
try:
|
||||
df = pl.read_parquet(out)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth5 parquet 读取失败: %s", e)
|
||||
return {}
|
||||
sealed_key = "sealed_down" if is_down else "sealed_up"
|
||||
# 封单量: 涨停=买一量, 跌停=卖一量
|
||||
vol_key = "ask1_vol" if is_down else "bid1_vol"
|
||||
result = {}
|
||||
for row in df.to_dicts():
|
||||
sym = row.get("symbol")
|
||||
if not sym:
|
||||
continue
|
||||
result[sym] = {
|
||||
"sealed": row.get(sealed_key),
|
||||
"vol": row.get(vol_key),
|
||||
"ready": True,
|
||||
"age": None, # 盘后定版, 无 age
|
||||
}
|
||||
return result
|
||||
|
||||
def is_sealed_ready(self, target_date: date) -> bool:
|
||||
"""sealed 数据是否就绪(供前端降级判定)。"""
|
||||
# 内存缓存对应的数据日 == 查询日 → 看内存就绪状态
|
||||
if self._sealed_date and target_date == self._sealed_date:
|
||||
return self._sealed_ready
|
||||
# 其他日期: 有 parquet 就 ready
|
||||
return self._persisted_for_date(target_date)
|
||||
|
||||
def get_sealed_age(self, target_date: date) -> float | None:
|
||||
"""返回 sealed 数据 age(秒), 盘后定版为 None。"""
|
||||
if self._sealed_date and target_date == self._sealed_date and self._sealed_ready and self._sealed_fetched_ts:
|
||||
return time.perf_counter() - self._sealed_fetched_ts
|
||||
return None
|
||||
|
||||
# ================================================================
|
||||
# 盘中轮询线程
|
||||
# ================================================================
|
||||
|
||||
def _poll_loop(self) -> None:
|
||||
"""盘中轮询: 按 capset 自适应间隔拉 depth, 更新内存缓存。"""
|
||||
while self._running:
|
||||
try:
|
||||
if self._is_trading_hours():
|
||||
self._poll_once()
|
||||
else:
|
||||
logger.debug("depth sealed: 非交易时段, 跳过")
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("depth sealed 轮询异常: %s", e)
|
||||
|
||||
# 等待下一轮(用 _running 检查保证能及时退出)
|
||||
interval = self._current_sleep_interval()
|
||||
waited = 0.0
|
||||
while self._running and waited < interval:
|
||||
time.sleep(0.5)
|
||||
waited += 0.5
|
||||
|
||||
def _poll_once(self) -> None:
|
||||
"""单次轮询: 算间隔(三层防护) → 拉取 → 检测系统接管通知。"""
|
||||
# 数当前涨跌停股
|
||||
n = self._count_limit_stocks()
|
||||
if n == 0:
|
||||
return
|
||||
|
||||
interval, taken_over, user_interval = self._compute_interval(n)
|
||||
|
||||
# 系统接管通知(状态切换时才推, 防刷屏)
|
||||
if taken_over and (self._last_taken_over is False or self._last_user_interval != user_interval):
|
||||
self._notify_takeover(n, user_interval, interval)
|
||||
self._last_taken_over = taken_over
|
||||
self._last_user_interval = user_interval
|
||||
|
||||
self._fetch_and_seal(persist=False)
|
||||
|
||||
def _current_sleep_interval(self) -> float:
|
||||
"""计算当前 sleep 间隔(供 _poll_loop 等待用)。"""
|
||||
n = self._count_limit_stocks()
|
||||
if n == 0:
|
||||
return 30.0 # 无涨跌停, 慢轮询
|
||||
interval, _, _ = self._compute_interval(n)
|
||||
return interval
|
||||
|
||||
# ================================================================
|
||||
# 三层防护节流
|
||||
# ================================================================
|
||||
|
||||
def _compute_interval(self, n_symbols: int) -> tuple[float, bool, float]:
|
||||
"""三层防护计算实际轮询间隔。
|
||||
|
||||
返回 (actual_interval, taken_over, user_interval)
|
||||
- actual_interval: 实际使用的间隔(秒)
|
||||
- taken_over: 是否被系统接管(用户设置会超限)
|
||||
- user_interval: 用户设置(经套餐 clamp 后)的间隔
|
||||
"""
|
||||
from app.services import preferences
|
||||
from app.tickflow.policy import tier_label
|
||||
|
||||
capset = self._get_capset()
|
||||
lim = capset.limits(__import__("app.tickflow.capabilities", fromlist=["Cap"]).Cap.DEPTH5_BATCH)
|
||||
batch_size = (lim.batch if lim and lim.batch else 100)
|
||||
rpm = (lim.rpm if lim and lim.rpm else 30)
|
||||
|
||||
# ① 套餐范围 clamp
|
||||
tier = tier_label().split()[0].split("+")[0].strip().lower()
|
||||
lo, hi = TIER_INTERVAL_RANGE.get(tier, DEFAULT_RANGE)
|
||||
raw_user = preferences.get_depth_polling_interval()
|
||||
user_interval = max(lo, min(hi, raw_user))
|
||||
|
||||
# ② 限速安全 clamp
|
||||
batches = max(1, math.ceil(n_symbols / batch_size))
|
||||
usable_rpm = rpm * RPM_MARGIN
|
||||
calls_per_min = usable_rpm / batches if batches > 0 else usable_rpm
|
||||
safe_interval = 60.0 / calls_per_min if calls_per_min > 0 else INTERVAL_HARD_MAX
|
||||
|
||||
# 实际: 取用户设置和安全的较大值
|
||||
actual = max(user_interval, safe_interval)
|
||||
# 硬上下限
|
||||
actual = max(INTERVAL_HARD_MIN, min(actual, INTERVAL_HARD_MAX))
|
||||
taken_over = safe_interval > user_interval
|
||||
|
||||
return actual, taken_over, user_interval
|
||||
|
||||
def _count_limit_stocks(self) -> int:
|
||||
"""数当前涨跌停股总数(供节流计算)。"""
|
||||
if not self._repo:
|
||||
return 0
|
||||
enriched, _ = self._repo.get_enriched_latest()
|
||||
if enriched.is_empty():
|
||||
return 0
|
||||
n = 0
|
||||
if "signal_limit_up" in enriched.columns:
|
||||
n += enriched.filter(pl.col("signal_limit_up").fill_null(False)).height
|
||||
if "signal_limit_down" in enriched.columns:
|
||||
n += enriched.filter(pl.col("signal_limit_down").fill_null(False)).height
|
||||
return n
|
||||
|
||||
# ================================================================
|
||||
# 通知
|
||||
# ================================================================
|
||||
|
||||
def _notify_takeover(self, n_stocks: int, user_interval: float, actual_interval: float) -> None:
|
||||
"""系统接管通知: 复用 quote_service 的 _pending_alerts 通道。"""
|
||||
if not self._app_state:
|
||||
return
|
||||
qs = getattr(self._app_state, "quote_service", None)
|
||||
if not qs:
|
||||
return
|
||||
msg = (f"五档轮询: 当前涨跌停 {n_stocks} 只, 您设置的 {user_interval:.0f} 秒间隔会超限, "
|
||||
f"系统已自动调整为 {actual_interval:.0f} 秒")
|
||||
alert = {
|
||||
"source": "depth",
|
||||
"type": "takeover",
|
||||
"message": msg,
|
||||
}
|
||||
try:
|
||||
with qs._lock:
|
||||
qs._pending_alerts.append(alert)
|
||||
qs._alert_event.set()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("depth 接管通知推送失败: %s", e)
|
||||
|
||||
def _notify_depth_updated(self, count: int) -> None:
|
||||
"""修正完成通知: set quote_service._depth_update_event, SSE 推 depth_updated 刷新连板梯队。"""
|
||||
if not self._app_state:
|
||||
return
|
||||
qs = getattr(self._app_state, "quote_service", None)
|
||||
if not qs:
|
||||
return
|
||||
try:
|
||||
qs.notify_depth_updated()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("depth 更新通知推送失败: %s", e)
|
||||
|
||||
# ================================================================
|
||||
# 工具
|
||||
# ================================================================
|
||||
|
||||
def _has_capability(self) -> bool:
|
||||
capset = self._get_capset()
|
||||
from app.tickflow.capabilities import Cap
|
||||
return capset.has(Cap.DEPTH5_BATCH)
|
||||
|
||||
def _get_capset(self):
|
||||
"""获取当前 capset(优先 app.state, 回退 detect)。"""
|
||||
if self._app_state:
|
||||
cs = getattr(self._app_state, "capabilities", None)
|
||||
if cs:
|
||||
return cs
|
||||
from app.tickflow.policy import detect_capabilities
|
||||
return detect_capabilities()
|
||||
|
||||
@staticmethod
|
||||
def _is_trading_hours() -> bool:
|
||||
now = datetime.now()
|
||||
t = now.time()
|
||||
morning = dt_time(9, 25) <= t <= dt_time(11, 35)
|
||||
afternoon = dt_time(12, 55) <= t <= dt_time(15, 5)
|
||||
return now.weekday() < 5 and (morning or afternoon)
|
||||
@@ -0,0 +1,514 @@
|
||||
"""扩展数据服务 — 配置管理 + 文件解析 + Parquet 存储。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import polars as pl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 配置模型
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ExtField:
|
||||
"""扩展字段定义。"""
|
||||
__slots__ = ("name", "dtype", "label")
|
||||
|
||||
def __init__(self, name: str, dtype: str = "string", label: str = "") -> None:
|
||||
self.name = name
|
||||
self.dtype = dtype # string | int | float | bool
|
||||
self.label = label or name
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {"name": self.name, "dtype": self.dtype, "label": self.label}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict) -> ExtField:
|
||||
return cls(d["name"], d.get("dtype", "string"), d.get("label", ""))
|
||||
|
||||
|
||||
class PullConfig:
|
||||
"""定时拉取配置。"""
|
||||
__slots__ = (
|
||||
"url", "method", "headers", "body", "response_path",
|
||||
"field_map", "schedule_minutes", "enabled",
|
||||
"last_run", "last_status", "last_message", "last_rows",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str = "",
|
||||
method: str = "GET",
|
||||
headers: dict[str, str] | None = None,
|
||||
body: str | None = None,
|
||||
response_path: str = "",
|
||||
field_map: dict[str, str] | None = None,
|
||||
schedule_minutes: int = 1440,
|
||||
enabled: bool = False,
|
||||
last_run: str | None = None,
|
||||
last_status: str | None = None,
|
||||
last_message: str | None = None,
|
||||
last_rows: int | None = None,
|
||||
) -> None:
|
||||
self.url = url
|
||||
self.method = method # GET | POST
|
||||
self.headers = headers or {}
|
||||
self.body = body # JSON string (POST body template)
|
||||
self.response_path = response_path # dot-path to rows array, e.g. "data.list"
|
||||
self.field_map = field_map or {} # external_name → config_field_name
|
||||
self.schedule_minutes = schedule_minutes
|
||||
self.enabled = enabled
|
||||
self.last_run = last_run
|
||||
self.last_status = last_status # "success" | "error"
|
||||
self.last_message = last_message
|
||||
self.last_rows = last_rows
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"url": self.url,
|
||||
"method": self.method,
|
||||
"headers": self.headers,
|
||||
"body": self.body,
|
||||
"response_path": self.response_path,
|
||||
"field_map": self.field_map,
|
||||
"schedule_minutes": self.schedule_minutes,
|
||||
"enabled": self.enabled,
|
||||
"last_run": self.last_run,
|
||||
"last_status": self.last_status,
|
||||
"last_message": self.last_message,
|
||||
"last_rows": self.last_rows,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict) -> PullConfig:
|
||||
if not d:
|
||||
return cls()
|
||||
return cls(
|
||||
url=d.get("url", ""),
|
||||
method=d.get("method", "GET"),
|
||||
headers=d.get("headers"),
|
||||
body=d.get("body"),
|
||||
response_path=d.get("response_path", ""),
|
||||
field_map=d.get("field_map"),
|
||||
schedule_minutes=d.get("schedule_minutes", 1440),
|
||||
enabled=d.get("enabled", False),
|
||||
last_run=d.get("last_run"),
|
||||
last_status=d.get("last_status"),
|
||||
last_message=d.get("last_message"),
|
||||
last_rows=d.get("last_rows"),
|
||||
)
|
||||
|
||||
|
||||
class ExtConfig:
|
||||
"""一个扩展数据源的完整配置。"""
|
||||
__slots__ = (
|
||||
"id", "label", "mode", "fields", "description",
|
||||
"symbol_map", "code_map",
|
||||
"created_at", "updated_at", "pull",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
id: str,
|
||||
label: str,
|
||||
mode: Literal["snapshot", "timeseries"],
|
||||
fields: list[ExtField],
|
||||
description: str = "",
|
||||
symbol_map: dict | None = None,
|
||||
code_map: dict | None = None,
|
||||
created_at: str | None = None,
|
||||
updated_at: str | None = None,
|
||||
pull: PullConfig | None = None,
|
||||
) -> None:
|
||||
self.id = id
|
||||
self.label = label
|
||||
self.mode = mode
|
||||
self.fields = fields
|
||||
self.description = description
|
||||
# 映射关系: {"type": "mapped", "col": "原始列名"} 或 {"type": "computed", "from": "symbol|code", "method": "strip_exchange|append_exchange"}
|
||||
self.symbol_map = symbol_map or {}
|
||||
self.code_map = code_map or {}
|
||||
self.created_at = created_at or datetime.now().isoformat()
|
||||
self.updated_at = updated_at or datetime.now().isoformat()
|
||||
self.pull = pull
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
d = {
|
||||
"id": self.id,
|
||||
"label": self.label,
|
||||
"mode": self.mode,
|
||||
"fields": [f.to_dict() for f in self.fields],
|
||||
"description": self.description,
|
||||
"symbol_map": self.symbol_map,
|
||||
"code_map": self.code_map,
|
||||
"created_at": self.created_at,
|
||||
"updated_at": self.updated_at,
|
||||
}
|
||||
if self.pull:
|
||||
d["pull"] = self.pull.to_dict()
|
||||
return d
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: dict) -> ExtConfig:
|
||||
return cls(
|
||||
id=d["id"],
|
||||
label=d["label"],
|
||||
mode=d["mode"],
|
||||
fields=[ExtField.from_dict(f) for f in d.get("fields", [])],
|
||||
description=d.get("description", ""),
|
||||
symbol_map=d.get("symbol_map"),
|
||||
code_map=d.get("code_map"),
|
||||
created_at=d.get("created_at"),
|
||||
updated_at=d.get("updated_at"),
|
||||
pull=PullConfig.from_dict(d["pull"]) if d.get("pull") else None,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 配置持久化
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class ExtConfigStore:
|
||||
"""扩展数据配置文件读写 — 每个表独立目录 data/ext/{config_id}/config.json。"""
|
||||
|
||||
def __init__(self, data_dir: Path) -> None:
|
||||
self._base = data_dir / "ext_data"
|
||||
|
||||
def _config_path(self, config_id: str) -> Path:
|
||||
return self._base / config_id / "config.json"
|
||||
|
||||
def load_all(self) -> list[ExtConfig]:
|
||||
# 兼容旧版: 如果目录为空且旧配置文件存在则迁移
|
||||
if not self._base.exists() or not any(self._base.iterdir()):
|
||||
old = self._base.parent / "ext_configs.json"
|
||||
if not old.exists():
|
||||
old = self._base.parent / "ext_configs.json.bak"
|
||||
if old.exists():
|
||||
self._migrate_legacy(old)
|
||||
if not self._base.exists():
|
||||
return []
|
||||
configs = []
|
||||
for d in sorted(self._base.iterdir()):
|
||||
cp = d / "config.json"
|
||||
if d.is_dir() and cp.exists():
|
||||
try:
|
||||
raw = json.loads(cp.read_text(encoding="utf-8"))
|
||||
configs.append(ExtConfig.from_dict(raw))
|
||||
except Exception as e:
|
||||
logger.warning("扩展表配置解析失败 %s: %s", cp, e)
|
||||
return configs
|
||||
|
||||
def get(self, config_id: str) -> ExtConfig | None:
|
||||
cp = self._config_path(config_id)
|
||||
if not cp.exists():
|
||||
return None
|
||||
try:
|
||||
raw = json.loads(cp.read_text(encoding="utf-8"))
|
||||
return ExtConfig.from_dict(raw)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def upsert(self, config: ExtConfig) -> None:
|
||||
config.updated_at = datetime.now().isoformat()
|
||||
cp = self._config_path(config.id)
|
||||
cp.parent.mkdir(parents=True, exist_ok=True)
|
||||
cp.write_text(
|
||||
json.dumps(config.to_dict(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def delete(self, config_id: str) -> bool:
|
||||
import shutil
|
||||
cp = self._config_path(config_id)
|
||||
if not cp.exists():
|
||||
return False
|
||||
shutil.rmtree(cp.parent, ignore_errors=True)
|
||||
return True
|
||||
|
||||
def _migrate_legacy(self, old_path: Path) -> None:
|
||||
"""一次性迁移旧版 ext_configs.json 到独立目录结构。"""
|
||||
try:
|
||||
raw = json.loads(old_path.read_text(encoding="utf-8"))
|
||||
configs = [ExtConfig.from_dict(d) for d in raw]
|
||||
for c in configs:
|
||||
cp = self._config_path(c.id)
|
||||
cp.parent.mkdir(parents=True, exist_ok=True)
|
||||
cp.write_text(
|
||||
json.dumps(c.to_dict(), ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
# 迁移完成后重命名旧文件作为备份
|
||||
backup = old_path.with_suffix(".json.bak")
|
||||
old_path.rename(backup)
|
||||
logger.info("ext_configs.json 已迁移至 ext/ (备份: %s)", backup.name)
|
||||
except Exception as e:
|
||||
logger.warning("ext_configs 迁移失败: %s", e)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CSV / Excel 解析 → Parquet 写入
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_POLARS_DTYPE_MAP = {
|
||||
"string": pl.Utf8,
|
||||
"int": pl.Int64,
|
||||
"float": pl.Float64,
|
||||
"bool": pl.Boolean,
|
||||
}
|
||||
|
||||
|
||||
def build_code_lookup(data_dir: Path) -> dict[str, str]:
|
||||
"""从 instruments 维表构建 code → symbol 映射。"""
|
||||
path = data_dir / "instruments" / "instruments.parquet"
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
df = pl.read_parquet(path, columns=["code", "symbol"])
|
||||
return dict(zip(df["code"].to_list(), df["symbol"].to_list()))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def normalize_symbol(series: pl.Series, lookup: dict[str, str] | None = None) -> pl.Series:
|
||||
"""将 symbol 列标准化为 代码.交易所 格式。
|
||||
|
||||
优先使用 instruments 维表查找 code → symbol,确保 100% 准确。
|
||||
查不到时按规则兜底:6开头 → .SH,其余 → .SZ。
|
||||
"""
|
||||
_lookup = lookup or {}
|
||||
|
||||
def _fix_one(val: str) -> str:
|
||||
if not val:
|
||||
return val
|
||||
val = val.strip()
|
||||
# 已经是标准格式(含 .),直接返回
|
||||
if "." in val:
|
||||
return val
|
||||
# 纯6位数字代码 → 优先查维表
|
||||
if len(val) == 6 and val.isdigit():
|
||||
mapped = _lookup.get(val)
|
||||
if mapped:
|
||||
return mapped
|
||||
# 兜底规则
|
||||
if val.startswith(("6",)):
|
||||
return f"{val}.SH"
|
||||
else:
|
||||
return f"{val}.SZ"
|
||||
return val
|
||||
|
||||
return series.map_elements(_fix_one, return_dtype=pl.Utf8)
|
||||
|
||||
|
||||
def ensure_utf8_csv(file_path: Path) -> Path:
|
||||
"""确保 CSV 文件以 UTF-8 编码可读,非 UTF-8(如 GBK/GB18030)则转换。
|
||||
|
||||
国内行情软件(同花顺/东财/通达信)和 Windows 中文 Excel 导出的 CSV 多为
|
||||
GBK 系编码,Polars 的 read_csv 默认按 UTF-8 解析会抛 "invalid utf-8 sequence"。
|
||||
这里在交给 Polars 前做一次编码规范化。
|
||||
|
||||
返回值:若已是 UTF-8 则返回原路径;否则在同目录写一个 *.utf8 文件并返回它
|
||||
(调用方用临时目录,随目录一起清理)。
|
||||
"""
|
||||
raw = file_path.read_bytes()
|
||||
# BOM 处理:UTF-8-SIG 等带 BOM 文件直接交给 Polars(它认识 BOM)
|
||||
try:
|
||||
raw.decode("utf-8")
|
||||
return file_path # 已是合法 UTF-8
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
# 依次尝试常见中文编码,第一个能完整解码的即为命中
|
||||
for enc in ("gb18030", "gbk", "gb2312", "big5"):
|
||||
try:
|
||||
text = raw.decode(enc)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
out_path = file_path.with_suffix(file_path.suffix + ".utf8")
|
||||
out_path.write_text(text, encoding="utf-8")
|
||||
logger.info("CSV 编码转换 %s → %s (%s)", file_path.name, out_path.name, enc)
|
||||
return out_path
|
||||
# 都无法解码:返回原路径,让 Polars 抛出更精确的原始错误
|
||||
return file_path
|
||||
|
||||
|
||||
def parse_upload_file(file_path: Path, symbol_col: str = "symbol", data_dir: Path | None = None) -> pl.DataFrame:
|
||||
"""解析上传的 CSV / Excel 文件为 Polars DataFrame。"""
|
||||
suffix = file_path.suffix.lower()
|
||||
if suffix == ".csv":
|
||||
df = pl.read_csv(ensure_utf8_csv(file_path), infer_schema_length=10000)
|
||||
elif suffix in (".xlsx", ".xls"):
|
||||
df = pl.read_excel(file_path)
|
||||
else:
|
||||
raise ValueError(f"不支持的文件格式: {suffix}")
|
||||
|
||||
if symbol_col not in df.columns:
|
||||
# 尝试模糊匹配
|
||||
candidates = [c for c in df.columns if c.lower() in ("symbol", "code", "代码", "标的")]
|
||||
if candidates:
|
||||
df = df.rename({candidates[0]: symbol_col})
|
||||
else:
|
||||
raise ValueError(f"未找到标的代码列 (symbol),可选列: {df.columns}")
|
||||
|
||||
# 确保 symbol 列为字符串并标准化
|
||||
lookup = build_code_lookup(data_dir) if data_dir else None
|
||||
df = df.with_columns(normalize_symbol(df[symbol_col].cast(pl.Utf8), lookup))
|
||||
return df
|
||||
|
||||
|
||||
def cast_df_to_schema(df: pl.DataFrame, fields: list[ExtField]) -> pl.DataFrame:
|
||||
"""按配置的字段类型转换 DataFrame 列类型。"""
|
||||
for f in fields:
|
||||
if f.name in df.columns:
|
||||
target = _POLARS_DTYPE_MAP.get(f.dtype, pl.Utf8)
|
||||
df = df.with_columns(pl.col(f.name).cast(target))
|
||||
return df
|
||||
|
||||
|
||||
def _config_dir(config_id: str, data_dir: Path) -> Path:
|
||||
"""返回扩展配置的根目录 data/ext_data/{config_id}/。"""
|
||||
return data_dir / "ext_data" / config_id
|
||||
|
||||
|
||||
def write_ext_parquet(
|
||||
df: pl.DataFrame,
|
||||
config: ExtConfig,
|
||||
data_dir: Path,
|
||||
snapshot_date: date | None = None,
|
||||
) -> int:
|
||||
"""将 DataFrame 写入扩展数据 Parquet。
|
||||
|
||||
目录结构:
|
||||
- snapshot: data/ext_data/{id}/part.parquet(与 config.json 同级,覆盖写)
|
||||
- timeseries: data/ext_data/{id}/timeseries/date=xxx/part.parquet(按日分区)
|
||||
|
||||
Returns:
|
||||
写入行数。
|
||||
"""
|
||||
snap = snapshot_date or date.today()
|
||||
cfg_dir = _config_dir(config.id, data_dir)
|
||||
|
||||
# 标准化 symbol 列: 用维表查找 → 准确匹配交易所
|
||||
if "symbol" in df.columns:
|
||||
lookup = build_code_lookup(data_dir)
|
||||
df = df.with_columns(normalize_symbol(df["symbol"], lookup))
|
||||
|
||||
if config.mode == "snapshot":
|
||||
# 快照: 与 config.json 同级,直接覆盖
|
||||
cfg_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_path = cfg_dir / "part.parquet"
|
||||
|
||||
# 如果已有文件,合并去重后覆盖
|
||||
if out_path.exists():
|
||||
try:
|
||||
existing = pl.read_parquet(out_path)
|
||||
key = "symbol" if "symbol" in df.columns else df.columns[0]
|
||||
df = pl.concat([existing, df]).unique(subset=[key], keep="last")
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
# 时序: timeseries/ 下按日期分区
|
||||
out_dir = cfg_dir / "timeseries" / f"date={snap}"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_path = out_dir / "part.parquet"
|
||||
|
||||
# 如果已有文件,合并去重
|
||||
if out_path.exists():
|
||||
try:
|
||||
existing = pl.read_parquet(out_path)
|
||||
key = "symbol" if "symbol" in df.columns else df.columns[0]
|
||||
df = pl.concat([existing, df]).unique(subset=[key], keep="last")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
df = cast_df_to_schema(df, config.fields)
|
||||
df.write_parquet(out_path)
|
||||
logger.info("扩展表写入: %s → %s (%d 行)", config.id, out_path, len(df))
|
||||
return len(df)
|
||||
|
||||
|
||||
def delete_ext_parquet(config_id: str, data_dir: Path) -> None:
|
||||
"""删除扩展数据源关联的所有 Parquet 数据(保留 config.json)。
|
||||
|
||||
- snapshot: 删除 ext_data/{id}/part.parquet
|
||||
- timeseries: 删除 ext_data/{id}/timeseries/ 目录
|
||||
"""
|
||||
cfg_dir = _config_dir(config_id, data_dir)
|
||||
# 删除快照文件
|
||||
snap = cfg_dir / "part.parquet"
|
||||
if snap.exists():
|
||||
snap.unlink()
|
||||
# 删除时序目录
|
||||
ts_dir = cfg_dir / "timeseries"
|
||||
if ts_dir.exists():
|
||||
import shutil
|
||||
shutil.rmtree(ts_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def fix_symbol_format(config: ExtConfig, data_dir: Path) -> int:
|
||||
"""扫描该扩展配置的所有 Parquet 文件,将 symbol 列标准化为 代码.交易所 格式。
|
||||
|
||||
- snapshot: 扫描 ext_data/{id}/part.parquet
|
||||
- timeseries: 扫描 ext_data/{id}/timeseries/date=xxx/part.parquet
|
||||
|
||||
Returns:
|
||||
修复的文件数。
|
||||
"""
|
||||
cfg_dir = _config_dir(config.id, data_dir)
|
||||
if not cfg_dir.exists():
|
||||
return 0
|
||||
|
||||
# 收集需要扫描的 parquet 文件列表
|
||||
parquet_files: list[Path] = []
|
||||
if config.mode == "snapshot":
|
||||
p = cfg_dir / "part.parquet"
|
||||
if p.exists():
|
||||
parquet_files.append(p)
|
||||
else:
|
||||
ts_dir = cfg_dir / "timeseries"
|
||||
if ts_dir.exists():
|
||||
for part_dir in sorted(ts_dir.iterdir()):
|
||||
if not part_dir.is_dir() or not part_dir.name.startswith("date="):
|
||||
continue
|
||||
p = part_dir / "part.parquet"
|
||||
if p.exists():
|
||||
parquet_files.append(p)
|
||||
|
||||
fixed = 0
|
||||
lookup = build_code_lookup(data_dir)
|
||||
for parquet_path in parquet_files:
|
||||
try:
|
||||
df = pl.read_parquet(parquet_path)
|
||||
if "symbol" not in df.columns:
|
||||
continue
|
||||
old = df["symbol"].to_list()
|
||||
df = df.with_columns(normalize_symbol(df["symbol"], lookup))
|
||||
new = df["symbol"].to_list()
|
||||
if old != new:
|
||||
df.write_parquet(parquet_path)
|
||||
fixed += 1
|
||||
logger.info("代码格式修复: %s/%s (%d 行)", config.id, parquet_path.parent.name, len(df))
|
||||
except Exception as e:
|
||||
logger.warning("代码格式修复跳过 %s: %s", parquet_path, e)
|
||||
|
||||
return fixed
|
||||
|
||||
|
||||
def rows_to_parquet(
|
||||
rows: list[dict],
|
||||
config: ExtConfig,
|
||||
data_dir: Path,
|
||||
snapshot_date: date | None = None,
|
||||
) -> int:
|
||||
"""将 JSON 行列表转为 DataFrame 写入 Parquet,复用 write_ext_parquet 的存储逻辑。
|
||||
|
||||
Returns:
|
||||
写入行数。
|
||||
"""
|
||||
df = pl.DataFrame(rows)
|
||||
if "symbol" in df.columns:
|
||||
df = df.with_columns(pl.col("symbol").cast(pl.Utf8))
|
||||
return write_ext_parquet(df, config, data_dir, snapshot_date=snapshot_date)
|
||||
@@ -0,0 +1,216 @@
|
||||
"""扩展数据定时拉取引擎 — 从外部 API 拉取数据写入 Parquet。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from datetime import date, datetime, timezone
|
||||
from functools import reduce
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.services.ext_data import (
|
||||
ExtConfig,
|
||||
ExtConfigStore,
|
||||
PullConfig,
|
||||
rows_to_parquet,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 响应解析
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _extract_rows(data: Any, path: str) -> list[dict]:
|
||||
"""按 dot-path 从 JSON 响应中提取行数组。
|
||||
|
||||
例: path="data.list" → response["data"]["list"]
|
||||
如果 path 为空,直接将 data 视为数组。
|
||||
"""
|
||||
if not path:
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
raise ValueError("response_path 为空但响应不是数组")
|
||||
|
||||
keys = path.split(".")
|
||||
current = data
|
||||
for key in keys:
|
||||
if isinstance(current, dict):
|
||||
if key not in current:
|
||||
raise ValueError(f"响应中不存在路径 '{path}',缺失键 '{key}'")
|
||||
current = current[key]
|
||||
elif isinstance(current, list):
|
||||
try:
|
||||
current = current[int(key)]
|
||||
except (ValueError, IndexError) as e:
|
||||
raise ValueError(f"响应路径 '{path}' 解析失败: {e}") from e
|
||||
else:
|
||||
raise ValueError(f"响应路径 '{path}' 中间值不是 dict/list: {type(current)}")
|
||||
|
||||
if not isinstance(current, list):
|
||||
raise ValueError(f"路径 '{path}' 指向的不是数组,而是 {type(current)}")
|
||||
|
||||
return current
|
||||
|
||||
|
||||
def _apply_field_map(rows: list[dict], field_map: dict[str, str]) -> list[dict]:
|
||||
"""将外部字段名映射为内部配置字段名。field_map: {外部名: 内部名}。"""
|
||||
if not field_map:
|
||||
return rows
|
||||
mapped = []
|
||||
for row in rows:
|
||||
new_row: dict = {}
|
||||
for k, v in row.items():
|
||||
mapped_key = field_map.get(k, k)
|
||||
new_row[mapped_key] = v
|
||||
mapped.append(new_row)
|
||||
return mapped
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 拉取执行
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def fetch_and_ingest(
|
||||
config: ExtConfig,
|
||||
data_dir,
|
||||
) -> tuple[int, str]:
|
||||
"""执行一次拉取: 请求外部 API → 解析响应 → 写入 Parquet。
|
||||
|
||||
Returns:
|
||||
(rows_written, date_str)
|
||||
"""
|
||||
pull = config.pull
|
||||
if not pull or not pull.url:
|
||||
raise ValueError("拉取未配置或 URL 为空")
|
||||
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
headers = pull.headers or {}
|
||||
kwargs: dict[str, Any] = {"headers": headers}
|
||||
|
||||
if pull.method.upper() == "POST" and pull.body:
|
||||
kwargs["content"] = pull.body
|
||||
if "content-type" not in {k.lower() for k in headers}:
|
||||
kwargs["headers"]["Content-Type"] = "application/json"
|
||||
|
||||
resp = await client.request(pull.method.upper(), pull.url, **kwargs)
|
||||
resp.raise_for_status()
|
||||
|
||||
# 解析 JSON
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
raise ValueError(f"响应不是有效 JSON: {e}") from e
|
||||
|
||||
# 提取行
|
||||
rows = _extract_rows(data, pull.response_path)
|
||||
if not rows:
|
||||
raise ValueError("提取到的行数为 0")
|
||||
|
||||
# 字段映射
|
||||
rows = _apply_field_map(rows, pull.field_map)
|
||||
|
||||
# 校验 symbol 列
|
||||
if rows and "symbol" not in rows[0]:
|
||||
raise ValueError("数据行中缺少 symbol 字段,请配置 field_map 映射")
|
||||
|
||||
# 写入
|
||||
snap = date.today()
|
||||
n = rows_to_parquet(rows, config, data_dir, snapshot_date=snap)
|
||||
return n, snap.isoformat()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 调度器
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class PullScheduler:
|
||||
"""后台调度器:为每个启用了 pull 的 ExtConfig 维护定时任务。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._tasks: dict[str, asyncio.Task] = {}
|
||||
self._running = False
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def start(self, data_dir) -> None:
|
||||
"""启动调度(在 lifespan startup 调用)。"""
|
||||
self._running = True
|
||||
self._data_dir = data_dir
|
||||
logger.info("PullScheduler started")
|
||||
|
||||
def stop(self) -> None:
|
||||
"""停止所有任务。"""
|
||||
self._running = False
|
||||
for task in self._tasks.values():
|
||||
task.cancel()
|
||||
self._tasks.clear()
|
||||
logger.info("PullScheduler stopped")
|
||||
|
||||
def refresh(self, data_dir) -> None:
|
||||
"""重新加载配置,更新调度任务(增/删/改)。"""
|
||||
self._data_dir = data_dir
|
||||
store = ExtConfigStore(data_dir)
|
||||
configs = store.load_all()
|
||||
|
||||
active_ids: set[str] = set()
|
||||
|
||||
for config in configs:
|
||||
if not config.pull or not config.pull.enabled or not config.pull.url:
|
||||
continue
|
||||
active_ids.add(config.id)
|
||||
if config.id not in self._tasks:
|
||||
# 新增调度
|
||||
task = asyncio.create_task(self._run_loop(config))
|
||||
self._tasks[config.id] = task
|
||||
logger.info("PullScheduler: scheduled %s (every %d min)", config.id, config.pull.schedule_minutes)
|
||||
|
||||
# 移除不再活跃的
|
||||
for cid in list(self._tasks):
|
||||
if cid not in active_ids:
|
||||
self._tasks[cid].cancel()
|
||||
del self._tasks[cid]
|
||||
logger.info("PullScheduler: removed %s", cid)
|
||||
|
||||
async def _run_loop(self, config: ExtConfig) -> None:
|
||||
"""单个配置的定时拉取循环。"""
|
||||
try:
|
||||
while self._running:
|
||||
pull = config.pull
|
||||
if not pull:
|
||||
break
|
||||
interval = max(pull.schedule_minutes * 60, 60) # 至少 60s
|
||||
await asyncio.sleep(interval)
|
||||
if not self._running:
|
||||
break
|
||||
try:
|
||||
# 重新加载最新配置(用户可能中途修改)
|
||||
store = ExtConfigStore(self._data_dir)
|
||||
fresh = store.get(config.id)
|
||||
if not fresh or not fresh.pull or not fresh.pull.enabled:
|
||||
break
|
||||
n, d = await fetch_and_ingest(fresh, self._data_dir)
|
||||
fresh.pull.last_run = datetime.now(timezone.utc).isoformat()
|
||||
fresh.pull.last_status = "success"
|
||||
fresh.pull.last_message = f"{n} rows @ {d}"
|
||||
fresh.pull.last_rows = n
|
||||
store.upsert(fresh)
|
||||
logger.info("PullScheduler: %s success, %d rows", config.id, n)
|
||||
except Exception as e:
|
||||
store = ExtConfigStore(self._data_dir)
|
||||
fresh = store.get(config.id)
|
||||
if fresh and fresh.pull:
|
||||
fresh.pull.last_run = datetime.now(timezone.utc).isoformat()
|
||||
fresh.pull.last_status = "error"
|
||||
fresh.pull.last_message = str(e)[:200]
|
||||
store.upsert(fresh)
|
||||
logger.warning("PullScheduler: %s error: %s", config.id, e)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
|
||||
# 全局单例
|
||||
pull_scheduler = PullScheduler()
|
||||
@@ -0,0 +1,224 @@
|
||||
"""向前扩展历史数据 — 完全独立于 daily_pipeline 的盘后管道。
|
||||
|
||||
用户从日 K 卡片手动触发,指定往前补的时长 (x 天/月/年)。
|
||||
流程:
|
||||
1. 获取当前最早日期
|
||||
2. 向前拉日 K batch (start = 最早日期 - offset, end = 最早日期)
|
||||
3. 向前拉除权因子 (同范围)
|
||||
4. 全量重算 enriched
|
||||
5. 刷新视图 + 缓存
|
||||
|
||||
⚠️ 本模块不导入 daily_pipeline 的任何函数,只复用基础设施:
|
||||
- kline_sync.sync_and_persist_daily_batch / sync_adj_factor
|
||||
- indicators.pipeline.run_pipeline
|
||||
- pipeline_jobs.JobStore
|
||||
- tickflow.repository.KlineRepository
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from datetime import date, datetime, timedelta
|
||||
|
||||
from app.services import kline_sync
|
||||
from app.services.pipeline_jobs import job_store
|
||||
from app.tickflow.capabilities import Cap, CapabilitySet
|
||||
from app.tickflow.repository import KlineRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _noop(stage: str, pct: int, msg: str, **kwargs) -> None: # noqa: ARG001
|
||||
pass
|
||||
|
||||
|
||||
def _invalidate(table: str | None = None) -> None:
|
||||
from app.api.data import invalidate_data_cache
|
||||
invalidate_data_cache(table)
|
||||
|
||||
|
||||
def _resolve_universe(capset: CapabilitySet) -> list[str]:
|
||||
"""解析标的池 — 与 daily_pipeline 独立的副本。"""
|
||||
if capset.has(Cap.KLINE_DAILY_BATCH):
|
||||
try:
|
||||
from app.tickflow.pools import get_pool
|
||||
all_a = get_pool("CN_Equity_A", refresh=True)
|
||||
if all_a:
|
||||
return sorted(all_a)
|
||||
except Exception as e:
|
||||
logger.warning("CN_Equity_A pool unavailable: %s", e)
|
||||
|
||||
from app.tickflow.pools import DEMO_SYMBOLS, get_pool as _get_pool
|
||||
from app.config import settings
|
||||
from pathlib import Path
|
||||
import polars as pl
|
||||
base: set[str] = set(DEMO_SYMBOLS)
|
||||
base.update(_get_pool("watchlist"))
|
||||
d = Path(settings.data_dir)
|
||||
inst_path = d / "instruments" / "instruments.parquet"
|
||||
if inst_path.exists():
|
||||
try:
|
||||
inst = pl.read_parquet(inst_path, columns=["symbol"])
|
||||
base.update(inst["symbol"].to_list())
|
||||
except Exception as e:
|
||||
logger.warning("instruments supplement failed: %s", e)
|
||||
return sorted(base)
|
||||
|
||||
|
||||
def _refresh_single_view(repo: KlineRepository, name: str) -> None:
|
||||
"""刷新单个 DuckDB 视图。"""
|
||||
d = repo.store.data_dir.as_posix()
|
||||
paths = {
|
||||
"kline_daily": f"{d}/kline_daily/**/*.parquet",
|
||||
"kline_enriched": f"{d}/kline_daily_enriched/**/*.parquet",
|
||||
"kline_minute": f"{d}/kline_minute/**/*.parquet",
|
||||
"adj_factor": f"{d}/adj_factor/**/*.parquet",
|
||||
"instruments": f"{d}/instruments/**/*.parquet",
|
||||
}
|
||||
path = paths.get(name)
|
||||
if not path:
|
||||
return
|
||||
try:
|
||||
repo.db.execute(
|
||||
f"CREATE OR REPLACE VIEW {name} AS "
|
||||
f"SELECT * FROM read_parquet('{path}', union_by_name=true)"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("refresh view %s failed: %s", name, e)
|
||||
|
||||
|
||||
def compute_offset(value: int, unit: str) -> timedelta:
|
||||
"""将用户输入的 value + unit 转成 timedelta。"""
|
||||
if unit == "day":
|
||||
return timedelta(days=value)
|
||||
elif unit == "month":
|
||||
return timedelta(days=value * 30)
|
||||
elif unit == "year":
|
||||
return timedelta(days=value * 365)
|
||||
else:
|
||||
raise ValueError(f"不支持的单位: {unit}")
|
||||
|
||||
|
||||
def run_extend_history(
|
||||
repo: KlineRepository,
|
||||
capset: CapabilitySet,
|
||||
value: int,
|
||||
unit: str,
|
||||
on_progress: Callable | None = None,
|
||||
) -> dict:
|
||||
"""向前扩展历史数据的主函数。
|
||||
|
||||
完全独立于 daily_pipeline.run_now(),不调用其任何逻辑。
|
||||
返回结果 dict 供 job_store 记录。
|
||||
"""
|
||||
emit = on_progress or _noop
|
||||
|
||||
# 0. 计算时间偏移
|
||||
offset = compute_offset(value, unit)
|
||||
today = date.today()
|
||||
|
||||
# 1. 获取当前最早日期
|
||||
emit("extend_history", 2, "检查当前数据范围…")
|
||||
earliest = repo.earliest_daily_date()
|
||||
|
||||
if not earliest:
|
||||
return {"error": "本地无日K数据,请先执行一次完整同步"}
|
||||
|
||||
new_start = earliest - offset
|
||||
# 不能超过今天
|
||||
if new_start >= earliest:
|
||||
return {"error": "扩展范围无效,请增大时间跨度"}
|
||||
|
||||
# 2. 解析标的池
|
||||
emit("extend_history", 5, "解析标的池…")
|
||||
universe = _resolve_universe(capset)
|
||||
if not universe:
|
||||
return {"error": "标的池为空"}
|
||||
emit("extend_history", 8, f"标的池: {len(universe)} 只")
|
||||
|
||||
start_str = new_start.strftime("%Y-%m-%d")
|
||||
end_str = earliest.strftime("%Y-%m-%d")
|
||||
|
||||
# 3. 拉日 K
|
||||
emit("extend_history", 10, f"获取日K [{start_str} ~ {end_str}]…")
|
||||
logger.info("extend_history: daily K [%s ~ %s], %d symbols", start_str, end_str, len(universe))
|
||||
|
||||
def _daily_chunk(cur: int, tot: int) -> None:
|
||||
emit("extend_history", 10 + int(35 * cur / tot),
|
||||
f"日K 批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
|
||||
|
||||
written_daily = kline_sync.sync_and_persist_daily_batch(
|
||||
universe, repo, capset,
|
||||
start_date=datetime.combine(new_start, datetime.min.time()),
|
||||
end_date=datetime.combine(earliest, datetime.min.time()),
|
||||
on_chunk_done=_daily_chunk,
|
||||
)
|
||||
emit("extend_history", 45, f"日K 完成,写入 {written_daily} 行")
|
||||
logger.info("extend_history: daily K done, %d rows", written_daily)
|
||||
_refresh_single_view(repo, "kline_daily")
|
||||
_invalidate("daily")
|
||||
|
||||
# 4. 拉除权因子 (新范围)
|
||||
written_adj = 0
|
||||
adj_start = datetime.combine(new_start, datetime.min.time())
|
||||
adj_end = datetime.combine(today, datetime.min.time())
|
||||
adj_start_str = new_start.strftime("%Y-%m-%d")
|
||||
adj_end_str = today.strftime("%Y-%m-%d")
|
||||
|
||||
if capset.has(Cap.ADJ_FACTOR):
|
||||
emit("extend_history", 48, f"获取除权因子 [{adj_start_str} ~ {adj_end_str}]…")
|
||||
logger.info("extend_history: adj_factor [%s ~ %s]", adj_start_str, adj_end_str)
|
||||
|
||||
def _adj_chunk(cur: int, tot: int) -> None:
|
||||
emit("extend_history", 48 + int(10 * cur / tot),
|
||||
f"除权因子批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
|
||||
|
||||
written_adj, _affected = kline_sync.sync_adj_factor(
|
||||
universe, repo, capset,
|
||||
start_time=adj_start, end_time=adj_end,
|
||||
on_chunk_done=_adj_chunk,
|
||||
)
|
||||
emit("extend_history", 60, f"除权因子完成,{written_adj} 行")
|
||||
logger.info("extend_history: adj_factor done, %d rows", written_adj)
|
||||
_refresh_single_view(repo, "adj_factor")
|
||||
_invalidate("adj_factor")
|
||||
else:
|
||||
emit("extend_history", 60, "除权因子跳过(无权限)")
|
||||
logger.info("extend_history: adj_factor skipped, no ADJ_FACTOR capability")
|
||||
|
||||
# 5. 全量重算 enriched
|
||||
emit("extend_history", 65, "全量计算 enriched…")
|
||||
logger.info("extend_history: full enriched rebuild start")
|
||||
|
||||
from app.indicators.pipeline import run_pipeline
|
||||
written_enriched = run_pipeline()
|
||||
|
||||
enriched_dir = repo.store.data_dir / "kline_daily_enriched"
|
||||
enriched_days = len(list(enriched_dir.glob("date=*"))) if enriched_dir.exists() else 0
|
||||
emit("extend_history", 92, f"enriched 完成,覆盖 {enriched_days} 天")
|
||||
logger.info("extend_history: enriched done, %d days", enriched_days)
|
||||
_refresh_single_view(repo, "kline_enriched")
|
||||
_invalidate("enriched")
|
||||
|
||||
# 6. 刷新视图
|
||||
emit("extend_history", 95, "刷新视图…")
|
||||
_refresh_single_view(repo, "kline_daily")
|
||||
_refresh_single_view(repo, "kline_enriched")
|
||||
_refresh_single_view(repo, "adj_factor")
|
||||
_invalidate(None)
|
||||
|
||||
# 7. 统计结果
|
||||
daily_dir = repo.store.data_dir / "kline_daily"
|
||||
daily_days = len(list(daily_dir.glob("date=*"))) if daily_dir.exists() else 0
|
||||
|
||||
emit("extend_history", 100, f"完成,已扩展至 {new_start}")
|
||||
|
||||
return {
|
||||
"earliest_before": earliest.isoformat(),
|
||||
"earliest_after": new_start.isoformat(),
|
||||
"daily_rows": written_daily,
|
||||
"daily_days": daily_days,
|
||||
"adj_factor_rows": written_adj,
|
||||
"enriched_days": enriched_days,
|
||||
"universe_size": len(universe),
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
"""财务数据独立同步服务。
|
||||
|
||||
解耦于 K-line 管道, 自有调度 + 自有存储。
|
||||
能力门控: Cap.FINANCIAL (Expert 套餐)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.tickflow.capabilities import Cap, CapabilitySet
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 每个 API 请求最多 100 个标的
|
||||
_BATCH_SIZE = 100
|
||||
|
||||
# 4 张财务表
|
||||
FINANCIAL_TABLES = ("metrics", "income", "balance_sheet", "cash_flow")
|
||||
|
||||
|
||||
# ================================================================
|
||||
# 同步函数
|
||||
# ================================================================
|
||||
|
||||
def _get_symbols(data_dir: Path) -> list[str]:
|
||||
"""从 instruments 表获取标的列表。"""
|
||||
inst_path = data_dir / "instruments" / "instruments.parquet"
|
||||
if not inst_path.exists():
|
||||
return []
|
||||
try:
|
||||
df = pl.read_parquet(inst_path, columns=["symbol"])
|
||||
return df["symbol"].to_list()
|
||||
except Exception as e:
|
||||
logger.warning("读取 instruments 失败: %s", e)
|
||||
return []
|
||||
|
||||
|
||||
def _sync_table(
|
||||
table: str,
|
||||
symbols: list[str],
|
||||
data_dir: Path,
|
||||
capset: CapabilitySet,
|
||||
latest_only: bool = True,
|
||||
) -> int:
|
||||
"""同步单张财务表。返回写入的行数。"""
|
||||
if not capset.has(Cap.FINANCIAL):
|
||||
logger.info("sync_%s skipped: no FINANCIAL capability", table)
|
||||
return 0
|
||||
if not symbols:
|
||||
logger.warning("sync_%s skipped: no symbols", table)
|
||||
return 0
|
||||
|
||||
from app.tickflow.client import get_client
|
||||
tf = get_client()
|
||||
|
||||
# 分批拉取
|
||||
api_method = {
|
||||
"metrics": tf.financials.metrics,
|
||||
"income": tf.financials.income,
|
||||
"balance_sheet": tf.financials.balance_sheet,
|
||||
"cash_flow": tf.financials.cash_flow,
|
||||
}[table]
|
||||
|
||||
all_records: list[dict] = []
|
||||
total_batches = (len(symbols) + _BATCH_SIZE - 1) // _BATCH_SIZE
|
||||
|
||||
for i in range(0, len(symbols), _BATCH_SIZE):
|
||||
chunk = symbols[i : i + _BATCH_SIZE]
|
||||
batch_num = i // _BATCH_SIZE + 1
|
||||
try:
|
||||
data = api_method(chunk, latest=latest_only)
|
||||
# data 格式: { "600519.SH": [record, ...], ... }
|
||||
if isinstance(data, dict):
|
||||
for sym, records in data.items():
|
||||
if isinstance(records, list):
|
||||
for rec in records:
|
||||
if isinstance(rec, dict):
|
||||
rec["symbol"] = sym
|
||||
all_records.append(rec)
|
||||
logger.debug("sync_%s batch %d/%d: %d records", table, batch_num, total_batches, len(data) if isinstance(data, dict) else 0)
|
||||
except Exception as e:
|
||||
logger.warning("sync_%s batch %d/%d failed: %s", table, batch_num, total_batches, e)
|
||||
|
||||
if not all_records:
|
||||
return 0
|
||||
|
||||
df = pl.DataFrame(all_records)
|
||||
if df.is_empty():
|
||||
return 0
|
||||
|
||||
# 确保 symbol 列存在
|
||||
if "symbol" not in df.columns:
|
||||
return 0
|
||||
|
||||
# 写入 Parquet (全量覆盖)
|
||||
out_dir = data_dir / "financials" / table
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_file = out_dir / "part.parquet"
|
||||
df.write_parquet(out_file)
|
||||
|
||||
logger.info("sync_%s done: %d records written", table, len(df))
|
||||
return len(df)
|
||||
|
||||
|
||||
def sync_metrics(data_dir: Path, capset: CapabilitySet) -> int:
|
||||
"""同步核心财务指标 (metrics)。"""
|
||||
symbols = _get_symbols(data_dir)
|
||||
return _sync_table("metrics", symbols, data_dir, capset, latest_only=True)
|
||||
|
||||
|
||||
def sync_income(data_dir: Path, capset: CapabilitySet) -> int:
|
||||
"""同步利润表。"""
|
||||
symbols = _get_symbols(data_dir)
|
||||
return _sync_table("income", symbols, data_dir, capset, latest_only=True)
|
||||
|
||||
|
||||
def sync_balance_sheet(data_dir: Path, capset: CapabilitySet) -> int:
|
||||
"""同步资产负债表。"""
|
||||
symbols = _get_symbols(data_dir)
|
||||
return _sync_table("balance_sheet", symbols, data_dir, capset, latest_only=True)
|
||||
|
||||
|
||||
def sync_cash_flow(data_dir: Path, capset: CapabilitySet) -> int:
|
||||
"""同步现金流量表。"""
|
||||
symbols = _get_symbols(data_dir)
|
||||
return _sync_table("cash_flow", symbols, data_dir, capset, latest_only=True)
|
||||
|
||||
|
||||
def sync_all(data_dir: Path, capset: CapabilitySet) -> dict[str, int]:
|
||||
"""同步所有财务表。返回 {table: rows}。"""
|
||||
if not capset.has(Cap.FINANCIAL):
|
||||
logger.info("sync_all financials skipped: no FINANCIAL capability")
|
||||
return {}
|
||||
|
||||
symbols = _get_symbols(data_dir)
|
||||
results: dict[str, int] = {}
|
||||
for table in FINANCIAL_TABLES:
|
||||
results[table] = _sync_table(table, symbols, data_dir, capset, latest_only=True)
|
||||
|
||||
# 同步完成后注册 DuckDB 视图
|
||||
_refresh_financials_views(data_dir)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# ================================================================
|
||||
# DuckDB 视图
|
||||
# ================================================================
|
||||
|
||||
def _refresh_financials_views(data_dir: Path) -> None:
|
||||
"""刷新财务表 DuckDB 视图 (在 DataStore.db 上注册)。"""
|
||||
d = data_dir.as_posix()
|
||||
views = {
|
||||
"financials_metrics": f"{d}/financials/metrics/*.parquet",
|
||||
"financials_income": f"{d}/financials/income/*.parquet",
|
||||
"financials_balance_sheet": f"{d}/financials/balance_sheet/*.parquet",
|
||||
"financials_cash_flow": f"{d}/financials/cash_flow/*.parquet",
|
||||
}
|
||||
for name, path in views.items():
|
||||
out = data_dir / "financials" / name.replace("financials_", "") / "part.parquet"
|
||||
if not out.exists():
|
||||
continue
|
||||
# 视图注册需要由 DataStore 完成,这里只做日志
|
||||
logger.debug("financial parquet ready: %s (%d rows)", name, out.stat().st_size)
|
||||
|
||||
|
||||
def get_financial_df(data_dir: Path, table: str) -> pl.DataFrame:
|
||||
"""读取本地财务 Parquet。"""
|
||||
path = data_dir / "financials" / table / "part.parquet"
|
||||
if not path.exists():
|
||||
return pl.DataFrame()
|
||||
try:
|
||||
return pl.read_parquet(path)
|
||||
except Exception as e:
|
||||
logger.warning("读取 financials/%s 失败: %s", table, e)
|
||||
return pl.DataFrame()
|
||||
|
||||
|
||||
# ================================================================
|
||||
# 调度器
|
||||
# ================================================================
|
||||
|
||||
class FinancialScheduler:
|
||||
"""独立调度器: 每周同步 metrics, 每季度同步三张报表。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._task: asyncio.Task | None = None
|
||||
self._running = False
|
||||
self._data_dir: Path | None = None
|
||||
self._capset: CapabilitySet | None = None
|
||||
self._lock = threading.Lock()
|
||||
self._last_sync: dict[str, str] = {} # {table: iso_timestamp}
|
||||
# 手动同步(run_now)是否正在进行。前端据此显示"同步中"并防重复点击。
|
||||
self._is_syncing = False
|
||||
|
||||
def start(self, data_dir: Path, capset: CapabilitySet) -> None:
|
||||
if not capset.has(Cap.FINANCIAL):
|
||||
logger.info("FinancialScheduler skipped: no FINANCIAL capability")
|
||||
return
|
||||
self._data_dir = data_dir
|
||||
self._capset = capset
|
||||
self._running = True
|
||||
self._task = asyncio.create_task(self._run_loop())
|
||||
logger.info("FinancialScheduler started")
|
||||
|
||||
def stop(self) -> None:
|
||||
self._running = False
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
self._task = None
|
||||
logger.info("FinancialScheduler stopped")
|
||||
|
||||
def update(self, data_dir: Path, capset: CapabilitySet) -> None:
|
||||
"""运行时更新数据目录和能力集。
|
||||
|
||||
用户在设置页更换/清除 Key 后,能力集可能变化,无需重启服务即可让
|
||||
财务调度器生效或失效。
|
||||
"""
|
||||
had_financial = self._capset is not None and self._capset.has(Cap.FINANCIAL)
|
||||
has_financial = capset.has(Cap.FINANCIAL)
|
||||
self._data_dir = data_dir
|
||||
self._capset = capset
|
||||
|
||||
if has_financial and not self._running:
|
||||
self.start(data_dir, capset)
|
||||
elif had_financial and not has_financial and self._running:
|
||||
self.stop()
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
"""每周执行一次 metrics 同步。"""
|
||||
try:
|
||||
while self._running:
|
||||
# 首次启动等 60s, 之后每 7 天执行一次
|
||||
await asyncio.sleep(60)
|
||||
if not self._running:
|
||||
break
|
||||
|
||||
# 每周: 只同步 metrics
|
||||
try:
|
||||
rows = sync_metrics(self._data_dir, self._capset)
|
||||
self._last_sync["metrics"] = datetime.now(timezone.utc).isoformat()
|
||||
logger.info("FinancialScheduler: metrics synced, %d rows", rows)
|
||||
except Exception as e:
|
||||
logger.warning("FinancialScheduler: metrics sync failed: %s", e)
|
||||
|
||||
# 等待下一次 (7天)
|
||||
for _ in range(7 * 24 * 60): # 每分钟检查一次 _running
|
||||
if not self._running:
|
||||
break
|
||||
await asyncio.sleep(60)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
def run_now(self, table: str | None = None) -> dict[str, int]:
|
||||
"""手动触发同步。table=None 同步全部。
|
||||
|
||||
用 _is_syncing 标志防并发:若已有同步在进行,本次直接跳过,
|
||||
避免重复请求拖慢服务端 / 触发上游限流。
|
||||
"""
|
||||
if not self._capset or not self._capset.has(Cap.FINANCIAL):
|
||||
return {}
|
||||
with self._lock:
|
||||
if self._is_syncing:
|
||||
logger.info("financial sync skipped: already running")
|
||||
return {"_skipped": 1}
|
||||
self._is_syncing = True
|
||||
try:
|
||||
if table:
|
||||
fn = {
|
||||
"metrics": sync_metrics,
|
||||
"income": sync_income,
|
||||
"balance_sheet": sync_balance_sheet,
|
||||
"cash_flow": sync_cash_flow,
|
||||
}.get(table)
|
||||
if not fn:
|
||||
return {}
|
||||
rows = fn(self._data_dir, self._capset)
|
||||
self._last_sync[table] = datetime.now(timezone.utc).isoformat()
|
||||
return {table: rows}
|
||||
else:
|
||||
# 全部同步: 逐表执行, 每张完成立即更新 last_sync,
|
||||
# 让前端轮询 /status 能看到进度递增 (而非等全部完成才一次性更新)。
|
||||
symbols = _get_symbols(self._data_dir)
|
||||
result: dict[str, int] = {}
|
||||
for t in FINANCIAL_TABLES:
|
||||
result[t] = _sync_table(t, symbols, self._data_dir, self._capset, latest_only=True)
|
||||
self._last_sync[t] = datetime.now(timezone.utc).isoformat()
|
||||
_refresh_financials_views(self._data_dir)
|
||||
return result
|
||||
finally:
|
||||
with self._lock:
|
||||
self._is_syncing = False
|
||||
|
||||
@property
|
||||
def is_syncing(self) -> bool:
|
||||
"""手动同步是否正在进行(供 /status 返回,前端据此显示"同步中")。"""
|
||||
with self._lock:
|
||||
return self._is_syncing
|
||||
|
||||
@property
|
||||
def last_sync(self) -> dict[str, str]:
|
||||
return dict(self._last_sync)
|
||||
|
||||
|
||||
# 全局单例
|
||||
financial_scheduler = FinancialScheduler()
|
||||
@@ -0,0 +1,146 @@
|
||||
"""指数数据同步服务。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import gc
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.indicators.pipeline import compute_enriched
|
||||
from app.services import kline_sync, preferences
|
||||
from app.tickflow.capabilities import Cap, CapabilitySet
|
||||
from app.tickflow.client import get_client
|
||||
from app.tickflow.repository import KlineRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _quotes_to_index_instruments(resp) -> pl.DataFrame:
|
||||
"""将数据源 quotes 响应规范为指数 instruments。"""
|
||||
if resp is None:
|
||||
return pl.DataFrame()
|
||||
|
||||
if isinstance(resp, pl.DataFrame):
|
||||
df = resp
|
||||
elif hasattr(resp, "columns"):
|
||||
df = pl.from_pandas(resp.reset_index() if hasattr(resp, "reset_index") else resp)
|
||||
else:
|
||||
rows: list[dict] = []
|
||||
for q in resp or []:
|
||||
item = q if isinstance(q, dict) else {}
|
||||
ext = item.get("ext") or {}
|
||||
symbol = item.get("symbol")
|
||||
if not symbol:
|
||||
continue
|
||||
rows.append({
|
||||
"symbol": str(symbol),
|
||||
"name": ext.get("name") or item.get("name") or str(symbol),
|
||||
})
|
||||
df = pl.DataFrame(rows)
|
||||
|
||||
if df.is_empty() or "symbol" not in df.columns:
|
||||
return pl.DataFrame()
|
||||
|
||||
rename = {"ts_code": "symbol"}
|
||||
df = df.rename({k: v for k, v in rename.items() if k in df.columns})
|
||||
|
||||
if "name" not in df.columns:
|
||||
if "ext" in df.columns:
|
||||
df = df.with_columns(pl.col("symbol").cast(pl.Utf8).alias("name"))
|
||||
else:
|
||||
df = df.with_columns(pl.col("symbol").cast(pl.Utf8).alias("name"))
|
||||
|
||||
result = df.select([
|
||||
pl.col("symbol").cast(pl.Utf8),
|
||||
pl.col("name").cast(pl.Utf8),
|
||||
]).with_columns([
|
||||
pl.col("symbol").str.split(".").list.first().alias("code"),
|
||||
pl.lit("index").alias("asset_type"),
|
||||
])
|
||||
return result.unique(subset=["symbol"], keep="last").sort("symbol")
|
||||
|
||||
|
||||
def sync_index_instruments(repo: KlineRepository) -> int:
|
||||
"""同步 CN_Index 指数标的维表,返回指数数量。"""
|
||||
tf = get_client()
|
||||
resp = None
|
||||
errors: list[str] = []
|
||||
for kwargs in (
|
||||
{"universes": ["CN_Index"]},
|
||||
{"universes": ["CN_Index"], "as_dataframe": False},
|
||||
):
|
||||
try:
|
||||
resp = tf.quotes.get_by_universes(**kwargs)
|
||||
if resp is not None and len(resp) > 0:
|
||||
break
|
||||
except Exception as e: # noqa: BLE001
|
||||
errors.append(str(e))
|
||||
resp = None
|
||||
|
||||
if resp is None or len(resp) == 0:
|
||||
logger.warning("CN_Index universe returned empty: %s", "; ".join(errors))
|
||||
return 0
|
||||
|
||||
instruments = _quotes_to_index_instruments(resp)
|
||||
if instruments.is_empty():
|
||||
return 0
|
||||
repo.save_index_instruments(instruments)
|
||||
repo.refresh_index_views()
|
||||
return instruments.height
|
||||
|
||||
|
||||
def sync_and_persist_index_daily(
|
||||
repo: KlineRepository,
|
||||
capset: CapabilitySet,
|
||||
count: int | None = None,
|
||||
start_date: datetime | None = None,
|
||||
end_date: datetime | None = None,
|
||||
) -> int:
|
||||
"""同步指数日K到独立 parquet,并计算指数 enriched。"""
|
||||
if not capset.has(Cap.KLINE_DAILY_BATCH):
|
||||
return 0
|
||||
|
||||
instruments = repo.get_index_instruments()
|
||||
if instruments.is_empty():
|
||||
sync_index_instruments(repo)
|
||||
instruments = repo.get_index_instruments()
|
||||
if instruments.is_empty() or "symbol" not in instruments.columns:
|
||||
return 0
|
||||
|
||||
symbols = sorted(set(instruments["symbol"].to_list()))
|
||||
lim = capset.limits(Cap.KLINE_DAILY_BATCH)
|
||||
batch_size = preferences.get_index_daily_batch_size()
|
||||
if lim and lim.batch:
|
||||
batch_size = min(batch_size, lim.batch)
|
||||
rpm = lim.rpm if lim else None
|
||||
|
||||
end_time = end_date or datetime.now()
|
||||
start_time = start_date or (end_time - timedelta(days=365))
|
||||
|
||||
total_rows = 0
|
||||
interval = (60.0 / rpm) if rpm else 0
|
||||
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
|
||||
for i, chunk in enumerate(chunks):
|
||||
if i > 0 and interval > 0 and len(chunks) > rpm:
|
||||
import time
|
||||
time.sleep(interval)
|
||||
raw = kline_sync.sync_daily_batch(
|
||||
chunk,
|
||||
count=count,
|
||||
batch_size=None,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
if raw.is_empty():
|
||||
continue
|
||||
|
||||
repo.append_index_daily(raw)
|
||||
enriched = compute_enriched(raw, factors=None, instruments=None)
|
||||
repo.append_index_enriched(enriched)
|
||||
total_rows += raw.height
|
||||
logger.info("index daily synced: %d/%d chunks, +%d rows", i + 1, len(chunks), raw.height)
|
||||
del raw, enriched
|
||||
gc.collect()
|
||||
repo.refresh_index_views()
|
||||
return total_rows
|
||||
@@ -0,0 +1,122 @@
|
||||
"""标的维表同步服务。
|
||||
|
||||
盘前 9:10 调用 tf.exchanges.get_instruments("SH"/"SZ"/"BJ", type="stock")
|
||||
获取全量标的元数据,flatten ext 字段,写入 instruments.parquet。
|
||||
|
||||
Starter+ 盘后可用 quotes.get(universes) 顺便补充 name。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.tickflow.client import get_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_EXCHANGES = ["SH", "SZ", "BJ"]
|
||||
|
||||
|
||||
def _flatten_instruments(items: list[dict]) -> list[dict]:
|
||||
"""把 SDK 返回的 Instrument 列表 flatten 成扁平行。"""
|
||||
rows = []
|
||||
for item in items:
|
||||
row = {
|
||||
"symbol": item.get("symbol"),
|
||||
"name": item.get("name"),
|
||||
"code": item.get("code"),
|
||||
"exchange": item.get("exchange"),
|
||||
"region": item.get("region"),
|
||||
"type": item.get("type"),
|
||||
}
|
||||
ext = item.get("ext") or {}
|
||||
row["listing_date"] = ext.get("listing_date")
|
||||
row["total_shares"] = ext.get("total_shares")
|
||||
row["float_shares"] = ext.get("float_shares")
|
||||
row["tick_size"] = ext.get("tick_size")
|
||||
row["limit_up"] = ext.get("limit_up")
|
||||
row["limit_down"] = ext.get("limit_down")
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def sync_instruments(data_dir: Path) -> int:
|
||||
"""全量同步标的维表 → data/instruments/instruments.parquet。
|
||||
|
||||
返回写入的行数。
|
||||
"""
|
||||
tf = get_client()
|
||||
all_rows: list[dict] = []
|
||||
|
||||
for ex in _EXCHANGES:
|
||||
try:
|
||||
items = tf.exchanges.get_instruments(ex, instrument_type="stock")
|
||||
if items:
|
||||
all_rows.extend(_flatten_instruments(items))
|
||||
logger.info("instruments %s: %d stocks", ex, len(items))
|
||||
except Exception as e:
|
||||
logger.warning("get_instruments(%s) failed: %s", ex, e)
|
||||
|
||||
if not all_rows:
|
||||
return 0
|
||||
|
||||
df = pl.DataFrame(all_rows)
|
||||
df = df.with_columns(pl.lit(date.today()).alias("as_of"))
|
||||
|
||||
out = data_dir / "instruments" / "instruments.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
df.write_parquet(out)
|
||||
|
||||
logger.info("instruments synced: %d rows → %s", df.height, out)
|
||||
return df.height
|
||||
|
||||
|
||||
def enrich_names_from_quotes(
|
||||
data_dir: Path,
|
||||
quotes_data: list[dict],
|
||||
) -> int:
|
||||
"""从 quotes 响应中提取 name,更新 instruments 维表(兜底补充)。
|
||||
|
||||
盘后 quotes.get(universes) 返回的数据中包含 ext.name,
|
||||
用来补充 instruments 中可能缺失的 name。
|
||||
"""
|
||||
if not quotes_data:
|
||||
return 0
|
||||
|
||||
# 构建 symbol → name 映射
|
||||
name_map: dict[str, str] = {}
|
||||
for q in quotes_data:
|
||||
symbol = q.get("symbol", "")
|
||||
ext = q.get("ext") or {}
|
||||
name = ext.get("name") or q.get("name", "")
|
||||
if symbol and name:
|
||||
name_map[symbol] = name
|
||||
|
||||
if not name_map:
|
||||
return 0
|
||||
|
||||
inst_path = data_dir / "instruments" / "instruments.parquet"
|
||||
if not inst_path.exists():
|
||||
return 0
|
||||
|
||||
df = pl.read_parquet(inst_path)
|
||||
|
||||
# 只更新空 name 的行
|
||||
updates = pl.DataFrame({
|
||||
"symbol": list(name_map.keys()),
|
||||
"_new_name": list(name_map.values()),
|
||||
})
|
||||
df = df.join(updates, on="symbol", how="left")
|
||||
df = df.with_columns(
|
||||
pl.when(pl.col("name").is_null() | (pl.col("name") == ""))
|
||||
.then(pl.col("_new_name"))
|
||||
.otherwise(pl.col("name"))
|
||||
.alias("name"),
|
||||
).drop("_new_name")
|
||||
|
||||
df.write_parquet(inst_path)
|
||||
logger.info("instruments name enriched from quotes: %d names", len(name_map))
|
||||
return len(name_map)
|
||||
@@ -0,0 +1,617 @@
|
||||
"""日 K 同步服务(§7.7 Step 1)。
|
||||
|
||||
调度器在 capability 允许下,把符号集合的日 K 批量同步到本地 Parquet。
|
||||
策略:
|
||||
- 日 K 仅使用 `kline.daily.batch`
|
||||
- 除权因子仅使用 `adj_factor`
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.indicators.pipeline import filter_halt_days
|
||||
from app.tickflow.capabilities import Cap, CapabilitySet
|
||||
from app.tickflow.client import get_client
|
||||
from app.tickflow.repository import KlineRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# 标准列(无论 SDK 返回什么形状,我们把它规范成这套)
|
||||
CANONICAL_DAILY_COLS = [
|
||||
"symbol", "date", "open", "high", "low", "close", "volume", "amount",
|
||||
]
|
||||
|
||||
|
||||
def _normalize_daily(df_in, default_symbol: str | None = None) -> pl.DataFrame:
|
||||
"""把 SDK 返回的 pandas/任意 DataFrame 规范成 canonical 列。"""
|
||||
if df_in is None or len(df_in) == 0:
|
||||
return pl.DataFrame()
|
||||
|
||||
if not isinstance(df_in, pl.DataFrame):
|
||||
df = pl.from_pandas(df_in.reset_index() if hasattr(df_in, "reset_index") else df_in)
|
||||
else:
|
||||
df = df_in
|
||||
|
||||
# 兼容字段名差异
|
||||
rename_map = {
|
||||
"ts_code": "symbol",
|
||||
"trade_date": "date",
|
||||
"vol": "volume",
|
||||
"amt": "amount",
|
||||
"datetime": "date",
|
||||
}
|
||||
df = df.rename({k: v for k, v in rename_map.items() if k in df.columns})
|
||||
|
||||
if "symbol" not in df.columns and default_symbol is not None:
|
||||
df = df.with_columns(pl.lit(default_symbol).alias("symbol"))
|
||||
|
||||
# 类型规范
|
||||
if "date" in df.columns and df.schema["date"] != pl.Date:
|
||||
df = df.with_columns(pl.col("date").cast(pl.Date, strict=False))
|
||||
|
||||
for col in ("open", "high", "low", "close"):
|
||||
if col in df.columns:
|
||||
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
|
||||
for col in ("volume", "amount"):
|
||||
if col in df.columns:
|
||||
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
|
||||
|
||||
# 过滤停牌日 (open/high 为 0; close 可能被填充为前收盘价, 不能用全零判断)
|
||||
df = filter_halt_days(df)
|
||||
|
||||
# 只保留 canonical 列
|
||||
keep = [c for c in CANONICAL_DAILY_COLS if c in df.columns]
|
||||
return df.select(keep)
|
||||
|
||||
|
||||
def sync_daily_batch(symbols: list[str],
|
||||
count: int | None = None,
|
||||
batch_size: int | None = None,
|
||||
rpm: int | None = None,
|
||||
start_time: datetime | None = None,
|
||||
end_time: datetime | None = None,
|
||||
on_chunk_done: Callable[[int, int], None] | None = None) -> pl.DataFrame:
|
||||
"""批量拉取多股日 K。
|
||||
|
||||
优先使用 start_time / end_time 区间 + count=10000,确保覆盖完整时间段。
|
||||
仅传 count 时按条数回溯。
|
||||
"""
|
||||
tf = get_client()
|
||||
out: list[pl.DataFrame] = []
|
||||
interval = (60.0 / rpm) if rpm else 0
|
||||
|
||||
if batch_size is None:
|
||||
chunks = [symbols]
|
||||
else:
|
||||
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
if i > 0 and interval > 0 and len(chunks) > rpm:
|
||||
time.sleep(interval)
|
||||
try:
|
||||
if start_time and end_time:
|
||||
raw = tf.klines.batch(
|
||||
chunk, period="1d", adjust="none",
|
||||
start_time=_datetime_to_ms(start_time),
|
||||
end_time=_datetime_to_ms(end_time),
|
||||
count=10000,
|
||||
as_dataframe=True, show_progress=False,
|
||||
)
|
||||
else:
|
||||
raw = tf.klines.batch(chunk, period="1d", count=count or 250, adjust="none",
|
||||
as_dataframe=True, show_progress=False)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("batch fetch failed for %d symbols: %s", len(chunk), e)
|
||||
continue
|
||||
|
||||
# 兼容两种形态:dict[sym → df] 和扁平 df
|
||||
if isinstance(raw, dict):
|
||||
for sym, sub in raw.items():
|
||||
if sub is None or len(sub) == 0:
|
||||
continue
|
||||
out.append(_normalize_daily(sub, default_symbol=sym))
|
||||
elif raw is not None and len(raw) > 0:
|
||||
out.append(_normalize_daily(raw))
|
||||
|
||||
if on_chunk_done:
|
||||
on_chunk_done(i + 1, len(chunks))
|
||||
|
||||
if not out:
|
||||
return pl.DataFrame()
|
||||
return pl.concat(out, how="diagonal_relaxed")
|
||||
|
||||
|
||||
def sync_and_persist_daily_batch(
|
||||
symbols: list[str],
|
||||
repo: KlineRepository,
|
||||
capset: CapabilitySet,
|
||||
count: int | None = None,
|
||||
start_date: datetime | None = None,
|
||||
end_date: datetime | None = None,
|
||||
on_chunk_done: Callable[[int, int], None] | None = None,
|
||||
) -> int:
|
||||
"""批量同步日 K 并落到 Parquet。返回写入的行数。
|
||||
|
||||
start_date/end_date: 外部传入的时间范围(由 pipeline 根据已有数据计算)。
|
||||
未传入时默认拉最近 1 年。
|
||||
"""
|
||||
if not symbols or not capset.has(Cap.KLINE_DAILY_BATCH):
|
||||
return 0
|
||||
|
||||
lim = capset.limits(Cap.KLINE_DAILY_BATCH)
|
||||
batch_size = lim.batch if lim and lim.batch else 100
|
||||
rpm = lim.rpm if lim else None
|
||||
|
||||
end_time = end_date or datetime.now()
|
||||
start_time = start_date or (end_time - timedelta(days=365))
|
||||
|
||||
df = sync_daily_batch(
|
||||
symbols, count=count, batch_size=batch_size, rpm=rpm,
|
||||
start_time=start_time, end_time=end_time,
|
||||
on_chunk_done=on_chunk_done,
|
||||
)
|
||||
|
||||
if df.is_empty():
|
||||
return 0
|
||||
|
||||
repo.append_daily(df)
|
||||
|
||||
try:
|
||||
d = repo.store.data_dir.as_posix()
|
||||
repo.db.execute(
|
||||
f"""CREATE OR REPLACE VIEW kline_daily AS
|
||||
SELECT * FROM read_parquet('{d}/kline_daily/**/*.parquet', union_by_name=true)"""
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("refresh view failed: %s", e)
|
||||
|
||||
return df.height
|
||||
|
||||
|
||||
def sync_daily_by_quotes(repo: KlineRepository) -> int:
|
||||
"""用实时行情接口拉全市场当日数据,覆写 kline_daily 今天分区。
|
||||
|
||||
一个请求覆盖 ~5500 只股票,比 batch K-line 快几个数量级。
|
||||
返回写入的行数。
|
||||
"""
|
||||
from datetime import date as _date
|
||||
|
||||
from app.tickflow.client import get_client
|
||||
|
||||
tf = get_client()
|
||||
try:
|
||||
resp = tf.quotes.get_by_universes(universes=["CN_Equity_A"])
|
||||
except Exception as e:
|
||||
logger.warning("get_by_universes failed: %s", e)
|
||||
return 0
|
||||
|
||||
if not resp:
|
||||
logger.warning("get_by_universes returned empty")
|
||||
return 0
|
||||
|
||||
records = []
|
||||
for q in resp:
|
||||
ext = q.get("ext") or {}
|
||||
records.append({
|
||||
"symbol": q.get("symbol"),
|
||||
"open": q.get("open"),
|
||||
"high": q.get("high"),
|
||||
"low": q.get("low"),
|
||||
"close": q.get("last_price"),
|
||||
"volume": q.get("volume"),
|
||||
"amount": q.get("amount"),
|
||||
})
|
||||
|
||||
df = pl.DataFrame(records)
|
||||
if df.is_empty():
|
||||
return 0
|
||||
|
||||
today = _date.today()
|
||||
daily_df = df.with_columns(pl.lit(today).cast(pl.Date).alias("date"))
|
||||
|
||||
# 过滤停牌 (open/high 为 0; close 可能被填充为前收盘价, 不能用全零判断)
|
||||
daily_df = filter_halt_days(daily_df)
|
||||
|
||||
repo.flush_live_daily(daily_df)
|
||||
logger.info("sync_daily_by_quotes: %d symbols flushed for %s", daily_df.height, today)
|
||||
return daily_df.height
|
||||
|
||||
|
||||
def sync_adj_factor(symbols: list[str], repo: KlineRepository,
|
||||
capset: CapabilitySet,
|
||||
start_time: datetime | None = None,
|
||||
end_time: datetime | None = None,
|
||||
on_chunk_done: Callable[[int, int], None] | None = None) -> tuple[int, list[str]]:
|
||||
"""同步除权因子(Starter+)。SDK 接口:`tf.klines.ex_factors(symbols=...)`。
|
||||
|
||||
支持增量: 传 start_time/end_time 只拉取该时间范围内的新除权事件。
|
||||
返回 (写入行数, 受影响的 symbol 列表) — 供 enriched 局部重算使用。
|
||||
"""
|
||||
if not capset.has(Cap.ADJ_FACTOR) or not symbols:
|
||||
return 0, []
|
||||
|
||||
tf = get_client()
|
||||
lim = capset.limits(Cap.ADJ_FACTOR)
|
||||
batch_size = lim.batch if lim and lim.batch else 50
|
||||
rpm = lim.rpm if lim else 30
|
||||
interval = 60.0 / rpm if rpm else 0
|
||||
|
||||
# 构建 SDK 参数
|
||||
sdk_kwargs: dict = {"as_dataframe": True, "batch_size": batch_size, "show_progress": False}
|
||||
if start_time:
|
||||
sdk_kwargs["start_time"] = _datetime_to_ms(start_time)
|
||||
if end_time:
|
||||
sdk_kwargs["end_time"] = _datetime_to_ms(end_time)
|
||||
|
||||
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
|
||||
all_dfs: list[pl.DataFrame] = []
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
if i > 0 and interval > 0 and len(chunks) > rpm:
|
||||
time.sleep(interval)
|
||||
try:
|
||||
raw = tf.klines.ex_factors(chunk, **sdk_kwargs)
|
||||
if raw is not None and len(raw) > 0:
|
||||
all_dfs.append(pl.from_pandas(
|
||||
raw.reset_index() if hasattr(raw, "reset_index") else raw
|
||||
))
|
||||
logger.debug("adj_factor chunk %d/%d: %d symbols", i + 1, len(chunks), len(chunk))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("adj_factor chunk %d failed: %s", i + 1, e)
|
||||
|
||||
if on_chunk_done:
|
||||
on_chunk_done(i + 1, len(chunks))
|
||||
|
||||
if not all_dfs:
|
||||
return 0, []
|
||||
|
||||
new_data = pl.concat(all_dfs, how="diagonal_relaxed") if len(all_dfs) > 1 else all_dfs[0]
|
||||
|
||||
# 提取受影响的 symbol 列表(合并前)
|
||||
affected = new_data["symbol"].unique().to_list()
|
||||
|
||||
out = repo.store.data_dir / "adj_factor" / "all.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if out.exists():
|
||||
existing = pl.read_parquet(out)
|
||||
before = existing.height
|
||||
merged = pl.concat([existing, new_data]).unique(
|
||||
subset=["symbol", "trade_date"], keep="last",
|
||||
).sort(["symbol", "trade_date"])
|
||||
merged.write_parquet(out)
|
||||
added = merged.height - before
|
||||
logger.info("adj_factor merged: %d total (+%d new), %d/%d symbols",
|
||||
merged.height, added, new_data.height, len(symbols))
|
||||
return added, affected
|
||||
else:
|
||||
new_data.sort(["symbol", "trade_date"]).write_parquet(out)
|
||||
logger.info("adj_factor synced: %d rows (%d symbols)", new_data.height, len(symbols))
|
||||
return new_data.height, affected
|
||||
|
||||
|
||||
# ===== 分钟 K 同步 =====
|
||||
|
||||
CANONICAL_MINUTE_COLS = [
|
||||
"symbol", "datetime", "open", "high", "low", "close", "volume", "amount",
|
||||
]
|
||||
|
||||
|
||||
def _normalize_minute(df_in, default_symbol: str | None = None) -> pl.DataFrame:
|
||||
"""把 SDK 返回的分钟 K 数据规范成 canonical 列。"""
|
||||
if df_in is None or len(df_in) == 0:
|
||||
return pl.DataFrame()
|
||||
|
||||
if not isinstance(df_in, pl.DataFrame):
|
||||
df = pl.from_pandas(df_in.reset_index() if hasattr(df_in, "reset_index") else df_in)
|
||||
else:
|
||||
df = df_in
|
||||
|
||||
rename_map = {
|
||||
"ts_code": "symbol",
|
||||
"vol": "volume",
|
||||
"amt": "amount",
|
||||
}
|
||||
df = df.rename({k: v for k, v in rename_map.items() if k in df.columns})
|
||||
|
||||
# datetime 列:优先用 timestamp(毫秒精度),其次 trade_time
|
||||
if "timestamp" in df.columns:
|
||||
df = df.with_columns(
|
||||
pl.from_epoch("timestamp", time_unit="ms").alias("datetime"),
|
||||
).drop("timestamp")
|
||||
for drop_col in ("trade_time", "trade_date"):
|
||||
if drop_col in df.columns:
|
||||
df = df.drop(drop_col)
|
||||
elif "trade_time" in df.columns:
|
||||
df = df.rename({"trade_time": "datetime"})
|
||||
if "trade_date" in df.columns:
|
||||
df = df.drop("trade_date")
|
||||
elif "trade_date" in df.columns:
|
||||
df = df.rename({"trade_date": "datetime"})
|
||||
|
||||
if "symbol" not in df.columns and default_symbol is not None:
|
||||
df = df.with_columns(pl.lit(default_symbol).alias("symbol"))
|
||||
|
||||
# 类型规范:统一转 Datetime('us')
|
||||
if "datetime" in df.columns:
|
||||
dt_type = df.schema["datetime"]
|
||||
if not isinstance(dt_type, pl.Datetime) or dt_type.time_unit != "us":
|
||||
df = df.with_columns(pl.col("datetime").cast(pl.Datetime("us"), strict=False))
|
||||
|
||||
for col in ("open", "high", "low", "close"):
|
||||
if col in df.columns:
|
||||
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
|
||||
for col in ("volume", "amount"):
|
||||
if col in df.columns:
|
||||
df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
|
||||
|
||||
keep = [c for c in CANONICAL_MINUTE_COLS if c in df.columns]
|
||||
return df.select(keep)
|
||||
|
||||
|
||||
def _datetime_to_ms(dt: datetime) -> int:
|
||||
"""datetime → 毫秒时间戳 (供 SDK start_time / end_time 使用)。"""
|
||||
return int(dt.timestamp() * 1000)
|
||||
|
||||
|
||||
def sync_minute_batch(
|
||||
symbols: list[str],
|
||||
start_time: datetime | None = None,
|
||||
end_time: datetime | None = None,
|
||||
count: int | None = None,
|
||||
batch_size: int | None = None,
|
||||
rpm: int | None = None,
|
||||
on_chunk_done: Callable[[int, int], None] | None = None,
|
||||
) -> pl.DataFrame:
|
||||
"""批量拉取多股分钟 K。
|
||||
|
||||
优先使用 start_time / end_time 区间, 确保所有标的覆盖同一时间段。
|
||||
count 仅作为 fallback 保留。
|
||||
on_chunk_done(current, total) 每个 chunk 完成后回调。
|
||||
"""
|
||||
tf = get_client()
|
||||
out: list[pl.DataFrame] = []
|
||||
interval = (60.0 / rpm) if rpm else 0
|
||||
|
||||
if batch_size is None:
|
||||
chunks = [symbols]
|
||||
else:
|
||||
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
|
||||
|
||||
for i, chunk in enumerate(chunks):
|
||||
if i > 0 and interval > 0 and len(chunks) > rpm:
|
||||
time.sleep(interval)
|
||||
try:
|
||||
if start_time and end_time:
|
||||
raw = tf.klines.batch(
|
||||
chunk, period="1m",
|
||||
start_time=_datetime_to_ms(start_time),
|
||||
end_time=_datetime_to_ms(end_time),
|
||||
count=10000,
|
||||
as_dataframe=True, show_progress=False,
|
||||
)
|
||||
else:
|
||||
raw = tf.klines.batch(chunk, period="1m", count=count or 1200,
|
||||
as_dataframe=True, show_progress=False)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("minute batch fetch failed for %d symbols: %s", len(chunk), e)
|
||||
continue
|
||||
|
||||
if isinstance(raw, dict):
|
||||
for sym, sub in raw.items():
|
||||
if sub is None or len(sub) == 0:
|
||||
continue
|
||||
out.append(_normalize_minute(sub, default_symbol=sym))
|
||||
elif raw is not None and len(raw) > 0:
|
||||
out.append(_normalize_minute(raw))
|
||||
|
||||
if on_chunk_done:
|
||||
on_chunk_done(i + 1, len(chunks))
|
||||
|
||||
if not out:
|
||||
return pl.DataFrame()
|
||||
return pl.concat(out, how="diagonal_relaxed")
|
||||
|
||||
|
||||
def fetch_minute_single(symbol: str, trade_date: date) -> pl.DataFrame:
|
||||
"""从数据源实时拉取单股单日分钟 K(不写入本地)。"""
|
||||
from datetime import datetime
|
||||
start_time = datetime(trade_date.year, trade_date.month, trade_date.day, 9, 25, 0)
|
||||
end_time = datetime(trade_date.year, trade_date.month, trade_date.day, 15, 5, 0)
|
||||
tf = get_client()
|
||||
try:
|
||||
raw = tf.klines.batch(
|
||||
[symbol], period="1m",
|
||||
start_time=_datetime_to_ms(start_time),
|
||||
end_time=_datetime_to_ms(end_time),
|
||||
count=10000,
|
||||
as_dataframe=True, show_progress=False,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("fetch_minute_single(%s, %s) failed: %s", symbol, trade_date, e)
|
||||
return pl.DataFrame()
|
||||
|
||||
if isinstance(raw, dict):
|
||||
sub = raw.get(symbol)
|
||||
return _normalize_minute(sub) if sub is not None and len(sub) > 0 else pl.DataFrame()
|
||||
if raw is not None and len(raw) > 0:
|
||||
return _normalize_minute(raw)
|
||||
return pl.DataFrame()
|
||||
|
||||
|
||||
def _latest_minute_datetime(repo: KlineRepository) -> datetime | None:
|
||||
"""本地分钟 K 数据的最新时间。"""
|
||||
try:
|
||||
res = repo.execute_one("SELECT max(datetime) FROM kline_minute")
|
||||
if res and res[0]:
|
||||
d = res[0]
|
||||
if isinstance(d, datetime):
|
||||
return d
|
||||
return datetime.fromisoformat(str(d))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _cleanup_null_datetime_minute(repo: KlineRepository) -> None:
|
||||
"""检测并清除 datetime 全为 null 的旧版分钟 K 数据(迁移用)。"""
|
||||
minute_dir = repo.store.data_dir / "kline_minute"
|
||||
if not minute_dir.exists():
|
||||
return
|
||||
try:
|
||||
row = repo.execute_one(
|
||||
"SELECT count(*) AS total, count(datetime) AS non_null FROM kline_minute"
|
||||
)
|
||||
if row and row[0] > 0 and (row[1] is None or row[1] == 0):
|
||||
# 全部 datetime 为 null — 清除所有分钟 K parquet
|
||||
n = 0
|
||||
for f in minute_dir.rglob("*.parquet"):
|
||||
f.unlink()
|
||||
n += 1
|
||||
logger.info("cleaned %d corrupted minute-K parquet files (null datetime)", n)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("minute cleanup check failed: %s", e)
|
||||
|
||||
|
||||
def _migrate_symbol_to_date_partition(repo: KlineRepository) -> None:
|
||||
"""将旧版 symbol= 分区迁移为 date= 分区。迁移完成后删除旧目录。"""
|
||||
minute_dir = repo.store.data_dir / "kline_minute"
|
||||
if not minute_dir.exists():
|
||||
return
|
||||
|
||||
old_dirs = [d for d in minute_dir.iterdir() if d.is_dir() and d.name.startswith("symbol=")]
|
||||
if not old_dirs:
|
||||
return
|
||||
|
||||
logger.info("migrating %d symbol-partitioned minute-K dirs to date partition…", len(old_dirs))
|
||||
|
||||
all_frames: list[pl.DataFrame] = []
|
||||
for sym_dir in old_dirs:
|
||||
for pq in sym_dir.glob("*.parquet"):
|
||||
try:
|
||||
df = pl.read_parquet(pq)
|
||||
if "datetime" in df.columns:
|
||||
df = df.filter(pl.col("datetime").is_not_null())
|
||||
if not df.is_empty():
|
||||
all_frames.append(df)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
if not all_frames:
|
||||
# 数据全部不可用,直接删旧目录
|
||||
for d in old_dirs:
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
for f in d.rglob("*"):
|
||||
if f.is_file():
|
||||
f.unlink()
|
||||
d.rmdir()
|
||||
return
|
||||
|
||||
combined = pl.concat(all_frames, how="diagonal_relaxed")
|
||||
combined = combined.unique(subset=["symbol", "datetime"], keep="last")
|
||||
|
||||
# 按日期写新分区
|
||||
combined = combined.with_columns(pl.col("datetime").dt.date().alias("_trade_date"))
|
||||
for day_df in combined.partition_by("_trade_date"):
|
||||
trade_date = day_df["_trade_date"][0]
|
||||
out = minute_dir / f"date={trade_date}" / "part.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
day_df = day_df.drop("_trade_date").sort("symbol", "datetime")
|
||||
day_df.write_parquet(out)
|
||||
|
||||
# 删旧目录
|
||||
for d in old_dirs:
|
||||
for f in d.rglob("*"):
|
||||
if f.is_file():
|
||||
f.unlink()
|
||||
# 移除空目录
|
||||
try:
|
||||
d.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
logger.info("minute-K migration done: %d rows migrated", combined.height)
|
||||
|
||||
|
||||
def sync_and_persist_minute(
|
||||
symbols: list[str],
|
||||
repo: KlineRepository,
|
||||
capset: CapabilitySet,
|
||||
days: int = 5,
|
||||
on_chunk_done: Callable[[int, int], None] | None = None,
|
||||
) -> int:
|
||||
"""同步分钟 K 并存到 Parquet(仅 raw,不前复权)。返回写入行数。
|
||||
|
||||
使用 start_time / end_time 区间拉取, 确保所有标的覆盖同一时间段。
|
||||
on_chunk_done(current, total) 每个 chunk 完成后回调。
|
||||
"""
|
||||
if not symbols or not capset.has(Cap.KLINE_MINUTE_BATCH):
|
||||
return 0
|
||||
|
||||
# 迁移:旧版 _normalize_minute 未转换 timestamp→datetime,导致全部 datetime 为 null
|
||||
# 检测到后直接清除(这些数据无法使用)
|
||||
_cleanup_null_datetime_minute(repo)
|
||||
|
||||
# 迁移:旧版按 symbol= 分区转为 date= 分区
|
||||
_migrate_symbol_to_date_partition(repo)
|
||||
|
||||
now = datetime.now()
|
||||
|
||||
# 计算时间区间: 首次拉取回溯 N 天, 增量从最后数据时间开始
|
||||
last_dt = _latest_minute_datetime(repo)
|
||||
if last_dt:
|
||||
start_time = last_dt
|
||||
else:
|
||||
start_time = now - timedelta(days=days)
|
||||
end_time = now
|
||||
|
||||
lim = capset.limits(Cap.KLINE_MINUTE_BATCH)
|
||||
batch_size = lim.batch if lim and lim.batch else 100
|
||||
rpm = lim.rpm if lim else 30
|
||||
|
||||
df = sync_minute_batch(symbols, start_time=start_time, end_time=end_time,
|
||||
batch_size=batch_size, rpm=rpm,
|
||||
on_chunk_done=on_chunk_done)
|
||||
if df.is_empty():
|
||||
return 0
|
||||
|
||||
# 按日期分区写: data/kline_minute/date={YYYY-MM-DD}/part.parquet
|
||||
df = df.with_columns(
|
||||
pl.col("datetime").dt.date().alias("_trade_date")
|
||||
)
|
||||
written = 0
|
||||
for day_df in df.partition_by("_trade_date"):
|
||||
trade_date = day_df["_trade_date"][0]
|
||||
out = repo.store.data_dir / "kline_minute" / f"date={trade_date}" / "part.parquet"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
if out.exists():
|
||||
existing = pl.read_parquet(out)
|
||||
if "datetime" in existing.columns:
|
||||
existing = existing.filter(pl.col("datetime").is_not_null())
|
||||
day_df = pl.concat([existing, day_df.drop("_trade_date")]).unique(
|
||||
subset=["symbol", "datetime"], keep="last",
|
||||
)
|
||||
else:
|
||||
day_df = day_df.drop("_trade_date")
|
||||
day_df = day_df.sort("symbol", "datetime")
|
||||
day_df.write_parquet(out)
|
||||
written += day_df.height
|
||||
|
||||
# 刷新视图
|
||||
try:
|
||||
d = repo.store.data_dir.as_posix()
|
||||
repo.db.execute(
|
||||
f"""CREATE OR REPLACE VIEW kline_minute AS
|
||||
SELECT * FROM read_parquet('{d}/kline_minute/**/*.parquet', union_by_name=true)"""
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("refresh kline_minute view failed: %s", e)
|
||||
|
||||
logger.info("minute K synced: %d rows (%d symbols)", written, len(symbols))
|
||||
return written
|
||||
@@ -0,0 +1,112 @@
|
||||
"""系统通知适配器 — 调用操作系统原生通知命令。
|
||||
|
||||
职责: 把后端产生的告警事件推送到操作系统通知中心。
|
||||
窗口最小化 / 被遮挡 / 后台运行时都能弹通知。
|
||||
|
||||
平台实现:
|
||||
- macOS: osascript (系统已内置)
|
||||
- Linux: notify-send (系统已内置)
|
||||
- Windows: 暂不支持原生通知中心 (无额外依赖实现)
|
||||
|
||||
设计: 失败静默降级, 绝不因通知失败阻断告警主流程 (落盘 / SSE 推送)。
|
||||
通知去重不在本层做, 复用 MonitorRuleEngine 的 cooldown 逻辑。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 单次通知最长字符 (避免某些平台截断报错)
|
||||
_MAX_LEN = 200
|
||||
|
||||
# 避免重复探测平台能力, 缓存一次
|
||||
_backend_cache: str | None = None
|
||||
|
||||
|
||||
def _detect_backend() -> str | None:
|
||||
"""探测当前平台可用的通知后端。"""
|
||||
global _backend_cache
|
||||
if _backend_cache is not None:
|
||||
return _backend_cache if _backend_cache != "none" else None
|
||||
|
||||
backend = None
|
||||
if sys.platform == "darwin":
|
||||
backend = "osascript"
|
||||
elif sys.platform.startswith("linux"):
|
||||
backend = "notify-send"
|
||||
|
||||
_backend_cache = backend or "none"
|
||||
return backend
|
||||
|
||||
|
||||
def _truncate(text: str) -> str:
|
||||
"""截断超长文本, 避免平台通知上限报错。"""
|
||||
text = (text or "").strip()
|
||||
return text[:_MAX_LEN] + ("…" if len(text) > _MAX_LEN else "")
|
||||
|
||||
|
||||
def notify(title: str, message: str, icon: Path | None = None) -> bool:
|
||||
"""推送一条系统通知。
|
||||
|
||||
Args:
|
||||
title: 通知标题
|
||||
message: 通知正文
|
||||
icon: 可选图标路径 (部分平台支持)
|
||||
|
||||
Returns:
|
||||
True=成功送达, False=失败或无可用后端。
|
||||
失败静默, 不抛异常 (通知是辅助通道, 不能阻断告警主流程)。
|
||||
"""
|
||||
title = _truncate(title)
|
||||
message = _truncate(message)
|
||||
if not title:
|
||||
return False
|
||||
|
||||
backend = _detect_backend()
|
||||
if backend is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
if backend == "osascript":
|
||||
return _notify_osascript(title, message)
|
||||
if backend == "notify-send":
|
||||
return _notify_notify_send(title, message, icon)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("系统通知失败 (%s): %s", backend, e)
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def _notify_osascript(title: str, message: str) -> bool:
|
||||
"""macOS 通知 (osascript) — 调用系统 AppleScript。"""
|
||||
# 转义双引号, 避免 AppleScript 注入
|
||||
safe_title = title.replace('"', '\\"')
|
||||
safe_msg = message.replace('"', '\\"')
|
||||
script = (
|
||||
f'display notification "{safe_msg}" with title "{safe_title}"'
|
||||
)
|
||||
result = subprocess.run( # noqa: S603, S607
|
||||
["osascript", "-e", script],
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def _notify_notify_send(title: str, message: str, icon: Path | None) -> bool:
|
||||
"""Linux 通知 (notify-send) — freedesktop.org 标准。"""
|
||||
args = ["notify-send", title]
|
||||
if icon and Path(icon).exists():
|
||||
args.extend(["--icon", str(icon)])
|
||||
args.append(message)
|
||||
result = subprocess.run( # noqa: S603
|
||||
args,
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
)
|
||||
return result.returncode == 0
|
||||
@@ -0,0 +1,250 @@
|
||||
"""异步盘后管道任务注册表 — 每个 job 独立 JSON 文件。
|
||||
|
||||
设计:
|
||||
- job_store/ 文件夹,每个 job 一个 {id}.json,最多保留 max_jobs 个文件
|
||||
- running/pending 状态的 job 仅存内存(高频读写)
|
||||
- succeeded/failed 后写入独立文件并从内存释放
|
||||
- 列表查询 = 内存中的活跃 job + 磁盘文件扫描,按时间排序
|
||||
- 单个查询 = 内存优先,没有则读磁盘
|
||||
- 创建新 job 前检查文件数量,>= max_jobs 时删除最老的文件
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
JobStatus = Literal["pending", "running", "succeeded", "failed"]
|
||||
|
||||
|
||||
def _default_store_dir() -> Path:
|
||||
from app.config import settings
|
||||
return settings.data_dir / "job_store"
|
||||
|
||||
|
||||
_STORE_DIR = _default_store_dir()
|
||||
|
||||
|
||||
class JobStore:
|
||||
def __init__(self, max_jobs: int = 50, store_dir: Path = _STORE_DIR) -> None:
|
||||
self._max_jobs = max_jobs
|
||||
self._store_dir = store_dir
|
||||
self._active_jobs: dict[str, dict[str, Any]] = {} # running/pending
|
||||
self._active_id: str | None = None
|
||||
self._lock = threading.Lock()
|
||||
self._store_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ===== persistence =====
|
||||
|
||||
def _write_file(self, job: dict[str, Any]) -> None:
|
||||
"""将终态 job 写入独立 JSON 文件。"""
|
||||
path = self._store_dir / f"{job['id']}.json"
|
||||
try:
|
||||
path.write_text(
|
||||
json.dumps(job, ensure_ascii=False, indent=None),
|
||||
encoding="utf-8",
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("failed to write job file %s", path)
|
||||
|
||||
def _read_file(self, job_id: str) -> dict[str, Any] | None:
|
||||
"""从磁盘读取单个 job 文件。"""
|
||||
path = self._store_dir / f"{job_id}.json"
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(path.read_text("utf-8"))
|
||||
except Exception:
|
||||
logger.warning("failed to read job file %s", path)
|
||||
return None
|
||||
|
||||
def _delete_oldest(self) -> None:
|
||||
"""删除最老的 job 文件,保持文件数量 < max_jobs。"""
|
||||
try:
|
||||
files = sorted(self._store_dir.glob("*.json"), key=lambda f: f.stat().st_mtime)
|
||||
except Exception:
|
||||
return
|
||||
while len(files) >= self._max_jobs:
|
||||
oldest = files.pop(0)
|
||||
try:
|
||||
oldest.unlink()
|
||||
except Exception:
|
||||
logger.warning("failed to delete old job file %s", oldest)
|
||||
|
||||
def _job_files_sorted(self) -> list[dict[str, Any]]:
|
||||
"""扫描磁盘上所有 job 文件,按 started_at 从新到旧排序。"""
|
||||
jobs: list[dict[str, Any]] = []
|
||||
for f in self._store_dir.glob("*.json"):
|
||||
try:
|
||||
jobs.append(json.loads(f.read_text("utf-8")))
|
||||
except Exception:
|
||||
continue
|
||||
jobs.sort(key=lambda j: j.get("started_at") or "", reverse=True)
|
||||
return jobs
|
||||
|
||||
# ===== lifecycle =====
|
||||
|
||||
def create(self) -> str:
|
||||
with self._lock:
|
||||
if self._active_id and self._active_jobs.get(self._active_id, {}).get("status") == "running":
|
||||
return self._active_id
|
||||
|
||||
job_id = uuid.uuid4().hex[:10]
|
||||
self._active_jobs[job_id] = {
|
||||
"id": job_id,
|
||||
"status": "pending",
|
||||
"stage": "init",
|
||||
"progress": 0,
|
||||
"stage_pct": 0,
|
||||
"log": [],
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"duration_s": None,
|
||||
"result": None,
|
||||
"error": None,
|
||||
}
|
||||
self._active_id = job_id
|
||||
return job_id
|
||||
|
||||
def start(self, job_id: str) -> None:
|
||||
with self._lock:
|
||||
j = self._active_jobs.get(job_id)
|
||||
if not j:
|
||||
return
|
||||
j["status"] = "running"
|
||||
j["started_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
|
||||
def succeed(self, job_id: str, result: Any) -> None:
|
||||
with self._lock:
|
||||
j = self._active_jobs.pop(job_id, None)
|
||||
if not j:
|
||||
return
|
||||
j["status"] = "succeeded"
|
||||
j["finished_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
j["progress"] = 100
|
||||
j["result"] = result
|
||||
j["duration_s"] = _duration_s(j)
|
||||
if self._active_id == job_id:
|
||||
self._active_id = None
|
||||
self._delete_oldest()
|
||||
self._write_file(j)
|
||||
|
||||
def fail(self, job_id: str, error: str) -> None:
|
||||
with self._lock:
|
||||
j = self._active_jobs.pop(job_id, None)
|
||||
if not j:
|
||||
return
|
||||
j["status"] = "failed"
|
||||
j["finished_at"] = datetime.utcnow().isoformat(timespec="seconds") + "Z"
|
||||
j["error"] = error
|
||||
j["duration_s"] = _duration_s(j)
|
||||
if self._active_id == job_id:
|
||||
self._active_id = None
|
||||
self._delete_oldest()
|
||||
self._write_file(j)
|
||||
|
||||
# ===== progress =====
|
||||
|
||||
def progress(self, job_id: str, stage: str, pct: int, msg: str,
|
||||
stage_pct: int | None = None, skip_log: bool = False) -> None:
|
||||
with self._lock:
|
||||
j = self._active_jobs.get(job_id)
|
||||
if not j:
|
||||
return
|
||||
j["stage"] = stage
|
||||
j["progress"] = max(0, min(100, int(pct)))
|
||||
if stage_pct is not None:
|
||||
j["stage_pct"] = max(0, min(100, int(stage_pct)))
|
||||
elif j["stage"] != stage:
|
||||
j["stage_pct"] = 0
|
||||
entry = {
|
||||
"ts": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
"stage": stage,
|
||||
"msg": msg,
|
||||
}
|
||||
if skip_log:
|
||||
entry["_skip"] = True
|
||||
if skip_log and j["log"] and j["log"][-1].get("stage") == stage and j["log"][-1].get("_skip"):
|
||||
j["log"][-1] = entry
|
||||
else:
|
||||
j["log"].append(entry)
|
||||
if len(j["log"]) > 200:
|
||||
j["log"] = j["log"][-200:]
|
||||
|
||||
# ===== query =====
|
||||
|
||||
def get(self, job_id: str) -> dict[str, Any] | None:
|
||||
# 内存中的活跃 job 优先
|
||||
j = self._active_jobs.get(job_id)
|
||||
if j:
|
||||
return j
|
||||
# 否则从磁盘读
|
||||
return self._read_file(job_id)
|
||||
|
||||
def list_recent(self, limit: int = 20) -> list[dict[str, Any]]:
|
||||
# 合并: 内存中的活跃 job + 磁盘文件
|
||||
all_jobs: list[dict[str, Any]] = list(self._active_jobs.values())
|
||||
all_jobs.extend(self._job_files_sorted())
|
||||
# 按 started_at 从新到旧排序,去重(理论上不会有重复)
|
||||
seen: set[str] = set()
|
||||
result: list[dict[str, Any]] = []
|
||||
for j in sorted(all_jobs, key=lambda x: x.get("started_at") or "", reverse=True):
|
||||
jid = j["id"]
|
||||
if jid in seen:
|
||||
continue
|
||||
seen.add(jid)
|
||||
result.append(_summary(j))
|
||||
if len(result) >= limit:
|
||||
break
|
||||
return result
|
||||
|
||||
def active_id(self) -> str | None:
|
||||
return self._active_id
|
||||
|
||||
def clear(self) -> None:
|
||||
"""清空所有任务(内存 + 磁盘文件)。"""
|
||||
with self._lock:
|
||||
self._active_jobs.clear()
|
||||
self._active_id = None
|
||||
for f in self._store_dir.glob("*.json"):
|
||||
try:
|
||||
f.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _summary(j: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": j["id"],
|
||||
"status": j["status"],
|
||||
"stage": j["stage"],
|
||||
"progress": j["progress"],
|
||||
"stage_pct": j.get("stage_pct", 0),
|
||||
"started_at": j["started_at"],
|
||||
"finished_at": j["finished_at"],
|
||||
"duration_s": j["duration_s"],
|
||||
"result": j["result"],
|
||||
"error": j["error"],
|
||||
}
|
||||
|
||||
|
||||
def _duration_s(j: dict[str, Any]) -> float | None:
|
||||
if not j.get("started_at") or not j.get("finished_at"):
|
||||
return None
|
||||
try:
|
||||
s = datetime.fromisoformat(j["started_at"])
|
||||
e = datetime.fromisoformat(j["finished_at"])
|
||||
return round((e - s).total_seconds(), 2)
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
|
||||
|
||||
# 进程内单例
|
||||
job_store = JobStore()
|
||||
@@ -0,0 +1,313 @@
|
||||
"""用户偏好设置持久化。
|
||||
|
||||
存储位置: data/user_data/preferences.json
|
||||
沿用 secrets_store 的 merge-write 模式,但不做 chmod 0600 (非敏感数据)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _path() -> Path:
|
||||
from app.config import settings
|
||||
p = settings.data_dir / "user_data" / "preferences.json"
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def load() -> dict:
|
||||
p = _path()
|
||||
if p.exists():
|
||||
try:
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("preferences.json malformed: %s", e)
|
||||
return {}
|
||||
|
||||
|
||||
def save(updates: dict) -> dict:
|
||||
"""合并写入。返回新内容。"""
|
||||
current = load()
|
||||
current.update(updates)
|
||||
_path().write_text(
|
||||
json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8",
|
||||
)
|
||||
return current
|
||||
|
||||
|
||||
def get_realtime_quotes_enabled() -> bool:
|
||||
return load().get("realtime_quotes_enabled", False)
|
||||
|
||||
|
||||
def get_indices_nav_pinned() -> bool:
|
||||
"""侧栏指数报价卡片是否固定显示。默认 True(常驻)。
|
||||
关闭后,卡片跟随实时行情开关(仅实时开时显示)。"""
|
||||
return load().get("indices_nav_pinned", True)
|
||||
|
||||
|
||||
def get_realtime_quote_interval() -> float:
|
||||
return load().get("realtime_quote_interval", 10.0)
|
||||
|
||||
|
||||
def set_realtime_quote_interval(interval: float) -> float:
|
||||
"""保存行情轮询间隔(不在此做 min/max 校验,由调用方按档位限制)。"""
|
||||
current = load()
|
||||
current["realtime_quote_interval"] = interval
|
||||
_path().write_text(
|
||||
json.dumps(current, indent=2, ensure_ascii=False), encoding="utf-8",
|
||||
)
|
||||
return interval
|
||||
|
||||
|
||||
def get_minute_sync_enabled() -> bool:
|
||||
return load().get("minute_sync_enabled", False)
|
||||
|
||||
|
||||
def get_minute_sync_days() -> int:
|
||||
return max(1, min(30, load().get("minute_sync_days", 5)))
|
||||
|
||||
|
||||
def get_pipeline_schedule() -> dict:
|
||||
"""返回盘后管道调度时间 {"hour": 15, "minute": 30}。"""
|
||||
d = load().get("pipeline_schedule", {"hour": 15, "minute": 30})
|
||||
return {"hour": d.get("hour", 15), "minute": d.get("minute", 30)}
|
||||
|
||||
|
||||
def set_pipeline_schedule(hour: int, minute: int) -> dict:
|
||||
h = max(0, min(23, hour))
|
||||
m = max(0, min(59, minute))
|
||||
# 盘后不早于 15:00
|
||||
if h * 60 + m < 15 * 60:
|
||||
h, m = 15, 0
|
||||
save({"pipeline_schedule": {"hour": h, "minute": m}})
|
||||
return {"hour": h, "minute": m}
|
||||
|
||||
|
||||
def get_instruments_schedule() -> dict:
|
||||
"""返回盘前标的维表调度时间 {"hour": 9, "minute": 10}。"""
|
||||
d = load().get("instruments_schedule", {"hour": 9, "minute": 10})
|
||||
return {"hour": d.get("hour", 9), "minute": d.get("minute", 10)}
|
||||
|
||||
|
||||
def set_instruments_schedule(hour: int, minute: int) -> dict:
|
||||
h = max(0, min(23, hour))
|
||||
m = max(0, min(59, minute))
|
||||
# 盘前不晚于 09:15
|
||||
if h * 60 + m > 9 * 60 + 15:
|
||||
h, m = 9, 15
|
||||
save({"instruments_schedule": {"hour": h, "minute": m}})
|
||||
return {"hour": h, "minute": m}
|
||||
|
||||
|
||||
def get_enriched_batch_size() -> int:
|
||||
"""返回 enriched 全量计算每批 symbol 数量。"""
|
||||
return max(1, min(10000, load().get("enriched_batch_size", 1000)))
|
||||
|
||||
|
||||
def set_enriched_batch_size(size: int) -> int:
|
||||
"""保存 enriched 全量计算批次大小。"""
|
||||
size = max(10, min(6000, size))
|
||||
save({"enriched_batch_size": size})
|
||||
return size
|
||||
|
||||
|
||||
def get_index_daily_batch_size() -> int:
|
||||
"""返回指数日 K 同步每批 symbol 数量。"""
|
||||
return max(1, min(10000, load().get("index_daily_batch_size", 100)))
|
||||
|
||||
|
||||
def set_index_daily_batch_size(size: int) -> int:
|
||||
"""保存指数日 K 同步批次大小。"""
|
||||
size = max(1, min(10000, size))
|
||||
save({"index_daily_batch_size": size})
|
||||
return size
|
||||
|
||||
|
||||
# ── 五档盘口 sealed(真假涨停) 配置 ──────────────────────
|
||||
|
||||
def get_limit_ladder_monitor_enabled() -> bool:
|
||||
"""连板梯队 5 档监控开关。关闭时 depth 不轮询(连板梯队降级显示)。"""
|
||||
return load().get("limit_ladder_monitor_enabled", False)
|
||||
|
||||
|
||||
def get_depth_polling_interval() -> float:
|
||||
"""depth 盘中轮询间隔(秒)。默认 20(Pro/Expert 都适用)。"""
|
||||
return float(load().get("depth_polling_interval", 20.0))
|
||||
|
||||
|
||||
def set_depth_polling_interval(interval: float) -> float:
|
||||
"""保存 depth 轮询间隔。套餐范围 clamp 由 depth_service 按档位做。"""
|
||||
interval = max(1.0, min(600.0, float(interval)))
|
||||
save({"depth_polling_interval": interval})
|
||||
return interval
|
||||
|
||||
|
||||
def get_depth_finalize_time() -> dict:
|
||||
"""盘后 sealed 定版时间 {"hour": 15, "minute": 2}。范围 15:01~18:00。"""
|
||||
d = load().get("depth_finalize_time", {"hour": 15, "minute": 2})
|
||||
return {"hour": d.get("hour", 15), "minute": d.get("minute", 2)}
|
||||
|
||||
|
||||
def set_depth_finalize_time(hour: int, minute: int) -> dict:
|
||||
"""保存盘后 sealed 定版时间,强制范围 15:01~18:00。"""
|
||||
h = max(0, min(23, hour))
|
||||
m = max(0, min(59, minute))
|
||||
# 下限 15:01, 上限 18:00
|
||||
if h * 60 + m < 15 * 60 + 1:
|
||||
h, m = 15, 1
|
||||
if h * 60 + m > 18 * 60:
|
||||
h, m = 18, 0
|
||||
save({"depth_finalize_time": {"hour": h, "minute": m}})
|
||||
return {"hour": h, "minute": m}
|
||||
|
||||
|
||||
|
||||
# ===== 实时监控 =====
|
||||
|
||||
# 页面 SSE 刷新配置: { "watchlist": true, "monitor": true, ... }
|
||||
# 可刷新的页面列表及其默认值
|
||||
SSE_REFRESH_PAGES_DEFAULT = {
|
||||
"watchlist": True,
|
||||
"limit-ladder": False,
|
||||
}
|
||||
|
||||
SIDEBAR_INDEX_SYMBOLS_DEFAULT = ["000001.SH", "399001.SZ", "399006.SZ", "000680.SH"]
|
||||
|
||||
|
||||
def get_sse_refresh_pages() -> dict[str, bool]:
|
||||
"""返回每个页面的 SSE 刷新开关。"""
|
||||
stored = load().get("sse_refresh_pages", {})
|
||||
# 合并默认值 (新增页面自动出现)
|
||||
result = dict(SSE_REFRESH_PAGES_DEFAULT)
|
||||
result.update(stored)
|
||||
return result
|
||||
|
||||
|
||||
def set_sse_refresh_pages(pages: dict[str, bool]) -> dict[str, bool]:
|
||||
"""保存页面 SSE 刷新配置。"""
|
||||
save({"sse_refresh_pages": pages})
|
||||
return get_sse_refresh_pages()
|
||||
|
||||
|
||||
def get_sidebar_index_symbols() -> list[str]:
|
||||
"""返回左侧菜单显示的指数代码。"""
|
||||
stored = load().get("sidebar_index_symbols", SIDEBAR_INDEX_SYMBOLS_DEFAULT)
|
||||
allowed = set(SIDEBAR_INDEX_SYMBOLS_DEFAULT)
|
||||
return [s for s in stored if s in allowed]
|
||||
|
||||
|
||||
def get_strategy_monitor_enabled() -> bool:
|
||||
"""策略告警评估总开关。"""
|
||||
return load().get("strategy_monitor_enabled", False)
|
||||
|
||||
|
||||
def get_system_notify_enabled() -> bool:
|
||||
"""系统通知开关 — 开启后监控告警同时推送到操作系统通知中心。"""
|
||||
return load().get("system_notify_enabled", False)
|
||||
|
||||
|
||||
def set_system_notify_enabled(enabled: bool) -> bool:
|
||||
"""保存系统通知开关。"""
|
||||
save({"system_notify_enabled": bool(enabled)})
|
||||
return bool(enabled)
|
||||
|
||||
|
||||
def get_screener_auto_run() -> bool:
|
||||
"""选股页进入时是否自动运行所有策略 (获取命中数)。默认开。"""
|
||||
return load().get("screener_auto_run", True)
|
||||
|
||||
|
||||
def get_strategy_monitor_ids() -> list[str]:
|
||||
"""返回监控池中的策略 ID。"""
|
||||
return load().get("strategy_monitor_ids", [])
|
||||
|
||||
|
||||
def set_realtime_monitor_config(cfg: dict) -> dict:
|
||||
"""批量更新实时监控配置。"""
|
||||
updates = {}
|
||||
if "sse_refresh_pages" in cfg:
|
||||
updates["sse_refresh_pages"] = cfg["sse_refresh_pages"]
|
||||
if "strategy_monitor_enabled" in cfg:
|
||||
updates["strategy_monitor_enabled"] = cfg["strategy_monitor_enabled"]
|
||||
if "strategy_monitor_ids" in cfg:
|
||||
updates["strategy_monitor_ids"] = cfg["strategy_monitor_ids"]
|
||||
if "sidebar_index_symbols" in cfg:
|
||||
allowed = set(SIDEBAR_INDEX_SYMBOLS_DEFAULT)
|
||||
updates["sidebar_index_symbols"] = [s for s in cfg["sidebar_index_symbols"] if s in allowed]
|
||||
if "screener_auto_run" in cfg:
|
||||
updates["screener_auto_run"] = bool(cfg["screener_auto_run"])
|
||||
if updates:
|
||||
save(updates)
|
||||
return get_realtime_monitor_config()
|
||||
|
||||
|
||||
def get_realtime_monitor_config() -> dict:
|
||||
"""返回完整的实时监控配置。"""
|
||||
return {
|
||||
"sse_refresh_pages": get_sse_refresh_pages(),
|
||||
"strategy_monitor_enabled": get_strategy_monitor_enabled(),
|
||||
"strategy_monitor_ids": get_strategy_monitor_ids(),
|
||||
"sidebar_index_symbols": get_sidebar_index_symbols(),
|
||||
"screener_auto_run": get_screener_auto_run(),
|
||||
}
|
||||
|
||||
|
||||
def get_nav_order() -> list[str]:
|
||||
"""返回左侧菜单的自定义排序(内置页面 path + 扩展分析菜单 id)。"""
|
||||
return load().get("nav_order", [])
|
||||
|
||||
|
||||
def set_nav_order(order: list[str]) -> list[str]:
|
||||
"""保存左侧菜单排序。"""
|
||||
save({"nav_order": order})
|
||||
return get_nav_order()
|
||||
|
||||
|
||||
def get_nav_hidden() -> list[str]:
|
||||
"""返回左侧菜单中隐藏的项 id 列表。"""
|
||||
return load().get("nav_hidden", [])
|
||||
|
||||
|
||||
def set_nav_hidden(hidden: list[str]) -> list[str]:
|
||||
"""保存左侧菜单隐藏项。"""
|
||||
save({"nav_hidden": hidden})
|
||||
return get_nav_hidden()
|
||||
|
||||
|
||||
def get_watchlist_columns() -> list[dict] | None:
|
||||
"""返回自选列表列配置。"""
|
||||
return load().get("watchlist_columns")
|
||||
|
||||
|
||||
def set_watchlist_columns(columns: list[dict]) -> list[dict]:
|
||||
"""保存自选列表列配置。"""
|
||||
save({"watchlist_columns": columns})
|
||||
return columns
|
||||
|
||||
|
||||
def get_screener_result_columns() -> list[dict] | None:
|
||||
"""返回策略结果列表列配置。"""
|
||||
return load().get("screener_result_columns")
|
||||
|
||||
|
||||
def set_screener_result_columns(columns: list[dict]) -> list[dict]:
|
||||
"""保存策略结果列表列配置。"""
|
||||
save({"screener_result_columns": columns})
|
||||
return columns
|
||||
|
||||
|
||||
# ===== 首次使用引导 =====
|
||||
|
||||
def get_onboarding_completed() -> bool:
|
||||
"""是否已完成首次使用向导。默认 False(新用户)。"""
|
||||
return bool(load().get("onboarding_completed", False))
|
||||
|
||||
|
||||
def set_onboarding_completed(done: bool = True) -> bool:
|
||||
"""标记首次使用向导完成状态。"""
|
||||
save({"onboarding_completed": bool(done)})
|
||||
return bool(done)
|
||||
@@ -0,0 +1,779 @@
|
||||
"""全局实时行情服务。
|
||||
|
||||
集中管理全市场行情拉取 + enriched 缓存,供盘中选股、自选股等所有模块复用。
|
||||
|
||||
架构:
|
||||
- 后台线程轮询数据源 get_by_universes(["CN_Equity_A", "CN_Index"])
|
||||
- 拉取行情 → 写 kline_daily (不复权) + 增量计算 enriched → 写盘 + 更新缓存
|
||||
- _enriched_cache 是唯一的盘中数据源 (OHLCV + 全套技术指标)
|
||||
- _live_agg_cache 是递推状态 (只加载一次, 盘中不变)
|
||||
|
||||
数据流 (每轮 ~15s):
|
||||
1. API 拉取 → raw_records (临时变量)
|
||||
2. raw_records → 写 kline_daily (不复权原始价格)
|
||||
3. raw_records → 更新 _enriched_cache 的 OHLCV
|
||||
4. 增量计算 enriched 指标 (~50ms)
|
||||
5. 写 kline_daily_enriched + 替换 _enriched_cache
|
||||
6. 通知 SSE
|
||||
|
||||
生命周期:
|
||||
- 服务启动时读取 preferences,若 enabled 则自动启动线程
|
||||
- 运行中可通过 API 切换开关
|
||||
- 关闭时停止线程
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from datetime import date, datetime, time as dt_time
|
||||
|
||||
import polars as pl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class QuoteService:
|
||||
"""全局实时行情服务 — 单例。"""
|
||||
|
||||
CORE_INDEX_SYMBOLS = ("000001.SH", "399001.SZ", "399006.SZ", "000680.SH")
|
||||
|
||||
# 档位 → 最小轮询间隔 (秒)
|
||||
TIER_MIN_INTERVAL = {
|
||||
"expert": 1.0,
|
||||
"pro": 2.0,
|
||||
"starter": 3.0,
|
||||
}
|
||||
DEFAULT_INTERVAL = 10.0
|
||||
MAX_INTERVAL = 60.0
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self._running = False
|
||||
self._enabled = False # 全局开关 (持久化到 preferences)
|
||||
self._interval = self.DEFAULT_INTERVAL
|
||||
self._thread: threading.Thread | None = None
|
||||
self._repo = None # 延迟注入, 避免循环导入
|
||||
self._update_event = threading.Event() # SSE 通知: 行情更新后 set
|
||||
self._alert_event = threading.Event() # SSE 通知: 有告警时 set
|
||||
self._depth_update_event = threading.Event() # SSE 通知: depth 五档修正后 set (刷新连板梯队)
|
||||
self._pending_alerts: list[dict] = [] # 待推送的告警
|
||||
self._max_pending_alerts: int = 1000 # 背压上限: 超出丢弃最旧
|
||||
self._strategy_monitor = None # 延迟注入
|
||||
self._app_state = None # 延迟注入 (FastAPI app.state)
|
||||
|
||||
# 拉取元信息 (给 SSE / status 用)
|
||||
self._fetch_time: float = 0.0 # perf_counter (用于计算 quote_age_ms)
|
||||
self._fetch_ms: float = 0.0 # 拉取耗时 (毫秒)
|
||||
self._fetched_at: float = 0.0 # 拉取完成的 Unix 时间戳 (毫秒)
|
||||
self._symbol_count: int = 0
|
||||
self._index_symbol_count: int = 0
|
||||
self._index_quotes_cache: pl.DataFrame | None = None
|
||||
|
||||
# ================================================================
|
||||
# 生命周期
|
||||
# ================================================================
|
||||
|
||||
def start(self, interval: float = 0.0) -> None:
|
||||
"""启动后台行情轮询线程。"""
|
||||
if self._running:
|
||||
return
|
||||
if interval <= 0:
|
||||
from app.services import preferences
|
||||
interval = preferences.get_realtime_quote_interval()
|
||||
self._interval = self._clamp_interval(interval)
|
||||
self._running = True
|
||||
self._enabled = True
|
||||
self._thread = threading.Thread(target=self._poll_loop, daemon=True)
|
||||
self._thread.start()
|
||||
self._save_enabled(True)
|
||||
logger.info("行情服务已启动, 轮询间隔 %.1fs", self._interval)
|
||||
|
||||
def stop(self) -> None:
|
||||
"""停止后台行情轮询线程。"""
|
||||
self._running = False
|
||||
self._enabled = False
|
||||
if self._thread:
|
||||
self._thread.join(timeout=10)
|
||||
self._thread = None
|
||||
self._save_enabled(False)
|
||||
logger.info("行情服务已停止")
|
||||
|
||||
def enable(self) -> bool:
|
||||
"""开启自动行情 (不立即启动线程,等下一个交易时段)。
|
||||
|
||||
none/free 档无实时行情权限,拒绝开启并返回 False;
|
||||
starter+ 正常启动。返回值表示是否真正开启。
|
||||
"""
|
||||
if not self.is_realtime_allowed():
|
||||
logger.warning("实时行情开启被拒:当前档位(none/free)无实时行情权限")
|
||||
return False
|
||||
self._enabled = True
|
||||
self._save_enabled(True)
|
||||
if not self._running:
|
||||
from app.services import preferences
|
||||
self._interval = self._clamp_interval(preferences.get_realtime_quote_interval())
|
||||
self._running = True
|
||||
self._thread = threading.Thread(target=self._poll_loop, daemon=True)
|
||||
self._thread.start()
|
||||
logger.info("行情服务已启用, 轮询间隔 %.1fs", self._interval)
|
||||
|
||||
def disable(self) -> None:
|
||||
"""关闭自动行情。"""
|
||||
self.stop()
|
||||
logger.info("行情服务已关闭")
|
||||
|
||||
def boot_check(self) -> None:
|
||||
"""启动时检查 preferences,若 enabled 则自动启动。
|
||||
|
||||
none/free 档无实时行情权限:即使 preferences 标记为 enabled,
|
||||
也不启动,并同步 preferences 为关闭(避免 UI 误显示已开启)。
|
||||
"""
|
||||
from app.services import preferences
|
||||
if not self.is_realtime_allowed():
|
||||
if preferences.get_realtime_quotes_enabled():
|
||||
self._save_enabled(False)
|
||||
logger.info("实时行情未启动:当前档位(none/free)无实时行情权限")
|
||||
return
|
||||
if preferences.get_realtime_quotes_enabled():
|
||||
self.start()
|
||||
|
||||
def set_repo(self, repo) -> None:
|
||||
"""注入 KlineRepository, 用于实时落盘。"""
|
||||
self._repo = repo
|
||||
|
||||
def set_app_state(self, app_state) -> None:
|
||||
"""注入 FastAPI app.state, 用于获取 strategy_monitor 等单例。"""
|
||||
self._app_state = app_state
|
||||
|
||||
def set_interval(self, interval: float) -> float:
|
||||
"""运行时更新轮询间隔(立即生效)。"""
|
||||
clamped = self._clamp_interval(interval)
|
||||
self._interval = clamped
|
||||
from app.services import preferences
|
||||
preferences.set_realtime_quote_interval(clamped)
|
||||
logger.info("轮询间隔已更新为 %.1fs", clamped)
|
||||
return clamped
|
||||
|
||||
def get_min_interval(self) -> float:
|
||||
"""返回当前档位允许的最小间隔。"""
|
||||
return self._tier_min_interval()
|
||||
|
||||
def wait_for_update(self, timeout: float = 30.0) -> bool:
|
||||
"""阻塞等待下一次行情更新 (供 SSE 线程使用)。"""
|
||||
self._update_event.clear()
|
||||
return self._update_event.wait(timeout=timeout)
|
||||
|
||||
def wait_for_alert(self, timeout: float = 30.0) -> bool:
|
||||
"""阻塞等待告警 (供 SSE 线程使用)。"""
|
||||
self._alert_event.clear()
|
||||
return self._alert_event.wait(timeout=timeout)
|
||||
|
||||
def notify_depth_updated(self) -> None:
|
||||
"""五档盘口修正完成后调用: 通知 SSE 推送 depth_updated, 触发连板梯队刷新。
|
||||
|
||||
与行情/告警通道独立 — 只刷新连板梯队, 不连带刷新 watchlist 等。
|
||||
"""
|
||||
self._depth_update_event.set()
|
||||
|
||||
def wait_for_depth_update(self, timeout: float = 30.0) -> bool:
|
||||
"""阻塞等待 depth 修正 (供 SSE 线程使用)。"""
|
||||
self._depth_update_event.clear()
|
||||
return self._depth_update_event.wait(timeout=timeout)
|
||||
|
||||
def pop_alerts(self) -> list[dict]:
|
||||
"""取走所有待推送的告警 (线程安全)。"""
|
||||
with self._lock:
|
||||
alerts = self._pending_alerts
|
||||
self._pending_alerts = []
|
||||
return alerts
|
||||
|
||||
# ================================================================
|
||||
# 档位感知间隔限制
|
||||
# ================================================================
|
||||
|
||||
@staticmethod
|
||||
def _current_tier() -> str:
|
||||
"""获取当前档位名(小写)。"""
|
||||
from app.tickflow.policy import tier_label
|
||||
return tier_label().split()[0].split("+")[0].strip().lower()
|
||||
|
||||
@classmethod
|
||||
def is_realtime_allowed(cls) -> bool:
|
||||
"""当前档位是否允许使用实时行情。
|
||||
|
||||
none/free 档走 free-api 服务器,无实时行情权限 → 不允许;
|
||||
starter+ 付费档走付费端点,有实时行情 → 允许。
|
||||
"""
|
||||
return cls._current_tier() not in ("none", "free")
|
||||
|
||||
@classmethod
|
||||
def _tier_min_interval(cls) -> float:
|
||||
tier = cls._current_tier()
|
||||
return cls.TIER_MIN_INTERVAL.get(tier, cls.DEFAULT_INTERVAL)
|
||||
|
||||
def _clamp_interval(self, interval: float) -> float:
|
||||
return max(self._tier_min_interval(), min(self.MAX_INTERVAL, interval))
|
||||
|
||||
# ================================================================
|
||||
# 行情数据访问
|
||||
# ================================================================
|
||||
|
||||
def get_enriched_today(self) -> tuple[pl.DataFrame, date | None]:
|
||||
"""返回今天 enriched 数据 + 日期 (线程安全)。
|
||||
|
||||
所有页面统一通过此方法获取实时行情 + 技术指标。
|
||||
"""
|
||||
if not self._repo:
|
||||
return pl.DataFrame(), None
|
||||
return self._repo.get_enriched_latest()
|
||||
|
||||
def get_quotes_compat(self) -> pl.DataFrame:
|
||||
"""兼容接口: 返回行情 DataFrame (用于盘中选股等需要 last_price/prev_close 的场景)。
|
||||
|
||||
从 _enriched_cache 取 today 的数据, 只选行情基础列, 补上 last_price 别名。
|
||||
不返回指标列, 避免 JOIN live_agg 时列名冲突。
|
||||
"""
|
||||
df, _ = self.get_enriched_today()
|
||||
if df.is_empty():
|
||||
return df
|
||||
|
||||
# 只取盘中选股需要的行情基础列
|
||||
keep = [c for c in [
|
||||
"symbol", "close", "open", "high", "low", "volume", "amount",
|
||||
"prev_close", "change_pct", "change_amount", "amplitude", "turnover_rate",
|
||||
] if c in df.columns]
|
||||
df = df.select(keep)
|
||||
|
||||
# enriched 的 close 等价于 last_price
|
||||
if "close" in df.columns and "last_price" not in df.columns:
|
||||
df = df.with_columns(pl.col("close").alias("last_price"))
|
||||
return df
|
||||
|
||||
def get_index_quotes(self, symbols: list[str] | None = None) -> pl.DataFrame:
|
||||
"""返回实时指数行情缓存。不会触发数据源请求。"""
|
||||
with self._lock:
|
||||
df = self._index_quotes_cache.clone() if self._index_quotes_cache is not None else pl.DataFrame()
|
||||
if df.is_empty():
|
||||
return df
|
||||
if symbols:
|
||||
return df.filter(pl.col("symbol").is_in(symbols))
|
||||
return df
|
||||
|
||||
def status(self) -> dict:
|
||||
"""返回行情服务状态。"""
|
||||
age = (time.perf_counter() - self._fetch_time) * 1000 if self._fetch_time else -1
|
||||
return {
|
||||
"enabled": self._enabled,
|
||||
"running": self._running,
|
||||
"interval_s": self._interval,
|
||||
"symbol_count": self._symbol_count,
|
||||
"index_symbol_count": self._index_symbol_count,
|
||||
"quote_age_ms": round(age, 0) if age >= 0 else None,
|
||||
"is_trading_hours": self._is_trading_hours(),
|
||||
"last_fetch_ms": round(self._fetched_at, 0) if self._fetched_at else None,
|
||||
}
|
||||
|
||||
def refresh(self) -> dict:
|
||||
"""手动触发一次行情拉取。"""
|
||||
self._fetch_quotes()
|
||||
return self.status()
|
||||
|
||||
# ================================================================
|
||||
# 后台轮询
|
||||
# ================================================================
|
||||
|
||||
def _poll_loop(self) -> None:
|
||||
while self._running and self._enabled:
|
||||
try:
|
||||
if self._is_trading_hours():
|
||||
self._fetch_quotes()
|
||||
else:
|
||||
logger.debug("非交易时段, 跳过行情轮询")
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("行情轮询异常: %s", e)
|
||||
|
||||
waited = 0.0
|
||||
while self._running and self._enabled and waited < self._interval:
|
||||
time.sleep(0.5)
|
||||
waited += 0.5
|
||||
|
||||
def _fetch_quotes(self) -> None:
|
||||
"""拉取全市场行情 → 写 daily + 计算 enriched + 更新缓存。"""
|
||||
from app.tickflow.client import get_client
|
||||
|
||||
tf = get_client()
|
||||
t0 = time.perf_counter()
|
||||
now_ts = time.perf_counter()
|
||||
|
||||
try:
|
||||
all_index_symbols = set(self._repo.get_index_symbol_set()) if self._repo else set()
|
||||
all_index_symbols.update(self.CORE_INDEX_SYMBOLS)
|
||||
resp = tf.quotes.get_by_universes(universes=["CN_Equity_A", "CN_Index"])
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("行情拉取失败: %s", e)
|
||||
return
|
||||
|
||||
if not resp:
|
||||
logger.warning("行情数据为空")
|
||||
return
|
||||
|
||||
# ---- 解析 API 响应 (临时变量, 用完丢弃) ----
|
||||
records = []
|
||||
for q in resp:
|
||||
ext = q.get("ext") or {}
|
||||
last_price = q.get("last_price")
|
||||
prev_close = q.get("prev_close")
|
||||
change_amount = ext.get("change_amount")
|
||||
change_pct = ext.get("change_pct")
|
||||
if change_amount is None and last_price is not None and prev_close is not None:
|
||||
change_amount = float(last_price) - float(prev_close)
|
||||
if change_pct is None and change_amount is not None and prev_close not in (None, 0):
|
||||
change_pct = float(change_amount) / float(prev_close) * 100
|
||||
records.append({
|
||||
"symbol": q.get("symbol"),
|
||||
"name": q.get("name") or ext.get("name"),
|
||||
"last_price": last_price,
|
||||
"prev_close": prev_close,
|
||||
"open": q.get("open"),
|
||||
"high": q.get("high"),
|
||||
"low": q.get("low"),
|
||||
"volume": q.get("volume"),
|
||||
"amount": q.get("amount"),
|
||||
"change_pct": change_pct,
|
||||
"change_amount": change_amount,
|
||||
"amplitude": ext.get("amplitude"),
|
||||
"turnover_rate": ext.get("turnover_rate"),
|
||||
"timestamp": q.get("timestamp"),
|
||||
"session": q.get("session"),
|
||||
})
|
||||
|
||||
index_records = [r for r in records if r.get("symbol") in all_index_symbols]
|
||||
stock_records = [r for r in records if r.get("symbol") not in all_index_symbols]
|
||||
|
||||
fetch_ms = (time.perf_counter() - t0) * 1000
|
||||
fetched_at = time.time() * 1000
|
||||
|
||||
# ---- 更新元信息 ----
|
||||
with self._lock:
|
||||
self._fetch_time = now_ts
|
||||
self._fetch_ms = fetch_ms
|
||||
self._fetched_at = fetched_at
|
||||
self._symbol_count = len(stock_records)
|
||||
self._index_symbol_count = len(index_records)
|
||||
self._index_quotes_cache = self._build_index_quotes(index_records)
|
||||
|
||||
logger.info("行情刷新: %d 只股票, %d 只指数, 耗时 %.0fms", len(stock_records), len(index_records), fetch_ms)
|
||||
|
||||
# ---- 写 kline_daily (不复权原始价格, 只有 OHLCV) ----
|
||||
daily_df = self._build_daily(stock_records)
|
||||
if not daily_df.is_empty() and self._repo:
|
||||
try:
|
||||
self._repo.flush_live_daily(daily_df)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("日K写盘失败: %s", e)
|
||||
|
||||
# ---- 构建 API 直接值的补充表 (不写 daily, 只用于 enriched 计算) ----
|
||||
quote_extra = self._build_quote_extra(stock_records)
|
||||
|
||||
# ---- 增量计算 enriched + 写盘 + 更新缓存 ----
|
||||
if not daily_df.is_empty() and self._repo:
|
||||
self._flush_live_enriched(daily_df, quote_extra)
|
||||
|
||||
# ---- 通知 SSE ----
|
||||
self._update_event.set()
|
||||
|
||||
# ---- 策略监控 + 告警评估 ----
|
||||
self._evaluate_monitors(daily_df, quote_extra)
|
||||
|
||||
# ================================================================
|
||||
# 工具
|
||||
# ================================================================
|
||||
|
||||
@staticmethod
|
||||
def _build_daily(records: list[dict]) -> pl.DataFrame:
|
||||
"""将 API records 转为日K格式 DataFrame (只有 OHLCV, 写 kline_daily 用)。"""
|
||||
if not records:
|
||||
return pl.DataFrame()
|
||||
df = pl.DataFrame(records)
|
||||
cols_map = {
|
||||
"symbol": "symbol",
|
||||
"last_price": "close",
|
||||
"open": "open",
|
||||
"high": "high",
|
||||
"low": "low",
|
||||
"volume": "volume",
|
||||
"amount": "amount",
|
||||
}
|
||||
select_exprs = []
|
||||
for src, dst in cols_map.items():
|
||||
if src in df.columns:
|
||||
select_exprs.append(pl.col(src).alias(dst))
|
||||
if not select_exprs:
|
||||
return pl.DataFrame()
|
||||
result = df.select(select_exprs).with_columns(
|
||||
pl.lit(date.today()).cast(pl.Date).alias("date"),
|
||||
)
|
||||
# 修复: API 在非交易时段可能返回 open/high/low=0 或 null,
|
||||
# 导致蜡烛从 0 开始。用 close 填充这些异常值。
|
||||
for col in ("open", "high", "low"):
|
||||
if col in result.columns:
|
||||
result = result.with_columns(
|
||||
pl.when((pl.col(col) == 0) | pl.col(col).is_null())
|
||||
.then(pl.col("close"))
|
||||
.otherwise(pl.col(col))
|
||||
.alias(col)
|
||||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _build_quote_extra(records: list[dict]) -> pl.DataFrame:
|
||||
"""构建 API 直接提供的补充字段 (不写 daily, 只传给 enriched 计算)。
|
||||
|
||||
包含: prev_close, change_pct, change_amount, amplitude, turnover_rate。
|
||||
"""
|
||||
if not records:
|
||||
return pl.DataFrame()
|
||||
df = pl.DataFrame(records)
|
||||
keep = [c for c in [
|
||||
"symbol", "prev_close", "change_pct", "change_amount",
|
||||
"amplitude", "turnover_rate",
|
||||
] if c in df.columns]
|
||||
if not keep or "symbol" not in keep:
|
||||
return pl.DataFrame()
|
||||
return df.select(keep)
|
||||
|
||||
@staticmethod
|
||||
def _build_index_quotes(records: list[dict]) -> pl.DataFrame:
|
||||
"""构建指数实时行情缓存,不落股票 parquet。
|
||||
|
||||
注意: API 返回的 change_pct/amplitude 是小数 (0.0366 = 3.66%),
|
||||
统一转成百分比输出, 与 _fallback_index_quotes_from_daily 口径一致
|
||||
(前端指数侧不×100, 直接 toFixed(2)% 展示)。
|
||||
"""
|
||||
if not records:
|
||||
return pl.DataFrame()
|
||||
df = pl.DataFrame(records)
|
||||
keep = [c for c in [
|
||||
"symbol", "name", "last_price", "prev_close", "open", "high", "low",
|
||||
"volume", "amount", "change_pct", "change_amount", "amplitude", "timestamp", "session",
|
||||
] if c in df.columns]
|
||||
if not keep or "symbol" not in keep:
|
||||
return pl.DataFrame()
|
||||
df = df.select(keep)
|
||||
# change_pct / amplitude: 小数 → 百分比 (统一指数展示口径)
|
||||
for col in ("change_pct", "amplitude"):
|
||||
if col in df.columns:
|
||||
df = df.with_columns((pl.col(col).cast(pl.Float64) * 100).alias(col))
|
||||
if "last_price" in df.columns and "close" not in df.columns:
|
||||
df = df.with_columns(pl.col("last_price").alias("close"))
|
||||
return df
|
||||
|
||||
@staticmethod
|
||||
def _is_trading_hours() -> bool:
|
||||
now = datetime.now()
|
||||
t = now.time()
|
||||
morning = dt_time(9, 15) <= t <= dt_time(11, 35)
|
||||
afternoon = dt_time(12, 55) <= t <= dt_time(15, 5)
|
||||
return now.weekday() < 5 and (morning or afternoon)
|
||||
|
||||
@staticmethod
|
||||
def _save_enabled(enabled: bool) -> None:
|
||||
from app.services import preferences
|
||||
preferences.save({"realtime_quotes_enabled": enabled})
|
||||
|
||||
# ================================================================
|
||||
# 策略监控
|
||||
# ================================================================
|
||||
|
||||
def _evaluate_monitors(self, daily_df: pl.DataFrame, quote_extra: pl.DataFrame | None) -> None:
|
||||
"""行情更新后评估统一监控规则引擎,并刷新策略结果缓存。"""
|
||||
try:
|
||||
# 获取 enriched 数据 (刚算好的)
|
||||
enriched_today, enriched_date = self.get_enriched_today()
|
||||
if enriched_today.is_empty():
|
||||
return
|
||||
|
||||
all_alerts: list[dict] = []
|
||||
|
||||
# 通用监控规则评估 (统一引擎: signal/price/market/strategy)
|
||||
if self._app_state:
|
||||
engine = getattr(self._app_state, "monitor_engine", None)
|
||||
if engine and engine.rule_count > 0:
|
||||
# 预构建 symbol → name 映射 (enriched 已 drop name 列, 引擎触发时回填用)
|
||||
try:
|
||||
inst_df = self._app_state.repo.get_instruments()
|
||||
if not inst_df.is_empty() and "symbol" in inst_df.columns and "name" in inst_df.columns:
|
||||
engine.set_name_map({
|
||||
row["symbol"]: row["name"]
|
||||
for row in inst_df.select(["symbol", "name"]).iter_rows(named=True)
|
||||
if row.get("name")
|
||||
})
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("name_map 构建失败 (不影响监控): %s", e)
|
||||
rule_events = engine.evaluate(enriched_today)
|
||||
if rule_events:
|
||||
# 落盘到 alerts.jsonl
|
||||
try:
|
||||
from app.services import alert_store
|
||||
alert_store.append_many(
|
||||
self._app_state.repo.store.data_dir, rule_events,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("告警落盘失败: %s", e)
|
||||
# 转为 SSE 推送格式 (兼容旧 alert schema)
|
||||
for ev in rule_events:
|
||||
all_alerts.append({
|
||||
"source": ev["source"],
|
||||
"type": ev["type"],
|
||||
"rule_id": ev.get("rule_id"),
|
||||
"strategy_id": ev.get("rule_id") if ev["source"] == "strategy" else None,
|
||||
"symbol": ev["symbol"],
|
||||
"name": ev["name"],
|
||||
"message": ev["message"],
|
||||
"price": ev["price"],
|
||||
"change_pct": ev["change_pct"],
|
||||
"signals": ev["signals"],
|
||||
"severity": ev.get("severity", "info"),
|
||||
})
|
||||
|
||||
# 刷新策略结果缓存 (实时行情开启时,每轮行情更新后自动重算)
|
||||
if self._enabled and self._app_state:
|
||||
self._refresh_strategy_cache(enriched_today, enriched_date)
|
||||
|
||||
# 推入待推送队列 + 通知 SSE (含背压保护)
|
||||
if all_alerts:
|
||||
with self._lock:
|
||||
self._pending_alerts.extend(all_alerts)
|
||||
# 背压: 超出上限丢弃最旧
|
||||
if len(self._pending_alerts) > self._max_pending_alerts:
|
||||
overflow = len(self._pending_alerts) - self._max_pending_alerts
|
||||
self._pending_alerts = self._pending_alerts[overflow:]
|
||||
self._alert_event.set()
|
||||
logger.info("监控评估完成: %d 条通知", len(all_alerts))
|
||||
|
||||
# 系统通知 (可选通道, 由 preferences 开关控制)。
|
||||
# cooldown 去重已在 MonitorRuleEngine 做过, 这里只负责转发。
|
||||
self._maybe_send_system_notifications(all_alerts)
|
||||
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("监控评估失败: %s", e)
|
||||
|
||||
def _maybe_send_system_notifications(self, all_alerts: list[dict]) -> None:
|
||||
"""把告警转发到操作系统通知中心 (由 preferences 开关控制)。
|
||||
|
||||
- 开关关闭: 直接返回
|
||||
- 开关开启: 逐条发系统通知; 失败静默, 不阻断主流程
|
||||
- 去重: 复用 MonitorRuleEngine 的 cooldown, 此处不重复去重
|
||||
- 批量策略事件 (symbol="") 聚合为一条通知, 避免刷屏
|
||||
"""
|
||||
try:
|
||||
from app.services import preferences
|
||||
from app.services import notify_adapter
|
||||
|
||||
if not preferences.get_system_notify_enabled():
|
||||
return
|
||||
|
||||
for ev in all_alerts:
|
||||
# 通知标题: 用 source 分类 (策略/信号/价格/异动)
|
||||
source = ev.get("source", "")
|
||||
source_label = {
|
||||
"strategy": "策略", "signal": "信号",
|
||||
"price": "价格", "market": "异动",
|
||||
}.get(source, source or "通知")
|
||||
|
||||
name = ev.get("name") or ""
|
||||
symbol = ev.get("symbol") or ""
|
||||
message = ev.get("message") or ""
|
||||
|
||||
# 正文: 优先用现成 message, 拼上 symbol/name 让用户一眼定位
|
||||
if symbol:
|
||||
body = f"{symbol} {name} {message}".strip()
|
||||
else:
|
||||
body = message or name
|
||||
|
||||
title = f"Stock Panel · {source_label}"
|
||||
notify_adapter.notify(title, body)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.debug("系统通知发送异常 (不影响告警主流程): %s", e)
|
||||
|
||||
def _refresh_strategy_cache(self, enriched_today: pl.DataFrame, enriched_date: date | None) -> None:
|
||||
"""利用已计算好的 enriched 数据,运行策略池并写入缓存。"""
|
||||
import math
|
||||
from dataclasses import asdict
|
||||
from app.services import strategy_cache
|
||||
from app.services.screener import PRESET_STRATEGIES, ScreenerService
|
||||
from app.strategy import config as strategy_config
|
||||
|
||||
try:
|
||||
if enriched_date is None:
|
||||
return
|
||||
as_of = enriched_date
|
||||
data_dir = self._repo.store.data_dir
|
||||
svc = ScreenerService(self._repo)
|
||||
engine = getattr(self._app_state, "strategy_engine", None)
|
||||
|
||||
# 确定要运行的策略: 策略监控池中的策略
|
||||
monitor_ids = self._get_monitor_pool_ids()
|
||||
if not monitor_ids:
|
||||
return
|
||||
|
||||
# 一次加载所有 override
|
||||
all_overrides = strategy_config.list_overrides(data_dir)
|
||||
|
||||
# 历史策略: 只在需要时加载
|
||||
shared_history = None
|
||||
history_strats = []
|
||||
if engine:
|
||||
id_set = set(monitor_ids)
|
||||
history_strats = [
|
||||
(sid, s) for sid, s in engine._strategies.items()
|
||||
if s.filter_history_fn and sid in id_set
|
||||
]
|
||||
if history_strats:
|
||||
max_lb = max(s.lookback_days for _, s in history_strats)
|
||||
shared_history = svc._load_enriched_history(as_of, max(1, max_lb))
|
||||
|
||||
results: dict[str, dict] = {}
|
||||
for sid in monitor_ids:
|
||||
try:
|
||||
overrides = all_overrides.get(sid, {})
|
||||
bf = overrides.get("basic_filter") if overrides else None
|
||||
dl = overrides.get("display_limit") if overrides else None
|
||||
if dl is None and overrides and "display_limit" in overrides:
|
||||
dl = 0
|
||||
|
||||
if sid in PRESET_STRATEGIES:
|
||||
r = svc.run_preset(sid, as_of=as_of, precomputed=enriched_today, basic_filter=bf, display_limit=dl)
|
||||
elif engine:
|
||||
r = engine.run(
|
||||
sid, as_of, overrides=overrides or None,
|
||||
precomputed=enriched_today, precomputed_history=shared_history,
|
||||
)
|
||||
if dl is not None and dl > 0:
|
||||
r.rows = r.rows[:dl]
|
||||
r.total = min(r.total, dl)
|
||||
else:
|
||||
continue
|
||||
|
||||
# sanitize NaN/Inf
|
||||
rows = []
|
||||
for row_dict in asdict(r).get("rows", []):
|
||||
for k, v in list(row_dict.items()):
|
||||
if isinstance(v, float) and not math.isfinite(v):
|
||||
row_dict[k] = None
|
||||
rows.append(row_dict)
|
||||
results[sid] = {"total": r.total, "as_of": str(as_of), "rows": rows}
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
|
||||
if results:
|
||||
strategy_cache.write_cache(data_dir, str(as_of), results)
|
||||
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("策略缓存刷新失败: %s", e)
|
||||
|
||||
def _get_monitor_pool_ids(self) -> list[str]:
|
||||
"""获取策略监控池中的策略 ID 列表。"""
|
||||
from app.services import preferences
|
||||
ids = preferences.get_strategy_monitor_ids()
|
||||
if not ids:
|
||||
return []
|
||||
return [sid for sid in ids if sid]
|
||||
|
||||
@staticmethod
|
||||
def _get_strategy_monitor():
|
||||
"""获取 StrategyMonitorService — 不再使用, 改用 _app_state 注入。"""
|
||||
return None
|
||||
|
||||
# ================================================================
|
||||
# enriched 增量计算
|
||||
# ================================================================
|
||||
|
||||
def _flush_live_enriched(self, daily_df: pl.DataFrame, quote_extra: pl.DataFrame = None) -> None:
|
||||
"""增量计算今天的 enriched: 用昨天的递推状态 + 今天 OHLCV → 只算今天 5500 行。
|
||||
|
||||
quote_extra: API 直接提供的补充字段 (prev_close, change_pct 等),
|
||||
不写 daily, 直接传给 compute_enriched_today 避免重复计算。
|
||||
"""
|
||||
try:
|
||||
today = date.today()
|
||||
t0 = time.perf_counter()
|
||||
|
||||
# ---- 尝试增量路径 ----
|
||||
live_agg = self._repo.get_live_agg()
|
||||
prev_enriched, prev_date = self._repo.get_enriched_latest()
|
||||
|
||||
use_incremental = (
|
||||
not live_agg.is_empty()
|
||||
and not prev_enriched.is_empty()
|
||||
and prev_date is not None
|
||||
)
|
||||
|
||||
if use_incremental:
|
||||
from app.indicators.pipeline import compute_enriched_today
|
||||
instruments = self._repo.get_instruments()
|
||||
# 将 API 直接提供的补充字段 JOIN 到 daily_df
|
||||
today_ohlcv = daily_df
|
||||
if quote_extra is not None and not quote_extra.is_empty():
|
||||
today_ohlcv = daily_df.join(quote_extra, on="symbol", how="left")
|
||||
enriched_today = compute_enriched_today(
|
||||
live_agg=live_agg,
|
||||
prev_enriched=prev_enriched,
|
||||
today_ohlcv=today_ohlcv,
|
||||
instruments=instruments,
|
||||
)
|
||||
if enriched_today.is_empty():
|
||||
logger.warning("增量计算结果为空, 回退到全量计算")
|
||||
use_incremental = False
|
||||
|
||||
# ---- 全量回退路径 ----
|
||||
if not use_incremental:
|
||||
from datetime import timedelta
|
||||
from app.indicators.pipeline import compute_enriched
|
||||
|
||||
logger.info("enriched 全量计算 (live_agg=%s, 上次日期=%s)",
|
||||
"ok" if not live_agg.is_empty() else "空", prev_date)
|
||||
|
||||
cutoff = today - timedelta(days=90)
|
||||
daily_glob = str(self._repo.store.data_dir / "kline_daily" / "**" / "*.parquet")
|
||||
ohlcv_cols = ["symbol", "date", "open", "high", "low", "close", "volume", "amount"]
|
||||
hist_df = (
|
||||
pl.scan_parquet(daily_glob)
|
||||
.filter(pl.col("date") >= cutoff)
|
||||
.sort(["symbol", "date"])
|
||||
.collect()
|
||||
)
|
||||
if hist_df.is_empty():
|
||||
return
|
||||
|
||||
hist_cols = [c for c in ohlcv_cols if c in hist_df.columns]
|
||||
hist_df = hist_df.select(hist_cols).filter(pl.col("date") != today)
|
||||
daily_ohlcv = daily_df.select([c for c in ohlcv_cols if c in daily_df.columns])
|
||||
full_df = pl.concat([hist_df, daily_ohlcv], how="diagonal_relaxed")
|
||||
full_df = full_df.sort(["symbol", "date"])
|
||||
|
||||
factor_path = self._repo.store.data_dir / "adj_factor" / "all.parquet"
|
||||
factors = pl.DataFrame()
|
||||
if factor_path.exists():
|
||||
try:
|
||||
factors = pl.read_parquet(factor_path)
|
||||
except Exception:
|
||||
pass
|
||||
instruments = self._repo.get_instruments()
|
||||
|
||||
enriched_full = compute_enriched(full_df, factors=factors, instruments=instruments)
|
||||
enriched_today = enriched_full.filter(pl.col("date") == today)
|
||||
|
||||
if enriched_today.is_empty():
|
||||
return
|
||||
|
||||
# ---- 写盘 + 更新缓存 ----
|
||||
self._repo.flush_live_enriched(enriched_today)
|
||||
|
||||
elapsed = time.perf_counter() - t0
|
||||
mode_label = "增量" if use_incremental else "全量"
|
||||
logger.info("enriched %s: %d 只, %s, 耗时 %.0fms",
|
||||
mode_label, len(enriched_today), today, elapsed * 1000)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("enriched 计算失败: %s", e)
|
||||
@@ -0,0 +1,593 @@
|
||||
"""Screener 服务(§6.3)。
|
||||
|
||||
性能优化:
|
||||
- enriched parquet 仅存 14 列基础数据, 指标和信号即时计算
|
||||
- preset 策略: 从内存缓存或即时计算获取完整指标, ~10-50ms
|
||||
- custom SQL: DuckDB (用户传 SQL WHERE 字符串), ~10-50ms
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, timedelta
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.tickflow.repository import KlineRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── 进程级历史数据缓存 (避免 run_all 每次重新扫描 parquet + 计算指标) ──
|
||||
_history_cache: dict[tuple[date, int], tuple[float, pl.DataFrame]] = {}
|
||||
_HISTORY_CACHE_TTL = 120.0 # 秒
|
||||
|
||||
|
||||
# 内置预设策略 — Polars 表达式方式
|
||||
PRESET_STRATEGIES: dict[str, dict] = {
|
||||
"trend_breakout": {
|
||||
"name": "趋势突破",
|
||||
"description": "MA60 上方 + 60 日新高 + 量能 ≥ 2 倍均量",
|
||||
"filter": (
|
||||
(pl.col("close") > pl.col("ma60"))
|
||||
& pl.col("signal_n_day_high").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= 2.0)
|
||||
),
|
||||
"order_by": "momentum_60d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"ma_golden_cross": {
|
||||
"name": "MA 金叉",
|
||||
"description": "MA5 上穿 MA20 当日触发,量能配合",
|
||||
"filter": (
|
||||
pl.col("signal_ma_golden_5_20").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= 1.2)
|
||||
& (pl.col("close") > pl.col("ma60"))
|
||||
),
|
||||
"order_by": "momentum_20d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"macd_golden": {
|
||||
"name": "MACD 金叉放量",
|
||||
"description": "MACD 金叉当日 + 量能放大",
|
||||
"filter": (
|
||||
pl.col("signal_macd_golden").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= 1.5)
|
||||
),
|
||||
"order_by": "momentum_60d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"volume_price_surge": {
|
||||
"name": "量价齐升",
|
||||
"description": "突破 MA20 + 放量 + 收阳",
|
||||
"filter": (
|
||||
pl.col("signal_ma20_breakout").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= 2.0)
|
||||
& (pl.col("close") > pl.col("open"))
|
||||
),
|
||||
"order_by": "vol_ratio_5d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"low_volatility_leader": {
|
||||
"name": "低波动龙头",
|
||||
"description": "20 日动量为正 + 年化波动 < 30% + MA20 上方",
|
||||
"filter": (
|
||||
(pl.col("momentum_20d") > 0)
|
||||
& (pl.col("annual_vol_20d") < 0.30)
|
||||
& (pl.col("close") > pl.col("ma20"))
|
||||
),
|
||||
"order_by": "momentum_60d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"broken_board_recovery": {
|
||||
"name": "断板反包",
|
||||
"description": "连板 ≥2 后断板 1-2 天,出现放量反包信号",
|
||||
"filter": (
|
||||
pl.col("signal_limit_up").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= 1.5)
|
||||
& (pl.col("change_pct") > 0.03)
|
||||
),
|
||||
"order_by": "change_pct",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"oversold_bounce": {
|
||||
"name": "超跌反弹",
|
||||
"description": "RSI14 < 30 超卖区 + 当日收阳 + 放量,抄底信号",
|
||||
"filter": (
|
||||
(pl.col("rsi_14") < 30)
|
||||
& (pl.col("close") > pl.col("open"))
|
||||
& (pl.col("vol_ratio_5d") >= 1.2)
|
||||
),
|
||||
"order_by": "rsi_14",
|
||||
"descending": False,
|
||||
"limit": 100,
|
||||
},
|
||||
"boll_breakout": {
|
||||
"name": "布林突破",
|
||||
"description": "突破布林上轨 + 放量,强势加速信号",
|
||||
"filter": (
|
||||
pl.col("signal_boll_breakout_upper").fill_null(False)
|
||||
& (pl.col("vol_ratio_5d") >= 1.5)
|
||||
),
|
||||
"order_by": "vol_ratio_5d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"bullish_alignment": {
|
||||
"name": "均线多头",
|
||||
"description": "MA5 > MA10 > MA20 > MA60 多头排列 + 短期动量为正",
|
||||
"filter": (
|
||||
(pl.col("ma5") > pl.col("ma10"))
|
||||
& (pl.col("ma10") > pl.col("ma20"))
|
||||
& (pl.col("ma20") > pl.col("ma60"))
|
||||
& (pl.col("momentum_20d") > 0)
|
||||
),
|
||||
"order_by": "momentum_60d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"consecutive_limit_ups": {
|
||||
"name": "连板股",
|
||||
"description": "当日涨停且连续涨停 ≥ 2 天,强势追涨",
|
||||
"filter": (
|
||||
pl.col("signal_limit_up").fill_null(False)
|
||||
& (pl.col("consecutive_limit_ups") >= 2)
|
||||
),
|
||||
"order_by": "consecutive_limit_ups",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"pullback_to_support": {
|
||||
"name": "缩量回踩",
|
||||
"description": "回踩 MA20 附近 + 缩量 + 中期趋势向上",
|
||||
"filter": (
|
||||
(pl.col("close") > pl.col("ma20") * 0.98)
|
||||
& (pl.col("close") < pl.col("ma20") * 1.02)
|
||||
& (pl.col("vol_ratio_5d") < 0.8)
|
||||
& (pl.col("close") > pl.col("ma60"))
|
||||
& (pl.col("momentum_20d") > 0)
|
||||
),
|
||||
"order_by": "momentum_60d",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
"n_day_low_reversal": {
|
||||
"name": "新低反转",
|
||||
"description": "触及 60 日新低后当日收阳放量,反转信号",
|
||||
"filter": (
|
||||
pl.col("signal_n_day_low").fill_null(False)
|
||||
& (pl.col("close") > pl.col("open"))
|
||||
& (pl.col("vol_ratio_5d") >= 1.5)
|
||||
),
|
||||
"order_by": "change_pct",
|
||||
"descending": True,
|
||||
"limit": 100,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScreenerResult:
|
||||
as_of: date
|
||||
strategy: str | None
|
||||
rows: list[dict] = field(default_factory=list)
|
||||
total: int = 0
|
||||
elapsed_ms: float = 0.0
|
||||
|
||||
|
||||
class ScreenerService:
|
||||
def __init__(self, repo: KlineRepository) -> None:
|
||||
self.repo = repo
|
||||
|
||||
@staticmethod
|
||||
def clear_history_cache() -> None:
|
||||
"""清空进程级 _history_cache (TTL 缓存)。
|
||||
|
||||
清除数据后调用, 避免内存里的旧历史窗口残留导致策略/看板仍命中旧数据。
|
||||
"""
|
||||
_history_cache.clear()
|
||||
|
||||
def _load_enriched_for_date(self, target_date: date) -> pl.DataFrame:
|
||||
"""从 enriched parquet 读取指定日期的基础数据并即时计算完整指标+信号。
|
||||
|
||||
enriched parquet 仅存 14 列。读取后需要即时计算 ma/ema/macd/kdj/rsi/boll/momentum/signal 等列。
|
||||
对于最新日, 优先使用内存缓存 (已包含完整指标)。
|
||||
"""
|
||||
# 优先使用 repo 最新日缓存
|
||||
cache, cache_date = self.repo.get_enriched_latest()
|
||||
if cache is not None and not cache.is_empty() and cache_date == target_date:
|
||||
df = cache
|
||||
# JOIN instruments
|
||||
df_i = self.repo.get_instruments()
|
||||
if not df_i.is_empty():
|
||||
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"] if c in df_i.columns]
|
||||
if "name" not in df.columns:
|
||||
df = df.join(df_i.select(inst_cols), on="symbol", how="left")
|
||||
return df
|
||||
|
||||
# 尝试从 repo 级预计算历史缓存中提取目标日期
|
||||
cached_hist = self.repo.get_enriched_history(target_date, 1)
|
||||
if cached_hist is not None and not cached_hist.is_empty() and "date" in cached_hist.columns:
|
||||
df = cached_hist.filter(pl.col("date") == target_date)
|
||||
if not df.is_empty():
|
||||
logger.debug("_load_enriched_for_date: repo history cache for %s", target_date)
|
||||
# JOIN instruments
|
||||
df_i = self.repo.get_instruments()
|
||||
if not df_i.is_empty():
|
||||
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"] if c in df_i.columns]
|
||||
if "name" not in df.columns:
|
||||
df = df.join(df_i.select(inst_cols), on="symbol", how="left")
|
||||
return df
|
||||
|
||||
# 历史日期: 从 parquet 读取 14 列, 即时计算指标 (慢路径)
|
||||
enriched_dir = self.repo.store.data_dir / "kline_daily_enriched"
|
||||
ds = target_date.isoformat()
|
||||
target_parquet = enriched_dir / f"date={ds}" / "part.parquet"
|
||||
|
||||
if not target_parquet.exists():
|
||||
return pl.DataFrame()
|
||||
|
||||
try:
|
||||
df = pl.read_parquet(target_parquet)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("load_enriched_for_date failed: %s", e)
|
||||
return pl.DataFrame()
|
||||
|
||||
if df.is_empty():
|
||||
return df
|
||||
|
||||
# 即时计算指标: 需要加载历史窗口作 warmup
|
||||
df_full = self._compute_enriched_full(df, target_date)
|
||||
return df_full
|
||||
|
||||
def _compute_enriched_full(self, df_target: pl.DataFrame, target_date: date) -> pl.DataFrame:
|
||||
"""从 14 列基础数据即时计算完整 enriched (含全部指标和信号)。
|
||||
|
||||
读取历史数据作为指标计算的 warmup, 计算完成后只返回目标日期的行。
|
||||
"""
|
||||
from app.indicators.pipeline import compute_indicators, compute_signals, compute_limit_signals
|
||||
|
||||
# 加载 warmup 历史 (目标日期前 ~120 天)
|
||||
enriched_dir = self.repo.store.data_dir / "kline_daily_enriched"
|
||||
start = target_date - timedelta(days=150)
|
||||
read_cols = ["symbol", "date", "open", "high", "low", "close", "volume",
|
||||
"amount", "raw_close", "raw_high", "raw_low"]
|
||||
|
||||
try:
|
||||
lf = (
|
||||
pl.scan_parquet(str(enriched_dir / "**" / "*.parquet"))
|
||||
.filter(
|
||||
(pl.col("date") >= start)
|
||||
& (pl.col("date") <= target_date)
|
||||
)
|
||||
.sort(["symbol", "date"])
|
||||
)
|
||||
available = [c for c in read_cols if c in lf.schema]
|
||||
df_hist = lf.select(available).collect()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("warmup history load failed: %s", e)
|
||||
df_hist = df_target
|
||||
|
||||
if df_hist.is_empty():
|
||||
df_hist = df_target
|
||||
|
||||
# 计算指标
|
||||
df_full = compute_indicators(df_hist)
|
||||
df_full = compute_signals(df_full)
|
||||
|
||||
# 计算涨跌停信号 (需要 instruments)
|
||||
instruments = self.repo.get_instruments()
|
||||
if instruments is not None and not instruments.is_empty():
|
||||
df_full = compute_limit_signals(df_full, instruments)
|
||||
|
||||
# 只保留目标日期
|
||||
df_result = df_full.filter(pl.col("date") == target_date)
|
||||
|
||||
# JOIN instruments (name, total_shares, float_shares)
|
||||
if not instruments.is_empty():
|
||||
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"] if c in instruments.columns]
|
||||
if "name" not in df_result.columns:
|
||||
df_result = df_result.join(instruments.select(inst_cols), on="symbol", how="left")
|
||||
|
||||
return df_result
|
||||
|
||||
def _load_enriched_history(self, target_date: date, lookback_days: int) -> pl.DataFrame:
|
||||
"""读取目标日期之前的基础行情数据, 供历史窗口策略使用。
|
||||
|
||||
优先从 repo 内存缓存获取 (启动时已预计算), 命中时 0ms。
|
||||
缓存 miss 时走 scan_parquet + compute_indicators 慢路径。
|
||||
"""
|
||||
# 优先级 1: repo 级预计算缓存 (启动时 _refresh_enriched 已计算完整历史)
|
||||
t0 = time.perf_counter()
|
||||
cached = self.repo.get_enriched_history(target_date, lookback_days)
|
||||
if cached is not None and not cached.is_empty():
|
||||
# JOIN instruments (repo 缓存不含 name 等列)
|
||||
instruments = self.repo.get_instruments()
|
||||
if instruments is not None and not instruments.is_empty() and "name" not in cached.columns:
|
||||
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"]
|
||||
if c in instruments.columns]
|
||||
cached = cached.join(instruments.select(inst_cols), on="symbol", how="left")
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
logger.info("_load_enriched_history(%s, %d): repo cache hit, %.1fms, %d rows",
|
||||
target_date, lookback_days, elapsed, len(cached))
|
||||
return cached
|
||||
|
||||
# 优先级 2: 进程级 history_cache (之前的 TTL 缓存)
|
||||
cache_key = (target_date, lookback_days)
|
||||
now = time.monotonic()
|
||||
ttl_cached = _history_cache.get(cache_key)
|
||||
if ttl_cached is not None:
|
||||
ts, cached_df = ttl_cached
|
||||
if now - ts < _HISTORY_CACHE_TTL:
|
||||
logger.debug("history TTL cache hit: %s lookback=%d", target_date, lookback_days)
|
||||
return cached_df
|
||||
del _history_cache[cache_key]
|
||||
|
||||
# 优先级 3: scan_parquet + compute_indicators (慢路径, ~5s)
|
||||
logger.warning("_load_enriched_history cache miss, computing indicators (%s, %d)...",
|
||||
target_date, lookback_days)
|
||||
from app.indicators.pipeline import compute_indicators, compute_signals, compute_limit_signals
|
||||
|
||||
warmup = 60
|
||||
start = target_date - timedelta(days=min((lookback_days + warmup) * 2, 180))
|
||||
|
||||
enriched_dir = self.repo.store.data_dir / "kline_daily_enriched"
|
||||
read_cols = ["symbol", "date", "open", "high", "low", "close", "volume",
|
||||
"amount", "raw_close", "raw_high", "raw_low"]
|
||||
|
||||
try:
|
||||
lf = (
|
||||
pl.scan_parquet(str(enriched_dir / "**" / "*.parquet"))
|
||||
.filter((pl.col("date") >= start) & (pl.col("date") <= target_date))
|
||||
.sort(["symbol", "date"])
|
||||
)
|
||||
available = [c for c in read_cols if c in lf.collect_schema().names()]
|
||||
df_hist = lf.select(available).collect()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("load_enriched_history failed: %s", e)
|
||||
return pl.DataFrame()
|
||||
|
||||
if df_hist.is_empty():
|
||||
return pl.DataFrame()
|
||||
|
||||
df_full = compute_indicators(df_hist)
|
||||
df_full = compute_signals(df_full)
|
||||
|
||||
instruments = self.repo.get_instruments()
|
||||
if instruments is not None and not instruments.is_empty():
|
||||
df_full = compute_limit_signals(df_full, instruments)
|
||||
|
||||
if instruments is not None and not instruments.is_empty():
|
||||
inst_cols = [c for c in ["symbol", "name", "total_shares", "float_shares"] if c in instruments.columns]
|
||||
if "name" not in df_full.columns:
|
||||
df_full = df_full.join(instruments.select(inst_cols), on="symbol", how="left")
|
||||
|
||||
# 裁剪掉 warmup 部分, 只保留 lookback 范围 (减少 group_by 开销)
|
||||
lookback_start = target_date - timedelta(days=lookback_days)
|
||||
if "date" in df_full.columns:
|
||||
df_full = df_full.filter(pl.col("date") >= lookback_start)
|
||||
|
||||
df_full = df_full.sort(["symbol", "date"])
|
||||
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
logger.info("_load_enriched_history(%s, %d): computed in %.1fms, %d rows",
|
||||
target_date, lookback_days, elapsed, len(df_full))
|
||||
|
||||
_history_cache[cache_key] = (now, df_full)
|
||||
if len(_history_cache) > 10:
|
||||
expired = [k for k, (ts, _) in _history_cache.items() if now - ts > _HISTORY_CACHE_TTL]
|
||||
for k in expired:
|
||||
del _history_cache[k]
|
||||
|
||||
return df_full
|
||||
|
||||
def run(
|
||||
self,
|
||||
as_of: date,
|
||||
conditions: list[str],
|
||||
order_by: str | None = None,
|
||||
limit: int = 30,
|
||||
pool: list[str] | None = None,
|
||||
) -> ScreenerResult:
|
||||
"""自定义 SQL 条件选股。
|
||||
|
||||
先通过 Polars 即时计算完整指标, 再用 DuckDB 做 SQL WHERE 过滤。
|
||||
kline_enriched DuckDB 视图只有 14 列, 不能直接用于指标过滤。
|
||||
"""
|
||||
t0 = time.perf_counter()
|
||||
|
||||
if not conditions:
|
||||
return ScreenerResult(as_of=as_of, strategy=None)
|
||||
|
||||
# 从即时计算获取完整 enriched 数据
|
||||
df = self._load_enriched_for_date(as_of)
|
||||
if df.is_empty():
|
||||
return ScreenerResult(as_of=as_of, strategy=None)
|
||||
|
||||
# Pool 过滤
|
||||
if pool:
|
||||
df = df.filter(pl.col("symbol").is_in(pool))
|
||||
|
||||
# 用 DuckDB 做 SQL 过滤 (注册临时视图)
|
||||
try:
|
||||
import duckdb
|
||||
con = duckdb.connect(database=":memory:")
|
||||
con.register("enriched", df.to_arrow())
|
||||
where = " AND ".join(f"({c})" for c in conditions)
|
||||
sql = f"SELECT * FROM enriched WHERE {where}"
|
||||
if order_by:
|
||||
sql += f" ORDER BY {order_by}"
|
||||
if limit:
|
||||
sql += f" LIMIT {limit}"
|
||||
df_result = con.execute(sql).pl()
|
||||
con.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("screener SQL query failed: %s", e)
|
||||
df_result = pl.DataFrame()
|
||||
|
||||
rows = df_result.to_dicts() if not df_result.is_empty() else []
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
|
||||
return ScreenerResult(
|
||||
as_of=as_of,
|
||||
strategy=None,
|
||||
rows=rows,
|
||||
total=len(rows),
|
||||
elapsed_ms=elapsed,
|
||||
)
|
||||
|
||||
def run_preset(
|
||||
self,
|
||||
strategy_id: str,
|
||||
as_of: date,
|
||||
pool: list[str] | None = None,
|
||||
precomputed: pl.DataFrame | None = None,
|
||||
basic_filter: dict | None = None,
|
||||
display_limit: int | None = None,
|
||||
) -> ScreenerResult:
|
||||
"""预设策略选股 — 从 enriched 读取预计算好的指标列后过滤。
|
||||
|
||||
- precomputed 不为空: 直接复用(run_all 场景)
|
||||
- precomputed 为空: 从 enriched 读目标日期
|
||||
- basic_filter: 用户保存的基础参数过滤(boards、价格等)
|
||||
"""
|
||||
t0 = time.perf_counter()
|
||||
|
||||
strat = PRESET_STRATEGIES.get(strategy_id)
|
||||
if not strat:
|
||||
raise ValueError(f"unknown strategy: {strategy_id}")
|
||||
|
||||
if precomputed is not None and not precomputed.is_empty():
|
||||
df = precomputed
|
||||
else:
|
||||
df = self._load_enriched_for_date(as_of)
|
||||
if df.is_empty():
|
||||
return ScreenerResult(as_of=as_of, strategy=strategy_id)
|
||||
|
||||
# 应用用户基础参数过滤(boards、价格区间等)
|
||||
if basic_filter and basic_filter.get("enabled", True):
|
||||
df = self._apply_basic_filter(df, basic_filter)
|
||||
|
||||
# 应用策略过滤
|
||||
df = df.filter(strat["filter"])
|
||||
|
||||
# 应用 pool
|
||||
if pool:
|
||||
df = df.filter(pl.col("symbol").is_in(pool))
|
||||
|
||||
# 排序 + 限制
|
||||
order_col = strat["order_by"]
|
||||
if order_col in df.columns:
|
||||
df = df.sort(order_col, descending=strat.get("descending", True))
|
||||
|
||||
# display_limit: None=不限制, 0=全部, N=前N个
|
||||
if display_limit == 0:
|
||||
limit = None # 不限制
|
||||
elif display_limit is not None:
|
||||
limit = display_limit
|
||||
else:
|
||||
limit = None # 未配置时默认不限制
|
||||
if limit is not None and limit > 0:
|
||||
df = df.head(limit)
|
||||
|
||||
# 基于排序列生成 0-100 评分 (与 StrategyEngine 统一)
|
||||
if order_col in df.columns and not df.is_empty():
|
||||
col_vals = df[order_col].cast(pl.Float64)
|
||||
col_min = col_vals.min()
|
||||
col_max = col_vals.max()
|
||||
col_range = col_max - col_min
|
||||
if col_range and col_range > 0:
|
||||
normalized = (col_vals - col_min) / col_range
|
||||
else:
|
||||
normalized = pl.Series("norm", [0.5] * len(df))
|
||||
if not strat.get("descending", True):
|
||||
normalized = 1.0 - normalized
|
||||
df = df.with_columns((normalized * 100).alias("score"))
|
||||
|
||||
rows = df.to_dicts()
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
|
||||
# sanitize
|
||||
for r in rows:
|
||||
for k, v in list(r.items()):
|
||||
if isinstance(v, float) and (v != v or abs(v) == float("inf")):
|
||||
r[k] = None
|
||||
|
||||
return ScreenerResult(
|
||||
as_of=as_of,
|
||||
strategy=strategy_id,
|
||||
rows=rows,
|
||||
total=len(rows),
|
||||
elapsed_ms=elapsed,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _apply_basic_filter(df: pl.DataFrame, bf: dict) -> pl.DataFrame:
|
||||
"""应用用户基础参数过滤(boards、价格区间、市值等)"""
|
||||
exprs: list[pl.Expr] = []
|
||||
if bf.get("price_min") is not None:
|
||||
exprs.append(pl.col("close") >= bf["price_min"])
|
||||
if bf.get("price_max") is not None:
|
||||
exprs.append(pl.col("close") <= bf["price_max"])
|
||||
if bf.get("float_cap_min") is not None and "float_shares" in df.columns:
|
||||
exprs.append(pl.col("close") * pl.col("float_shares") >= bf["float_cap_min"])
|
||||
if bf.get("float_cap_max") is not None and "float_shares" in df.columns:
|
||||
exprs.append(pl.col("close") * pl.col("float_shares") <= bf["float_cap_max"])
|
||||
if bf.get("amount_min") is not None:
|
||||
exprs.append(pl.col("amount") >= bf["amount_min"])
|
||||
if bf.get("amount_max") is not None:
|
||||
exprs.append(pl.col("amount") <= bf["amount_max"])
|
||||
if bf.get("turnover_min") is not None and "turnover_rate" in df.columns:
|
||||
exprs.append(pl.col("turnover_rate") >= bf["turnover_min"])
|
||||
if bf.get("turnover_max") is not None and "turnover_rate" in df.columns:
|
||||
exprs.append(pl.col("turnover_rate") <= bf["turnover_max"])
|
||||
if bf.get("exclude_st") and "name" in df.columns:
|
||||
exprs.append(~pl.col("name").str.contains("(?i)ST|\\*ST|退"))
|
||||
# 板块过滤
|
||||
boards = bf.get("boards")
|
||||
if boards and isinstance(boards, list) and len(boards) > 0:
|
||||
board_exprs: list[pl.Expr] = []
|
||||
for b in boards:
|
||||
if b == "沪主板":
|
||||
board_exprs.append(pl.col("symbol").str.starts_with("60"))
|
||||
elif b == "深主板":
|
||||
board_exprs.append(
|
||||
pl.col("symbol").str.starts_with("00")
|
||||
| pl.col("symbol").str.starts_with("001")
|
||||
)
|
||||
elif b == "创业板":
|
||||
board_exprs.append(
|
||||
pl.col("symbol").str.starts_with("300")
|
||||
| pl.col("symbol").str.starts_with("301")
|
||||
)
|
||||
elif b == "科创板":
|
||||
board_exprs.append(pl.col("symbol").str.starts_with("688"))
|
||||
elif b == "北交所":
|
||||
board_exprs.append(pl.col("symbol").str.contains(r"\.BJ$"))
|
||||
if board_exprs:
|
||||
exprs.append(pl.any_horizontal(board_exprs))
|
||||
if exprs:
|
||||
return df.filter(pl.all_horizontal(exprs))
|
||||
return df
|
||||
|
||||
def latest_date(self) -> date | None:
|
||||
d = self.repo.enriched_latest_date()
|
||||
if d:
|
||||
return d
|
||||
# 回退 DuckDB
|
||||
try:
|
||||
res = self.repo.execute_one(
|
||||
"SELECT max(date) FROM kline_enriched",
|
||||
)
|
||||
if res and res[0]:
|
||||
d = res[0]
|
||||
return d if isinstance(d, date) else date.fromisoformat(str(d))
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
return None
|
||||
@@ -0,0 +1,150 @@
|
||||
"""策略结果缓存 — 写入本地文件,供策略页面秒加载。
|
||||
|
||||
缓存结构:
|
||||
{
|
||||
"as_of": "2024-01-15",
|
||||
"results": { strategy_id: { total, as_of, rows } },
|
||||
"today_ever_matched": { strategy_id: [symbol, ...] }, // 今日曾命中 symbol 并集
|
||||
"today_ever_rows": { strategy_id: { symbol: row_data } },// 今日曾命中的完整行数据
|
||||
"updated_at": 1705324800000 # Unix ms
|
||||
}
|
||||
|
||||
文件路径: data/user_data/strategy_cache.json
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _json_default(obj: Any) -> Any:
|
||||
"""处理 date/datetime 等 JSON 不认识的类型。"""
|
||||
if isinstance(obj, date):
|
||||
return obj.isoformat()
|
||||
if isinstance(obj, datetime):
|
||||
return obj.isoformat()
|
||||
raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable")
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CACHE_FILENAME = "strategy_cache.json"
|
||||
|
||||
|
||||
def _cache_path(data_dir: Path) -> Path:
|
||||
return data_dir / "user_data" / _CACHE_FILENAME
|
||||
|
||||
|
||||
def _enriched_parquet_path(data_dir: Path, as_of: str) -> Path:
|
||||
"""返回 enriched parquet 文件路径。"""
|
||||
return data_dir / "kline_daily_enriched" / f"date={as_of}" / "part.parquet"
|
||||
|
||||
|
||||
def _get_enriched_mtime(data_dir: Path, as_of: str) -> float | None:
|
||||
"""返回 enriched parquet 文件的 mtime (秒)。文件不存在返回 None。"""
|
||||
p = _enriched_parquet_path(data_dir, as_of)
|
||||
try:
|
||||
return p.stat().st_mtime
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def read_cache(data_dir: Path) -> dict | None:
|
||||
"""读取策略缓存文件。返回 None 表示无缓存、读取失败或 enriched 数据已更新导致缓存过期。"""
|
||||
path = _cache_path(data_dir)
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if not text.strip():
|
||||
return None
|
||||
cached = json.loads(text)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("读取策略缓存失败: %s", e)
|
||||
return None
|
||||
|
||||
# 校验 enriched mtime: 数据文件变化 → 缓存过期
|
||||
as_of = cached.get("as_of")
|
||||
stored_mtime = cached.get("enriched_mtime")
|
||||
if as_of and stored_mtime:
|
||||
current_mtime = _get_enriched_mtime(data_dir, as_of)
|
||||
if current_mtime is not None and current_mtime != stored_mtime:
|
||||
logger.info("策略缓存过期: enriched 数据已更新 (as_of=%s)", as_of)
|
||||
return None
|
||||
|
||||
return cached
|
||||
|
||||
|
||||
def _rows_to_symbol_map(rows: list[dict]) -> dict[str, dict]:
|
||||
"""将 rows 列表转为 {symbol: row_data} 映射。"""
|
||||
result: dict[str, dict] = {}
|
||||
for row in rows:
|
||||
sym = row.get("symbol")
|
||||
if sym:
|
||||
result[sym] = row
|
||||
return result
|
||||
|
||||
|
||||
def write_cache(
|
||||
data_dir: Path,
|
||||
as_of: str,
|
||||
results: dict[str, Any],
|
||||
) -> None:
|
||||
"""将策略结果写入缓存文件,同时更新今日曾命中集合。
|
||||
|
||||
- 日期变更时重置 today_ever_matched 和 today_ever_rows
|
||||
- 同一天内合并 (并集) 之前曾命中的 symbol,并用最新行数据更新
|
||||
"""
|
||||
path = _cache_path(data_dir)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 读取旧缓存
|
||||
old = read_cache(data_dir)
|
||||
old_as_of = old.get("as_of") if old else None
|
||||
old_ever_rows: dict[str, dict[str, dict]] = old.get("today_ever_rows", {}) if old else {}
|
||||
|
||||
# 当前命中的行数据 → symbol 映射
|
||||
current_row_maps: dict[str, dict[str, dict]] = {}
|
||||
for sid, r in results.items():
|
||||
current_row_maps[sid] = _rows_to_symbol_map(r.get("rows", []))
|
||||
|
||||
if old_as_of and old_as_of == as_of and old_ever_rows:
|
||||
# 同一天: 合并 — 用当前行数据更新旧数据 (保持最新价格等)
|
||||
merged_rows: dict[str, dict[str, dict]] = {}
|
||||
all_keys = set(old_ever_rows.keys()) | set(current_row_maps.keys())
|
||||
for sid in all_keys:
|
||||
old_map = old_ever_rows.get(sid, {})
|
||||
cur_map = current_row_maps.get(sid, {})
|
||||
# 以旧数据为基础,用当前数据覆盖 (当前数据更新鲜)
|
||||
combined = {**old_map, **cur_map}
|
||||
merged_rows[sid] = combined
|
||||
today_ever_rows = merged_rows
|
||||
else:
|
||||
# 新的一天或首次写入
|
||||
today_ever_rows = current_row_maps
|
||||
|
||||
# 从 ever_rows 提取 symbol 列表 (用于快速计数)
|
||||
today_ever_matched = {sid: sorted(maps.keys()) for sid, maps in today_ever_rows.items()}
|
||||
|
||||
# 记录 enriched parquet 文件的 mtime,用于后续校验缓存是否过期
|
||||
enriched_mtime = _get_enriched_mtime(data_dir, as_of)
|
||||
|
||||
payload = {
|
||||
"as_of": as_of,
|
||||
"results": results,
|
||||
"today_ever_matched": today_ever_matched,
|
||||
"today_ever_rows": today_ever_rows,
|
||||
"enriched_mtime": enriched_mtime,
|
||||
"updated_at": int(time.time() * 1000),
|
||||
}
|
||||
try:
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, default=_json_default), encoding="utf-8")
|
||||
total_rows = sum(len(r.get("rows", [])) for r in results.values())
|
||||
total_ever = sum(len(v) for v in today_ever_matched.values())
|
||||
logger.info("策略缓存已写入: %s, %d 策略, %d 命中, %d 曾命中", as_of, len(results), total_rows, total_ever)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("写入策略缓存失败: %s", e)
|
||||
@@ -0,0 +1,130 @@
|
||||
"""自选股服务(§6.1)。
|
||||
|
||||
存储:`data/user_data/watchlist.parquet`,字段 symbol + added_at + note。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
|
||||
from app.config import settings
|
||||
from app.tickflow.capabilities import Cap, CapabilitySet
|
||||
from app.tickflow.client import get_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _path() -> Path:
|
||||
p = settings.data_dir / "user_data" / "watchlist.parquet"
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
return p
|
||||
|
||||
|
||||
def list_symbols() -> list[dict]:
|
||||
p = _path()
|
||||
if not p.exists():
|
||||
return []
|
||||
df = pl.read_parquet(p)
|
||||
if df.is_empty():
|
||||
return []
|
||||
return df.to_dicts()
|
||||
|
||||
|
||||
def add(symbol: str, note: str = "") -> list[dict]:
|
||||
p = _path()
|
||||
if p.exists():
|
||||
df = pl.read_parquet(p)
|
||||
# 已存在则先移除,后面重新插入到最前面
|
||||
if symbol in df["symbol"].to_list():
|
||||
df = df.filter(pl.col("symbol") != symbol)
|
||||
else:
|
||||
df = pl.DataFrame(schema={"symbol": pl.Utf8, "added_at": pl.Utf8, "note": pl.Utf8})
|
||||
|
||||
new_row = pl.DataFrame({
|
||||
"symbol": [symbol],
|
||||
"added_at": [datetime.utcnow().isoformat(timespec="seconds")],
|
||||
"note": [note],
|
||||
})
|
||||
out = pl.concat([new_row, df], how="diagonal_relaxed")
|
||||
out.write_parquet(p)
|
||||
return out.to_dicts()
|
||||
|
||||
|
||||
def remove(symbol: str) -> list[dict]:
|
||||
p = _path()
|
||||
if not p.exists():
|
||||
return []
|
||||
df = pl.read_parquet(p)
|
||||
df = df.filter(pl.col("symbol") != symbol)
|
||||
df.write_parquet(p)
|
||||
return df.to_dicts()
|
||||
|
||||
|
||||
def clear() -> int:
|
||||
"""清空自选列表。返回移除的数量。"""
|
||||
p = _path()
|
||||
if not p.exists():
|
||||
return 0
|
||||
df = pl.read_parquet(p)
|
||||
count = df.height
|
||||
if count > 0:
|
||||
pl.DataFrame(schema={"symbol": pl.Utf8, "added_at": pl.Utf8, "note": pl.Utf8}).write_parquet(p)
|
||||
return count
|
||||
|
||||
|
||||
def fetch_quotes(symbols: list[str], capset: CapabilitySet, timeout_s: float = 8.0) -> list[dict]:
|
||||
"""拉取实时行情。
|
||||
|
||||
优先用 quote.batch;否则降级为 quote.by_symbol 单股请求。
|
||||
timeout_s: 单批次请求超时(秒),防止 API 卡死阻塞整个请求。
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout
|
||||
|
||||
if not symbols:
|
||||
return []
|
||||
|
||||
tf = get_client()
|
||||
quotes: list[dict] = []
|
||||
|
||||
# 走 batch
|
||||
batch_size = 5
|
||||
if capset.has(Cap.QUOTE_BATCH):
|
||||
lim = capset.limits(Cap.QUOTE_BATCH)
|
||||
batch_size = lim.batch if lim and lim.batch else 50
|
||||
elif capset.has(Cap.QUOTE_BY_SYMBOL):
|
||||
lim = capset.limits(Cap.QUOTE_BY_SYMBOL)
|
||||
batch_size = lim.batch if lim and lim.batch else 5
|
||||
else:
|
||||
# 无任何实时行情能力(none/free 档走 free-api 服务器,不提供实时行情)
|
||||
# 提前返回空,避免发起注定失败的请求
|
||||
return []
|
||||
|
||||
chunks = [symbols[i:i + batch_size] for i in range(0, len(symbols), batch_size)]
|
||||
|
||||
# 用线程池为每个批次加超时保护
|
||||
pool = ThreadPoolExecutor(max_workers=1)
|
||||
for chunk in chunks:
|
||||
try:
|
||||
future = pool.submit(tf.quotes.get, symbols=chunk, as_dataframe=True)
|
||||
raw = future.result(timeout=timeout_s)
|
||||
if raw is None or len(raw) == 0:
|
||||
continue
|
||||
df = pl.from_pandas(raw)
|
||||
rename_map = {
|
||||
"last_price": "price",
|
||||
"ext.change_pct": "pct",
|
||||
"ext.name": "name",
|
||||
}
|
||||
df = df.rename({k: v for k, v in rename_map.items() if k in df.columns})
|
||||
quotes.extend(df.to_dicts())
|
||||
except FuturesTimeout:
|
||||
logger.warning("quote fetch timeout (%.1fs) for %d symbols", timeout_s, len(chunk))
|
||||
break # 超时后不再尝试后续批次
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("quote fetch failed for %d symbols: %s", len(chunk), e)
|
||||
pool.shutdown(wait=False)
|
||||
|
||||
return quotes
|
||||
Reference in New Issue
Block a user