823 lines
32 KiB
Python
823 lines
32 KiB
Python
"""K 线 / 同步 API。"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from datetime import date, timedelta
|
|
from typing import Optional
|
|
|
|
from fastapi import APIRouter, HTTPException, Query, Request
|
|
|
|
from app.indicators.pipeline import compute_enriched, compute_enriched_single
|
|
from app.services import kline_sync
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
router = APIRouter(prefix="/api/kline", tags=["kline"])
|
|
|
|
|
|
@router.get("/instruments/search")
|
|
def search_instruments(
|
|
request: Request,
|
|
q: str = Query("", min_length=0, max_length=50, description="搜索关键词"),
|
|
limit: int = Query(20, ge=1, le=50),
|
|
):
|
|
"""模糊搜索标的 (代码 / 名称)。从内存 instruments 缓存中查。"""
|
|
repo = request.app.state.repo
|
|
df = repo.get_instruments()
|
|
if df.is_empty() or not q.strip():
|
|
return {"results": []}
|
|
|
|
keyword = q.strip().upper()
|
|
import polars as pl
|
|
|
|
# code/symbol 前缀优先，再 name 包含匹配
|
|
prefix_mask = (
|
|
pl.col("code").str.starts_with(keyword)
|
|
| pl.col("symbol").str.to_uppercase().str.starts_with(keyword)
|
|
)
|
|
contains_mask = (
|
|
pl.col("code").str.contains(keyword, literal=True)
|
|
| pl.col("symbol").str.to_uppercase().str.contains(keyword, literal=True)
|
|
| pl.col("name").str.contains(keyword, literal=True)
|
|
)
|
|
|
|
# 前缀匹配优先，剩余名额用包含匹配补充
|
|
prefix_hits = df.filter(prefix_mask).head(limit)
|
|
if prefix_hits.height >= limit:
|
|
matched = prefix_hits
|
|
else:
|
|
remaining = limit - prefix_hits.height
|
|
# 排除已匹配的 symbol
|
|
prefix_symbols = set(prefix_hits["symbol"].to_list()) if not prefix_hits.is_empty() else set()
|
|
contain_hits = df.filter(contains_mask & ~pl.col("symbol").is_in(prefix_symbols)).head(remaining)
|
|
matched = pl.concat([prefix_hits, contain_hits]) if not prefix_hits.is_empty() else contain_hits
|
|
rows = matched.select(["symbol", "name", "code"]).to_dicts()
|
|
return {"results": rows}
|
|
|
|
|
|
@router.post("/instruments/names")
|
|
def instruments_names(request: Request, symbols: list[str]):
|
|
"""批量查股票名称。传入 symbol 列表, 返回 {symbol: name}。"""
|
|
if not symbols:
|
|
return {"names": {}}
|
|
repo = request.app.state.repo
|
|
df = repo.get_instruments()
|
|
if df.is_empty():
|
|
return {"names": {}}
|
|
import polars as pl
|
|
matched = df.filter(pl.col("symbol").is_in(symbols)).select(["symbol", "name"])
|
|
names = {row["symbol"]: row["name"] for row in matched.iter_rows(named=True)}
|
|
return {"names": names}
|
|
|
|
|
|
def _get_stock_info(repo, symbol: str) -> dict:
|
|
"""从 instruments 视图查标的名称 + 股本。"""
|
|
try:
|
|
row = repo.execute_one(
|
|
"SELECT name, total_shares, float_shares FROM instruments WHERE symbol = ? LIMIT 1",
|
|
[symbol],
|
|
)
|
|
except Exception: # noqa: BLE001
|
|
return {}
|
|
if not row:
|
|
return {}
|
|
return {
|
|
"name": row[0],
|
|
"total_shares": row[1],
|
|
"float_shares": row[2],
|
|
}
|
|
|
|
|
|
@router.get("/daily")
|
|
def get_daily(
|
|
request: Request,
|
|
symbol: str = Query(..., description="标的代码,如 000001.SZ"),
|
|
days: int = Query(120, ge=10, le=2000),
|
|
start_date: Optional[str] = Query(None, description="起始日期 YYYY-MM-DD, 优先于 days"),
|
|
end_date: Optional[str] = Query(None, description="截止日期 YYYY-MM-DD, 默认今天"),
|
|
ext_columns: Optional[str] = Query(None, description="逗号分隔的 ext 列: config_id.field_name"),
|
|
):
|
|
"""读取本地 enriched 表中某只股票的日 K。
|
|
|
|
- 若 QuoteService 有实时行情, 追加/覆盖今日实时蜡烛
|
|
- Free 用户: 若 enriched 表里没有该股票, 实时拉取 + 本地算 enriched 返回
|
|
- ext_columns: 可选，动态 LEFT JOIN 扩展数据表，结果平铺到 stock_info.ext 下
|
|
(key 为 "{config_id}__{field_name}")，供日K信息条等场景展示自定义字段
|
|
"""
|
|
import polars as pl
|
|
|
|
repo = request.app.state.repo
|
|
end = date.fromisoformat(end_date) if end_date else date.today()
|
|
if start_date:
|
|
start = date.fromisoformat(start_date)
|
|
else:
|
|
start = end - timedelta(days=days)
|
|
|
|
stock_info = _get_stock_info(repo, symbol)
|
|
stock_name = stock_info.get("name")
|
|
|
|
# 从 enriched 表读取 (已含前复权 OHLCV + 技术指标 + 信号)
|
|
df = repo.get_daily(symbol, start, end)
|
|
|
|
if df.is_empty():
|
|
try:
|
|
raw = kline_sync.sync_daily_batch([symbol], count=days + 30)
|
|
except Exception as e:
|
|
raise HTTPException(status_code=502, detail=f"TickFlow fetch failed: {e}") from e
|
|
if raw.is_empty():
|
|
return {"symbol": symbol, "name": stock_name, "stock_info": stock_info, "rows": []}
|
|
# 拉除权因子做前复权 (Starter+ 有权限), 否则空 df → compute_enriched 退回未复权
|
|
factors = pl.DataFrame()
|
|
capset = getattr(request.app.state, "capabilities", None)
|
|
try:
|
|
from app.tickflow.capabilities import Cap
|
|
if capset and capset.has(Cap.ADJ_FACTOR):
|
|
factors = kline_sync.fetch_adj_factor_single(symbol)
|
|
except Exception as e: # noqa: BLE001
|
|
logger.debug("单股除权因子拉取失败 %s: %s", symbol, e)
|
|
enriched = compute_enriched(raw, factors=factors)
|
|
rows = enriched.tail(days).to_dicts()
|
|
# 即使 live 模式也尝试追加实时蜡烛
|
|
rows = _maybe_inject_live_candle(request, symbol, rows)
|
|
resp = {"symbol": symbol, "name": stock_name, "stock_info": stock_info, "rows": rows, "source": "live"}
|
|
return _attach_ext(resp, repo, symbol, ext_columns)
|
|
|
|
rows = df.to_dicts()
|
|
|
|
# 追加/覆盖今日实时蜡烛
|
|
rows = _maybe_inject_live_candle(request, symbol, rows)
|
|
|
|
resp = {"symbol": symbol, "name": stock_name, "stock_info": stock_info, "rows": rows, "source": "enriched"}
|
|
return _attach_ext(resp, repo, symbol, ext_columns)
|
|
|
|
|
|
def _attach_ext(resp: dict, repo, symbol: str, ext_columns: Optional[str]) -> dict:
|
|
"""按 ext_columns 规格为单只股票 LEFT JOIN 扩展数据，平铺到 stock_info['ext']。
|
|
|
|
key 形如 "{config_id}__{field_name}"，与自选列表 enriched 接口保持一致。
|
|
JOIN 逻辑参考 watchlist.watchlist_enriched；任何 ext 表/字段缺失都静默跳过。
|
|
"""
|
|
if not ext_columns or not ext_columns.strip():
|
|
return resp
|
|
|
|
specs: list[tuple[str, str]] = []
|
|
for part in ext_columns.split(","):
|
|
part = part.strip()
|
|
if "." not in part:
|
|
continue
|
|
config_id, field_name = part.split(".", 1)
|
|
config_id, field_name = config_id.strip(), field_name.strip()
|
|
if config_id and field_name:
|
|
specs.append((config_id, field_name))
|
|
if not specs:
|
|
return resp
|
|
|
|
import polars as pl
|
|
data_dir = repo.store.data_dir
|
|
try:
|
|
from app.services.ext_data import ExtConfigStore
|
|
from app.api.ext_data import _read_ext_dataframe
|
|
ext_store = ExtConfigStore(data_dir)
|
|
configs = {c.id: c for c in ext_store.load_all()}
|
|
except Exception: # noqa: BLE001
|
|
configs = {}
|
|
|
|
ext_values: dict = {}
|
|
for config_id, field_name in specs:
|
|
ext_col_name = f"{config_id}__{field_name}"
|
|
value = None
|
|
try:
|
|
cfg = configs.get(config_id)
|
|
if cfg:
|
|
ext_df, _ = _read_ext_dataframe(cfg, data_dir)
|
|
else:
|
|
ext_df = pl.from_arrow(
|
|
repo.store.db.query(
|
|
f'SELECT symbol, "{field_name}" FROM ext_{config_id}'
|
|
).arrow()
|
|
)
|
|
if not ext_df.is_empty() and "symbol" in ext_df.columns and field_name in ext_df.columns:
|
|
# 时序表取最新分区，避免一个 symbol 多行
|
|
row = (
|
|
ext_df
|
|
.select(["symbol", field_name])
|
|
.unique(subset=["symbol"], keep="last")
|
|
.filter(pl.col("symbol") == symbol)
|
|
)
|
|
if not row.is_empty():
|
|
value = row[field_name][0]
|
|
except Exception as e: # noqa: BLE001
|
|
logger.debug("kline ext join failed for %s.%s: %s", config_id, field_name, e)
|
|
ext_values[ext_col_name] = value
|
|
|
|
stock_info = dict(resp.get("stock_info") or {})
|
|
stock_info["ext"] = ext_values
|
|
resp["stock_info"] = stock_info
|
|
return resp
|
|
|
|
|
|
def _maybe_inject_live_candle(request: Request, symbol: str, rows: list[dict]) -> list[dict]:
|
|
"""如果 QuoteService 有实时 enriched 数据, 用实时数据生成今日蜡烛并追加/覆盖。"""
|
|
qs = getattr(request.app.state, "quote_service", None)
|
|
if not qs:
|
|
return rows
|
|
|
|
df_today, enriched_date = qs.get_enriched_today()
|
|
if df_today.is_empty():
|
|
return rows
|
|
|
|
# 非交易日（周末/假日）缓存的行情日期 != 今天，跳过注入避免产生重复蜡烛
|
|
if not enriched_date or enriched_date != date.today():
|
|
return rows
|
|
|
|
# 查找该 symbol 的实时 enriched 行
|
|
import polars as pl
|
|
try:
|
|
q = df_today.filter(pl.col("symbol") == symbol).to_dicts()
|
|
if not q:
|
|
return rows
|
|
q = q[0]
|
|
except Exception: # noqa: BLE001
|
|
return rows
|
|
|
|
close_price = q.get("close")
|
|
if not close_price or close_price <= 0:
|
|
return rows
|
|
|
|
today_str = str(enriched_date)
|
|
|
|
# enriched 行已包含 OHLCV + 全套指标, 直接用它
|
|
# 修复: API 在非交易时段可能返回 open/high/low=0, 用 close 填充避免异常蜡烛
|
|
raw_open = q.get("open")
|
|
raw_high = q.get("high")
|
|
raw_low = q.get("low")
|
|
live_row: dict = {
|
|
"date": today_str,
|
|
"symbol": symbol,
|
|
"open": raw_open if raw_open and raw_open > 0 else close_price,
|
|
"high": raw_high if raw_high and raw_high > 0 else close_price,
|
|
"low": raw_low if raw_low and raw_low > 0 else close_price,
|
|
"close": close_price,
|
|
"volume": q.get("volume"),
|
|
"amount": q.get("amount"),
|
|
"change_pct": q.get("change_pct"),
|
|
"is_live": True,
|
|
}
|
|
# 补上 enriched 的技术指标字段
|
|
for key in ("ma5", "ma10", "ma20", "ma30", "ma60",
|
|
"macd_dif", "macd_dea", "macd_hist",
|
|
"kdj_k", "kdj_d", "kdj_j",
|
|
"boll_upper", "boll_lower",
|
|
"rsi_6", "rsi_14", "rsi_24",
|
|
"atr_14", "vol_ratio_5d"):
|
|
if key in q and q[key] is not None:
|
|
live_row[key] = q[key]
|
|
|
|
# 如果已有今天的 enriched 行, 覆盖; 否则追加
|
|
found = False
|
|
for i, r in enumerate(rows):
|
|
if str(r.get("date")) == today_str:
|
|
r.update(live_row)
|
|
found = True
|
|
break
|
|
|
|
if not found:
|
|
rows.append(live_row)
|
|
|
|
return rows
|
|
|
|
|
|
class DailyBatchRequest:
|
|
"""批量日K请求。"""
|
|
symbols: list[str]
|
|
days: int = 12
|
|
|
|
|
|
@router.post("/daily-batch")
|
|
def get_daily_batch(request: Request, body: dict):
|
|
"""批量获取多只股票最近 N 天日K (OHLCV)。
|
|
|
|
用于自选列表迷你蜡烛图等场景，只返回基础列，不返回全部 enriched 指标。
|
|
"""
|
|
symbols = body.get("symbols", [])
|
|
days = body.get("days", 12)
|
|
if not symbols:
|
|
return {"data": {}}
|
|
days = max(5, min(60, days))
|
|
|
|
repo = request.app.state.repo
|
|
import polars as pl
|
|
from datetime import date, timedelta
|
|
|
|
end = date.today()
|
|
start = end - timedelta(days=days * 2) # 多取一些确保交易日够
|
|
|
|
cols = ["symbol", "date", "open", "high", "low", "close", "volume"]
|
|
df = repo.get_daily_batch(symbols, start, end, columns=cols)
|
|
|
|
if df.is_empty():
|
|
return {"data": {}}
|
|
|
|
# 按 symbol 分组, 每只取最近 N 条
|
|
result: dict[str, list[dict]] = {}
|
|
for sym in symbols:
|
|
sub = df.filter(pl.col("symbol") == sym).sort("date").tail(days)
|
|
if not sub.is_empty():
|
|
result[sym] = sub.to_dicts()
|
|
|
|
return {"data": result}
|
|
|
|
|
|
@router.get("/minute")
|
|
def get_minute(
|
|
request: Request,
|
|
symbol: str = Query(..., description="标的代码"),
|
|
trade_date: date | None = Query(None, alias="date", description="交易日期, 默认最新"),
|
|
):
|
|
"""读取某只股票某天的分钟 K 线。
|
|
|
|
- 本地有完整数据(240条) → 直接返回
|
|
- 本地无数据或不完整 → 从 TickFlow 实时拉取返回（不写入）
|
|
"""
|
|
repo = request.app.state.repo
|
|
stock_info = _get_stock_info(repo, symbol)
|
|
stock_name = stock_info.get("name")
|
|
|
|
if trade_date is None:
|
|
trade_date = repo.latest_minute_date(symbol)
|
|
if trade_date is None:
|
|
# 本地无任何分钟K，尝试从 TickFlow 拉取当天
|
|
trade_date = date.today()
|
|
df = kline_sync.fetch_minute_single(symbol, trade_date)
|
|
return {
|
|
"symbol": symbol, "name": stock_name, "stock_info": stock_info,
|
|
"date": str(trade_date), "rows": df.to_dicts(), "source": "live",
|
|
}
|
|
|
|
df = repo.get_minute(symbol, trade_date)
|
|
|
|
# 完整交易日应有 240 条分钟K；如果是今天(盘中)，期望条数按已交易分钟估算
|
|
expected = 240
|
|
today = date.today()
|
|
if trade_date == today:
|
|
from datetime import datetime as _dt
|
|
now = _dt.now()
|
|
h, m = now.hour, now.minute
|
|
if h < 9 or (h == 9 and m < 30):
|
|
expected = 0 # 还没开盘
|
|
elif h < 12 or (h == 12 and m == 0):
|
|
expected = (h - 9) * 60 + m - 30 # 9:30 起
|
|
elif h < 13:
|
|
expected = 120 # 午休
|
|
elif h < 15:
|
|
expected = 120 + (h - 13) * 60 + m
|
|
else:
|
|
expected = 240
|
|
|
|
is_complete = not df.is_empty() and len(df) >= expected * 0.9 # 允许 10% 容差
|
|
|
|
if is_complete:
|
|
return {
|
|
"symbol": symbol, "name": stock_name, "stock_info": stock_info,
|
|
"date": str(trade_date), "rows": df.to_dicts(), "source": "local",
|
|
}
|
|
|
|
# 本地不完整或无数据 → 从 TickFlow 实时拉取
|
|
live_df = kline_sync.fetch_minute_single(symbol, trade_date)
|
|
return {
|
|
"symbol": symbol, "name": stock_name, "stock_info": stock_info,
|
|
"date": str(trade_date), "rows": live_df.to_dicts(),
|
|
"source": "live" if not live_df.is_empty() else "none",
|
|
}
|
|
|
|
|
|
@router.post("/sync")
|
|
def sync_symbol(
|
|
request: Request,
|
|
symbol: str = Query(...),
|
|
days: int = Query(250, ge=10, le=2000),
|
|
):
|
|
"""手动触发单股同步(Free 用户在 K 线页用)。"""
|
|
repo = request.app.state.repo
|
|
capset = request.app.state.capabilities
|
|
n = kline_sync.sync_and_persist_daily_batch([symbol], repo, capset, count=days)
|
|
return {"symbol": symbol, "rows_written": n}
|
|
|
|
|
|
@router.post("/sync_batch")
|
|
def sync_batch(
|
|
request: Request,
|
|
symbols: list[str],
|
|
days: int = Query(250, ge=10, le=2000),
|
|
):
|
|
repo = request.app.state.repo
|
|
capset = request.app.state.capabilities
|
|
n = kline_sync.sync_and_persist_daily_batch(symbols, repo, capset, count=days)
|
|
return {"symbols": symbols, "rows_written": n}
|
|
|
|
|
|
@router.post("/refresh_views")
|
|
def refresh_views(request: Request):
|
|
"""刷新所有 DuckDB 视图(解决视图状态不一致问题)。"""
|
|
from app.jobs.daily_pipeline import _refresh_views
|
|
repo = request.app.state.repo
|
|
_refresh_views(repo)
|
|
return {"status": "ok"}
|
|
|
|
|
|
@router.post("/sync_minute")
|
|
async def sync_minute(request: Request):
|
|
"""手动触发分钟 K 同步(全市场)。返回 pipeline job_id 可轮询进度。"""
|
|
import asyncio
|
|
|
|
from app.services.pipeline_jobs import job_store
|
|
from app.api.data import invalidate_storage_cache
|
|
from app.services.preferences import get_minute_sync_days
|
|
from app.tickflow.capabilities import Cap
|
|
from app.tickflow.pools import get_pool
|
|
|
|
repo = request.app.state.repo
|
|
capset = request.app.state.capabilities
|
|
|
|
if not capset.has(Cap.KLINE_MINUTE_BATCH):
|
|
raise HTTPException(status_code=403, detail="需要 Pro+ 权限")
|
|
|
|
job_id = job_store.create()
|
|
existing = job_store.get(job_id)
|
|
if existing and existing["status"] == "running":
|
|
return {"status": "reused", "job_id": job_id}
|
|
|
|
async def task() -> None:
|
|
job_store.start(job_id)
|
|
loop = asyncio.get_event_loop()
|
|
|
|
def progress(stage: str, pct: int, msg: str) -> None:
|
|
job_store.progress(job_id, stage, pct, msg)
|
|
|
|
try:
|
|
progress("sync_minute", 5, "解析标的池…")
|
|
universe = sorted(set(get_pool("watchlist")) | set(get_pool("CN_Equity_A")))
|
|
# 补充 instruments 全量标的，覆盖北交所、新股等
|
|
inst_path = repo.store.data_dir / "instruments" / "instruments.parquet"
|
|
if inst_path.exists():
|
|
try:
|
|
import polars as pl
|
|
inst = pl.read_parquet(inst_path, columns=["symbol"])
|
|
universe = sorted(set(universe) | set(inst["symbol"].to_list()))
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
progress("sync_minute", 10, f"标的池 {len(universe)} 只")
|
|
|
|
days = get_minute_sync_days()
|
|
|
|
def _run():
|
|
return kline_sync.sync_and_persist_minute(universe, repo, capset, days=days)
|
|
|
|
written = await loop.run_in_executor(_long_task_executor, _run)
|
|
|
|
# 刷新视图
|
|
from app.jobs.daily_pipeline import _refresh_single_view
|
|
_refresh_single_view(repo, "kline_minute")
|
|
|
|
progress("done", 100, f"分钟 K 同步完成,{written} 行")
|
|
job_store.succeed(job_id, {"minute_rows": written, "universe_size": len(universe)})
|
|
invalidate_storage_cache()
|
|
except Exception as e: # noqa: BLE001
|
|
job_store.fail(job_id, str(e))
|
|
invalidate_storage_cache()
|
|
|
|
asyncio.create_task(task())
|
|
return {"status": "started", "job_id": job_id}
|
|
|
|
|
|
@router.post("/extend_history")
|
|
async def extend_history(request: Request):
|
|
"""向前扩展历史日K数据 — 独立于盘后管道。
|
|
|
|
body: { "value": int, "unit": "day"|"month"|"year" }
|
|
返回 job_id,可轮询 /api/pipeline/jobs 查看进度。
|
|
"""
|
|
import asyncio
|
|
import traceback as _tb
|
|
try:
|
|
body = await request.json()
|
|
value = body.get("value")
|
|
unit = body.get("unit", "month")
|
|
if not value or value <= 0:
|
|
raise HTTPException(status_code=400, detail="value 必须为正整数")
|
|
if unit not in ("day", "month", "year"):
|
|
raise HTTPException(status_code=400, detail="unit 只支持 day/month/year")
|
|
|
|
repo = request.app.state.repo
|
|
capset = request.app.state.capabilities
|
|
|
|
from app.tickflow.capabilities import Cap
|
|
if not capset.has(Cap.KLINE_DAILY_BATCH):
|
|
raise HTTPException(status_code=403, detail="需要 Pro+ 权限 (batch K-line)")
|
|
|
|
from app.services.extend_history import run_extend_history
|
|
from app.services.pipeline_jobs import job_store
|
|
from app.api.data import invalidate_storage_cache
|
|
|
|
job_id = job_store.create()
|
|
existing = job_store.get(job_id)
|
|
if existing and existing["status"] == "running":
|
|
return {"status": "reused", "job_id": job_id}
|
|
|
|
async def task() -> None:
|
|
job_store.start(job_id)
|
|
loop = asyncio.get_event_loop()
|
|
|
|
def progress(stage: str, pct: int, msg: str,
|
|
stage_pct: int | None = None, skip_log: bool = False) -> None:
|
|
job_store.progress(job_id, stage, pct, msg,
|
|
stage_pct=stage_pct, skip_log=skip_log)
|
|
|
|
try:
|
|
result = await loop.run_in_executor(
|
|
_long_task_executor,
|
|
lambda: run_extend_history(repo, capset, value, unit, on_progress=progress),
|
|
)
|
|
if "error" in result:
|
|
job_store.fail(job_id, result["error"])
|
|
else:
|
|
job_store.succeed(job_id, result)
|
|
invalidate_storage_cache()
|
|
except Exception as e:
|
|
logger.exception("extend_history failed: job_id=%s", job_id)
|
|
job_store.fail(job_id, str(e))
|
|
invalidate_storage_cache()
|
|
|
|
asyncio.create_task(task())
|
|
return {"status": "started", "job_id": job_id}
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error("extend_history error: %s\n%s", e, _tb.format_exc())
|
|
raise HTTPException(status_code=500, detail=str(e)) from e
|
|
|
|
|
|
@router.post("/rebuild_enriched")
|
|
async def rebuild_enriched(request: Request):
|
|
"""全量重算 enriched 表 — 不获取任何数据,仅基于已有 kline_daily + adj_factor 重算复权+指标。
|
|
|
|
返回 job_id,可轮询 /api/pipeline/jobs 查看进度。
|
|
"""
|
|
import asyncio
|
|
try:
|
|
repo = request.app.state.repo
|
|
|
|
from app.services.pipeline_jobs import job_store
|
|
from app.api.data import invalidate_storage_cache
|
|
|
|
job_id = job_store.create()
|
|
existing = job_store.get(job_id)
|
|
if existing and existing["status"] == "running":
|
|
return {"status": "reused", "job_id": job_id}
|
|
|
|
async def task() -> None:
|
|
job_store.start(job_id)
|
|
loop = asyncio.get_event_loop()
|
|
|
|
def progress(stage: str, pct: int, msg: str,
|
|
stage_pct: int | None = None, skip_log: bool = False) -> None:
|
|
job_store.progress(job_id, stage, pct, msg,
|
|
stage_pct=stage_pct, skip_log=skip_log)
|
|
|
|
try:
|
|
progress("rebuild_enriched", 10, "全量计算 enriched…")
|
|
from app.indicators.pipeline import run_pipeline
|
|
|
|
def _batch_progress(cur: int, tot: int) -> None:
|
|
pct = 10 + int(85 * cur / tot)
|
|
progress("rebuild_enriched", pct,
|
|
f"计算指标 批次 {cur}/{tot}",
|
|
stage_pct=int(100 * cur / tot), skip_log=True)
|
|
|
|
written = await loop.run_in_executor(
|
|
_long_task_executor,
|
|
lambda: run_pipeline(on_batch_done=_batch_progress),
|
|
)
|
|
|
|
enriched_dir = repo.store.data_dir / "kline_daily_enriched"
|
|
enriched_days = len(list(enriched_dir.glob("date=*"))) if enriched_dir.exists() else 0
|
|
|
|
# 刷新视图
|
|
d = repo.store.data_dir.as_posix()
|
|
for view_name, glob in [
|
|
("kline_enriched", f"{d}/kline_daily_enriched/**/*.parquet"),
|
|
]:
|
|
try:
|
|
repo.db.execute(
|
|
f"CREATE OR REPLACE VIEW {view_name} AS "
|
|
f"SELECT * FROM read_parquet('{glob}', union_by_name=true)"
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
progress("rebuild_enriched", 100, f"完成,覆盖 {enriched_days} 天")
|
|
job_store.succeed(job_id, {
|
|
"enriched_days": enriched_days,
|
|
"enriched_rows": written,
|
|
})
|
|
invalidate_storage_cache()
|
|
except Exception as e:
|
|
logger.exception("rebuild_enriched failed: job_id=%s", job_id)
|
|
job_store.fail(job_id, str(e))
|
|
invalidate_storage_cache()
|
|
|
|
asyncio.create_task(task())
|
|
return {"status": "started", "job_id": job_id}
|
|
except Exception as e:
|
|
import traceback as _tb
|
|
logger.error("rebuild_enriched error: %s\n%s", e, _tb.format_exc())
|
|
raise HTTPException(status_code=500, detail=str(e)) from e
|
|
|
|
|
|
# 长时间任务专用线程池（隔离于 FastAPI 默认线程池，防止阻塞请求处理）
|
|
import concurrent.futures as _cf
|
|
_long_task_executor = _cf.ThreadPoolExecutor(max_workers=2, thread_name_prefix="long-task")
|
|
|
|
|
|
@router.post("/extend_minute_history")
|
|
async def extend_minute_history(request: Request):
|
|
"""向前扩展分钟K历史数据 — 仅拉数据,不做任何后续处理。
|
|
|
|
body: { "value": int, "unit": "day"|"month" }
|
|
- day 单位:1~15 天(所有有分钟K权限的套餐可用)
|
|
- month 单位:1~6 月(每月按 30 天计,即最多 180 天)—— 仅 Expert+ 可用
|
|
返回 job_id,可轮询 /api/pipeline/jobs 查看进度。
|
|
"""
|
|
import asyncio
|
|
import traceback as _tb
|
|
try:
|
|
body = await request.json()
|
|
value = body.get("value")
|
|
unit = body.get("unit", "day")
|
|
if not value or value <= 0:
|
|
raise HTTPException(status_code=400, detail="value 必须为正整数")
|
|
if unit not in ("day", "month"):
|
|
raise HTTPException(status_code=400, detail="unit 只支持 day/month")
|
|
|
|
repo = request.app.state.repo
|
|
capset = request.app.state.capabilities
|
|
|
|
from app.tickflow.capabilities import Cap
|
|
if not capset.has(Cap.KLINE_MINUTE_BATCH):
|
|
raise HTTPException(status_code=403, detail="需要 Pro+ 权限 (batch minute K-line)")
|
|
|
|
# month 单位(按月扩展更长的分钟K历史)仅 Expert+ 开放;Pro 仅可用 day
|
|
if unit == "month":
|
|
from app.tickflow.policy import tier_label
|
|
base_tier = tier_label().split()[0].split("+")[0].strip().lower()
|
|
if base_tier != "expert":
|
|
raise HTTPException(
|
|
status_code=403,
|
|
detail="按月扩展分钟K历史需要 Expert 及以上套餐",
|
|
)
|
|
|
|
# 计算天数上限:day 最多 15 天;month 最多 6 月(180 天)
|
|
from datetime import timedelta
|
|
if unit == "month":
|
|
total_days = min(value * 30, 180)
|
|
else:
|
|
total_days = min(value, 15)
|
|
|
|
if total_days <= 0:
|
|
raise HTTPException(status_code=400, detail="扩展范围无效")
|
|
|
|
from app.services.pipeline_jobs import job_store
|
|
from app.api.data import invalidate_storage_cache
|
|
|
|
job_id = job_store.create()
|
|
existing = job_store.get(job_id)
|
|
if existing and existing["status"] == "running":
|
|
return {"status": "reused", "job_id": job_id}
|
|
|
|
async def task() -> None:
|
|
job_store.start(job_id)
|
|
loop = asyncio.get_event_loop()
|
|
|
|
def progress(stage: str, pct: int, msg: str,
|
|
stage_pct: int | None = None, skip_log: bool = False) -> None:
|
|
job_store.progress(job_id, stage, pct, msg,
|
|
stage_pct=stage_pct, skip_log=skip_log)
|
|
|
|
try:
|
|
# 获取当前最早日期
|
|
earliest = repo.earliest_minute_date()
|
|
if not earliest:
|
|
# 本地无分钟K数据 → 以今天为基准往前获取
|
|
from datetime import date as _date
|
|
latest = _date.today()
|
|
else:
|
|
latest = earliest
|
|
|
|
new_start = latest - timedelta(days=total_days)
|
|
if new_start >= latest:
|
|
job_store.fail(job_id, "扩展范围无效")
|
|
invalidate_storage_cache()
|
|
return
|
|
|
|
start_str = new_start.strftime("%Y-%m-%d")
|
|
end_str = latest.strftime("%Y-%m-%d")
|
|
|
|
progress("extend_minute", 5, "解析标的池…")
|
|
universe = _resolve_minute_universe(capset, repo)
|
|
progress("extend_minute", 8, f"标的池: {len(universe)} 只")
|
|
|
|
from app.tickflow.capabilities import Cap
|
|
|
|
lim = capset.limits(Cap.KLINE_MINUTE_BATCH)
|
|
batch_size = lim.batch if lim and lim.batch else 100
|
|
rpm = lim.rpm if lim else 30
|
|
|
|
def _run():
|
|
"""全部在 executor 线程里完成,避免阻塞事件循环。"""
|
|
from app.services.kline_sync import sync_minute_batch
|
|
from datetime import datetime as _dt
|
|
|
|
def _chunk(cur: int, tot: int) -> None:
|
|
progress("extend_minute", 8 + int(85 * cur / tot),
|
|
f"分钟K 批次 {cur}/{tot}", stage_pct=int(100 * cur / tot), skip_log=True)
|
|
|
|
df = sync_minute_batch(
|
|
universe,
|
|
start_time=_dt.combine(new_start, _dt.min.time()),
|
|
end_time=_dt.combine(latest, _dt.min.time()),
|
|
batch_size=batch_size, rpm=rpm,
|
|
on_chunk_done=_chunk,
|
|
)
|
|
|
|
written = 0
|
|
day_count = 0
|
|
if not df.is_empty():
|
|
import polars as pl
|
|
df = df.with_columns(pl.col("datetime").dt.date().alias("_trade_date"))
|
|
for day_df in df.partition_by("_trade_date"):
|
|
trade_date = day_df["_trade_date"][0]
|
|
out = repo.store.data_dir / "kline_minute" / f"date={trade_date}" / "part.parquet"
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
if out.exists():
|
|
existing_df = pl.read_parquet(out)
|
|
if "datetime" in existing_df.columns:
|
|
existing_df = existing_df.filter(pl.col("datetime").is_not_null())
|
|
day_df = pl.concat([existing_df, day_df.drop("_trade_date")]).unique(
|
|
subset=["symbol", "datetime"], keep="last",
|
|
)
|
|
else:
|
|
day_df = day_df.drop("_trade_date")
|
|
day_df = day_df.sort("symbol", "datetime")
|
|
day_df.write_parquet(out)
|
|
written += day_df.height
|
|
day_count += 1
|
|
|
|
# 刷新视图
|
|
d = repo.store.data_dir.as_posix()
|
|
try:
|
|
repo.db.execute(
|
|
f"CREATE OR REPLACE VIEW kline_minute AS "
|
|
f"SELECT * FROM read_parquet('{d}/kline_minute/**/*.parquet', union_by_name=true)"
|
|
)
|
|
except Exception:
|
|
pass
|
|
return written, day_count
|
|
|
|
progress("extend_minute", 10, f"获取分钟K [{start_str} ~ {end_str}]…")
|
|
written, day_count = await loop.run_in_executor(_long_task_executor, _run)
|
|
|
|
progress("extend_minute", 95, f"分钟K 完成,{day_count} 天")
|
|
job_store.succeed(job_id, {
|
|
"minute_days": day_count,
|
|
"universe_size": len(universe),
|
|
"earliest_before": (earliest or latest).isoformat(),
|
|
"earliest_after": new_start.isoformat(),
|
|
})
|
|
invalidate_storage_cache()
|
|
except Exception as e:
|
|
logger.exception("extend_minute_history failed: job_id=%s", job_id)
|
|
job_store.fail(job_id, str(e))
|
|
invalidate_storage_cache()
|
|
|
|
asyncio.create_task(task())
|
|
return {"status": "started", "job_id": job_id}
|
|
except HTTPException:
|
|
raise
|
|
except Exception as e:
|
|
logger.error("extend_minute_history error: %s\n%s", e, _tb.format_exc())
|
|
raise HTTPException(status_code=500, detail=str(e)) from e
|
|
|
|
|
|
def _resolve_minute_universe(capset, repo) -> list[str]:
|
|
"""分钟K标的池解析。"""
|
|
from app.tickflow.capabilities import Cap
|
|
if capset.has(Cap.KLINE_MINUTE_BATCH):
|
|
try:
|
|
from app.tickflow.pools import get_pool
|
|
all_a = get_pool("CN_Equity_A", refresh=True)
|
|
if all_a:
|
|
return sorted(all_a)
|
|
except Exception:
|
|
pass
|
|
return []
|