重置项目

refer/backend/app/data_providers/__init__.py

@@ -0,0 +1,8 @@
+"""Market data provider abstraction.
+
+Providers normalize external data sources into the internal parquet schema.
+"""
+from app.data_providers.base import AssetType, MarketDataProvider, ProviderCapabilities
+from app.data_providers.registry import get_provider
+
+__all__ = ["AssetType", "MarketDataProvider", "ProviderCapabilities", "get_provider"]

refer/backend/app/data_providers/base.py

@@ -0,0 +1,68 @@
+"""Provider contracts for external market data sources.
+
+The first implementation wraps TickFlow. Other providers (Tushare/AkShare/etc.)
+should return the same normalized Polars schemas so storage, indicators and
+backtests stay data-source agnostic.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Literal, Protocol
+
+import polars as pl
+
+AssetType = Literal["stock", "index", "etf"]
+
+
+@dataclass(frozen=True)
+class ProviderCapabilities:
+    instruments: bool = False
+    daily: bool = False
+    adj_factor: bool = False
+    minute: bool = False
+    realtime: bool = False
+    financial: bool = False
+
+
+class MarketDataProvider(Protocol):
+    name: str
+    capabilities: ProviderCapabilities
+
+    def get_instruments(self, asset_type: AssetType) -> pl.DataFrame:
+        """Return normalized instruments: symbol/name/code/exchange/asset_type/source."""
+
+    def get_daily(
+        self,
+        symbols: list[str],
+        start_time: datetime | None,
+        end_time: datetime | None,
+        asset_type: AssetType,
+    ) -> pl.DataFrame:
+        """Return normalized daily K rows."""
+
+    def get_adj_factors(
+        self,
+        symbols: list[str],
+        start_time: datetime | None,
+        end_time: datetime | None,
+        asset_type: AssetType,
+    ) -> pl.DataFrame:
+        """Return normalized adjustment factors: symbol/trade_date/ex_factor."""
+
+    def get_minute(
+        self,
+        symbols: list[str],
+        start_time: datetime | None,
+        end_time: datetime | None,
+        asset_type: AssetType,
+        freq: str = "1m",
+    ) -> pl.DataFrame:
+        """Return normalized minute K rows. Implementations may return empty."""
+
+    def get_realtime(
+        self,
+        universes: list[str] | None = None,
+        symbols: list[str] | None = None,
+    ) -> pl.DataFrame:
+        """Return normalized realtime quotes. Implementations may return empty."""

refer/backend/app/data_providers/normalizer.py

@@ -0,0 +1,99 @@
+"""Normalize provider responses into internal Polars schemas."""
+from __future__ import annotations
+
+import polars as pl
+
+from app.indicators.pipeline import filter_halt_days
+
+DAILY_COLS = ["symbol", "date", "open", "high", "low", "close", "volume", "amount"]
+ADJ_FACTOR_COLS = ["symbol", "trade_date", "ex_factor"]
+INSTRUMENT_COLS = ["symbol", "name", "code", "exchange", "asset_type", "source"]
+
+
+def to_polars(data) -> pl.DataFrame:
+    if data is None:
+        return pl.DataFrame()
+    if isinstance(data, pl.DataFrame):
+        return data
+    if isinstance(data, dict):
+        rows: list[dict] = []
+        for sym, values in data.items():
+            for item in values or []:
+                row = dict(item or {})
+                row.setdefault("symbol", sym)
+                rows.append(row)
+        return pl.DataFrame(rows) if rows else pl.DataFrame()
+    if hasattr(data, "reset_index"):
+        return pl.from_pandas(data.reset_index())
+    try:
+        return pl.DataFrame(data)
+    except Exception:  # noqa: BLE001
+        return pl.DataFrame()
+
+
+def normalize_daily(data, default_symbol: str | None = None, source: str = "tickflow") -> pl.DataFrame:  # noqa: ARG001
+    df = to_polars(data)
+    if df.is_empty():
+        return df
+    rename_map = {
+        "ts_code": "symbol",
+        "trade_date": "date",
+        "datetime": "date",
+        "vol": "volume",
+        "amt": "amount",
+    }
+    df = df.rename({k: v for k, v in rename_map.items() if k in df.columns})
+    if "symbol" not in df.columns and default_symbol:
+        df = df.with_columns(pl.lit(default_symbol).alias("symbol"))
+    if "date" in df.columns and df.schema["date"] != pl.Date:
+        df = df.with_columns(pl.col("date").cast(pl.Date, strict=False))
+    for col in ("open", "high", "low", "close", "volume", "amount"):
+        if col in df.columns:
+            df = df.with_columns(pl.col(col).cast(pl.Float64, strict=False))
+    df = filter_halt_days(df)
+    keep = [c for c in DAILY_COLS if c in df.columns]
+    return df.select(keep) if keep else pl.DataFrame()
+
+
+def normalize_adj_factors(data, source: str = "tickflow") -> pl.DataFrame:  # noqa: ARG001
+    df = to_polars(data)
+    if df.is_empty():
+        return df
+    rename_map = {
+        "timestamp": "trade_date",
+        "date": "trade_date",
+        "adj_factor": "ex_factor",
+    }
+    df = df.rename({k: v for k, v in rename_map.items() if k in df.columns})
+    if "trade_date" in df.columns:
+        if df.schema["trade_date"] in {pl.Int64, pl.Int32, pl.UInt64, pl.UInt32, pl.Float64, pl.Float32}:
+            df = df.with_columns(
+                pl.from_epoch(pl.col("trade_date").cast(pl.Int64), time_unit="ms").dt.date().alias("trade_date")
+            )
+        else:
+            df = df.with_columns(pl.col("trade_date").cast(pl.Date, strict=False))
+    if "ex_factor" in df.columns:
+        df = df.with_columns(pl.col("ex_factor").cast(pl.Float64, strict=False))
+    keep = [c for c in ADJ_FACTOR_COLS if c in df.columns]
+    return df.select(keep).drop_nulls() if len(keep) == len(ADJ_FACTOR_COLS) else pl.DataFrame()
+
+
+def normalize_instruments(rows: list[dict], asset_type: str, source: str = "tickflow") -> pl.DataFrame:
+    if not rows:
+        return pl.DataFrame()
+    out: list[dict] = []
+    for item in rows:
+        symbol = item.get("symbol")
+        if not symbol:
+            continue
+        out.append({
+            "symbol": str(symbol),
+            "name": item.get("name") or str(symbol),
+            "code": item.get("code") or str(symbol).split(".")[0],
+            "exchange": item.get("exchange"),
+            "asset_type": asset_type,
+            "source": source,
+        })
+    if not out:
+        return pl.DataFrame()
+    return pl.DataFrame(out).select(INSTRUMENT_COLS).unique(subset=["symbol"], keep="last").sort("symbol")

refer/backend/app/data_providers/registry.py

@@ -0,0 +1,15 @@
+"""Provider registry."""
+from __future__ import annotations
+
+from app.data_providers.tickflow_provider import TickFlowProvider
+
+_PROVIDERS = {
+    "tickflow": TickFlowProvider,
+}
+
+
+def get_provider(name: str = "tickflow"):
+    provider_cls = _PROVIDERS.get((name or "tickflow").lower())
+    if provider_cls is None:
+        raise ValueError(f"Unsupported data provider: {name}")
+    return provider_cls()

refer/backend/app/data_providers/schemas.py

@@ -0,0 +1,18 @@
+"""Internal provider schema column lists."""
+from __future__ import annotations
+
+DAILY_COLUMNS = [
+    "symbol", "asset_type", "source", "date", "open", "high", "low", "close",
+    "volume", "amount", "pre_close", "change_pct",
+]
+
+ADJ_FACTOR_COLUMNS = ["symbol", "asset_type", "source", "trade_date", "ex_factor"]
+
+INSTRUMENT_COLUMNS = [
+    "symbol", "name", "exchange", "asset_type", "source", "list_date", "status",
+]
+
+MINUTE_COLUMNS = [
+    "symbol", "asset_type", "source", "datetime", "open", "high", "low", "close",
+    "volume", "amount", "freq",
+]

refer/backend/app/data_providers/tickflow_provider.py

@@ -0,0 +1,120 @@
+"""TickFlow provider implementation."""
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+
+import polars as pl
+
+from app.data_providers.base import AssetType, ProviderCapabilities
+from app.data_providers.normalizer import normalize_adj_factors, normalize_daily, normalize_instruments
+from app.tickflow.client import get_client
+
+logger = logging.getLogger(__name__)
+
+_EXCHANGES = ["SH", "SZ", "BJ"]
+
+
+class TickFlowProvider:
+    name = "tickflow"
+    capabilities = ProviderCapabilities(
+        instruments=True,
+        daily=True,
+        adj_factor=True,
+        minute=True,
+        realtime=True,
+        financial=True,
+    )
+
+    def get_instruments(self, asset_type: AssetType) -> pl.DataFrame:
+        tf = get_client()
+        instrument_type = "stock" if asset_type == "stock" else asset_type
+        rows: list[dict] = []
+        for ex in _EXCHANGES:
+            try:
+                items = tf.exchanges.get_instruments(ex, instrument_type=instrument_type)
+                rows.extend([it for it in (items or []) if isinstance(it, dict)])
+            except Exception as e:  # noqa: BLE001
+                logger.warning("TickFlow instruments %s/%s failed: %s", ex, instrument_type, e)
+        return normalize_instruments(rows, asset_type=asset_type, source=self.name)
+
+    def get_daily(
+        self,
+        symbols: list[str],
+        start_time: datetime | None,
+        end_time: datetime | None,
+        asset_type: AssetType,  # noqa: ARG002
+    ) -> pl.DataFrame:
+        if not symbols:
+            return pl.DataFrame()
+        tf = get_client()
+        kwargs = {
+            "period": "1d",
+            "adjust": "none",
+            "count": 10000 if start_time and end_time else 250,
+            "as_dataframe": True,
+            "show_progress": False,
+        }
+        if start_time and end_time:
+            from app.services.kline_sync import _datetime_to_ms
+            kwargs["start_time"] = _datetime_to_ms(start_time)
+            kwargs["end_time"] = _datetime_to_ms(end_time)
+        raw = tf.klines.batch(symbols, **kwargs)
+        frames: list[pl.DataFrame] = []
+        if isinstance(raw, dict):
+            for sym, sub in raw.items():
+                normalized = normalize_daily(sub, default_symbol=sym, source=self.name)
+                if not normalized.is_empty():
+                    frames.append(normalized)
+        else:
+            normalized = normalize_daily(raw, source=self.name)
+            if not normalized.is_empty():
+                frames.append(normalized)
+        return pl.concat(frames, how="diagonal_relaxed") if frames else pl.DataFrame()
+
+    def get_adj_factors(
+        self,
+        symbols: list[str],
+        start_time: datetime | None,
+        end_time: datetime | None,
+        asset_type: AssetType,  # noqa: ARG002
+    ) -> pl.DataFrame:
+        if not symbols:
+            return pl.DataFrame()
+        tf = get_client()
+        kwargs = {"as_dataframe": False}
+        if start_time or end_time:
+            from app.services.kline_sync import _datetime_to_ms
+            if start_time:
+                kwargs["start_time"] = _datetime_to_ms(start_time)
+            if end_time:
+                kwargs["end_time"] = _datetime_to_ms(end_time)
+        raw = tf.klines.ex_factors(symbols, **kwargs)
+        return normalize_adj_factors(raw, source=self.name)
+
+    def get_minute(
+        self,
+        symbols: list[str],
+        start_time: datetime | None,
+        end_time: datetime | None,
+        asset_type: AssetType,  # noqa: ARG002
+        freq: str = "1m",  # noqa: ARG002
+    ) -> pl.DataFrame:
+        # Existing minute sync remains in app.services.kline_sync for now.
+        return pl.DataFrame()
+
+    def get_realtime(
+        self,
+        universes: list[str] | None = None,
+        symbols: list[str] | None = None,
+    ) -> pl.DataFrame:
+        tf = get_client()
+        if universes and symbols:
+            raise ValueError("TickFlow realtime accepts either universes or symbols, not both")
+        if universes:
+            resp = tf.quotes.get_by_universes(universes=universes)
+        elif symbols:
+            resp = tf.quotes.get(symbols=symbols)
+        else:
+            return pl.DataFrame()
+        return pl.DataFrame(resp or [])