@@ -0,0 +1,200 @@
|
|||||||
|
"""数据同步 — 将本地 data/ 打包推送到 serve 端。"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
from datetime import datetime
|
||||||
|
from io import BytesIO
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from fastapi import APIRouter, HTTPException, Request
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/data/sync", tags=["data-sync"])
|
||||||
|
|
||||||
|
# 允许同步的子目录 — 覆盖 serve 功能面板所需的数据
|
||||||
|
SYNCABLE_PARTS = [
|
||||||
|
"kline_daily",
|
||||||
|
"kline_daily_enriched",
|
||||||
|
"kline_index_daily",
|
||||||
|
"kline_index_enriched",
|
||||||
|
"kline_etf_daily",
|
||||||
|
"kline_etf_enriched",
|
||||||
|
"kline_etf_minute",
|
||||||
|
"kline_minute",
|
||||||
|
"adj_factor",
|
||||||
|
"adj_factor_etf",
|
||||||
|
"instruments",
|
||||||
|
"instruments_index",
|
||||||
|
"instruments_etf",
|
||||||
|
"financials",
|
||||||
|
"pools",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _part_size_info(data_dir: Path, part: str) -> dict:
|
||||||
|
"""获取指定子目录的文件数和大小。"""
|
||||||
|
part_dir = data_dir / part
|
||||||
|
if not part_dir.exists():
|
||||||
|
return {"file_count": 0, "size_bytes": 0, "last_modified": None}
|
||||||
|
files = list(part_dir.rglob("*.parquet"))
|
||||||
|
if not files:
|
||||||
|
return {"file_count": 0, "size_bytes": 0, "last_modified": None}
|
||||||
|
total = sum(f.stat().st_size for f in files)
|
||||||
|
mtimes = [f.stat().st_mtime for f in files]
|
||||||
|
last_ts = datetime.fromtimestamp(max(mtimes)).isoformat() if mtimes else None
|
||||||
|
return {"file_count": len(files), "size_bytes": total, "last_modified": last_ts}
|
||||||
|
|
||||||
|
|
||||||
|
def _pack_parts(data_dir: Path, parts: list[str]) -> bytes:
|
||||||
|
"""将指定子目录打包为 tar.gz 字节流。"""
|
||||||
|
buf = BytesIO()
|
||||||
|
with tarfile.open(fileobj=buf, mode="w:gz") as tar:
|
||||||
|
for part in parts:
|
||||||
|
part_dir = data_dir / part
|
||||||
|
if not part_dir.exists():
|
||||||
|
continue
|
||||||
|
for fpath in sorted(part_dir.rglob("*.parquet")):
|
||||||
|
rel = fpath.relative_to(data_dir)
|
||||||
|
tar.add(fpath, arcname=str(rel))
|
||||||
|
buf.seek(0)
|
||||||
|
return buf.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
class SyncConfig(BaseModel):
|
||||||
|
serve_url: str = ""
|
||||||
|
sync_key: str = ""
|
||||||
|
enable_auto: bool = False
|
||||||
|
interval_minutes: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
# ── 存储配置用的 JSON 文件 ──────────────────────────────────
|
||||||
|
CONFIG_FILE = "sync_config.json"
|
||||||
|
|
||||||
|
|
||||||
|
def _config_path(data_dir: Path) -> Path:
|
||||||
|
return data_dir / "user_data" / CONFIG_FILE
|
||||||
|
|
||||||
|
|
||||||
|
def _load_config(data_dir: Path) -> SyncConfig:
|
||||||
|
cp = _config_path(data_dir)
|
||||||
|
if cp.exists():
|
||||||
|
try:
|
||||||
|
return SyncConfig(**json.loads(cp.read_text()))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return SyncConfig()
|
||||||
|
|
||||||
|
|
||||||
|
def _save_config(data_dir: Path, cfg: SyncConfig) -> None:
|
||||||
|
cp = _config_path(data_dir)
|
||||||
|
cp.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
cp.write_text(cfg.model_dump_json(indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
# ── 路由 ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/status")
|
||||||
|
def sync_status(request: Request):
|
||||||
|
"""各数据目录的本地状态 + 同步配置。"""
|
||||||
|
data_dir: Path = request.app.state.datastore.data_dir
|
||||||
|
cfg = _load_config(data_dir)
|
||||||
|
parts = {}
|
||||||
|
for p in SYNCABLE_PARTS:
|
||||||
|
parts[p] = _part_size_info(data_dir, p)
|
||||||
|
return {
|
||||||
|
"parts": parts,
|
||||||
|
"config": cfg.model_dump(),
|
||||||
|
"last_sync": cfg.model_dump().get("_last_sync_time"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigReq(BaseModel):
|
||||||
|
serve_url: str
|
||||||
|
sync_key: str
|
||||||
|
enable_auto: bool = False
|
||||||
|
interval_minutes: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/config")
|
||||||
|
def update_sync_config(request: Request, body: ConfigReq):
|
||||||
|
"""更新同步配置。"""
|
||||||
|
data_dir: Path = request.app.state.datastore.data_dir
|
||||||
|
cfg = SyncConfig(
|
||||||
|
serve_url=body.serve_url.rstrip("/"),
|
||||||
|
sync_key=body.sync_key,
|
||||||
|
enable_auto=body.enable_auto,
|
||||||
|
interval_minutes=body.interval_minutes,
|
||||||
|
)
|
||||||
|
_save_config(data_dir, cfg)
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
|
||||||
|
class PushReq(BaseModel):
|
||||||
|
parts: list[str]
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/push")
|
||||||
|
async def push_sync(request: Request, body: PushReq):
|
||||||
|
"""打包指定 parts 推送到 serve 端。"""
|
||||||
|
data_dir: Path = request.app.state.datastore.data_dir
|
||||||
|
cfg = _load_config(data_dir)
|
||||||
|
if not cfg.serve_url:
|
||||||
|
raise HTTPException(status_code=400, detail="未配置 serve_url,请先在同步设置中配置")
|
||||||
|
|
||||||
|
requested = [p for p in body.parts if p in SYNCABLE_PARTS]
|
||||||
|
if not requested:
|
||||||
|
raise HTTPException(status_code=400, detail="未指定有效的同步目录")
|
||||||
|
|
||||||
|
# 检查有没有数据可同步
|
||||||
|
available = [p for p in requested if (data_dir / p).exists()]
|
||||||
|
if not available:
|
||||||
|
raise HTTPException(status_code=400, detail="所选目录均无数据可同步")
|
||||||
|
|
||||||
|
# 打包
|
||||||
|
tarball = _pack_parts(data_dir, available)
|
||||||
|
size_bytes = len(tarball)
|
||||||
|
logger.info("packed %d parts -> %d bytes", len(available), size_bytes)
|
||||||
|
|
||||||
|
# 推送
|
||||||
|
upload_url = f"{cfg.serve_url}/api/data/sync/upload"
|
||||||
|
headers = {"X-Sync-Key": cfg.sync_key} if cfg.sync_key else {}
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=600) as client:
|
||||||
|
resp = await client.post(
|
||||||
|
upload_url,
|
||||||
|
files={"file": ("sync.tar.gz", tarball, "application/gzip")},
|
||||||
|
data={"parts": ",".join(available)},
|
||||||
|
headers=headers,
|
||||||
|
)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
detail = resp.text[:500]
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=502,
|
||||||
|
detail=f"serve 返回异常 ({resp.status_code}): {detail}",
|
||||||
|
)
|
||||||
|
result = resp.json()
|
||||||
|
except httpx.RequestError as exc:
|
||||||
|
raise HTTPException(status_code=502, detail=f"连接 serve 失败: {exc}") from exc
|
||||||
|
|
||||||
|
# 记录上次同步时间
|
||||||
|
cfg_dict = cfg.model_dump()
|
||||||
|
cfg_dict["_last_sync_time"] = datetime.now().isoformat()
|
||||||
|
cfg = SyncConfig(**cfg_dict)
|
||||||
|
_save_config(data_dir, cfg)
|
||||||
|
|
||||||
|
logger.info("sync pushed: parts=%s files=%d", available, result.get("file_count", 0))
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"parts_sent": available,
|
||||||
|
"size_bytes": size_bytes,
|
||||||
|
"serve_response": result,
|
||||||
|
}
|
||||||
@@ -95,6 +95,10 @@ class Settings(BaseSettings):
|
|||||||
log_level: str = "INFO"
|
log_level: str = "INFO"
|
||||||
backtest_range_guard: bool = False
|
backtest_range_guard: bool = False
|
||||||
|
|
||||||
|
# Sync — 数据同步到 serve
|
||||||
|
sync_serve_url: str = ""
|
||||||
|
sync_key: str = ""
|
||||||
|
|
||||||
# Auth — 首次启动时预置访问密码(明文, 仅用于初始化, 详见 services/auth.bootstrap_from_env)
|
# Auth — 首次启动时预置访问密码(明文, 仅用于初始化, 详见 services/auth.bootstrap_from_env)
|
||||||
# 公网服务器部署时免去 SSH 端口转发设密码的麻烦。写入 auth.json(哈希)后即不再读取。
|
# 公网服务器部署时免去 SSH 端口转发设密码的麻烦。写入 auth.json(哈希)后即不再读取。
|
||||||
auth_password: str = ""
|
auth_password: str = ""
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from fastapi.responses import FileResponse, JSONResponse
|
|||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
|
|
||||||
from app import __version__
|
from app import __version__
|
||||||
from app.api import analysis, auth as auth_api, backtest, data, ext_data, financials, indices, intraday, kline, market_recap, monitor_rules, alerts, overview, pipeline, rps, screener, settings as settings_api, signals, stock_analysis, strategy, watchlist
|
from app.api import analysis, auth as auth_api, backtest, data, data_sync, ext_data, financials, indices, intraday, kline, market_recap, monitor_rules, alerts, overview, pipeline, rps, screener, settings as settings_api, signals, stock_analysis, strategy, watchlist
|
||||||
from app.api.routes import router as core_router
|
from app.api.routes import router as core_router
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.jobs import daily_pipeline
|
from app.jobs import daily_pipeline
|
||||||
@@ -207,7 +207,7 @@ app.add_middleware(
|
|||||||
# 2. 未设密码 + 公网 → 拒绝(403, 防裸奔也防抢占; 引导本机设密码)
|
# 2. 未设密码 + 公网 → 拒绝(403, 防裸奔也防抢占; 引导本机设密码)
|
||||||
# 3. 已设密码 → 检查 session, 无效则 401(前端跳登录)
|
# 3. 已设密码 → 检查 session, 无效则 401(前端跳登录)
|
||||||
# 白名单: /api/auth/* (设密码/登录本身)、/health 等探活。
|
# 白名单: /api/auth/* (设密码/登录本身)、/health 等探活。
|
||||||
_AUTH_WHITELIST_PREFIX = ("/api/auth/",)
|
_AUTH_WHITELIST_PREFIX = ("/api/auth/", "/api/data/sync/")
|
||||||
_AUTH_WHITELIST_EXACT = ("/health", "/api/health", "/openapi.json", "/docs", "/redoc")
|
_AUTH_WHITELIST_EXACT = ("/health", "/api/health", "/openapi.json", "/docs", "/redoc")
|
||||||
|
|
||||||
|
|
||||||
@@ -257,6 +257,7 @@ app.include_router(overview.router)
|
|||||||
app.include_router(analysis.router)
|
app.include_router(analysis.router)
|
||||||
app.include_router(pipeline.router)
|
app.include_router(pipeline.router)
|
||||||
app.include_router(data.router)
|
app.include_router(data.router)
|
||||||
|
app.include_router(data_sync.router)
|
||||||
app.include_router(ext_data.router)
|
app.include_router(ext_data.router)
|
||||||
app.include_router(financials.router)
|
app.include_router(financials.router)
|
||||||
app.include_router(stock_analysis.router)
|
app.include_router(stock_analysis.router)
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ services:
|
|||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
args:
|
args:
|
||||||
BACKEND_EXTRAS: ${BACKEND_EXTRAS:-}
|
BACKEND_EXTRAS: ${BACKEND_EXTRAS:-}
|
||||||
container_name: TickFlow_Stock_Panel
|
container_name: TickFlow_Local
|
||||||
ports:
|
ports:
|
||||||
- "${PORT:-3018}:3018"
|
- "${PORT:-3018}:3018"
|
||||||
env_file:
|
env_file:
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ import {
|
|||||||
BookOpenCheck,
|
BookOpenCheck,
|
||||||
ExternalLink,
|
ExternalLink,
|
||||||
X,
|
X,
|
||||||
|
Upload,
|
||||||
} from 'lucide-react'
|
} from 'lucide-react'
|
||||||
import { Logo } from './Logo'
|
import { Logo } from './Logo'
|
||||||
import { api, type IndexQuote } from '@/lib/api'
|
import { api, type IndexQuote } from '@/lib/api'
|
||||||
@@ -78,6 +79,7 @@ const nav = [
|
|||||||
{ to: '/indices', label: '指数', icon: BarChart3 },
|
{ to: '/indices', label: '指数', icon: BarChart3 },
|
||||||
{ to: '/trading', label: '交易', icon: Cable },
|
{ to: '/trading', label: '交易', icon: Cable },
|
||||||
{ to: '/data', label: '数据', icon: Database },
|
{ to: '/data', label: '数据', icon: Database },
|
||||||
|
{ to: '/sync', label: '数据同步', icon: Upload },
|
||||||
] as const
|
] as const
|
||||||
|
|
||||||
function fmtIndexValue(v: number | null | undefined) {
|
function fmtIndexValue(v: number | null | undefined) {
|
||||||
|
|||||||
@@ -1752,6 +1752,14 @@ export const api = {
|
|||||||
method: 'POST',
|
method: 'POST',
|
||||||
body: JSON.stringify({ strategy_id: strategyId, code }),
|
body: JSON.stringify({ strategy_id: strategyId, code }),
|
||||||
}),
|
}),
|
||||||
|
|
||||||
|
// ===== Sync =====
|
||||||
|
syncStatus: (): Promise<SyncStatusResponse> =>
|
||||||
|
request('/api/data/sync/status'),
|
||||||
|
syncConfig: (body: SyncConfigReq): Promise<{ ok: boolean }> =>
|
||||||
|
request('/api/data/sync/config', { method: 'POST', body: JSON.stringify(body) }),
|
||||||
|
syncPush: (parts: string[]): Promise<SyncPushResponse> =>
|
||||||
|
request('/api/data/sync/push', { method: 'POST', body: JSON.stringify({ parts }) }),
|
||||||
}
|
}
|
||||||
|
|
||||||
// ===== Pipeline =====
|
// ===== Pipeline =====
|
||||||
@@ -1931,3 +1939,39 @@ export interface AnalysisMenu {
|
|||||||
updated_at?: string | null
|
updated_at?: string | null
|
||||||
builtin?: boolean
|
builtin?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ===== Sync =====
|
||||||
|
export interface SyncPartInfo {
|
||||||
|
file_count: number
|
||||||
|
size_bytes: number
|
||||||
|
last_modified: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SyncStatusResponse {
|
||||||
|
parts: Record<string, SyncPartInfo>
|
||||||
|
config: {
|
||||||
|
serve_url: string
|
||||||
|
sync_key: string
|
||||||
|
enable_auto: boolean
|
||||||
|
interval_minutes: number
|
||||||
|
}
|
||||||
|
last_sync?: string | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SyncPushResponse {
|
||||||
|
ok: boolean
|
||||||
|
parts_sent: string[]
|
||||||
|
size_bytes: number
|
||||||
|
serve_response: {
|
||||||
|
ok: boolean
|
||||||
|
parts: string[]
|
||||||
|
file_count: number
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SyncConfigReq {
|
||||||
|
serve_url: string
|
||||||
|
sync_key: string
|
||||||
|
enable_auto?: boolean
|
||||||
|
interval_minutes?: number
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,310 @@
|
|||||||
|
import { useCallback, useEffect, useState } from 'react'
|
||||||
|
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
|
||||||
|
import {
|
||||||
|
CheckCircle2,
|
||||||
|
Cloud,
|
||||||
|
CloudOff,
|
||||||
|
Database,
|
||||||
|
HardDrive,
|
||||||
|
Loader2,
|
||||||
|
RefreshCw,
|
||||||
|
Save,
|
||||||
|
Settings,
|
||||||
|
Upload,
|
||||||
|
} from 'lucide-react'
|
||||||
|
import { toast } from '@/components/Toast'
|
||||||
|
import { PageHeader } from '@/components/PageHeader'
|
||||||
|
import { api, type SyncStatusResponse, type SyncPushResponse } from '@/lib/api'
|
||||||
|
import { cn } from '@/lib/cn'
|
||||||
|
|
||||||
|
type PartEntry = {
|
||||||
|
key: string
|
||||||
|
label: string
|
||||||
|
}
|
||||||
|
|
||||||
|
const ALL_PARTS: PartEntry[] = [
|
||||||
|
{ key: 'kline_daily', label: '日 K 线' },
|
||||||
|
{ key: 'kline_daily_enriched', label: '日 K 增强' },
|
||||||
|
{ key: 'kline_index_daily', label: '指数日 K' },
|
||||||
|
{ key: 'kline_index_enriched', label: '指数日 K 增强' },
|
||||||
|
{ key: 'kline_etf_daily', label: 'ETF 日 K' },
|
||||||
|
{ key: 'kline_etf_enriched', label: 'ETF 日 K 增强' },
|
||||||
|
{ key: 'kline_etf_minute', label: 'ETF 分钟 K' },
|
||||||
|
{ key: 'kline_minute', label: '分钟 K' },
|
||||||
|
{ key: 'adj_factor', label: '复权因子' },
|
||||||
|
{ key: 'adj_factor_etf', label: 'ETF 复权因子' },
|
||||||
|
{ key: 'instruments', label: '股票列表' },
|
||||||
|
{ key: 'instruments_index', label: '指数列表' },
|
||||||
|
{ key: 'instruments_etf', label: 'ETF 列表' },
|
||||||
|
{ key: 'financials', label: '财务数据' },
|
||||||
|
{ key: 'pools', label: '板块池' },
|
||||||
|
]
|
||||||
|
|
||||||
|
function fmtSize(bytes: number): string {
|
||||||
|
if (bytes === 0) return '—'
|
||||||
|
if (bytes < 1024) return `${bytes} B`
|
||||||
|
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`
|
||||||
|
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
|
||||||
|
}
|
||||||
|
|
||||||
|
export function Sync() {
|
||||||
|
const qc = useQueryClient()
|
||||||
|
const [showSettings, setShowSettings] = useState(false)
|
||||||
|
const [serveUrl, setServeUrl] = useState('')
|
||||||
|
const [syncKey, setSyncKey] = useState('')
|
||||||
|
const [selected, setSelected] = useState<Set<string>>(new Set(ALL_PARTS.map(p => p.key)))
|
||||||
|
|
||||||
|
// Status
|
||||||
|
const { data: status, isLoading, refetch } = useQuery<SyncStatusResponse>({
|
||||||
|
queryKey: ['sync-status'],
|
||||||
|
queryFn: api.syncStatus,
|
||||||
|
refetchInterval: 30000,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Init form from server config
|
||||||
|
useEffect(() => {
|
||||||
|
if (status?.config) {
|
||||||
|
setServeUrl(status.config.serve_url)
|
||||||
|
setSyncKey(status.config.sync_key)
|
||||||
|
}
|
||||||
|
}, [status?.config])
|
||||||
|
|
||||||
|
// Save config
|
||||||
|
const saveCfg = useMutation<{ ok: boolean }, Error, void>({
|
||||||
|
mutationFn: () => api.syncConfig({ serve_url: serveUrl, sync_key: syncKey }),
|
||||||
|
onSuccess: () => {
|
||||||
|
toast('同步配置已保存', 'success')
|
||||||
|
qc.invalidateQueries({ queryKey: ['sync-status'] })
|
||||||
|
setShowSettings(false)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
// Push sync
|
||||||
|
const push = useMutation<SyncPushResponse, Error, void>({
|
||||||
|
mutationFn: () => api.syncPush([...selected]),
|
||||||
|
onSuccess: (res) => {
|
||||||
|
if (res.ok) {
|
||||||
|
toast(
|
||||||
|
`同步完成: ${res.parts_sent.length} 个目录, ${res.serve_response.file_count} 个文件`,
|
||||||
|
'success',
|
||||||
|
)
|
||||||
|
qc.invalidateQueries({ queryKey: ['sync-status'] })
|
||||||
|
}
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
const togglePart = useCallback((key: string) => {
|
||||||
|
setSelected(prev => {
|
||||||
|
const next = new Set(prev)
|
||||||
|
if (next.has(key)) next.delete(key)
|
||||||
|
else next.add(key)
|
||||||
|
return next
|
||||||
|
})
|
||||||
|
}, [])
|
||||||
|
|
||||||
|
const selectAll = () => setSelected(new Set(ALL_PARTS.map(p => p.key)))
|
||||||
|
const selectNone = () => setSelected(new Set())
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="mx-auto max-w-5xl space-y-6 px-4 py-6">
|
||||||
|
<PageHeader
|
||||||
|
title="数据同步"
|
||||||
|
subtitle="将本地数据推送到 serve 服务器"
|
||||||
|
/>
|
||||||
|
|
||||||
|
{/* 连接配置 */}
|
||||||
|
<section className="rounded-card border border-border bg-surface p-5">
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
{status?.config.serve_url ? (
|
||||||
|
<Cloud className="h-5 w-5 text-accent" />
|
||||||
|
) : (
|
||||||
|
<CloudOff className="h-5 w-5 text-muted" />
|
||||||
|
)}
|
||||||
|
<div>
|
||||||
|
<div className="text-sm font-medium text-foreground">Serve 连接</div>
|
||||||
|
<div className="text-xs text-muted">
|
||||||
|
{status?.config.serve_url
|
||||||
|
? status.config.serve_url
|
||||||
|
: '未配置 — 点击右侧设置按钮'}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
{status?.last_sync && (
|
||||||
|
<span className="text-[10px] text-muted">
|
||||||
|
上次同步: {new Date(status.last_sync).toLocaleString('zh-CN')}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
<button
|
||||||
|
onClick={() => { setShowSettings(v => !v); refetch() }}
|
||||||
|
className="rounded-btn p-2 text-muted hover:text-foreground hover:bg-elevated transition-colors"
|
||||||
|
title="同步设置"
|
||||||
|
>
|
||||||
|
<Settings className="h-4 w-4" />
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={() => refetch()}
|
||||||
|
className="rounded-btn p-2 text-muted hover:text-foreground hover:bg-elevated transition-colors"
|
||||||
|
title="刷新状态"
|
||||||
|
>
|
||||||
|
<RefreshCw className={cn('h-4 w-4', isLoading && 'animate-spin')} />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{showSettings && (
|
||||||
|
<div className="mt-4 space-y-3 border-t border-border pt-4">
|
||||||
|
<div>
|
||||||
|
<label className="text-xs text-muted">Serve 地址</label>
|
||||||
|
<input
|
||||||
|
value={serveUrl}
|
||||||
|
onChange={e => setServeUrl(e.target.value)}
|
||||||
|
placeholder="http://your-server:3018"
|
||||||
|
className="mt-1 w-full rounded border border-border bg-base px-3 py-2 text-sm text-foreground placeholder:text-muted focus:border-accent focus:outline-none"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label className="text-xs text-muted">同步 Key(可选)</label>
|
||||||
|
<input
|
||||||
|
value={syncKey}
|
||||||
|
onChange={e => setSyncKey(e.target.value)}
|
||||||
|
placeholder="留空则不校验"
|
||||||
|
type="password"
|
||||||
|
className="mt-1 w-full rounded border border-border bg-base px-3 py-2 text-sm text-foreground placeholder:text-muted focus:border-accent focus:outline-none"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
onClick={() => saveCfg.mutate()}
|
||||||
|
disabled={saveCfg.isPending}
|
||||||
|
className="inline-flex items-center gap-2 rounded-btn bg-accent px-4 py-2 text-sm font-medium text-white hover:bg-accent/90 disabled:opacity-50 transition-colors"
|
||||||
|
>
|
||||||
|
{saveCfg.isPending ? (
|
||||||
|
<Loader2 className="h-4 w-4 animate-spin" />
|
||||||
|
) : (
|
||||||
|
<Save className="h-4 w-4" />
|
||||||
|
)}
|
||||||
|
保存配置
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* 数据选择 */}
|
||||||
|
<section className="rounded-card border border-border bg-surface p-5">
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<h3 className="text-sm font-medium text-foreground">选择要同步的数据</h3>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<button
|
||||||
|
onClick={selectAll}
|
||||||
|
className="text-xs text-accent hover:text-accent/80 transition-colors"
|
||||||
|
>
|
||||||
|
全选
|
||||||
|
</button>
|
||||||
|
<span className="text-xs text-muted">/</span>
|
||||||
|
<button
|
||||||
|
onClick={selectNone}
|
||||||
|
className="text-xs text-accent hover:text-accent/80 transition-colors"
|
||||||
|
>
|
||||||
|
取消
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="mt-3 grid grid-cols-2 gap-2">
|
||||||
|
{ALL_PARTS.map(({ key, label }) => {
|
||||||
|
const info = status?.parts?.[key]
|
||||||
|
const hasData = info && info.file_count > 0
|
||||||
|
return (
|
||||||
|
<label
|
||||||
|
key={key}
|
||||||
|
className={cn(
|
||||||
|
'flex items-center gap-3 rounded-lg border px-3 py-2.5 cursor-pointer transition-colors',
|
||||||
|
selected.has(key)
|
||||||
|
? 'border-accent/40 bg-accent/5'
|
||||||
|
: 'border-border bg-base hover:border-muted',
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={selected.has(key)}
|
||||||
|
onChange={() => togglePart(key)}
|
||||||
|
className="h-4 w-4 rounded border-border text-accent focus:ring-accent"
|
||||||
|
/>
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<div className="flex items-center gap-1.5">
|
||||||
|
<span className="text-sm text-foreground">{label}</span>
|
||||||
|
{hasData && (
|
||||||
|
<HardDrive className="h-3 w-3 text-accent shrink-0" />
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
{info && (
|
||||||
|
<div className="mt-0.5 text-[10px] text-muted">
|
||||||
|
{info.file_count > 0
|
||||||
|
? `${info.file_count} 文件 · ${fmtSize(info.size_bytes)}`
|
||||||
|
: '无数据'}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</label>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* 操作 */}
|
||||||
|
<section className="flex items-center justify-between rounded-card border border-border bg-surface p-5">
|
||||||
|
<div className="flex items-center gap-2 text-sm text-muted">
|
||||||
|
<Database className="h-4 w-4" />
|
||||||
|
<span>
|
||||||
|
已选 {selected.size} 个目录
|
||||||
|
{push.data && (
|
||||||
|
<span className="ml-2 text-bull">
|
||||||
|
<CheckCircle2 className="inline h-3.5 w-3.5 mr-0.5" />
|
||||||
|
上次同步成功
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
onClick={() => push.mutate()}
|
||||||
|
disabled={push.isPending || selected.size === 0}
|
||||||
|
className="inline-flex items-center gap-2 rounded-btn bg-accent px-5 py-2.5 text-sm font-medium text-white hover:bg-accent/90 disabled:opacity-50 transition-colors"
|
||||||
|
>
|
||||||
|
{push.isPending ? (
|
||||||
|
<>
|
||||||
|
<Loader2 className="h-4 w-4 animate-spin" />
|
||||||
|
同步中…
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Upload className="h-4 w-4" />
|
||||||
|
立即同步
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
{/* 同步日志 */}
|
||||||
|
{push.data && (
|
||||||
|
<section className="rounded-card border border-border bg-surface p-4">
|
||||||
|
<div className="text-xs text-muted font-mono leading-relaxed">
|
||||||
|
<div>✓ 同步完成</div>
|
||||||
|
<div> 目录: {push.data.parts_sent.join(', ')}</div>
|
||||||
|
<div> 数据包: {fmtSize(push.data.size_bytes)}</div>
|
||||||
|
<div> 文件数: {push.data.serve_response.file_count}</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{push.error && (
|
||||||
|
<section className="rounded-card border border-danger/30 bg-danger/5 p-4">
|
||||||
|
<div className="text-xs text-danger font-mono leading-relaxed">
|
||||||
|
<div>✗ 同步失败</div>
|
||||||
|
<div> {String(push.error)}</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -20,6 +20,7 @@ import { Branding } from './pages/Branding'
|
|||||||
import { Settings } from './pages/Settings'
|
import { Settings } from './pages/Settings'
|
||||||
import { Indices } from './pages/Indices'
|
import { Indices } from './pages/Indices'
|
||||||
import { Dev } from './pages/Dev'
|
import { Dev } from './pages/Dev'
|
||||||
|
import { Sync } from './pages/Sync'
|
||||||
import { useSettings } from './lib/useSharedQueries'
|
import { useSettings } from './lib/useSharedQueries'
|
||||||
import { Logo } from './components/Logo'
|
import { Logo } from './components/Logo'
|
||||||
|
|
||||||
@@ -77,6 +78,7 @@ export const router = createBrowserRouter([
|
|||||||
{ path: 'data', element: <Data /> },
|
{ path: 'data', element: <Data /> },
|
||||||
{ path: 'monitor', element: <Monitor /> },
|
{ path: 'monitor', element: <Monitor /> },
|
||||||
{ path: 'trading', element: <Trading /> },
|
{ path: 'trading', element: <Trading /> },
|
||||||
|
{ path: 'sync', element: <Sync /> },
|
||||||
{ path: 'limit-ladder', element: <LimitUpLadder /> },
|
{ path: 'limit-ladder', element: <LimitUpLadder /> },
|
||||||
{ path: 'indices', element: <Indices /> },
|
{ path: 'indices', element: <Indices /> },
|
||||||
{ path: 'branding', element: <Branding /> },
|
{ path: 'branding', element: <Branding /> },
|
||||||
|
|||||||
@@ -0,0 +1,134 @@
|
|||||||
|
"""数据同步 — 接收 local 端推送的 Parquet 数据包。"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
from io import BytesIO
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile, File, Form
|
||||||
|
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/data/sync", tags=["data-sync"])
|
||||||
|
|
||||||
|
# 允许同步的子目录列表
|
||||||
|
SYNCABLE_PARTS = {
|
||||||
|
"kline_daily", "kline_daily_enriched",
|
||||||
|
"kline_index_daily", "kline_index_enriched",
|
||||||
|
"kline_etf_daily", "kline_etf_enriched", "kline_etf_minute",
|
||||||
|
"kline_minute",
|
||||||
|
"adj_factor", "adj_factor_etf",
|
||||||
|
"instruments", "instruments_index", "instruments_etf",
|
||||||
|
"financials",
|
||||||
|
"pools",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _verify_sync_key(request: Request) -> None:
|
||||||
|
"""简单的鉴权 — 校验 X-Sync-Key 头。"""
|
||||||
|
key = request.headers.get("X-Sync-Key", "")
|
||||||
|
expected = getattr(settings, "sync_key", "")
|
||||||
|
if expected and key != expected:
|
||||||
|
raise HTTPException(status_code=403, detail="sync key mismatch")
|
||||||
|
|
||||||
|
|
||||||
|
def _refresh_views(request: Request) -> None:
|
||||||
|
"""重新注册 DuckDB 视图 + 刷新 Polars 缓存。"""
|
||||||
|
repo = getattr(request.app.state, "repo", None)
|
||||||
|
if repo is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
d = repo.store.data_dir.as_posix()
|
||||||
|
views = {
|
||||||
|
"kline_daily": f"{d}/kline_daily/**/*.parquet",
|
||||||
|
"kline_enriched": f"{d}/kline_daily_enriched/**/*.parquet",
|
||||||
|
"kline_index_daily": f"{d}/kline_index_daily/**/*.parquet",
|
||||||
|
"kline_index_enriched": f"{d}/kline_index_enriched/**/*.parquet",
|
||||||
|
"kline_etf_daily": f"{d}/kline_etf_daily/**/*.parquet",
|
||||||
|
"kline_etf_enriched": f"{d}/kline_etf_enriched/**/*.parquet",
|
||||||
|
"kline_etf_minute": f"{d}/kline_etf_minute/**/*.parquet",
|
||||||
|
"kline_minute": f"{d}/kline_minute/**/*.parquet",
|
||||||
|
"adj_factor": f"{d}/adj_factor/**/*.parquet",
|
||||||
|
"adj_factor_etf": f"{d}/adj_factor_etf/**/*.parquet",
|
||||||
|
"instruments": f"{d}/instruments/**/*.parquet",
|
||||||
|
"instruments_index": f"{d}/instruments_index/**/*.parquet",
|
||||||
|
"instruments_etf": f"{d}/instruments_etf/**/*.parquet",
|
||||||
|
}
|
||||||
|
for name, path in views.items():
|
||||||
|
try:
|
||||||
|
repo.store.db.execute(
|
||||||
|
f"CREATE OR REPLACE VIEW {name} AS "
|
||||||
|
f"SELECT * FROM read_parquet('{path}', union_by_name=true)"
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("view %s refresh failed: %s", name, exc)
|
||||||
|
repo.store._register_unified_views()
|
||||||
|
repo.clear_cache()
|
||||||
|
repo.refresh_cache()
|
||||||
|
# 清除 API 数据缓存
|
||||||
|
from app.api.data import invalidate_data_cache
|
||||||
|
invalidate_data_cache()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("cache refresh failed: %s", exc)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/upload")
|
||||||
|
async def upload_sync(
|
||||||
|
request: Request,
|
||||||
|
file: UploadFile = File(...),
|
||||||
|
parts: str = Form(""),
|
||||||
|
_auth: None = Depends(_verify_sync_key),
|
||||||
|
):
|
||||||
|
"""接收 local 端打包的 tar.gz 数据,解压写入 data/ 目录。"""
|
||||||
|
data_dir: Path = request.app.state.datastore.data_dir
|
||||||
|
|
||||||
|
# 解析要同步的 parts
|
||||||
|
requested = set(p.strip() for p in parts.split(",") if p.strip())
|
||||||
|
if not requested:
|
||||||
|
raise HTTPException(status_code=400, detail="parts 不能为空,逗号分隔子目录名")
|
||||||
|
|
||||||
|
invalid = requested - SYNCABLE_PARTS
|
||||||
|
if invalid:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"不支持的同步目录: {', '.join(sorted(invalid))}",
|
||||||
|
)
|
||||||
|
|
||||||
|
# 读取上传的 tar.gz
|
||||||
|
raw = await file.read()
|
||||||
|
if not raw:
|
||||||
|
raise HTTPException(status_code=400, detail="上传文件为空")
|
||||||
|
|
||||||
|
extracted = 0
|
||||||
|
try:
|
||||||
|
with tarfile.open(fileobj=BytesIO(raw), mode="r:gz") as tar:
|
||||||
|
for member in tar.getmembers():
|
||||||
|
# 安全校验: 防止路径穿越
|
||||||
|
member_path = Path(member.name)
|
||||||
|
if member_path.is_absolute() or ".." in member_path.parts:
|
||||||
|
logger.warning("skipping unsafe path: %s", member.name)
|
||||||
|
continue
|
||||||
|
# 确定属于哪个 part
|
||||||
|
top_dir = member_path.parts[0] if member_path.parts else ""
|
||||||
|
if top_dir not in requested:
|
||||||
|
continue
|
||||||
|
target = data_dir / member.name
|
||||||
|
if member.isfile():
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with tar.extractfile(member) as src:
|
||||||
|
if src is None:
|
||||||
|
continue
|
||||||
|
target.write_bytes(src.read())
|
||||||
|
extracted += 1
|
||||||
|
except tarfile.TarError as exc:
|
||||||
|
raise HTTPException(status_code=400, detail=f"压缩包解析失败: {exc}") from exc
|
||||||
|
|
||||||
|
# 刷新缓存
|
||||||
|
_refresh_views(request)
|
||||||
|
|
||||||
|
logger.info("sync uploaded: parts=%s files=%d", parts, extracted)
|
||||||
|
return {"ok": True, "parts": sorted(requested), "file_count": extracted}
|
||||||
@@ -97,6 +97,9 @@ class Settings(BaseSettings):
|
|||||||
# 公网服务器部署时免去 SSH 端口转发设密码的麻烦。写入 auth.json(哈希)后即不再读取。
|
# 公网服务器部署时免去 SSH 端口转发设密码的麻烦。写入 auth.json(哈希)后即不再读取。
|
||||||
auth_password: str = ""
|
auth_password: str = ""
|
||||||
|
|
||||||
|
# Sync — 数据同步鉴权 Key(local→serve 推送时校验 X-Sync-Key 头)
|
||||||
|
sync_key: str = ""
|
||||||
|
|
||||||
# Data — frozen: exe 同级 data/ 子目录; 非 frozen: 项目根 data/
|
# Data — frozen: exe 同级 data/ 子目录; 非 frozen: 项目根 data/
|
||||||
# (均可被环境变量 DATA_DIR 覆盖, pydantic-settings 自动注入)
|
# (均可被环境变量 DATA_DIR 覆盖, pydantic-settings 自动注入)
|
||||||
data_dir: Path = _user_data_root()
|
data_dir: Path = _user_data_root()
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from fastapi.responses import FileResponse, JSONResponse
|
|||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
|
|
||||||
from app import __version__
|
from app import __version__
|
||||||
from app.api import analysis, auth as auth_api, data, ext_data, financials, indices, kline, market_recap, alerts, overview, pipeline, rps, screener, settings as settings_api, signals, stock_analysis, strategy
|
from app.api import analysis, auth as auth_api, data, data_sync, ext_data, financials, indices, kline, market_recap, alerts, overview, pipeline, rps, screener, settings as settings_api, signals, stock_analysis, strategy
|
||||||
from app.api.routes import router as core_router
|
from app.api.routes import router as core_router
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.jobs import daily_pipeline
|
from app.jobs import daily_pipeline
|
||||||
@@ -141,7 +141,7 @@ app.add_middleware(
|
|||||||
# 2. 未设密码 + 公网 → 拒绝(403, 防裸奔也防抢占; 引导本机设密码)
|
# 2. 未设密码 + 公网 → 拒绝(403, 防裸奔也防抢占; 引导本机设密码)
|
||||||
# 3. 已设密码 → 检查 session, 无效则 401(前端跳登录)
|
# 3. 已设密码 → 检查 session, 无效则 401(前端跳登录)
|
||||||
# 白名单: /api/auth/* (设密码/登录本身)、/health 等探活。
|
# 白名单: /api/auth/* (设密码/登录本身)、/health 等探活。
|
||||||
_AUTH_WHITELIST_PREFIX = ("/api/auth/",)
|
_AUTH_WHITELIST_PREFIX = ("/api/auth/", "/api/data/sync/")
|
||||||
_AUTH_WHITELIST_EXACT = ("/health", "/api/health", "/openapi.json", "/docs", "/redoc")
|
_AUTH_WHITELIST_EXACT = ("/health", "/api/health", "/openapi.json", "/docs", "/redoc")
|
||||||
|
|
||||||
|
|
||||||
@@ -188,6 +188,7 @@ app.include_router(overview.router)
|
|||||||
app.include_router(analysis.router)
|
app.include_router(analysis.router)
|
||||||
app.include_router(pipeline.router)
|
app.include_router(pipeline.router)
|
||||||
app.include_router(data.router)
|
app.include_router(data.router)
|
||||||
|
app.include_router(data_sync.router)
|
||||||
app.include_router(ext_data.router)
|
app.include_router(ext_data.router)
|
||||||
app.include_router(financials.router)
|
app.include_router(financials.router)
|
||||||
app.include_router(stock_analysis.router)
|
app.include_router(stock_analysis.router)
|
||||||
|
|||||||
Reference in New Issue
Block a user