363 lines
10 KiB
Python
363 lines
10 KiB
Python
"""Data collector — fetch OHLCV from akshare and upsert into daily_bars."""
|
|
import threading
|
|
from datetime import date
|
|
from app.database import SessionLocal
|
|
from app.models import DailyBar, Contract
|
|
from app.engine.lock_strategy import compute_amp_5d
|
|
|
|
_sync_progress = {
|
|
"running": False,
|
|
"contract": "",
|
|
"contract_done": 0,
|
|
"contract_total": 0,
|
|
"date_done": 0,
|
|
"date_total": 0,
|
|
"rows": 0,
|
|
"finished": False,
|
|
}
|
|
_lock = threading.Lock()
|
|
|
|
|
|
def get_sync_progress() -> dict:
|
|
with _lock:
|
|
return dict(_sync_progress)
|
|
|
|
|
|
def start_sync_positions_background():
|
|
"""Start sync_position_rankings in a background thread. Returns immediately."""
|
|
with _lock:
|
|
if _sync_progress["running"]:
|
|
return False
|
|
_sync_progress.update({
|
|
"running": True,
|
|
"finished": False,
|
|
"contract": "",
|
|
"contract_done": 0,
|
|
"contract_total": 0,
|
|
"date_done": 0,
|
|
"date_total": 0,
|
|
"rows": 0,
|
|
})
|
|
|
|
def _run():
|
|
try:
|
|
_do_sync_with_progress()
|
|
finally:
|
|
with _lock:
|
|
_sync_progress["running"] = False
|
|
_sync_progress["finished"] = True
|
|
|
|
threading.Thread(target=_run, daemon=True).start()
|
|
return True
|
|
|
|
|
|
def _do_sync_with_progress():
|
|
from app.models import PositionRanking
|
|
|
|
db = SessionLocal()
|
|
try:
|
|
active_contracts = (
|
|
db.query(Contract).filter(Contract.is_active == True).all()
|
|
)
|
|
with _lock:
|
|
_sync_progress["contract_total"] = len(active_contracts)
|
|
|
|
for idx, c in enumerate(active_contracts):
|
|
with _lock:
|
|
_sync_progress.update({
|
|
"contract": c.code,
|
|
"contract_done": idx,
|
|
"date_done": 0,
|
|
"date_total": 0,
|
|
})
|
|
|
|
code = c.code.upper()
|
|
|
|
existing_dates = {
|
|
r[0] for r in
|
|
db.query(PositionRanking.date)
|
|
.filter(PositionRanking.contract_code == code)
|
|
.distinct()
|
|
.all()
|
|
}
|
|
|
|
bar_dates = [
|
|
r[0] for r in
|
|
db.query(DailyBar.date)
|
|
.filter(DailyBar.contract == code)
|
|
.order_by(DailyBar.date)
|
|
.all()
|
|
]
|
|
|
|
missing = [d for d in bar_dates if d not in existing_dates]
|
|
with _lock:
|
|
_sync_progress["date_total"] = len(missing)
|
|
|
|
for date_idx, d in enumerate(missing):
|
|
date_str = d.strftime("%Y%m%d")
|
|
rankings = fetch_position_rankings(code, date_str)
|
|
inserted = 0
|
|
for r in rankings:
|
|
db.add(PositionRanking(
|
|
contract_code=code,
|
|
institution=r["institution"],
|
|
data_type=r["data_type"],
|
|
date=d,
|
|
rank=r["rank"],
|
|
value=r["value"],
|
|
change=r["change"],
|
|
))
|
|
inserted += 1
|
|
db.flush()
|
|
with _lock:
|
|
_sync_progress["date_done"] = date_idx + 1
|
|
_sync_progress["rows"] += inserted
|
|
|
|
db.commit()
|
|
with _lock:
|
|
_sync_progress["contract_done"] = len(active_contracts)
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def fetch_contract_bars(contract_code: str, start_date: str | None = None) -> list[dict]:
|
|
"""Fetch daily OHLCV for a single contract from akshare.
|
|
|
|
Returns list of {date, open, close, high, low} dicts.
|
|
Only returns bars on or after start_date when provided.
|
|
"""
|
|
import akshare as ak
|
|
|
|
try:
|
|
df = ak.futures_zh_daily_sina(symbol=contract_code.upper())
|
|
except Exception as e:
|
|
print(f"[collector] akshare error for {contract_code}: {e}")
|
|
return []
|
|
|
|
if df is None or df.empty:
|
|
print(f"[collector] No data returned for {contract_code}")
|
|
return []
|
|
|
|
filter_date = date.fromisoformat(start_date) if start_date else None
|
|
|
|
bars = []
|
|
for _, row in df.iterrows():
|
|
try:
|
|
val = row["date"]
|
|
if hasattr(val, "strftime"):
|
|
d = val.date() if hasattr(val, "date") else val
|
|
else:
|
|
d = date.fromisoformat(str(val)[:10])
|
|
|
|
if filter_date and d < filter_date:
|
|
continue
|
|
|
|
bars.append({
|
|
"date": d,
|
|
"open": float(row["open"]),
|
|
"close": float(row["close"]),
|
|
"high": float(row["high"]),
|
|
"low": float(row["low"]),
|
|
})
|
|
except (KeyError, ValueError, TypeError) as e:
|
|
print(f"[collector] skip row: {e}")
|
|
continue
|
|
|
|
return bars
|
|
|
|
|
|
def sync_active_contracts() -> dict:
|
|
"""Sync all active contracts. Returns {contract_code: new_bars_count}."""
|
|
db = SessionLocal()
|
|
results = {}
|
|
|
|
try:
|
|
active_contracts = (
|
|
db.query(Contract).filter(Contract.is_active == True).all()
|
|
)
|
|
|
|
for c in active_contracts:
|
|
count = _sync_one(db, c.code)
|
|
results[c.code] = count
|
|
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
|
|
return results
|
|
|
|
|
|
def sync_one_contract(contract_code: str) -> int:
|
|
"""Sync a single contract. Returns number of new bars inserted."""
|
|
db = SessionLocal()
|
|
try:
|
|
count = _sync_one(db, contract_code.upper())
|
|
db.commit()
|
|
return count
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def _sync_one(db, contract_code: str) -> int:
|
|
"""Internal: sync one contract using existing session."""
|
|
latest = (
|
|
db.query(DailyBar.date)
|
|
.filter(DailyBar.contract == contract_code)
|
|
.order_by(DailyBar.date.desc())
|
|
.first()
|
|
)
|
|
start_date = latest[0].isoformat() if latest else None
|
|
|
|
bars = fetch_contract_bars(contract_code, start_date)
|
|
if not bars:
|
|
return 0
|
|
|
|
inserted = 0
|
|
min_date = None
|
|
for bar in bars:
|
|
existing = (
|
|
db.query(DailyBar)
|
|
.filter(
|
|
DailyBar.contract == contract_code,
|
|
DailyBar.date == bar["date"],
|
|
)
|
|
.first()
|
|
)
|
|
if not existing:
|
|
db.add(DailyBar(
|
|
contract=contract_code,
|
|
date=bar["date"],
|
|
open=bar["open"],
|
|
close=bar["close"],
|
|
high=bar["high"],
|
|
low=bar["low"],
|
|
))
|
|
inserted += 1
|
|
if min_date is None or bar["date"] < min_date:
|
|
min_date = bar["date"]
|
|
|
|
if inserted > 0:
|
|
db.flush()
|
|
_recompute_amp(db, contract_code, from_date=min_date)
|
|
|
|
return inserted
|
|
|
|
|
|
def fetch_position_rankings(contract_code: str, trade_date: str) -> list[dict]:
|
|
"""Fetch top-20 position rankings for a contract on a given date from akshare.
|
|
|
|
Calls the API 3 times (volume, long, short) and returns a unified list of dicts:
|
|
{data_type, rank, institution, value, change}
|
|
"""
|
|
import akshare as ak
|
|
|
|
results = []
|
|
for sym, dtype in [("成交量", "volume"), ("多单持仓", "long"), ("空单持仓", "short")]:
|
|
try:
|
|
df = ak.futures_hold_pos_sina(symbol=sym, contract=contract_code.upper(), date=trade_date)
|
|
except Exception as e:
|
|
print(f"[collector] position akshare error for {contract_code} {dtype}: {e}")
|
|
continue
|
|
|
|
if df is None or df.empty:
|
|
continue
|
|
|
|
for _, row in df.iterrows():
|
|
try:
|
|
results.append({
|
|
"data_type": dtype,
|
|
"rank": int(row["名次"]),
|
|
"institution": str(row["会员简称"]),
|
|
"value": int(row.iloc[2]),
|
|
"change": int(row["比上交易增减"]),
|
|
})
|
|
except (KeyError, ValueError, TypeError) as e:
|
|
print(f"[collector] position skip row: {e}")
|
|
continue
|
|
|
|
return results
|
|
|
|
|
|
def sync_position_rankings() -> dict:
|
|
"""Sync position rankings for all active contracts. Returns {contract_code: new_rows}."""
|
|
db = SessionLocal()
|
|
results = {}
|
|
|
|
try:
|
|
active_contracts = (
|
|
db.query(Contract).filter(Contract.is_active == True).all()
|
|
)
|
|
|
|
for c in active_contracts:
|
|
count = _sync_positions_for_contract(db, c.code)
|
|
results[c.code] = count
|
|
|
|
db.commit()
|
|
finally:
|
|
db.close()
|
|
|
|
return results
|
|
|
|
|
|
def _sync_positions_for_contract(db, contract_code: str) -> int:
|
|
"""Sync position rankings for all dates that have bars but no position data."""
|
|
from app.models import PositionRanking
|
|
|
|
code = contract_code.upper()
|
|
|
|
existing_dates = {
|
|
r[0] for r in
|
|
db.query(PositionRanking.date)
|
|
.filter(PositionRanking.contract_code == code)
|
|
.distinct()
|
|
.all()
|
|
}
|
|
|
|
bar_dates = [
|
|
r[0] for r in
|
|
db.query(DailyBar.date)
|
|
.filter(DailyBar.contract == code)
|
|
.order_by(DailyBar.date)
|
|
.all()
|
|
]
|
|
|
|
inserted = 0
|
|
for d in bar_dates:
|
|
if d in existing_dates:
|
|
continue
|
|
date_str = d.strftime("%Y%m%d")
|
|
rankings = fetch_position_rankings(code, date_str)
|
|
for r in rankings:
|
|
db.add(PositionRanking(
|
|
contract_code=code,
|
|
institution=r["institution"],
|
|
data_type=r["data_type"],
|
|
date=d,
|
|
rank=r["rank"],
|
|
value=r["value"],
|
|
change=r["change"],
|
|
))
|
|
inserted += 1
|
|
db.flush()
|
|
|
|
return inserted
|
|
|
|
|
|
def _recompute_amp(db, contract_code: str, from_date: date | None = None):
|
|
"""Recompute amp_5d for bars of a contract from from_date onwards."""
|
|
bars = (
|
|
db.query(DailyBar)
|
|
.filter(DailyBar.contract == contract_code)
|
|
.order_by(DailyBar.date)
|
|
.all()
|
|
)
|
|
start_idx = 0
|
|
if from_date:
|
|
for i, bar in enumerate(bars):
|
|
if bar.date >= from_date:
|
|
start_idx = i
|
|
break
|
|
for i, bar in enumerate(bars):
|
|
if i >= 5 and i >= start_idx:
|
|
bar.amp_5d = compute_amp_5d([b.diff for b in bars[i - 5 : i]])
|