feat(frontend): 添加交易看板，统一时区为CST
refactor(db): 移除所有SQLite相关代码，迁移至MySQL
This commit is contained in:
crypto_predictor committed
1 parent
5e0d6b18ef
commit
55453d3739
12 files changed
+2116
-501
No files matched your search
+77
-121
@@ -1,200 +1,156 @@
|
||||
import sqlite3
|
||||
import pymysql
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
from typing import Optional, List, Tuple
|
||||
from typing import Optional
|
||||
from contextlib import contextmanager
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
load_dotenv()
|
||||
|
||||
class DatabaseManager:
|
||||
"""Manages database for storing OHLCV data.
|
||||
Supports SQLite (local) and MySQL (remote).
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str = "./data/market_data.db"):
|
||||
self.db_path = Path(db_path)
|
||||
self.use_mysql = bool(os.getenv('MYSQL_HOST'))
|
||||
|
||||
if not self.use_mysql:
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
class DatabaseManager:
|
||||
"""管理 OHLCV 数据的 MySQL 数据库"""
|
||||
|
||||
def __init__(self, db_path: str = None):
|
||||
"""初始化数据库管理器 (db_path 参数已废弃，仅为兼容性保留)"""
|
||||
self._engine = None
|
||||
self.initialize_db()
|
||||
|
||||
def _get_sqlalchemy_engine(self):
|
||||
"""获取 SQLAlchemy 引擎（用于 pandas read_sql）"""
|
||||
if self._engine is None:
|
||||
host = os.getenv('MYSQL_HOST')
|
||||
port = os.getenv('MYSQL_PORT', 3306)
|
||||
user = os.getenv('MYSQL_USER')
|
||||
password = os.getenv('MYSQL_PASSWORD')
|
||||
database = os.getenv('MYSQL_DB')
|
||||
self._engine = create_engine(
|
||||
f"mysql+pymysql://{user}:{password}@{host}:{port}/{database}?charset=utf8mb4"
|
||||
)
|
||||
return self._engine
|
||||
|
||||
@contextmanager
|
||||
def get_connection(self):
|
||||
"""Context manager for database connection."""
|
||||
if self.use_mysql:
|
||||
conn = pymysql.connect(
|
||||
host=os.getenv('MYSQL_HOST'),
|
||||
port=int(os.getenv('MYSQL_PORT', 3306)),
|
||||
user=os.getenv('MYSQL_USER'),
|
||||
password=os.getenv('MYSQL_PASSWORD'),
|
||||
database=os.getenv('MYSQL_DB'),
|
||||
charset='utf8mb4',
|
||||
# pandas requires standard cursor or handling
|
||||
)
|
||||
else:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
|
||||
"""获取 MySQL 数据库连接"""
|
||||
conn = pymysql.connect(
|
||||
host=os.getenv('MYSQL_HOST'),
|
||||
port=int(os.getenv('MYSQL_PORT', 3306)),
|
||||
user=os.getenv('MYSQL_USER'),
|
||||
password=os.getenv('MYSQL_PASSWORD'),
|
||||
database=os.getenv('MYSQL_DB'),
|
||||
charset='utf8mb4',
|
||||
)
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def initialize_db(self):
|
||||
"""Create tables if they don't exist."""
|
||||
"""创建表（如果不存在）"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
if self.use_mysql:
|
||||
# MySQL Schema
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ohlcv (
|
||||
timestamp DATETIME,
|
||||
symbol VARCHAR(20),
|
||||
timeframe VARCHAR(10),
|
||||
open DOUBLE,
|
||||
high DOUBLE,
|
||||
low DOUBLE,
|
||||
close DOUBLE,
|
||||
volume DOUBLE,
|
||||
PRIMARY KEY (symbol, timeframe, timestamp),
|
||||
INDEX idx_ohlcv_time (symbol, timeframe, timestamp)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
""")
|
||||
else:
|
||||
# SQLite Schema
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ohlcv (
|
||||
timestamp DATETIME,
|
||||
symbol TEXT,
|
||||
timeframe TEXT,
|
||||
open REAL,
|
||||
high REAL,
|
||||
low REAL,
|
||||
close REAL,
|
||||
volume REAL,
|
||||
PRIMARY KEY (symbol, timeframe, timestamp)
|
||||
)
|
||||
""")
|
||||
cursor.execute("""
|
||||
CREATE INDEX IF NOT EXISTS idx_ohlcv_time
|
||||
ON ohlcv (symbol, timeframe, timestamp)
|
||||
""")
|
||||
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS ohlcv (
|
||||
timestamp DATETIME,
|
||||
symbol VARCHAR(20),
|
||||
timeframe VARCHAR(10),
|
||||
open DOUBLE,
|
||||
high DOUBLE,
|
||||
low DOUBLE,
|
||||
close DOUBLE,
|
||||
volume DOUBLE,
|
||||
PRIMARY KEY (symbol, timeframe, timestamp),
|
||||
INDEX idx_ohlcv_time (symbol, timeframe, timestamp)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
def save_ohlcv(self, df: pd.DataFrame, symbol: str, timeframe: str):
|
||||
"""Save OHLCV data to database (Upsert)."""
|
||||
"""保存 OHLCV 数据到数据库 (Upsert)"""
|
||||
if df.empty:
|
||||
return
|
||||
|
||||
# Ensure timestamp is datetime
|
||||
# 确保 timestamp 是 datetime 类型
|
||||
df = df.copy()
|
||||
if not pd.api.types.is_datetime64_any_dtype(df['timestamp']):
|
||||
df['timestamp'] = pd.to_datetime(df['timestamp'])
|
||||
|
||||
# Convert to string for SQLite compatibility (ISO 8601) or MySQL
|
||||
# 转换为字符串格式
|
||||
df['timestamp'] = df['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# Add metadata columns if they don't exist
|
||||
# 添加元数据列
|
||||
if 'symbol' not in df.columns:
|
||||
df['symbol'] = symbol
|
||||
if 'timeframe' not in df.columns:
|
||||
df['timeframe'] = timeframe
|
||||
|
||||
# Prepare data for insertion (ensure order)
|
||||
# records = df[['timestamp', 'symbol', 'timeframe', 'open', 'high', 'low', 'close', 'volume']].to_records(index=False)
|
||||
# to_records returns numpy record array, which sqlite3 adapts.
|
||||
# For MySQL/pymysql, we need list of tuples.
|
||||
|
||||
cols = ['timestamp', 'symbol', 'timeframe', 'open', 'high', 'low', 'close', 'volume']
|
||||
data = [tuple(row) for row in df[cols].values]
|
||||
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
if self.use_mysql:
|
||||
# MySQL uses %s and REPLACE INTO
|
||||
sql = """
|
||||
REPLACE INTO ohlcv
|
||||
(timestamp, symbol, timeframe, open, high, low, close, volume)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
"""
|
||||
cursor.executemany(sql, data)
|
||||
else:
|
||||
# SQLite uses ? and INSERT OR REPLACE
|
||||
sql = """
|
||||
INSERT OR REPLACE INTO ohlcv
|
||||
(timestamp, symbol, timeframe, open, high, low, close, volume)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"""
|
||||
# sqlite3 accepts both list of tuples and numpy records, but list of tuples is safer for compatibility
|
||||
cursor.executemany(sql, data)
|
||||
|
||||
sql = """
|
||||
REPLACE INTO ohlcv
|
||||
(timestamp, symbol, timeframe, open, high, low, close, volume)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
"""
|
||||
cursor.executemany(sql, data)
|
||||
conn.commit()
|
||||
|
||||
def get_ohlcv(self, symbol: str, timeframe: str, start_date: Optional[str] = None, end_date: Optional[str] = None) -> pd.DataFrame:
|
||||
"""Retrieve OHLCV data from database."""
|
||||
"""从数据库获取 OHLCV 数据"""
|
||||
|
||||
placeholder = '%s' if self.use_mysql else '?'
|
||||
|
||||
query = f"""
|
||||
query = """
|
||||
SELECT timestamp, open, high, low, close, volume
|
||||
FROM ohlcv
|
||||
WHERE symbol = {placeholder} AND timeframe = {placeholder}
|
||||
WHERE symbol = :p0 AND timeframe = :p1
|
||||
"""
|
||||
params = [symbol, timeframe]
|
||||
params = {'p0': symbol, 'p1': timeframe}
|
||||
param_idx = 2
|
||||
|
||||
if start_date:
|
||||
query += f" AND timestamp >= {placeholder}"
|
||||
params.append(start_date)
|
||||
query += f" AND timestamp >= :p{param_idx}"
|
||||
params[f'p{param_idx}'] = start_date
|
||||
param_idx += 1
|
||||
|
||||
if end_date:
|
||||
# Handle 'YYYY-MM-DD' vs full timestamp
|
||||
if len(str(end_date)) == 10: # YYYY-MM-DD
|
||||
# SQLite has date(), MySQL uses DATE() or just string comparison
|
||||
if self.use_mysql:
|
||||
query += f" AND timestamp < DATE_ADD({placeholder}, INTERVAL 1 DAY)"
|
||||
else:
|
||||
query += f" AND timestamp < date({placeholder}, '+1 day')"
|
||||
# 处理 'YYYY-MM-DD' 格式
|
||||
if len(str(end_date)) == 10:
|
||||
query += f" AND timestamp < DATE_ADD(:p{param_idx}, INTERVAL 1 DAY)"
|
||||
else:
|
||||
query += f" AND timestamp <= {placeholder}"
|
||||
params.append(end_date)
|
||||
query += f" AND timestamp <= :p{param_idx}"
|
||||
params[f'p{param_idx}'] = end_date
|
||||
|
||||
query += " ORDER BY timestamp ASC"
|
||||
|
||||
with self.get_connection() as conn:
|
||||
df = pd.read_sql_query(query, conn, params=params, parse_dates=['timestamp'])
|
||||
engine = self._get_sqlalchemy_engine()
|
||||
with engine.connect() as conn:
|
||||
df = pd.read_sql_query(text(query), conn, params=params, parse_dates=['timestamp'])
|
||||
|
||||
return df
|
||||
|
||||
def get_latest_timestamp(self, symbol: str, timeframe: str) -> Optional[pd.Timestamp]:
|
||||
"""Get the latest timestamp for a symbol/timeframe."""
|
||||
placeholder = '%s' if self.use_mysql else '?'
|
||||
"""获取指定 symbol/timeframe 的最新时间戳"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(f"""
|
||||
cursor.execute("""
|
||||
SELECT MAX(timestamp)
|
||||
FROM ohlcv
|
||||
WHERE symbol = {placeholder} AND timeframe = {placeholder}
|
||||
WHERE symbol = %s AND timeframe = %s
|
||||
""", (symbol, timeframe))
|
||||
result = cursor.fetchone()
|
||||
|
||||
return pd.to_datetime(result[0]) if result and result[0] else None
|
||||
|
||||
def get_earliest_timestamp(self, symbol: str, timeframe: str) -> Optional[pd.Timestamp]:
|
||||
"""Get the earliest timestamp for a symbol/timeframe."""
|
||||
placeholder = '%s' if self.use_mysql else '?'
|
||||
"""获取指定 symbol/timeframe 的最早时间戳"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(f"""
|
||||
cursor.execute("""
|
||||
SELECT MIN(timestamp)
|
||||
FROM ohlcv
|
||||
WHERE symbol = {placeholder} AND timeframe = {placeholder}
|
||||
WHERE symbol = %s AND timeframe = %s
|
||||
""", (symbol, timeframe))
|
||||
result = cursor.fetchone()
|
||||
|
||||
|
||||
Reference in new issue
Block a user