Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 crypto_predictor
|
d34b67f522 | ||
|
crypto_predictorandClaude Opus 4.6
|
84288a687a | ||
|
crypto_predictorandClaude Opus 4.6
|
e78836bf42 |
No files matched your search
@@ -12,7 +12,12 @@
|
||||
"Bash(git push:*)",
|
||||
"Bash(conda activate:*)",
|
||||
"WebSearch",
|
||||
"Bash(pip install:*)"
|
||||
"Bash(pip install:*)",
|
||||
"Bash(git status:*)",
|
||||
"Bash(git -C /Users/xxz/PycharmProjects/crypto_predictor check-ignore -v test_dlinear_jan_2026_create_labels.py)",
|
||||
"Bash(git -C /Users/xxz/PycharmProjects/crypto_predictor status)",
|
||||
"Bash(git -C /Users/xxz/PycharmProjects/crypto_predictor diff)",
|
||||
"Bash(git -C /Users/xxz/PycharmProjects/crypto_predictor log --oneline -5)"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## 项目概述
|
||||
|
||||
基于深度学习的 BTC 小时级涨跌预测系统，包含模型训练、FastAPI 后端、Vue 3 看板、Polymarket 自动交易。当前最佳模型 DLinear 准确率 53.10%。
|
||||
|
||||
## 环境与构建
|
||||
|
||||
```bash
|
||||
# Python 环境（必须使用 Conda，禁止 venv/poetry）
|
||||
conda activate crypto_pred_312
|
||||
|
||||
# 安装依赖（PyTorch 需根据平台单独安装，requirements.txt 不含 torch）
|
||||
pip install -r requirements.txt
|
||||
|
||||
# 数据库：仅支持 MySQL，禁止 SQLite
|
||||
# 配置：复制 .env.example 为 .env 并填写 MySQL 连接信息
|
||||
```
|
||||
|
||||
## 常用命令
|
||||
|
||||
```bash
|
||||
# 启动完整系统（API + 自动交易）
|
||||
python start_system.py
|
||||
|
||||
# 单独启动 API 服务
|
||||
python -m uvicorn api.main:app --host 0.0.0.0 --port 8000
|
||||
|
||||
# 单独启动自动交易
|
||||
python auto_trade_polymarket.py
|
||||
|
||||
# 训练 DLinear 分类模型
|
||||
python train_classification_full.py
|
||||
|
||||
# 测试模型（2026年数据）
|
||||
python test_models_feb_2026.py
|
||||
|
||||
# 集成预测
|
||||
python ensemble_predict.py
|
||||
|
||||
# 测试与检查
|
||||
pytest
|
||||
flake8 .
|
||||
```
|
||||
|
||||
## 架构
|
||||
|
||||
```
|
||||
数据流: Binance API → data/fetcher.py → data/preprocessor.py (技术指标) → data/dataset.py → 模型推理
|
||||
交易流: 模型预测 → auto_trade_polymarket.py → Polymarket 下单 → MySQL 记录
|
||||
服务流: FastAPI (api/main.py) → 前端看板 (api/static/) → http://localhost:8000
|
||||
```
|
||||
|
||||
### 核心模块
|
||||
|
||||
- **`data/`** — 数据层：`fetcher.py`(Binance数据获取)、`preprocessor.py`(特征工程: RSI+MACD)、`dataset.py`(PyTorch数据集)、`database.py`(市场数据MySQL)、`trading_db.py`(交易历史MySQL)
|
||||
- **`models/`** — 模型层：`dlinear/model.py`(主力模型，时序分解+线性层)、`nbeats/`、`base.py`(抽象基类)
|
||||
- **`models/checkpoints/dlinear_classification_2026/`** — 训练产物：`best_model.pt`、`preprocessor.pkl`、`test_2026_results.json`
|
||||
- **`training/`** — 训练层：`trainer.py`(训练循环，支持 MPS/CUDA/CPU)、`config.py`(超参数)、`evaluator.py`(评估)
|
||||
- **`api/`** — 服务层：`main.py`(FastAPI入口)、`models.py`(DB操作+时区转换)、`prediction_service.py`(预测服务)、`polymarket_client.py`(交易客户端)
|
||||
- **`start_system.py`** — 统一启动入口（同时启动 API 和自动交易）
|
||||
- **`auto_trade_polymarket.py`** — 自动交易主程序（每小时执行预测+下单）
|
||||
|
||||
### DLinear 模型结构
|
||||
|
||||
输入 (168h × 9特征) → MovingAvg 时序分解 → 季节性/趋势分量各经线性层 → MLP 聚合 → logits 输出 (BCEWithLogitsLoss)
|
||||
|
||||
## 开发约束
|
||||
|
||||
- **数据库**：仅 MySQL（`pymysql`），禁止 SQLite。连接使用 `data/database.py` 的 `DatabaseManager` 或 `api/models.py` 的 `get_db_connection`
|
||||
- **时区**：数据库存 UTC/ET，前端展示 CST（北京时间），转换用 `api.models.convert_to_cst()`
|
||||
- **代码注释**：中文，解释"为什么"
|
||||
- **命名**：变量/函数 `snake_case`，类 `PascalCase`
|
||||
- **代理访问 Binance**：需在 `.env` 配置 `SOCKS5_PROXY` 和 `USE_PROXY=true`
|
||||
- **日志**：交易日志在 `logs/polymarket_trading.log`（10MB 轮转）
|
||||
|
||||
## 禁止事项
|
||||
|
||||
- 禁止引入 SQLite 或恢复已删除的 SQLite 代码
|
||||
- 禁止在未确认的情况下升级核心库版本（torch, pandas 等）
|
||||
- 禁止删除看似无用但可能被外部脚本调用的代码
|
||||
+100
-12
@@ -35,6 +35,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from data.fetcher import BinanceDataFetcher
|
||||
from data.trading_db import TradingDatabase
|
||||
from api.models import PredictionModel
|
||||
|
||||
load_dotenv()
|
||||
|
||||
@@ -141,6 +142,26 @@ class ClassificationPreprocessor:
|
||||
|
||||
return np.array(X), np.array(y), df
|
||||
|
||||
def preprocess_for_inference(self, df: pd.DataFrame):
|
||||
"""推理专用预处理：不调用create_labels，整个df一次性处理
|
||||
|
||||
与test_dlinear_jan_2026.py逻辑一致：
|
||||
加特征 -> dropna -> scaler.transform -> 构建窗口
|
||||
"""
|
||||
df = self.add_features(df)
|
||||
df = df.dropna().reset_index(drop=True)
|
||||
|
||||
feature_cols = ['open', 'high', 'low', 'close', 'volume',
|
||||
'rsi', 'macd', 'macd_hist', 'macd_signal']
|
||||
|
||||
scaled_features = self.scaler.transform(df[feature_cols])
|
||||
|
||||
X = []
|
||||
for i in range(len(scaled_features) - self.lookback):
|
||||
X.append(scaled_features[i:i + self.lookback])
|
||||
|
||||
return np.array(X), df
|
||||
|
||||
|
||||
class PolymarketBTCTrader:
|
||||
"""Polymarket BTC Hourly交易器"""
|
||||
@@ -614,11 +635,23 @@ class PolymarketBTCTrader:
|
||||
logger.info(f" 上一K线时间: {prev_time} (用于预测)")
|
||||
logger.info(f" 上一收盘价: ${prev_close:,.2f}")
|
||||
logger.info(f" 当前K线时间: {current_time} (预测目标)")
|
||||
logger.info(f" 当前收盘价: ${current_close:,.2f} (实际结果)")
|
||||
logger.info(f" 当前收盘价: ${current_close:,.2f} (未收盘)")
|
||||
|
||||
# 用除最后一根外的数据进行预测
|
||||
# 回填上一小时的实际结果
|
||||
# prev_time的K线已收盘，用它的收盘价(prev_close) vs 再前一根的收盘价来判断涨跌
|
||||
self._update_previous_prediction(df)
|
||||
|
||||
# 用除最后一根外的数据进行预测（无create_labels，与test逻辑一致）
|
||||
df_for_prediction = df.iloc[:-1]
|
||||
X, _, _ = self.preprocessor.preprocess(df_for_prediction, fit=False)
|
||||
X, processed_df = self.preprocessor.preprocess_for_inference(df_for_prediction)
|
||||
|
||||
# 打印模型输入的最后一根K线数据
|
||||
last_input_row = df_for_prediction.iloc[-1]
|
||||
logger.info(f" 📌 模型输入末根K线: {last_input_row['timestamp']} "
|
||||
f"O:{last_input_row['open']:.2f} H:{last_input_row['high']:.2f} "
|
||||
f"L:{last_input_row['low']:.2f} C:{last_input_row['close']:.2f} "
|
||||
f"V:{last_input_row['volume']:.0f}")
|
||||
|
||||
X_last = torch.FloatTensor(X[-1:]).to(self.device)
|
||||
|
||||
with torch.no_grad():
|
||||
@@ -632,16 +665,71 @@ class PolymarketBTCTrader:
|
||||
logger.info(f"🤖 DLinear预测: {direction}")
|
||||
logger.info(f" 置信度: {confidence:.2%}")
|
||||
|
||||
# 显示预测是否正确 (用于验证)
|
||||
actual_up = current_close > prev_close
|
||||
if prediction == 1:
|
||||
correct = actual_up
|
||||
else:
|
||||
correct = not actual_up
|
||||
logger.info(f" 实际涨跌: {'📈 涨' if actual_up else '📉 跌'} ({'✅ 正确' if correct else '❌ 错误'})")
|
||||
# 保存预测结果到 model_predictions 表
|
||||
# 当前K线尚未收盘，actual_result 和 is_correct 留空，等收盘后更新
|
||||
try:
|
||||
et_tz = ZoneInfo('America/New_York')
|
||||
# current_time 是目标K线的UTC时间戳（当前正在进行的K线）
|
||||
current_time_utc = pd.Timestamp(current_time)
|
||||
current_time_et = current_time_utc.tz_localize('UTC').tz_convert(et_tz)
|
||||
|
||||
prediction_data = {
|
||||
"timestamp": current_time_utc.isoformat(),
|
||||
"timestamp_et": current_time_et.isoformat(),
|
||||
"date": current_time_et.strftime('%Y-%m-%d'),
|
||||
"hour": current_time_et.hour,
|
||||
"model_name": "DLinear",
|
||||
"prediction": prediction,
|
||||
"probability": probability,
|
||||
"actual_result": None,
|
||||
"is_correct": None,
|
||||
"open_price": float(df['open'].iloc[-1]),
|
||||
"close_price": None,
|
||||
}
|
||||
PredictionModel.save_prediction(prediction_data)
|
||||
logger.info(f"💾 预测记录已保存 (ET {current_time_et.strftime('%Y-%m-%d %H:%M')})")
|
||||
except Exception as e:
|
||||
logger.error(f"保存预测记录失败: {e}")
|
||||
|
||||
return prediction, probability
|
||||
|
||||
def _update_previous_prediction(self, df: pd.DataFrame):
|
||||
"""回填上一小时预测的实际结果
|
||||
|
||||
df.iloc[-2] 是上一小时已收盘K线，收盘价 vs 开盘价即为实际涨跌。
|
||||
"""
|
||||
if len(df) < 2:
|
||||
return
|
||||
|
||||
try:
|
||||
et_tz = ZoneInfo('America/New_York')
|
||||
prev_row = df.iloc[-2]
|
||||
prev_utc = pd.Timestamp(prev_row['timestamp'])
|
||||
prev_et = prev_utc.tz_localize('UTC').tz_convert(et_tz)
|
||||
prev_date = prev_et.strftime('%Y-%m-%d')
|
||||
prev_hour = prev_et.hour
|
||||
|
||||
prev_open = float(prev_row['open'])
|
||||
prev_close = float(prev_row['close'])
|
||||
actual_result = 1 if prev_close > prev_open else 0
|
||||
|
||||
updated = PredictionModel.update_actual_result(
|
||||
date=prev_date,
|
||||
hour=prev_hour,
|
||||
model_name="DLinear",
|
||||
actual_result=actual_result,
|
||||
close_price=prev_close
|
||||
)
|
||||
if updated:
|
||||
record = PredictionModel.get_prediction(prev_date, prev_hour, "DLinear")
|
||||
is_correct = record.get('is_correct') if record else None
|
||||
correct_str = '✅ 正确' if is_correct == 1 else '❌ 错误' if is_correct == 0 else '未知'
|
||||
logger.info(f"📝 回填上一小时结果: ET {prev_date} {prev_hour}:00, "
|
||||
f"开:{prev_open:.2f} 收:{prev_close:.2f} "
|
||||
f"实际{'涨' if actual_result == 1 else '跌'}, {correct_str}")
|
||||
except Exception as e:
|
||||
logger.error(f"回填上一小时结果失败: {e}")
|
||||
|
||||
def run_once(self) -> Dict:
|
||||
"""运行一次交易流程（不等待整点）"""
|
||||
logger.info("运行单次交易...")
|
||||
@@ -784,9 +872,9 @@ class PolymarketBTCTrader:
|
||||
logger.info(f" 上一K线: {prev_time} 收盘价: ${prev_close:,.2f}")
|
||||
logger.info(f" 当前K线: {current_time} 收盘价: ${current_close:,.2f}")
|
||||
|
||||
# 进行预测
|
||||
# 进行预测（无create_labels，与test逻辑一致）
|
||||
df_for_prediction = df.iloc[:-1]
|
||||
X, _, _ = self.preprocessor.preprocess(df_for_prediction, fit=False)
|
||||
X, _ = self.preprocessor.preprocess_for_inference(df_for_prediction)
|
||||
X_last = torch.FloatTensor(X[-1:]).to(self.device)
|
||||
|
||||
with torch.no_grad():
|
||||
|
||||
@@ -0,0 +1,380 @@
|
||||
"""对比 test 旧逻辑 vs auto_trade 逻辑的数据处理差异
|
||||
|
||||
两种预处理方式：
|
||||
- test旧逻辑：整个df一次性预处理，无create_labels，按索引遍历窗口
|
||||
- auto_trade逻辑：截取到当前K线的df，含create_labels，取X[-1]预测
|
||||
|
||||
对比内容：
|
||||
1. 预处理后的df行数差异
|
||||
2. 同一时间点的特征值差异（RSI、MACD等）
|
||||
3. scaler transform后的值差异
|
||||
4. 最终输入窗口的数值差异
|
||||
5. 模型输出logits/概率/预测结果差异
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pickle
|
||||
import pandas_ta as ta
|
||||
from datetime import timedelta
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
from data.fetcher import BinanceDataFetcher
|
||||
|
||||
# ==================== pickle反序列化需要的占位类 ====================
|
||||
|
||||
class ClassificationPreprocessor:
|
||||
"""pkl文件中保存的类，pickle.load时需要能找到这个类名"""
|
||||
def __init__(self, lookback=168, forecast_horizon=1):
|
||||
self.lookback = lookback
|
||||
self.forecast_horizon = forecast_horizon
|
||||
self.scaler = StandardScaler()
|
||||
self.is_fitted = False
|
||||
|
||||
# ==================== 两种预处理器 ====================
|
||||
|
||||
class TestPreprocessor:
|
||||
"""test旧逻辑：无create_labels"""
|
||||
def __init__(self, lookback=168):
|
||||
self.lookback = lookback
|
||||
self.scaler = StandardScaler()
|
||||
|
||||
def preprocess(self, df, fit=True):
|
||||
df = df.copy()
|
||||
df['rsi'] = ta.rsi(df['close'], length=14)
|
||||
macd = ta.macd(df['close'])
|
||||
if macd is not None and not macd.empty:
|
||||
df['macd'] = macd.iloc[:, 0]
|
||||
df['macd_hist'] = macd.iloc[:, 1]
|
||||
df['macd_signal'] = macd.iloc[:, 2]
|
||||
|
||||
df = df.dropna().reset_index(drop=True)
|
||||
|
||||
feature_cols = ['open', 'high', 'low', 'close', 'volume',
|
||||
'rsi', 'macd', 'macd_hist', 'macd_signal']
|
||||
|
||||
if fit:
|
||||
scaled_features = self.scaler.fit_transform(df[feature_cols])
|
||||
else:
|
||||
scaled_features = self.scaler.transform(df[feature_cols])
|
||||
|
||||
X = []
|
||||
valid_indices = []
|
||||
for i in range(len(scaled_features) - self.lookback):
|
||||
X.append(scaled_features[i:i + self.lookback])
|
||||
valid_indices.append(i + self.lookback - 1)
|
||||
|
||||
return np.array(X), df, valid_indices, scaled_features
|
||||
|
||||
|
||||
class AutoTradePreprocessor:
|
||||
"""auto_trade逻辑：含create_labels"""
|
||||
def __init__(self, lookback=168, forecast_horizon=1):
|
||||
self.lookback = lookback
|
||||
self.forecast_horizon = forecast_horizon
|
||||
self.scaler = StandardScaler()
|
||||
|
||||
def preprocess(self, df, fit=True):
|
||||
df = df.copy()
|
||||
df['rsi'] = ta.rsi(df['close'], length=14)
|
||||
macd = ta.macd(df['close'])
|
||||
if macd is not None and not macd.empty:
|
||||
df['macd'] = macd.iloc[:, 0]
|
||||
df['macd_hist'] = macd.iloc[:, 1]
|
||||
df['macd_signal'] = macd.iloc[:, 2]
|
||||
|
||||
# create_labels: shift + 砍最后一行
|
||||
df['future_price'] = df['close'].shift(-self.forecast_horizon)
|
||||
df['future_return'] = (df['future_price'] - df['close']) / df['close']
|
||||
df['label'] = (df['future_return'] > 0).astype(int)
|
||||
df = df[:-self.forecast_horizon]
|
||||
|
||||
df = df.dropna().reset_index(drop=True)
|
||||
|
||||
feature_cols = ['open', 'high', 'low', 'close', 'volume',
|
||||
'rsi', 'macd', 'macd_hist', 'macd_signal']
|
||||
|
||||
if fit:
|
||||
scaled_features = self.scaler.fit_transform(df[feature_cols])
|
||||
else:
|
||||
scaled_features = self.scaler.transform(df[feature_cols])
|
||||
|
||||
X = []
|
||||
for i in range(len(scaled_features) - self.lookback):
|
||||
X.append(scaled_features[i:i + self.lookback])
|
||||
|
||||
return np.array(X), df, scaled_features
|
||||
|
||||
|
||||
def utc_to_beijing(ts):
|
||||
return ts + timedelta(hours=8)
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 100)
|
||||
print("数据处理对比：test旧逻辑 vs auto_trade逻辑")
|
||||
print("=" * 100)
|
||||
|
||||
device = torch.device('mps' if torch.backends.mps.is_available() else 'cpu')
|
||||
|
||||
# 加载模型和训练时的scaler
|
||||
from models.dlinear.model import DLinear
|
||||
checkpoint = torch.load('./models/checkpoints/dlinear_classification_2026/best_model.pt',
|
||||
map_location=device, weights_only=False)
|
||||
model = DLinear(checkpoint['config']).to(device).eval()
|
||||
with open('./models/checkpoints/dlinear_classification_2026/preprocessor.pkl', 'rb') as f:
|
||||
trained_preprocessor = pickle.load(f)
|
||||
|
||||
# 提取训练时的scaler参数
|
||||
trained_scaler = trained_preprocessor.scaler
|
||||
print(f"训练时scaler均值: {trained_scaler.mean_}")
|
||||
print(f"训练时scaler方差: {trained_scaler.scale_}")
|
||||
|
||||
# 获取数据
|
||||
print("\n获取数据...")
|
||||
fetcher = BinanceDataFetcher()
|
||||
df_full = fetcher.fetch_ohlcv('BTC', '1h', start_date='2025-12-01', end_date='2026-03-01', use_cache=False)
|
||||
print(f"总行数: {len(df_full)}")
|
||||
|
||||
# 选几个2月10日（北京时间）的时间点做对比
|
||||
# 北京时间02-10 00:00 = UTC 02-09 16:00
|
||||
test_targets_utc = [
|
||||
'2026-02-09 16:00:00', # 北京 02-10 00:00
|
||||
'2026-02-09 20:00:00', # 北京 02-10 04:00
|
||||
'2026-02-10 00:00:00', # 北京 02-10 08:00
|
||||
'2026-02-10 04:00:00', # 北京 02-10 12:00
|
||||
'2026-02-10 08:00:00', # 北京 02-10 16:00
|
||||
'2026-02-10 12:00:00', # 北京 02-10 20:00
|
||||
]
|
||||
|
||||
feature_cols = ['open', 'high', 'low', 'close', 'volume',
|
||||
'rsi', 'macd', 'macd_hist', 'macd_signal']
|
||||
|
||||
for target_utc_str in test_targets_utc:
|
||||
target_utc = pd.Timestamp(target_utc_str)
|
||||
target_bj = utc_to_beijing(target_utc)
|
||||
|
||||
# 找到目标K线在df中的索引
|
||||
target_mask = df_full['timestamp'] == target_utc
|
||||
if not target_mask.any():
|
||||
print(f"\n⚠ 找不到 {target_utc_str}，跳过")
|
||||
continue
|
||||
|
||||
target_idx = df_full[target_mask].index[0]
|
||||
input_idx = target_idx - 1 # 输入的最后一根K线
|
||||
|
||||
input_row = df_full.iloc[input_idx]
|
||||
target_row = df_full.iloc[target_idx]
|
||||
|
||||
print(f"\n{'=' * 100}")
|
||||
print(f"目标K线: 北京{target_bj.strftime('%m-%d %H:%M')} (UTC {target_utc_str})")
|
||||
print(f"输入末根: UTC {input_row['timestamp']} O:{input_row['open']:.2f} H:{input_row['high']:.2f} L:{input_row['low']:.2f} C:{input_row['close']:.2f}")
|
||||
print(f"目标K线: UTC {target_row['timestamp']} 收盘:{target_row['close']:.2f}")
|
||||
print("-" * 100)
|
||||
|
||||
# ===== 方式A: test旧逻辑 =====
|
||||
# 对整个df一次性预处理，按索引找到对应窗口
|
||||
test_prep = TestPreprocessor(lookback=168)
|
||||
test_prep.scaler = trained_scaler # 使用训练时的scaler
|
||||
X_test, df_test, idx_test, scaled_test = test_prep.preprocess(df_full, fit=False)
|
||||
|
||||
# 找到input_idx对应的窗口
|
||||
# idx_test[i] = i + 168 - 1，所以 input_idx 在 df_test 中的位置需要查找
|
||||
# df_test 是 dropna 后 reset_index 的，timestamp 可能对应不同的行号
|
||||
test_input_mask = df_test['timestamp'] == input_row['timestamp']
|
||||
if test_input_mask.any():
|
||||
test_df_idx = df_test[test_input_mask].index[0]
|
||||
# 对应的窗口序号
|
||||
test_window_i = test_df_idx - 168 + 1
|
||||
if 0 <= test_window_i < len(X_test):
|
||||
X_test_window = X_test[test_window_i]
|
||||
else:
|
||||
X_test_window = None
|
||||
print(" [test] 窗口索引越界")
|
||||
else:
|
||||
test_df_idx = None
|
||||
X_test_window = None
|
||||
print(" [test] 在df_test中找不到对应时间")
|
||||
|
||||
# ===== 方式B: auto_trade逻辑 =====
|
||||
# 截取到input_idx为止，含create_labels
|
||||
df_up_to_t = df_full.iloc[:input_idx + 1]
|
||||
auto_prep = AutoTradePreprocessor(lookback=168)
|
||||
auto_prep.scaler = trained_scaler # 使用训练时的scaler
|
||||
X_auto, df_auto, scaled_auto = auto_prep.preprocess(df_up_to_t, fit=False)
|
||||
|
||||
if len(X_auto) > 0:
|
||||
X_auto_window = X_auto[-1] # 取最后一个窗口
|
||||
else:
|
||||
X_auto_window = None
|
||||
print(" [auto] X为空")
|
||||
|
||||
# ===== 对比 =====
|
||||
|
||||
# 1. df行数对比
|
||||
print(f"\n[1] 预处理后df行数")
|
||||
print(f" test: {len(df_test)} 行 (整个df, 无create_labels)")
|
||||
print(f" auto: {len(df_auto)} 行 (截取到input, 含create_labels砍尾)")
|
||||
|
||||
# 2. 最后一行时间对比
|
||||
print(f"\n[2] 预处理后df最后一行")
|
||||
print(f" test最后一行: {df_test.iloc[-1]['timestamp']} close={df_test.iloc[-1]['close']:.2f}")
|
||||
print(f" auto最后一行: {df_auto.iloc[-1]['timestamp']} close={df_auto.iloc[-1]['close']:.2f}")
|
||||
|
||||
# 3. 输入末根K线的原始特征对比
|
||||
if test_df_idx is not None:
|
||||
test_features = df_test.loc[test_df_idx, feature_cols]
|
||||
auto_last_row = df_auto.iloc[-1]
|
||||
auto_features = auto_last_row[feature_cols]
|
||||
|
||||
print(f"\n[3] 输入末根K线原始特征值")
|
||||
print(f" {'特征':<15} {'test值':>15} {'auto值':>15} {'差异':>15}")
|
||||
print(f" {'-'*60}")
|
||||
for col in feature_cols:
|
||||
tv = test_features[col]
|
||||
av = auto_features[col]
|
||||
diff = tv - av
|
||||
marker = " ⚠" if abs(diff) > 1e-6 else ""
|
||||
print(f" {col:<15} {tv:>15.6f} {av:>15.6f} {diff:>15.8f}{marker}")
|
||||
|
||||
# 4. scaler transform 后的值对比（输入末根K线）
|
||||
if test_df_idx is not None and X_test_window is not None and X_auto_window is not None:
|
||||
test_scaled_last = scaled_test[test_df_idx]
|
||||
auto_scaled_last = scaled_auto[-1]
|
||||
|
||||
print(f"\n[4] scaler transform 后的值（输入末根K线）")
|
||||
print(f" {'特征':<15} {'test值':>15} {'auto值':>15} {'差异':>15}")
|
||||
print(f" {'-'*60}")
|
||||
for j, col in enumerate(feature_cols):
|
||||
tv = test_scaled_last[j]
|
||||
av = auto_scaled_last[j]
|
||||
diff = tv - av
|
||||
marker = " ⚠" if abs(diff) > 1e-6 else ""
|
||||
print(f" {col:<15} {tv:>15.6f} {av:>15.6f} {diff:>15.8f}{marker}")
|
||||
|
||||
# 5. 输入窗口对比（整个168步窗口）
|
||||
if X_test_window is not None and X_auto_window is not None:
|
||||
window_diff = X_test_window - X_auto_window
|
||||
max_diff = np.max(np.abs(window_diff))
|
||||
mean_diff = np.mean(np.abs(window_diff))
|
||||
nonzero_count = np.count_nonzero(np.abs(window_diff) > 1e-6)
|
||||
|
||||
print(f"\n[5] 输入窗口对比 (168步 × 9特征 = 1512个值)")
|
||||
print(f" 最大绝对差异: {max_diff:.10f}")
|
||||
print(f" 平均绝对差异: {mean_diff:.10f}")
|
||||
print(f" 差异>1e-6的值: {nonzero_count} / 1512")
|
||||
|
||||
if nonzero_count > 0:
|
||||
# 找到差异最大的位置
|
||||
max_pos = np.unravel_index(np.argmax(np.abs(window_diff)), window_diff.shape)
|
||||
print(f" 最大差异位置: 步={max_pos[0]} 特征={feature_cols[max_pos[1]]}")
|
||||
print(f" test值: {X_test_window[max_pos]:.10f}")
|
||||
print(f" auto值: {X_auto_window[max_pos]:.10f}")
|
||||
|
||||
# 6. 模型推理对比
|
||||
if X_test_window is not None and X_auto_window is not None:
|
||||
with torch.no_grad():
|
||||
test_tensor = torch.FloatTensor(X_test_window[np.newaxis]).to(device)
|
||||
auto_tensor = torch.FloatTensor(X_auto_window[np.newaxis]).to(device)
|
||||
|
||||
test_logits = model(test_tensor).cpu().numpy().item()
|
||||
auto_logits = model(auto_tensor).cpu().numpy().item()
|
||||
|
||||
test_prob = 1 / (1 + np.exp(-test_logits))
|
||||
auto_prob = 1 / (1 + np.exp(-auto_logits))
|
||||
|
||||
test_pred = "涨" if test_prob > 0.5 else "跌"
|
||||
auto_pred = "涨" if auto_prob > 0.5 else "跌"
|
||||
|
||||
match = "✓ 一致" if test_pred == auto_pred else "✗ 不一致"
|
||||
|
||||
print(f"\n[6] 模型推理结果")
|
||||
print(f" {'':15} {'test':>15} {'auto':>15} {'差异':>15}")
|
||||
print(f" {'-'*60}")
|
||||
print(f" {'logits':<15} {test_logits:>15.8f} {auto_logits:>15.8f} {test_logits-auto_logits:>15.8f}")
|
||||
print(f" {'概率':<15} {test_prob:>15.8f} {auto_prob:>15.8f} {test_prob-auto_prob:>15.8f}")
|
||||
print(f" {'预测':<15} {test_pred:>15} {auto_pred:>15} {match:>15}")
|
||||
|
||||
# ===== 全量对比：2月所有小时 =====
|
||||
print(f"\n\n{'=' * 100}")
|
||||
print("全量对比：2月份所有小时预测结果差异统计")
|
||||
print("=" * 100)
|
||||
|
||||
# test方式：一次性预处理
|
||||
test_prep_full = TestPreprocessor(lookback=168)
|
||||
test_prep_full.scaler = trained_scaler
|
||||
X_test_full, df_test_full, idx_test_full, _ = test_prep_full.preprocess(df_full, fit=False)
|
||||
|
||||
# 构建 timestamp -> test窗口索引 的映射
|
||||
test_ts_map = {}
|
||||
for i, idx in enumerate(idx_test_full):
|
||||
ts = df_test_full.iloc[idx]['timestamp']
|
||||
test_ts_map[ts] = i
|
||||
|
||||
total = 0
|
||||
same = 0
|
||||
diff_list = []
|
||||
|
||||
for target_idx in range(1, len(df_full)):
|
||||
target_time_utc = df_full.iloc[target_idx]['timestamp']
|
||||
target_time_bj = utc_to_beijing(target_time_utc)
|
||||
if not (target_time_bj.year == 2026 and target_time_bj.month == 2):
|
||||
continue
|
||||
|
||||
input_idx = target_idx - 1
|
||||
input_ts = df_full.iloc[input_idx]['timestamp']
|
||||
|
||||
# test方式
|
||||
test_i = test_ts_map.get(input_ts)
|
||||
if test_i is None or test_i >= len(X_test_full):
|
||||
continue
|
||||
|
||||
# auto方式
|
||||
df_up_to_t = df_full.iloc[:input_idx + 1]
|
||||
auto_prep_tmp = AutoTradePreprocessor(lookback=168)
|
||||
auto_prep_tmp.scaler = trained_scaler
|
||||
X_auto_tmp, _, _ = auto_prep_tmp.preprocess(df_up_to_t, fit=False)
|
||||
if len(X_auto_tmp) == 0:
|
||||
continue
|
||||
|
||||
with torch.no_grad():
|
||||
test_logits = model(torch.FloatTensor(X_test_full[test_i:test_i+1]).to(device)).cpu().numpy().item()
|
||||
auto_logits = model(torch.FloatTensor(X_auto_tmp[-1:]).to(device)).cpu().numpy().item()
|
||||
|
||||
test_prob = 1 / (1 + np.exp(-test_logits))
|
||||
auto_prob = 1 / (1 + np.exp(-auto_logits))
|
||||
test_pred = 1 if test_prob > 0.5 else 0
|
||||
auto_pred = 1 if auto_prob > 0.5 else 0
|
||||
|
||||
total += 1
|
||||
if test_pred == auto_pred:
|
||||
same += 1
|
||||
else:
|
||||
diff_list.append({
|
||||
'time_bj': target_time_bj.strftime('%m-%d %H:%M'),
|
||||
'test_prob': test_prob,
|
||||
'auto_prob': auto_prob,
|
||||
'test_pred': '涨' if test_pred == 1 else '跌',
|
||||
'auto_pred': '涨' if auto_pred == 1 else '跌',
|
||||
})
|
||||
|
||||
print(f"总预测数: {total}")
|
||||
print(f"结果一致: {same} ({same/total:.1%})")
|
||||
print(f"结果不同: {len(diff_list)} ({len(diff_list)/total:.1%})")
|
||||
|
||||
if diff_list:
|
||||
print(f"\n不一致的时间点:")
|
||||
print(f" {'北京时间':<15} {'test概率':>10} {'test预测':>8} {'auto概率':>10} {'auto预测':>8}")
|
||||
print(f" {'-'*55}")
|
||||
for d in diff_list:
|
||||
print(f" {d['time_bj']:<15} {d['test_prob']:>10.6f} {d['test_pred']:>8} {d['auto_prob']:>10.6f} {d['auto_pred']:>8}")
|
||||
|
||||
print("=" * 100)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,305 @@
|
||||
"""2026年1月BTC数据 - DLinear 准确率测试 (create_labels版)
|
||||
|
||||
与test_dlinear_jan_2026.py对比：
|
||||
- 本脚本使用create_labels预处理方式（auto_trade旧逻辑）
|
||||
- 每个目标K线截取到input_idx的df，调用create_labels后取X[-1]预测
|
||||
- 用于验证create_labels对预测结果的影响
|
||||
|
||||
输入数据：预测K线的前一根（即用截止到t的数据预测t+1）
|
||||
价格对比：预测结果与预测K线(t+1)的收盘价做对比
|
||||
时间：币安数据UTC，输出转为北京时间(UTC+8)，按北京时间分天
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pickle
|
||||
import pandas_ta as ta
|
||||
from datetime import timedelta
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
from data.fetcher import BinanceDataFetcher
|
||||
|
||||
# ==================== 时区与美股判断 ====================
|
||||
|
||||
def utc_to_beijing(ts):
|
||||
"""UTC时间转北京时间(UTC+8)"""
|
||||
return ts + timedelta(hours=8)
|
||||
|
||||
def is_us_stock_open_by_utc(utc_ts):
|
||||
"""根据UTC时间判断美股当天是否开盘
|
||||
|
||||
美股交易日基于美东时间(ET)，UTC-5(冬令时)
|
||||
2026年1月在冬令时范围内（夏令时3月第二个周日开始）
|
||||
所以美东 = UTC - 5h
|
||||
"""
|
||||
et_ts = utc_ts - timedelta(hours=5)
|
||||
et_date = pd.Timestamp(et_ts.date())
|
||||
|
||||
us_holidays_jan_2026 = {
|
||||
pd.Timestamp('2026-01-01'), # 元旦 New Year's Day
|
||||
pd.Timestamp('2026-01-19'), # 马丁·路德·金日 MLK Day
|
||||
}
|
||||
|
||||
if et_date.weekday() >= 5:
|
||||
return False
|
||||
if et_date in us_holidays_jan_2026:
|
||||
return False
|
||||
return True
|
||||
|
||||
# ==================== pickle反序列化占位类 ====================
|
||||
|
||||
class ClassificationPreprocessor:
|
||||
"""pickle.load时需要能找到这个类名"""
|
||||
def __init__(self, lookback=168, forecast_horizon=1):
|
||||
self.lookback = lookback
|
||||
self.forecast_horizon = forecast_horizon
|
||||
self.scaler = StandardScaler()
|
||||
self.is_fitted = False
|
||||
|
||||
# ==================== create_labels预处理 ====================
|
||||
|
||||
def preprocess_with_create_labels(df, scaler, lookback=168, forecast_horizon=1):
|
||||
"""使用create_labels方式预处理（auto_trade旧逻辑）
|
||||
|
||||
步骤：加特征 -> create_labels(shift + 砍尾) -> dropna -> scaler.transform -> 构建窗口
|
||||
"""
|
||||
df = df.copy()
|
||||
# 加特征
|
||||
df['rsi'] = ta.rsi(df['close'], length=14)
|
||||
macd = ta.macd(df['close'])
|
||||
if macd is not None and not macd.empty:
|
||||
df['macd'] = macd.iloc[:, 0]
|
||||
df['macd_hist'] = macd.iloc[:, 1]
|
||||
df['macd_signal'] = macd.iloc[:, 2]
|
||||
|
||||
# create_labels: shift + 砍最后一行
|
||||
df['future_price'] = df['close'].shift(-forecast_horizon)
|
||||
df['future_return'] = (df['future_price'] - df['close']) / df['close']
|
||||
df['label'] = (df['future_return'] > 0).astype(int)
|
||||
df = df[:-forecast_horizon]
|
||||
|
||||
df = df.dropna().reset_index(drop=True)
|
||||
|
||||
feature_cols = ['open', 'high', 'low', 'close', 'volume',
|
||||
'rsi', 'macd', 'macd_hist', 'macd_signal']
|
||||
|
||||
scaled_features = scaler.transform(df[feature_cols])
|
||||
|
||||
X = []
|
||||
for i in range(len(scaled_features) - lookback):
|
||||
X.append(scaled_features[i:i + lookback])
|
||||
|
||||
return np.array(X), df
|
||||
|
||||
# ==================== 工具函数 ====================
|
||||
|
||||
def fmt_pred(val):
|
||||
"""格式化预测结果"""
|
||||
if val is None or (isinstance(val, float) and np.isnan(val)):
|
||||
return "N/A"
|
||||
return "涨" if int(val) == 1 else "跌"
|
||||
|
||||
def fmt_result(pred, actual):
|
||||
"""格式化对错"""
|
||||
if pred is None or (isinstance(pred, float) and np.isnan(pred)):
|
||||
return " "
|
||||
return "✓" if int(pred) == int(actual) else "✗"
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 90)
|
||||
print("2026年1月 BTC 1h 涨跌预测准确率测试 (DLinear - create_labels版)")
|
||||
print("输入：预测K线的前一根 | 对比：预测结果 vs 预测K线收盘价 | 时间：北京时间(UTC+8)")
|
||||
print("预处理：截取到input的df，调用create_labels后取X[-1]")
|
||||
print("=" * 90)
|
||||
|
||||
device = torch.device('mps' if torch.backends.mps.is_available() else 'cpu')
|
||||
print(f"使用设备: {device}")
|
||||
|
||||
# ========== 步骤1: 加载模型 ==========
|
||||
print("\n[1/4] 加载模型...")
|
||||
|
||||
from models.dlinear.model import DLinear
|
||||
checkpoint = torch.load('./models/checkpoints/dlinear_classification_2026/best_model.pt',
|
||||
map_location=device, weights_only=False)
|
||||
model = DLinear(checkpoint['config'])
|
||||
model.load_state_dict(checkpoint['model_state_dict'])
|
||||
model = model.to(device).eval()
|
||||
with open('./models/checkpoints/dlinear_classification_2026/preprocessor.pkl', 'rb') as f:
|
||||
preprocessor = pickle.load(f)
|
||||
print(" ✓ DLinear已加载")
|
||||
|
||||
# ========== 步骤2: 获取测试数据 ==========
|
||||
print("\n[2/4] 获取数据...")
|
||||
fetcher = BinanceDataFetcher()
|
||||
df = fetcher.fetch_ohlcv('BTC', '1h', start_date='2025-10-01', end_date='2026-02-01', use_cache=False)
|
||||
|
||||
print(f"数据范围(UTC): {df['timestamp'].min()} -> {df['timestamp'].max()}")
|
||||
print(f"总行数: {len(df)}")
|
||||
|
||||
# ========== 步骤3: 预处理与预测 ==========
|
||||
print("\n[3/4] 运行预测...")
|
||||
|
||||
lookback = preprocessor.lookback
|
||||
scaler = preprocessor.scaler
|
||||
|
||||
# 找出1月（北京时间）的所有预测目标K线索引
|
||||
jan_targets = []
|
||||
for target_idx in range(1, len(df)):
|
||||
target_time_utc = df.iloc[target_idx]['timestamp']
|
||||
target_time_bj = utc_to_beijing(target_time_utc)
|
||||
if target_time_bj.year == 2026 and target_time_bj.month == 1:
|
||||
jan_targets.append(target_idx)
|
||||
|
||||
print(f"1月（北京时间）目标K线数: {len(jan_targets)}")
|
||||
|
||||
results = []
|
||||
|
||||
for target_idx in jan_targets:
|
||||
target_time_utc = df.iloc[target_idx]['timestamp']
|
||||
target_time_bj = utc_to_beijing(target_time_utc)
|
||||
|
||||
# 输入K线索引（前一根）
|
||||
input_idx = target_idx - 1
|
||||
current_close = df.iloc[input_idx]['close']
|
||||
target_close = df.iloc[target_idx]['close']
|
||||
actual_label = 1 if target_close > current_close else 0
|
||||
|
||||
# --- DLinear 预测（create_labels方式：截取到input_idx，取X[-1]） ---
|
||||
dl_pred = None
|
||||
dl_prob = None
|
||||
|
||||
df_up_to_input = df.iloc[:input_idx + 1]
|
||||
X, _ = preprocess_with_create_labels(df_up_to_input, scaler, lookback)
|
||||
|
||||
if len(X) > 0:
|
||||
x_tensor = torch.FloatTensor(X[-1:]).to(device)
|
||||
with torch.no_grad():
|
||||
logits = model(x_tensor).cpu().numpy()
|
||||
prob = 1 / (1 + np.exp(-logits))
|
||||
dl_pred = int(prob.item() > 0.5)
|
||||
dl_prob = float(prob.item())
|
||||
|
||||
results.append({
|
||||
'time_utc': target_time_utc,
|
||||
'time_bj': target_time_bj,
|
||||
'current_close': current_close,
|
||||
'target_close': target_close,
|
||||
'actual': actual_label,
|
||||
'dlinear': dl_pred,
|
||||
'dl_prob': dl_prob,
|
||||
})
|
||||
|
||||
# ========== 步骤4: 输出结果 ==========
|
||||
if not results:
|
||||
print("没有找到符合条件的测试样本")
|
||||
return
|
||||
|
||||
df_res = pd.DataFrame(results)
|
||||
df_res['date_bj'] = df_res['time_bj'].dt.date
|
||||
df_res['us_open'] = df_res['time_utc'].apply(is_us_stock_open_by_utc)
|
||||
|
||||
total_samples = len(df_res)
|
||||
|
||||
print(f"\n[4/4] 准确率统计 (2026年1月，北京时间)")
|
||||
print(f"总样本数: {total_samples}")
|
||||
|
||||
# ========== 每小时详细数据 ==========
|
||||
print("\n" + "=" * 90)
|
||||
print("每小时详细数据")
|
||||
print("=" * 90)
|
||||
|
||||
hourly_header = (
|
||||
f"{'北京时间':<18} {'前收盘':>10} {'现收盘':>10} {'实际':>4} "
|
||||
f"{'DL':>3} {'':>1} {'概率':>6}"
|
||||
)
|
||||
|
||||
for date_bj, day_group in df_res.groupby('date_bj'):
|
||||
us_open = day_group['us_open'].iloc[0]
|
||||
us_str = "美股开盘" if us_open else "美股休市"
|
||||
|
||||
dl_day_correct = (day_group['dlinear'] == day_group['actual']).sum()
|
||||
dl_day_total = day_group['dlinear'].notna().sum()
|
||||
dl_day_acc = dl_day_correct / dl_day_total if dl_day_total > 0 else 0
|
||||
|
||||
print(f"\n--- {date_bj} ({us_str}) --- DL准确率:{dl_day_acc:.1%} ({dl_day_correct}/{dl_day_total})")
|
||||
print(hourly_header)
|
||||
print("-" * 90)
|
||||
|
||||
for _, row in day_group.iterrows():
|
||||
time_str = row['time_bj'].strftime('%m-%d %H:%M')
|
||||
actual_str = fmt_pred(row['actual'])
|
||||
dl_str = fmt_pred(row['dlinear'])
|
||||
dl_r = fmt_result(row['dlinear'], row['actual'])
|
||||
prob_str = f"{row['dl_prob']:.3f}" if row['dl_prob'] is not None else " N/A"
|
||||
|
||||
print(
|
||||
f"{time_str:<18} {row['current_close']:>10.2f} {row['target_close']:>10.2f} {actual_str:>4} "
|
||||
f"{dl_str:>3} {dl_r:>1} {prob_str:>6}"
|
||||
)
|
||||
|
||||
# ========== 每日准确率汇总表 ==========
|
||||
print("\n\n" + "=" * 90)
|
||||
print("每日准确率汇总 (北京时间)")
|
||||
print("=" * 90)
|
||||
|
||||
header = f"{'日期':<14} {'样本':>4} {'美股':>6} {'DLinear':>8}"
|
||||
print(header)
|
||||
print("-" * 90)
|
||||
|
||||
for date_bj, group in df_res.groupby('date_bj'):
|
||||
total = len(group)
|
||||
us_open = group['us_open'].iloc[0]
|
||||
us_str = "开盘" if us_open else "休市"
|
||||
|
||||
dl_valid = group.dropna(subset=['dlinear'])
|
||||
dl_acc = (dl_valid['dlinear'] == dl_valid['actual']).mean() if len(dl_valid) > 0 else 0
|
||||
dl_str = f"{dl_acc:.1%}"
|
||||
|
||||
print(f"{str(date_bj):<14} {total:>4} {us_str:>6} {dl_str:>8}")
|
||||
|
||||
print("-" * 90)
|
||||
|
||||
# --- 总准确率 ---
|
||||
dl_valid_all = df_res.dropna(subset=['dlinear'])
|
||||
dl_total = (dl_valid_all['dlinear'] == dl_valid_all['actual']).mean() if len(dl_valid_all) > 0 else 0
|
||||
|
||||
print(f"{'总计':<14} {total_samples:>4} {'':>6} {dl_total:.2%}")
|
||||
print("=" * 90)
|
||||
|
||||
# --- 美股开盘日 vs 休市日对比 ---
|
||||
print("\n[美股开盘日 vs 休市日 准确率对比]")
|
||||
print("-" * 60)
|
||||
for label, flag in [("开盘日", True), ("休市日", False)]:
|
||||
subset = df_res[df_res['us_open'] == flag]
|
||||
if len(subset) == 0:
|
||||
continue
|
||||
n = len(subset)
|
||||
dl_s = subset.dropna(subset=['dlinear'])
|
||||
dl_a = (dl_s['dlinear'] == dl_s['actual']).mean() if len(dl_s) > 0 else 0
|
||||
print(f" {label} ({n}样本): DLinear={dl_a:.2%}")
|
||||
|
||||
print("-" * 60)
|
||||
|
||||
# --- 预测分布统计 ---
|
||||
print("\n[预测分布统计]")
|
||||
print("-" * 60)
|
||||
actual_up = (df_res['actual'] == 1).sum()
|
||||
actual_down = (df_res['actual'] == 0).sum()
|
||||
print(f" 实际: 上涨={actual_up} ({actual_up/total_samples:.1%}) 下跌={actual_down} ({actual_down/total_samples:.1%})")
|
||||
|
||||
dl_valid = df_res['dlinear'].dropna()
|
||||
if len(dl_valid) > 0:
|
||||
up = (dl_valid == 1).sum()
|
||||
down = (dl_valid == 0).sum()
|
||||
print(f" DLinear: 预测上涨={up} ({up/len(dl_valid):.1%}) 预测下跌={down} ({down/len(dl_valid):.1%})")
|
||||
|
||||
print("=" * 90)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user