Initial commit: BTC 1h prediction model (53.10% accuracy)
🎯 项目成果： - 准确率: 53.10% (2026年测试集) - 任务类型: 二分类 (预测涨/跌) - 特征: OHLCV + RSI + MACD (9个特征) - 训练数据: 2020-2025年BTC小时数据 (52,351序列) - 测试数据: 2026年1-29日 (484序列) 📊 性能指标： - 精确率: 51.79% - 召回率: 49.36% - F1分数: 0.5054 - 预测分布: 非常平衡 (偏差仅-11个) 🔬 实验历史： - 尝试了5种方案 (回归/分类, 不同数据划分) - 分类任务优于回归任务 (53.10% vs 49.5%) - 全量训练避免了过拟合问题 📦 主要模型文件： - models/checkpoints/dlinear_classification_2026/ - best_model.pt (最佳模型) - preprocessor.pkl (预处理器) - test_2026_results.json (测试结果) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
crypto_predictorandClaude Sonnet 4.5 committed
commit
9bbdc8d42b
82 files changed
+8742
No files matched your search
@@ -0,0 +1,180 @@
|
||||
"""重新评估已训练模型（使用修复后的evaluator）"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
from models.dlinear.model import DLinear
|
||||
from data.fetcher import BinanceDataFetcher
|
||||
from data.preprocessor import CryptoPreprocessor, split_data
|
||||
from data.dataset import create_dataloaders
|
||||
from training.evaluator import evaluate_model
|
||||
import json
|
||||
|
||||
def main():
|
||||
print("="*80)
|
||||
print("重新评估模型（已修复Sigmoid Bug）")
|
||||
print("="*80)
|
||||
|
||||
# 加载模型
|
||||
print("\n[1/4] 加载已训练的模型...")
|
||||
model_path = './models/checkpoints/dlinear_btc_1h_forecast/best_model.pt'
|
||||
device = torch.device('mps' if torch.backends.mps.is_available() else 'cpu')
|
||||
|
||||
checkpoint = torch.load(model_path, map_location=device, weights_only=False)
|
||||
config = checkpoint['config']
|
||||
|
||||
model = DLinear(config)
|
||||
model.load_state_dict(checkpoint['model_state_dict'])
|
||||
model = model.to(device)
|
||||
model.eval()
|
||||
|
||||
print(f"✓ 模型已加载: {model_path}")
|
||||
print(f"✓ 设备: {device}")
|
||||
print(f"✓ 模型参数: {model.count_parameters():,}")
|
||||
|
||||
# 准备测试数据
|
||||
print("\n[2/4] 准备测试数据...")
|
||||
fetcher = BinanceDataFetcher()
|
||||
df = fetcher.fetch_ohlcv('BTC', '1h', use_cache=True)
|
||||
|
||||
preprocessor = CryptoPreprocessor(
|
||||
lookback=168,
|
||||
forecast_horizon=1, # 预测未来1小时
|
||||
price_change_threshold=0.0
|
||||
)
|
||||
X, y, _ = preprocessor.preprocess_pipeline(df, task='classification', fit=True)
|
||||
|
||||
_, _, test_data = split_data(X, y, train_split=0.7, val_split=0.15)
|
||||
_, _, test_loader = create_dataloaders(
|
||||
test_data, test_data, test_data,
|
||||
task='classification',
|
||||
batch_size=64
|
||||
)
|
||||
|
||||
print(f"✓ 测试集大小: {len(test_data[0])} 样本")
|
||||
print(f"✓ 真实标签分布: 下跌={np.sum(test_data[1]==0)}, 上涨={np.sum(test_data[1]==1)}")
|
||||
|
||||
# 先查看模型输出分布
|
||||
print("\n[3/4] 分析模型输出分布...")
|
||||
all_logits = []
|
||||
all_probs = []
|
||||
all_preds = []
|
||||
all_labels = []
|
||||
|
||||
with torch.no_grad():
|
||||
for batch_X, batch_y in test_loader:
|
||||
batch_X = batch_X.to(device)
|
||||
outputs = model(batch_X).squeeze()
|
||||
|
||||
# 模型输出logits
|
||||
logits = outputs.cpu().numpy()
|
||||
probs = torch.sigmoid(outputs).cpu().numpy()
|
||||
preds = (probs > 0.5).astype(int)
|
||||
|
||||
all_logits.extend(logits.flatten())
|
||||
all_probs.extend(probs.flatten())
|
||||
all_preds.extend(preds.flatten())
|
||||
all_labels.extend(batch_y.numpy().flatten())
|
||||
|
||||
all_logits = np.array(all_logits)
|
||||
all_probs = np.array(all_probs)
|
||||
all_preds = np.array(all_preds)
|
||||
all_labels = np.array(all_labels)
|
||||
|
||||
print(f"\n📊 模型输出统计（修复后）:")
|
||||
print(f" Logits范围: [{all_logits.min():.4f}, {all_logits.max():.4f}]")
|
||||
print(f" Logits均值: {all_logits.mean():.4f}")
|
||||
print(f" 概率范围: [{all_probs.min():.4f}, {all_probs.max():.4f}]")
|
||||
print(f" 概率均值: {all_probs.mean():.4f} ← 应该接近0.5")
|
||||
print(f" 预测为上涨: {np.sum(all_preds==1)}/{len(all_preds)} ({np.sum(all_preds==1)/len(all_preds)*100:.1f}%)")
|
||||
print(f" 预测为下跌: {np.sum(all_preds==0)}/{len(all_preds)} ({np.sum(all_preds==0)/len(all_preds)*100:.1f}%)")
|
||||
|
||||
# 正式评估
|
||||
print("\n[4/4] 使用修复后的evaluator进行完整评估...")
|
||||
metrics = evaluate_model(model, test_loader, task='classification', device=str(device))
|
||||
|
||||
# 保存结果
|
||||
results_path = './models/checkpoints/dlinear_btc_1h_forecast/test_metrics_fixed.json'
|
||||
with open(results_path, 'w') as f:
|
||||
json.dump(metrics, f, indent=2)
|
||||
|
||||
print(f"\n✓ 评估结果已保存至：{results_path}")
|
||||
|
||||
# 展示结果
|
||||
print("\n" + "="*80)
|
||||
print("最终评估结果（修复后）")
|
||||
print("="*80)
|
||||
|
||||
print(f"\n📊 分类指标:")
|
||||
print(f" 准确率: {metrics.get('accuracy', 0):.4f} ({metrics.get('accuracy', 0)*100:.2f}%)")
|
||||
print(f" F1分数: {metrics.get('f1', 0):.4f}")
|
||||
print(f" 精确率: {metrics.get('precision', 0):.4f}")
|
||||
print(f" 召回率: {metrics.get('recall', 0):.4f}")
|
||||
|
||||
if 'true_positives' in metrics:
|
||||
print(f"\n📈 混淆矩阵:")
|
||||
tp = metrics['true_positives']
|
||||
tn = metrics['true_negatives']
|
||||
fp = metrics['false_positives']
|
||||
fn = metrics['false_negatives']
|
||||
print(f" 真阳性（预测涨且真涨）: {tp}")
|
||||
print(f" 真阴性（预测跌且真跌）: {tn}")
|
||||
print(f" 假阳性（预测涨但真跌）: {fp}")
|
||||
print(f" 假阴性（预测跌但真涨）: {fn}")
|
||||
|
||||
total = tp + tn + fp + fn
|
||||
if total > 0:
|
||||
print(f"\n 预测分布:")
|
||||
print(f" 预测上涨: {tp + fp} ({(tp+fp)/total*100:.1f}%)")
|
||||
print(f" 预测下跌: {tn + fn} ({(tn+fn)/total*100:.1f}%)")
|
||||
|
||||
print(f"\n💰 回测结果:")
|
||||
print(f" 总收益率: {metrics.get('backtest_total_return', 0)*100:.2f}%")
|
||||
print(f" 最终资金: ${metrics.get('backtest_final_capital', 0):,.2f}")
|
||||
print(f" 夏普比率: {metrics.get('backtest_sharpe_ratio', 0):.4f}")
|
||||
max_dd = metrics.get('backtest_max_drawdown', 0)
|
||||
if np.isnan(max_dd):
|
||||
print(f" 最大回撤: N/A (无交易)")
|
||||
else:
|
||||
print(f" 最大回撤: {max_dd*100:.2f}%")
|
||||
print(f" 胜率: {metrics.get('backtest_win_rate', 0)*100:.2f}%")
|
||||
print(f" 交易次数: {metrics.get('backtest_num_trades', 0)}")
|
||||
|
||||
# 性能评估
|
||||
print(f"\n🎯 性能评估:")
|
||||
acc = metrics.get('accuracy', 0)
|
||||
if acc > 0.55:
|
||||
print(f" ✅ 准确率 {acc*100:.1f}% > 55%，模型有一定预测能力")
|
||||
elif acc > 0.50:
|
||||
print(f" ⚠️ 准确率 {acc*100:.1f}% 略优于随机（50%），预测能力较弱")
|
||||
else:
|
||||
print(f" ❌ 准确率 {acc*100:.1f}% ≤ 50%，模型预测能力不足")
|
||||
|
||||
total_return = metrics.get('backtest_total_return', 0)
|
||||
if total_return > 0.05:
|
||||
print(f" ✅ 回测收益 {total_return*100:.2f}% > 5%，策略有较好盈利潜力")
|
||||
elif total_return > 0:
|
||||
print(f" ⚠️ 回测收益 {total_return*100:.2f}% > 0，策略有微弱盈利")
|
||||
else:
|
||||
print(f" ❌ 回测收益 {total_return*100:.2f}% ≤ 0，策略亏损")
|
||||
|
||||
# 对比修复前后
|
||||
print(f"\n📊 修复前后对比:")
|
||||
print(f" 修复前（有Bug）:")
|
||||
print(f" - 平均置信度: 0.69% (错误)")
|
||||
print(f" - 预测上涨: 0个 (0%)")
|
||||
print(f" - 准确率: 50.00% (偏差)")
|
||||
print(f" 修复后（正确）:")
|
||||
print(f" - 平均置信度: {metrics.get('mean_confidence', 0)*100:.1f}%")
|
||||
print(f" - 预测上涨: {tp + fp}个 ({(tp+fp)/total*100:.1f}%)")
|
||||
print(f" - 准确率: {acc*100:.2f}%")
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("✓ 评估完成！")
|
||||
print("="*80)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user