crpto_predictor/evaluate_fixed.py

"""重新评估已训练模型（使用修复后的evaluator）"""

import sys
import os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

import torch
import numpy as np
from models.dlinear.model import DLinear
from data.fetcher import BinanceDataFetcher
from data.preprocessor import CryptoPreprocessor, split_data
from data.dataset import create_dataloaders
from training.evaluator import evaluate_model
import json

def main():
    print("="*80)
    print("重新评估模型（已修复Sigmoid Bug）")
    print("="*80)

    # 加载模型
    print("\n[1/4] 加载已训练的模型...")
    model_path = './models/checkpoints/dlinear_btc_1h_forecast/best_model.pt'
    device = torch.device('mps' if torch.backends.mps.is_available() else 'cpu')

    checkpoint = torch.load(model_path, map_location=device, weights_only=False)
    config = checkpoint['config']

    model = DLinear(config)
    model.load_state_dict(checkpoint['model_state_dict'])
    model = model.to(device)
    model.eval()

    print(f"✓ 模型已加载: {model_path}")
    print(f"✓ 设备: {device}")
    print(f"✓ 模型参数: {model.count_parameters():,}")

    # 准备测试数据
    print("\n[2/4] 准备测试数据...")
    fetcher = BinanceDataFetcher()
    df = fetcher.fetch_ohlcv('BTC', '1h', use_cache=True)

    preprocessor = CryptoPreprocessor(
        lookback=168,
        forecast_horizon=1,  # 预测未来1小时
        price_change_threshold=0.0
    )
    X, y, _ = preprocessor.preprocess_pipeline(df, task='classification', fit=True)

    _, _, test_data = split_data(X, y, train_split=0.7, val_split=0.15)
    _, _, test_loader = create_dataloaders(
        test_data, test_data, test_data,
        task='classification',
        batch_size=64
    )

    print(f"✓ 测试集大小: {len(test_data[0])} 样本")
    print(f"✓ 真实标签分布: 下跌={np.sum(test_data[1]==0)}, 上涨={np.sum(test_data[1]==1)}")

    # 先查看模型输出分布
    print("\n[3/4] 分析模型输出分布...")
    all_logits = []
    all_probs = []
    all_preds = []
    all_labels = []

    with torch.no_grad():
        for batch_X, batch_y in test_loader:
            batch_X = batch_X.to(device)
            outputs = model(batch_X).squeeze()

            # 模型输出logits
            logits = outputs.cpu().numpy()
            probs = torch.sigmoid(outputs).cpu().numpy()
            preds = (probs > 0.5).astype(int)

            all_logits.extend(logits.flatten())
            all_probs.extend(probs.flatten())
            all_preds.extend(preds.flatten())
            all_labels.extend(batch_y.numpy().flatten())

    all_logits = np.array(all_logits)
    all_probs = np.array(all_probs)
    all_preds = np.array(all_preds)
    all_labels = np.array(all_labels)

    print(f"\n📊 模型输出统计（修复后）:")
    print(f"  Logits范围: [{all_logits.min():.4f}, {all_logits.max():.4f}]")
    print(f"  Logits均值: {all_logits.mean():.4f}")
    print(f"  概率范围: [{all_probs.min():.4f}, {all_probs.max():.4f}]")
    print(f"  概率均值: {all_probs.mean():.4f} ← 应该接近0.5")
    print(f"  预测为上涨: {np.sum(all_preds==1)}/{len(all_preds)} ({np.sum(all_preds==1)/len(all_preds)*100:.1f}%)")
    print(f"  预测为下跌: {np.sum(all_preds==0)}/{len(all_preds)} ({np.sum(all_preds==0)/len(all_preds)*100:.1f}%)")

    # 正式评估
    print("\n[4/4] 使用修复后的evaluator进行完整评估...")
    metrics = evaluate_model(model, test_loader, task='classification', device=str(device))

    # 保存结果
    results_path = './models/checkpoints/dlinear_btc_1h_forecast/test_metrics_fixed.json'
    with open(results_path, 'w') as f:
        json.dump(metrics, f, indent=2)

    print(f"\n✓ 评估结果已保存至：{results_path}")

    # 展示结果
    print("\n" + "="*80)
    print("最终评估结果（修复后）")
    print("="*80)

    print(f"\n📊 分类指标:")
    print(f"  准确率:  {metrics.get('accuracy', 0):.4f} ({metrics.get('accuracy', 0)*100:.2f}%)")
    print(f"  F1分数:  {metrics.get('f1', 0):.4f}")
    print(f"  精确率:  {metrics.get('precision', 0):.4f}")
    print(f"  召回率:  {metrics.get('recall', 0):.4f}")

    if 'true_positives' in metrics:
        print(f"\n📈 混淆矩阵:")
        tp = metrics['true_positives']
        tn = metrics['true_negatives']
        fp = metrics['false_positives']
        fn = metrics['false_negatives']
        print(f"  真阳性（预测涨且真涨）:  {tp}")
        print(f"  真阴性（预测跌且真跌）:  {tn}")
        print(f"  假阳性（预测涨但真跌）:  {fp}")
        print(f"  假阴性（预测跌但真涨）:  {fn}")

        total = tp + tn + fp + fn
        if total > 0:
            print(f"\n  预测分布:")
            print(f"    预测上涨: {tp + fp} ({(tp+fp)/total*100:.1f}%)")
            print(f"    预测下跌: {tn + fn} ({(tn+fn)/total*100:.1f}%)")

    print(f"\n💰 回测结果:")
    print(f"  总收益率:    {metrics.get('backtest_total_return', 0)*100:.2f}%")
    print(f"  最终资金:    ${metrics.get('backtest_final_capital', 0):,.2f}")
    print(f"  夏普比率:    {metrics.get('backtest_sharpe_ratio', 0):.4f}")
    max_dd = metrics.get('backtest_max_drawdown', 0)
    if np.isnan(max_dd):
        print(f"  最大回撤:    N/A (无交易)")
    else:
        print(f"  最大回撤:    {max_dd*100:.2f}%")
    print(f"  胜率:        {metrics.get('backtest_win_rate', 0)*100:.2f}%")
    print(f"  交易次数:    {metrics.get('backtest_num_trades', 0)}")

    # 性能评估
    print(f"\n🎯 性能评估:")
    acc = metrics.get('accuracy', 0)
    if acc > 0.55:
        print(f"  ✅ 准确率 {acc*100:.1f}% > 55%，模型有一定预测能力")
    elif acc > 0.50:
        print(f"  ⚠️  准确率 {acc*100:.1f}% 略优于随机（50%），预测能力较弱")
    else:
        print(f"  ❌ 准确率 {acc*100:.1f}% ≤ 50%，模型预测能力不足")

    total_return = metrics.get('backtest_total_return', 0)
    if total_return > 0.05:
        print(f"  ✅ 回测收益 {total_return*100:.2f}% > 5%，策略有较好盈利潜力")
    elif total_return > 0:
        print(f"  ⚠️  回测收益 {total_return*100:.2f}% > 0，策略有微弱盈利")
    else:
        print(f"  ❌ 回测收益 {total_return*100:.2f}% ≤ 0，策略亏损")

    # 对比修复前后
    print(f"\n📊 修复前后对比:")
    print(f"  修复前（有Bug）:")
    print(f"    - 平均置信度: 0.69% (错误)")
    print(f"    - 预测上涨: 0个 (0%)")
    print(f"    - 准确率: 50.00% (偏差)")
    print(f"  修复后（正确）:")
    print(f"    - 平均置信度: {metrics.get('mean_confidence', 0)*100:.1f}%")
    print(f"    - 预测上涨: {tp + fp}个 ({(tp+fp)/total*100:.1f}%)")
    print(f"    - 准确率: {acc*100:.2f}%")

    print("\n" + "="*80)
    print("✓ 评估完成！")
    print("="*80)

if __name__ == "__main__":
    main()