Initial commit: BTC 1h prediction model (53.10% accuracy)
🎯 项目成果： - 准确率: 53.10% (2026年测试集) - 任务类型: 二分类 (预测涨/跌) - 特征: OHLCV + RSI + MACD (9个特征) - 训练数据: 2020-2025年BTC小时数据 (52,351序列) - 测试数据: 2026年1-29日 (484序列) 📊 性能指标： - 精确率: 51.79% - 召回率: 49.36% - F1分数: 0.5054 - 预测分布: 非常平衡 (偏差仅-11个) 🔬 实验历史： - 尝试了5种方案 (回归/分类, 不同数据划分) - 分类任务优于回归任务 (53.10% vs 49.5%) - 全量训练避免了过拟合问题 📦 主要模型文件： - models/checkpoints/dlinear_classification_2026/ - best_model.pt (最佳模型) - preprocessor.pkl (预处理器) - test_2026_results.json (测试结果) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
crypto_predictorandClaude Sonnet 4.5 committed
commit
9bbdc8d42b
82 files changed
+8742
No files matched your search
@@ -0,0 +1,296 @@
|
||||
"""Model evaluation metrics and utilities."""
|
||||
|
||||
from typing import Dict, Literal
|
||||
import numpy as np
|
||||
import torch
|
||||
from sklearn.metrics import (
|
||||
accuracy_score,
|
||||
precision_score,
|
||||
recall_score,
|
||||
f1_score,
|
||||
confusion_matrix,
|
||||
mean_squared_error,
|
||||
mean_absolute_error
|
||||
)
|
||||
|
||||
|
||||
class ModelEvaluator:
|
||||
"""Evaluate model performance with task-specific metrics."""
|
||||
|
||||
def __init__(self, task: Literal['classification', 'regression']):
|
||||
"""Initialize evaluator.
|
||||
|
||||
Args:
|
||||
task: Task type ('classification' or 'regression')
|
||||
"""
|
||||
self.task = task
|
||||
|
||||
def evaluate(
|
||||
self,
|
||||
y_true: np.ndarray,
|
||||
y_pred: np.ndarray,
|
||||
y_prob: np.ndarray = None
|
||||
) -> Dict[str, float]:
|
||||
"""Compute evaluation metrics.
|
||||
|
||||
Args:
|
||||
y_true: True labels
|
||||
y_pred: Predicted labels/values
|
||||
y_prob: Prediction probabilities (for classification)
|
||||
|
||||
Returns:
|
||||
Dictionary of metrics
|
||||
"""
|
||||
if self.task == 'classification':
|
||||
return self._evaluate_classification(y_true, y_pred, y_prob)
|
||||
else:
|
||||
return self._evaluate_regression(y_true, y_pred)
|
||||
|
||||
def _evaluate_classification(
|
||||
self,
|
||||
y_true: np.ndarray,
|
||||
y_pred: np.ndarray,
|
||||
y_prob: np.ndarray = None
|
||||
) -> Dict[str, float]:
|
||||
"""Compute classification metrics.
|
||||
|
||||
Args:
|
||||
y_true: True labels (0 or 1)
|
||||
y_pred: Predicted labels (0 or 1)
|
||||
y_prob: Prediction probabilities
|
||||
|
||||
Returns:
|
||||
Dictionary of classification metrics
|
||||
"""
|
||||
metrics = {
|
||||
'accuracy': accuracy_score(y_true, y_pred),
|
||||
'precision': precision_score(y_true, y_pred, zero_division=0),
|
||||
'recall': recall_score(y_true, y_pred, zero_division=0),
|
||||
'f1': f1_score(y_true, y_pred, zero_division=0)
|
||||
}
|
||||
|
||||
# Confusion matrix
|
||||
cm = confusion_matrix(y_true, y_pred)
|
||||
if cm.shape == (2, 2):
|
||||
tn, fp, fn, tp = cm.ravel()
|
||||
metrics['true_negatives'] = int(tn)
|
||||
metrics['false_positives'] = int(fp)
|
||||
metrics['false_negatives'] = int(fn)
|
||||
metrics['true_positives'] = int(tp)
|
||||
|
||||
# Specificity
|
||||
if (tn + fp) > 0:
|
||||
metrics['specificity'] = tn / (tn + fp)
|
||||
|
||||
# Directional accuracy (if probabilities provided)
|
||||
if y_prob is not None:
|
||||
metrics['mean_confidence'] = float(np.mean(y_prob))
|
||||
|
||||
return metrics
|
||||
|
||||
def _evaluate_regression(
|
||||
self,
|
||||
y_true: np.ndarray,
|
||||
y_pred: np.ndarray
|
||||
) -> Dict[str, float]:
|
||||
"""Compute regression metrics.
|
||||
|
||||
Args:
|
||||
y_true: True values
|
||||
y_pred: Predicted values
|
||||
|
||||
Returns:
|
||||
Dictionary of regression metrics
|
||||
"""
|
||||
# Basic regression metrics
|
||||
mse = mean_squared_error(y_true, y_pred)
|
||||
mae = mean_absolute_error(y_true, y_pred)
|
||||
rmse = np.sqrt(mse)
|
||||
|
||||
# MAPE (Mean Absolute Percentage Error)
|
||||
# Avoid division by zero
|
||||
epsilon = 1e-8
|
||||
mape = np.mean(np.abs((y_true - y_pred) / (np.abs(y_true) + epsilon))) * 100
|
||||
|
||||
# Directional accuracy (sign agreement)
|
||||
directional_accuracy = np.mean(np.sign(y_true) == np.sign(y_pred))
|
||||
|
||||
metrics = {
|
||||
'mse': float(mse),
|
||||
'rmse': float(rmse),
|
||||
'mae': float(mae),
|
||||
'mape': float(mape),
|
||||
'directional_accuracy': float(directional_accuracy)
|
||||
}
|
||||
|
||||
return metrics
|
||||
|
||||
def backtest_strategy(
|
||||
self,
|
||||
y_true: np.ndarray,
|
||||
y_pred: np.ndarray,
|
||||
initial_capital: float = 10000.0,
|
||||
transaction_cost: float = 0.001
|
||||
) -> Dict[str, float]:
|
||||
"""Simple backtesting with long/short strategy.
|
||||
|
||||
Args:
|
||||
y_true: True returns
|
||||
y_pred: Predicted labels (classification) or returns (regression)
|
||||
initial_capital: Starting capital
|
||||
transaction_cost: Transaction cost per trade (e.g., 0.001 = 0.1%)
|
||||
|
||||
Returns:
|
||||
Dictionary of backtest metrics
|
||||
"""
|
||||
capital = initial_capital
|
||||
positions = []
|
||||
returns_list = []
|
||||
|
||||
for i in range(len(y_pred)):
|
||||
if self.task == 'classification':
|
||||
# Classification: y_pred已经是0/1的标签（不是概率）
|
||||
# 1 = buy, 0 = sell/short
|
||||
position = 1 if y_pred[i] == 1 else -1
|
||||
else:
|
||||
# Regression: positive prediction = buy, negative = sell
|
||||
position = 1 if y_pred[i] > 0 else -1
|
||||
|
||||
# Calculate return based on position
|
||||
actual_return = y_true[i]
|
||||
strategy_return = position * actual_return
|
||||
|
||||
# Apply transaction cost if position changes
|
||||
if i > 0 and positions[-1] != position:
|
||||
strategy_return -= transaction_cost
|
||||
|
||||
# Update capital
|
||||
capital *= (1 + strategy_return)
|
||||
positions.append(position)
|
||||
returns_list.append(strategy_return)
|
||||
|
||||
# Calculate metrics
|
||||
returns_array = np.array(returns_list)
|
||||
total_return = (capital - initial_capital) / initial_capital
|
||||
|
||||
# Sharpe ratio (annualized, assuming hourly data)
|
||||
# For hourly data: sqrt(24 * 365) ≈ 93
|
||||
if len(returns_array) > 0 and returns_array.std() > 0:
|
||||
sharpe_ratio = (returns_array.mean() / returns_array.std()) * np.sqrt(93)
|
||||
else:
|
||||
sharpe_ratio = 0.0
|
||||
|
||||
# Maximum drawdown
|
||||
cumulative = np.cumprod(1 + returns_array)
|
||||
running_max = np.maximum.accumulate(cumulative)
|
||||
drawdown = (cumulative - running_max) / running_max
|
||||
max_drawdown = float(drawdown.min())
|
||||
|
||||
# Win rate
|
||||
win_rate = np.mean(returns_array > 0)
|
||||
|
||||
return {
|
||||
'total_return': float(total_return),
|
||||
'final_capital': float(capital),
|
||||
'sharpe_ratio': float(sharpe_ratio),
|
||||
'max_drawdown': float(max_drawdown),
|
||||
'win_rate': float(win_rate),
|
||||
'num_trades': int(np.sum(np.diff(positions) != 0))
|
||||
}
|
||||
|
||||
|
||||
def evaluate_model(
|
||||
model: torch.nn.Module,
|
||||
dataloader: torch.utils.data.DataLoader,
|
||||
task: Literal['classification', 'regression'],
|
||||
device: str = 'cpu'
|
||||
) -> Dict[str, float]:
|
||||
"""Evaluate a model on a dataset.
|
||||
|
||||
Args:
|
||||
model: PyTorch model
|
||||
dataloader: DataLoader with evaluation data
|
||||
task: Task type
|
||||
device: Device to run evaluation on
|
||||
|
||||
Returns:
|
||||
Dictionary of evaluation metrics
|
||||
"""
|
||||
model.eval()
|
||||
model = model.to(device)
|
||||
|
||||
all_preds = []
|
||||
all_probs = []
|
||||
all_labels = []
|
||||
|
||||
with torch.no_grad():
|
||||
for batch_X, batch_y in dataloader:
|
||||
batch_X = batch_X.to(device)
|
||||
batch_y = batch_y.to(device)
|
||||
|
||||
# Forward pass
|
||||
outputs = model(batch_X).squeeze()
|
||||
|
||||
if task == 'classification':
|
||||
# 注意：修复后的模型输出logits，需要应用sigmoid得到概率
|
||||
probs = torch.sigmoid(outputs).cpu().numpy()
|
||||
preds = (probs > 0.5).astype(int)
|
||||
# 确保是1维数组
|
||||
if probs.ndim == 0:
|
||||
probs = np.array([probs])
|
||||
if preds.ndim == 0:
|
||||
preds = np.array([preds])
|
||||
all_probs.extend(probs.flatten())
|
||||
all_preds.extend(preds.flatten())
|
||||
else:
|
||||
# Regression predictions
|
||||
preds = outputs.cpu().numpy()
|
||||
if preds.ndim == 0:
|
||||
preds = np.array([preds])
|
||||
all_preds.extend(preds.flatten())
|
||||
|
||||
# Labels
|
||||
labels = batch_y.cpu().numpy()
|
||||
if labels.ndim == 0:
|
||||
labels = np.array([labels])
|
||||
all_labels.extend(labels.flatten())
|
||||
|
||||
# Convert to arrays
|
||||
y_true = np.array(all_labels)
|
||||
y_pred = np.array(all_preds)
|
||||
y_prob = np.array(all_probs) if task == 'classification' else None
|
||||
|
||||
# Evaluate
|
||||
evaluator = ModelEvaluator(task)
|
||||
metrics = evaluator.evaluate(y_true, y_pred, y_prob)
|
||||
|
||||
# Add backtesting metrics
|
||||
backtest = evaluator.backtest_strategy(y_true, y_pred)
|
||||
metrics.update({f'backtest_{k}': v for k, v in backtest.items()})
|
||||
|
||||
return metrics
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Test classification evaluation
|
||||
y_true = np.array([0, 1, 1, 0, 1, 0, 1, 1, 0, 0])
|
||||
y_pred = np.array([0, 1, 1, 0, 0, 1, 1, 1, 0, 1])
|
||||
y_prob = np.array([0.3, 0.8, 0.9, 0.2, 0.4, 0.6, 0.7, 0.85, 0.1, 0.55])
|
||||
|
||||
evaluator = ModelEvaluator('classification')
|
||||
metrics = evaluator.evaluate(y_true, y_pred, y_prob)
|
||||
|
||||
print("Classification metrics:")
|
||||
for k, v in metrics.items():
|
||||
print(f" {k}: {v}")
|
||||
|
||||
# Test regression evaluation
|
||||
y_true_reg = np.random.randn(100) * 0.02
|
||||
y_pred_reg = y_true_reg + np.random.randn(100) * 0.01
|
||||
|
||||
evaluator_reg = ModelEvaluator('regression')
|
||||
metrics_reg = evaluator_reg.evaluate(y_true_reg, y_pred_reg)
|
||||
|
||||
print("\nRegression metrics:")
|
||||
for k, v in metrics_reg.items():
|
||||
print(f" {k}: {v}")
|
||||
Reference in new issue
Block a user