Initial commit: BTC 1h prediction model (53.10% accuracy)
🎯 项目成果： - 准确率: 53.10% (2026年测试集) - 任务类型: 二分类 (预测涨/跌) - 特征: OHLCV + RSI + MACD (9个特征) - 训练数据: 2020-2025年BTC小时数据 (52,351序列) - 测试数据: 2026年1-29日 (484序列) 📊 性能指标： - 精确率: 51.79% - 召回率: 49.36% - F1分数: 0.5054 - 预测分布: 非常平衡 (偏差仅-11个) 🔬 实验历史： - 尝试了5种方案 (回归/分类, 不同数据划分) - 分类任务优于回归任务 (53.10% vs 49.5%) - 全量训练避免了过拟合问题 📦 主要模型文件： - models/checkpoints/dlinear_classification_2026/ - best_model.pt (最佳模型) - preprocessor.pkl (预处理器) - test_2026_results.json (测试结果) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
crypto_predictorandClaude Sonnet 4.5 committed
commit
9bbdc8d42b
82 files changed
+8742
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Data fetching, preprocessing, and dataset utilities."""
|
||||
+147
@@ -0,0 +1,147 @@
|
||||
"""PyTorch Dataset classes for cryptocurrency time series data."""
|
||||
|
||||
from typing import Literal, Tuple
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.utils.data import Dataset
|
||||
|
||||
|
||||
class CryptoDataset(Dataset):
|
||||
"""PyTorch Dataset for cryptocurrency time series prediction."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
X: np.ndarray,
|
||||
y: np.ndarray,
|
||||
task: Literal['classification', 'regression'] = 'classification'
|
||||
):
|
||||
"""Initialize dataset.
|
||||
|
||||
Args:
|
||||
X: Feature sequences of shape (n_samples, lookback, n_features)
|
||||
y: Labels of shape (n_samples,)
|
||||
task: Task type ('classification' or 'regression')
|
||||
"""
|
||||
self.X = torch.FloatTensor(X)
|
||||
|
||||
if task == 'classification':
|
||||
self.y = torch.LongTensor(y)
|
||||
else: # regression
|
||||
self.y = torch.FloatTensor(y)
|
||||
|
||||
self.task = task
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""Return number of samples."""
|
||||
return len(self.X)
|
||||
|
||||
def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Get a sample.
|
||||
|
||||
Args:
|
||||
idx: Sample index
|
||||
|
||||
Returns:
|
||||
Tuple of (features, label)
|
||||
"""
|
||||
return self.X[idx], self.y[idx]
|
||||
|
||||
def get_feature_dim(self) -> int:
|
||||
"""Get number of features.
|
||||
|
||||
Returns:
|
||||
Number of features
|
||||
"""
|
||||
return self.X.shape[-1]
|
||||
|
||||
def get_sequence_length(self) -> int:
|
||||
"""Get sequence length (lookback).
|
||||
|
||||
Returns:
|
||||
Sequence length
|
||||
"""
|
||||
return self.X.shape[1]
|
||||
|
||||
|
||||
def create_dataloaders(
|
||||
train_data: Tuple[np.ndarray, np.ndarray],
|
||||
val_data: Tuple[np.ndarray, np.ndarray],
|
||||
test_data: Tuple[np.ndarray, np.ndarray],
|
||||
task: Literal['classification', 'regression'] = 'classification',
|
||||
batch_size: int = 64,
|
||||
num_workers: int = 0, # 0 for M1 compatibility
|
||||
pin_memory: bool = False # False for MPS
|
||||
) -> Tuple[torch.utils.data.DataLoader, ...]:
|
||||
"""Create DataLoaders for train/val/test sets.
|
||||
|
||||
Args:
|
||||
train_data: Tuple of (X_train, y_train)
|
||||
val_data: Tuple of (X_val, y_val)
|
||||
test_data: Tuple of (X_test, y_test)
|
||||
task: Task type
|
||||
batch_size: Batch size
|
||||
num_workers: Number of worker processes (0 for M1)
|
||||
pin_memory: Whether to pin memory (False for MPS)
|
||||
|
||||
Returns:
|
||||
Tuple of (train_loader, val_loader, test_loader)
|
||||
"""
|
||||
# Create datasets
|
||||
train_dataset = CryptoDataset(*train_data, task=task)
|
||||
val_dataset = CryptoDataset(*val_data, task=task)
|
||||
test_dataset = CryptoDataset(*test_data, task=task)
|
||||
|
||||
# Create dataloaders
|
||||
train_loader = torch.utils.data.DataLoader(
|
||||
train_dataset,
|
||||
batch_size=batch_size,
|
||||
shuffle=True,
|
||||
num_workers=num_workers,
|
||||
pin_memory=pin_memory
|
||||
)
|
||||
|
||||
val_loader = torch.utils.data.DataLoader(
|
||||
val_dataset,
|
||||
batch_size=batch_size,
|
||||
shuffle=False,
|
||||
num_workers=num_workers,
|
||||
pin_memory=pin_memory
|
||||
)
|
||||
|
||||
test_loader = torch.utils.data.DataLoader(
|
||||
test_dataset,
|
||||
batch_size=batch_size,
|
||||
shuffle=False,
|
||||
num_workers=num_workers,
|
||||
pin_memory=pin_memory
|
||||
)
|
||||
|
||||
return train_loader, val_loader, test_loader
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Test the dataset
|
||||
from fetcher import BinanceDataFetcher
|
||||
from preprocessor import CryptoPreprocessor, split_data
|
||||
|
||||
# Fetch and preprocess data
|
||||
fetcher = BinanceDataFetcher()
|
||||
df = fetcher.fetch_ohlcv('BTC', '1h')
|
||||
|
||||
preprocessor = CryptoPreprocessor(lookback=168)
|
||||
X, y, _ = preprocessor.preprocess_pipeline(df, task='classification')
|
||||
|
||||
# Split data
|
||||
train_data, val_data, test_data = split_data(X, y)
|
||||
|
||||
# Create dataloaders
|
||||
train_loader, val_loader, test_loader = create_dataloaders(
|
||||
train_data, val_data, test_data,
|
||||
batch_size=32
|
||||
)
|
||||
|
||||
# Test iteration
|
||||
for batch_X, batch_y in train_loader:
|
||||
print(f"Batch X shape: {batch_X.shape}")
|
||||
print(f"Batch y shape: {batch_y.shape}")
|
||||
break
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
"""从Binance获取加密货币OHLCV数据"""
|
||||
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Optional, Literal
|
||||
|
||||
import pandas as pd
|
||||
from binance.client import Client
|
||||
import pytz
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# 加载环境变量
|
||||
load_dotenv()
|
||||
|
||||
|
||||
class BinanceDataFetcher:
|
||||
"""从Binance公开API获取和缓存OHLCV数据"""
|
||||
|
||||
INTERVALS = {
|
||||
'1h': Client.KLINE_INTERVAL_1HOUR,
|
||||
'15m': Client.KLINE_INTERVAL_15MINUTE,
|
||||
}
|
||||
|
||||
def __init__(self, cache_dir: str = "./data/cache"):
|
||||
"""初始化Binance客户端和缓存目录
|
||||
|
||||
Args:
|
||||
cache_dir: 缓存下载数据的目录
|
||||
"""
|
||||
# 配置代理
|
||||
proxies = None
|
||||
use_proxy = os.getenv('USE_PROXY', 'false').lower() == 'true'
|
||||
|
||||
if use_proxy:
|
||||
socks5_proxy = os.getenv('SOCKS5_PROXY', 'socks5://127.0.0.1:7890')
|
||||
proxies = {
|
||||
'http': socks5_proxy,
|
||||
'https': socks5_proxy
|
||||
}
|
||||
print(f"✓ 使用SOCKS5代理: {socks5_proxy}")
|
||||
|
||||
try:
|
||||
# 初始化Binance客户端（带代理）
|
||||
if proxies:
|
||||
self.client = Client(requests_params={'proxies': proxies})
|
||||
else:
|
||||
self.client = Client()
|
||||
|
||||
# 测试连接
|
||||
self.client.ping()
|
||||
print("✓ 已成功连接到Binance API")
|
||||
except Exception as e:
|
||||
print(f"⚠ Binance API连接失败: {e}")
|
||||
print("提示：")
|
||||
print(" 1. 检查代理设置 (.env文件)")
|
||||
print(" 2. 确保代理服务正在运行")
|
||||
print(" 3. 或者使用已缓存的数据")
|
||||
self.client = None
|
||||
|
||||
self.cache_dir = Path(cache_dir)
|
||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def fetch_ohlcv(
|
||||
self,
|
||||
symbol: Literal['BTC', 'ETH', 'SOL'],
|
||||
timeframe: Literal['1h', '15m'],
|
||||
start_date: Optional[str] = None,
|
||||
end_date: Optional[str] = None,
|
||||
use_cache: bool = True
|
||||
) -> pd.DataFrame:
|
||||
"""获取加密货币对的OHLCV数据
|
||||
|
||||
Args:
|
||||
symbol: 加密货币符号 (BTC, ETH, SOL)
|
||||
timeframe: 时间框架间隔 (1h 或 15m)
|
||||
start_date: 开始日期，格式：'YYYY-MM-DD' (默认: 1年前)
|
||||
end_date: 结束日期，格式：'YYYY-MM-DD' (默认: 现在)
|
||||
use_cache: 是否使用缓存数据
|
||||
|
||||
Returns:
|
||||
包含以下列的DataFrame: timestamp, open, high, low, close, volume
|
||||
"""
|
||||
# 生成缓存文件名
|
||||
cache_file = self._get_cache_filename(symbol, timeframe, start_date, end_date)
|
||||
|
||||
# 检查缓存
|
||||
if use_cache and cache_file.exists():
|
||||
print(f"从缓存加载数据：{cache_file}")
|
||||
return pd.read_csv(cache_file, parse_dates=['timestamp'])
|
||||
|
||||
# 检查客户端是否可用
|
||||
if self.client is None:
|
||||
raise ConnectionError(
|
||||
"Binance客户端不可用。请使用VPN或使用已缓存的数据。"
|
||||
)
|
||||
|
||||
# 设置默认日期
|
||||
if end_date is None:
|
||||
end_dt = datetime.now(pytz.UTC)
|
||||
else:
|
||||
end_dt = datetime.strptime(end_date, '%Y-%m-%d').replace(tzinfo=pytz.UTC)
|
||||
|
||||
if start_date is None:
|
||||
start_dt = end_dt - timedelta(days=365) # 1年前
|
||||
else:
|
||||
start_dt = datetime.strptime(start_date, '%Y-%m-%d').replace(tzinfo=pytz.UTC)
|
||||
|
||||
# Binance使用USDT交易对
|
||||
pair = f"{symbol}USDT"
|
||||
interval = self.INTERVALS[timeframe]
|
||||
|
||||
print(f"获取 {symbol} {timeframe} 数据，时间范围：{start_dt.date()} 至 {end_dt.date()}...")
|
||||
|
||||
# 获取K线数据
|
||||
klines = self.client.get_historical_klines(
|
||||
symbol=pair,
|
||||
interval=interval,
|
||||
start_str=str(int(start_dt.timestamp() * 1000)),
|
||||
end_str=str(int(end_dt.timestamp() * 1000))
|
||||
)
|
||||
|
||||
if not klines:
|
||||
raise ValueError(f"未获取到 {symbol} {timeframe} 的数据")
|
||||
|
||||
# 转换为DataFrame
|
||||
df = pd.DataFrame(klines, columns=[
|
||||
'timestamp', 'open', 'high', 'low', 'close', 'volume',
|
||||
'close_time', 'quote_volume', 'trades', 'taker_buy_base',
|
||||
'taker_buy_quote', 'ignore'
|
||||
])
|
||||
|
||||
# 选择并转换列
|
||||
df = df[['timestamp', 'open', 'high', 'low', 'close', 'volume']].copy()
|
||||
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms')
|
||||
|
||||
for col in ['open', 'high', 'low', 'close', 'volume']:
|
||||
df[col] = df[col].astype(float)
|
||||
|
||||
# 按时间排序
|
||||
df = df.sort_values('timestamp').reset_index(drop=True)
|
||||
|
||||
print(f"已获取 {len(df)} 行数据，时间范围：{df['timestamp'].min()} 至 {df['timestamp'].max()}")
|
||||
|
||||
# 保存到缓存
|
||||
df.to_csv(cache_file, index=False)
|
||||
print(f"已缓存数据至：{cache_file}")
|
||||
|
||||
return df
|
||||
|
||||
def _get_cache_filename(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
start_date: Optional[str],
|
||||
end_date: Optional[str]
|
||||
) -> Path:
|
||||
"""根据参数生成缓存文件名"""
|
||||
start = start_date or "1y"
|
||||
end = end_date or "now"
|
||||
filename = f"{symbol}_{timeframe}_{start}_{end}.csv"
|
||||
return self.cache_dir / filename
|
||||
|
||||
def get_latest_price(self, symbol: Literal['BTC', 'ETH', 'SOL']) -> float:
|
||||
"""获取符号的当前价格
|
||||
|
||||
Args:
|
||||
symbol: 加密货币符号
|
||||
|
||||
Returns:
|
||||
当前USDT价格
|
||||
"""
|
||||
pair = f"{symbol}USDT"
|
||||
ticker = self.client.get_symbol_ticker(symbol=pair)
|
||||
return float(ticker['price'])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 测试数据获取器
|
||||
fetcher = BinanceDataFetcher()
|
||||
|
||||
# 获取BTC 1小时数据（1年）
|
||||
df = fetcher.fetch_ohlcv('BTC', '1h')
|
||||
print("\nBTC 1小时数据:")
|
||||
print(df.head())
|
||||
print(f"\n形状：{df.shape}")
|
||||
print(f"日期范围：{df['timestamp'].min()} 至 {df['timestamp'].max()}")
|
||||
|
||||
# 获取当前价格
|
||||
current_price = fetcher.get_latest_price('BTC')
|
||||
print(f"\n当前BTC价格：${current_price:,.2f}")
|
||||
@@ -0,0 +1,329 @@
|
||||
"""Data preprocessing: normalization, feature engineering, and sequence creation."""
|
||||
|
||||
from typing import Tuple, Optional, Literal
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
import pandas_ta as ta
|
||||
|
||||
|
||||
class CryptoPreprocessor:
|
||||
"""Preprocess cryptocurrency data with normalization and feature engineering."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
lookback: int = 168,
|
||||
forecast_horizon: int = 24,
|
||||
price_change_threshold: float = 0.0 # 改为0，使用真实涨跌
|
||||
):
|
||||
"""Initialize preprocessor.
|
||||
|
||||
Args:
|
||||
lookback: Number of historical timesteps to use
|
||||
forecast_horizon: Number of timesteps to predict ahead
|
||||
price_change_threshold: Threshold for binary classification (e.g., 0.015 = 1.5%)
|
||||
"""
|
||||
self.lookback = lookback
|
||||
self.forecast_horizon = forecast_horizon
|
||||
self.price_change_threshold = price_change_threshold
|
||||
self.scaler = MinMaxScaler()
|
||||
self.feature_columns = []
|
||||
self.is_fitted = False
|
||||
|
||||
def add_technical_indicators(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""添加技术指标作为特征
|
||||
|
||||
Args:
|
||||
df: 包含OHLCV数据的DataFrame
|
||||
|
||||
Returns:
|
||||
添加了技术指标的DataFrame
|
||||
"""
|
||||
df = df.copy()
|
||||
|
||||
# 使用pandas_ta库
|
||||
# RSI (14 periods)
|
||||
df['rsi'] = ta.rsi(df['close'], length=14)
|
||||
|
||||
# MACD
|
||||
macd = ta.macd(df['close'])
|
||||
if macd is not None and not macd.empty:
|
||||
# pandas-ta 返回的列名可能不同，动态获取
|
||||
macd_cols = macd.columns.tolist()
|
||||
if len(macd_cols) >= 3:
|
||||
df['macd'] = macd.iloc[:, 0] # MACD线
|
||||
df['macd_hist'] = macd.iloc[:, 1] # MACD柱
|
||||
df['macd_signal'] = macd.iloc[:, 2] # 信号线
|
||||
else:
|
||||
# 如果格式不对，使用None
|
||||
df['macd'] = None
|
||||
df['macd_signal'] = None
|
||||
df['macd_hist'] = None
|
||||
else:
|
||||
df['macd'] = None
|
||||
df['macd_signal'] = None
|
||||
df['macd_hist'] = None
|
||||
|
||||
# Bollinger Bands
|
||||
bbands = ta.bbands(df['close'], length=20)
|
||||
if bbands is not None and not bbands.empty:
|
||||
# pandas-ta 返回的列名可能不同，动态获取
|
||||
bb_cols = bbands.columns.tolist()
|
||||
if len(bb_cols) >= 3:
|
||||
df['bb_lower'] = bbands.iloc[:, 0] # 第一列是下轨
|
||||
df['bb_middle'] = bbands.iloc[:, 1] # 第二列是中轨
|
||||
df['bb_upper'] = bbands.iloc[:, 2] # 第三列是上轨
|
||||
df['bb_bandwidth'] = (df['bb_upper'] - df['bb_lower']) / (df['bb_middle'] + 1e-8)
|
||||
else:
|
||||
# 如果格式不对，手动计算
|
||||
sma = df['close'].rolling(window=20).mean()
|
||||
std = df['close'].rolling(window=20).std()
|
||||
df['bb_middle'] = sma
|
||||
df['bb_upper'] = sma + (std * 2)
|
||||
df['bb_lower'] = sma - (std * 2)
|
||||
df['bb_bandwidth'] = (df['bb_upper'] - df['bb_lower']) / (df['bb_middle'] + 1e-8)
|
||||
else:
|
||||
# pandas-ta不可用，手动计算布林带
|
||||
sma = df['close'].rolling(window=20).mean()
|
||||
std = df['close'].rolling(window=20).std()
|
||||
df['bb_middle'] = sma
|
||||
df['bb_upper'] = sma + (std * 2)
|
||||
df['bb_lower'] = sma - (std * 2)
|
||||
df['bb_bandwidth'] = (df['bb_upper'] - df['bb_lower']) / (df['bb_middle'] + 1e-8)
|
||||
|
||||
# Volume indicators
|
||||
df['volume_sma'] = ta.sma(df['volume'], length=20)
|
||||
df['volume_ratio'] = df['volume'] / (df['volume_sma'] + 1e-8)
|
||||
|
||||
# Price momentum
|
||||
df['returns'] = df['close'].pct_change()
|
||||
df['returns_5'] = df['close'].pct_change(5)
|
||||
df['returns_10'] = df['close'].pct_change(10)
|
||||
|
||||
# Volatility
|
||||
df['volatility'] = df['returns'].rolling(window=20).std()
|
||||
|
||||
return df
|
||||
|
||||
def add_time_features(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Add time-based features.
|
||||
|
||||
Args:
|
||||
df: DataFrame with timestamp column
|
||||
|
||||
Returns:
|
||||
DataFrame with added time features
|
||||
"""
|
||||
df = df.copy()
|
||||
|
||||
df['hour'] = df['timestamp'].dt.hour
|
||||
df['day_of_week'] = df['timestamp'].dt.dayofweek
|
||||
df['is_weekend'] = (df['day_of_week'] >= 5).astype(int)
|
||||
|
||||
# Cyclical encoding for hour
|
||||
df['hour_sin'] = np.sin(2 * np.pi * df['hour'] / 24)
|
||||
df['hour_cos'] = np.cos(2 * np.pi * df['hour'] / 24)
|
||||
|
||||
# Cyclical encoding for day of week
|
||||
df['dow_sin'] = np.sin(2 * np.pi * df['day_of_week'] / 7)
|
||||
df['dow_cos'] = np.cos(2 * np.pi * df['day_of_week'] / 7)
|
||||
|
||||
return df
|
||||
|
||||
def create_labels(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
task: Literal['classification', 'regression']
|
||||
) -> pd.DataFrame:
|
||||
"""Create target labels for classification or regression.
|
||||
|
||||
Args:
|
||||
df: DataFrame with price data
|
||||
task: Task type ('classification' or 'regression')
|
||||
|
||||
Returns:
|
||||
DataFrame with added label column
|
||||
"""
|
||||
df = df.copy()
|
||||
|
||||
# Calculate future returns
|
||||
df['future_price'] = df['close'].shift(-self.forecast_horizon)
|
||||
df['future_return'] = (df['future_price'] - df['close']) / df['close']
|
||||
|
||||
if task == 'classification':
|
||||
# Binary classification: 1 if price increases above threshold, 0 otherwise
|
||||
df['label'] = (df['future_return'] > self.price_change_threshold).astype(int)
|
||||
else: # regression
|
||||
# Regression: predict percentage return
|
||||
df['label'] = df['future_return']
|
||||
|
||||
# Drop rows without labels (last forecast_horizon rows)
|
||||
df = df[:-self.forecast_horizon].copy()
|
||||
|
||||
return df
|
||||
|
||||
def fit(self, df: pd.DataFrame) -> 'CryptoPreprocessor':
|
||||
"""Fit the scaler on training data.
|
||||
|
||||
Args:
|
||||
df: Training DataFrame with features
|
||||
|
||||
Returns:
|
||||
Self
|
||||
"""
|
||||
# Get feature columns (exclude timestamp and label)
|
||||
exclude_cols = ['timestamp', 'label', 'future_price', 'future_return']
|
||||
self.feature_columns = [col for col in df.columns if col not in exclude_cols]
|
||||
|
||||
# Fit scaler
|
||||
self.scaler.fit(df[self.feature_columns].fillna(0))
|
||||
self.is_fitted = True
|
||||
|
||||
return self
|
||||
|
||||
def transform(self, df: pd.DataFrame) -> np.ndarray:
|
||||
"""Transform data using fitted scaler.
|
||||
|
||||
Args:
|
||||
df: DataFrame with features
|
||||
|
||||
Returns:
|
||||
Scaled feature array
|
||||
"""
|
||||
if not self.is_fitted:
|
||||
raise ValueError("Preprocessor must be fitted before transform")
|
||||
|
||||
# Fill NaN values
|
||||
df_filled = df[self.feature_columns].fillna(0)
|
||||
|
||||
# Scale features
|
||||
scaled = self.scaler.transform(df_filled)
|
||||
|
||||
return scaled
|
||||
|
||||
def fit_transform(self, df: pd.DataFrame) -> np.ndarray:
|
||||
"""Fit and transform in one step.
|
||||
|
||||
Args:
|
||||
df: DataFrame with features
|
||||
|
||||
Returns:
|
||||
Scaled feature array
|
||||
"""
|
||||
return self.fit(df).transform(df)
|
||||
|
||||
def create_sequences(
|
||||
self,
|
||||
features: np.ndarray,
|
||||
labels: np.ndarray
|
||||
) -> Tuple[np.ndarray, np.ndarray]:
|
||||
"""Create sequences for time series prediction.
|
||||
|
||||
Args:
|
||||
features: Feature array (n_samples, n_features)
|
||||
labels: Label array (n_samples,)
|
||||
|
||||
Returns:
|
||||
Tuple of (X, y) where X is (n_sequences, lookback, n_features)
|
||||
and y is (n_sequences,)
|
||||
"""
|
||||
X, y = [], []
|
||||
|
||||
for i in range(len(features) - self.lookback):
|
||||
X.append(features[i:i + self.lookback])
|
||||
y.append(labels[i + self.lookback])
|
||||
|
||||
return np.array(X), np.array(y)
|
||||
|
||||
def preprocess_pipeline(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
task: Literal['classification', 'regression'],
|
||||
fit: bool = True
|
||||
) -> Tuple[np.ndarray, np.ndarray, pd.DataFrame]:
|
||||
"""Full preprocessing pipeline.
|
||||
|
||||
Args:
|
||||
df: Raw OHLCV DataFrame
|
||||
task: Task type
|
||||
fit: Whether to fit the scaler (True for training data)
|
||||
|
||||
Returns:
|
||||
Tuple of (X, y, processed_df) where X is sequences, y is labels,
|
||||
and processed_df is the DataFrame after feature engineering
|
||||
"""
|
||||
# Add features
|
||||
df = self.add_technical_indicators(df)
|
||||
df = self.add_time_features(df)
|
||||
df = self.create_labels(df, task)
|
||||
|
||||
# Drop initial NaN rows from indicators
|
||||
df = df.dropna().reset_index(drop=True)
|
||||
|
||||
# Scale features
|
||||
if fit:
|
||||
features = self.fit_transform(df)
|
||||
else:
|
||||
features = self.transform(df)
|
||||
|
||||
# Get labels
|
||||
labels = df['label'].values
|
||||
|
||||
# Create sequences
|
||||
X, y = self.create_sequences(features, labels)
|
||||
|
||||
print(f"Created {len(X)} sequences with shape {X.shape}")
|
||||
|
||||
return X, y, df
|
||||
|
||||
|
||||
def split_data(
|
||||
X: np.ndarray,
|
||||
y: np.ndarray,
|
||||
train_split: float = 0.7,
|
||||
val_split: float = 0.15
|
||||
) -> Tuple[Tuple[np.ndarray, np.ndarray], ...]:
|
||||
"""Split data into train/val/test sets using time-based split.
|
||||
|
||||
Args:
|
||||
X: Feature sequences
|
||||
y: Labels
|
||||
train_split: Fraction for training
|
||||
val_split: Fraction for validation
|
||||
|
||||
Returns:
|
||||
Tuple of ((X_train, y_train), (X_val, y_val), (X_test, y_test))
|
||||
"""
|
||||
n = len(X)
|
||||
train_end = int(n * train_split)
|
||||
val_end = int(n * (train_split + val_split))
|
||||
|
||||
X_train, y_train = X[:train_end], y[:train_end]
|
||||
X_val, y_val = X[train_end:val_end], y[train_end:val_end]
|
||||
X_test, y_test = X[val_end:], y[val_end:]
|
||||
|
||||
print(f"Train: {len(X_train)} samples")
|
||||
print(f"Val: {len(X_val)} samples")
|
||||
print(f"Test: {len(X_test)} samples")
|
||||
|
||||
return (X_train, y_train), (X_val, y_val), (X_test, y_test)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Test the preprocessor
|
||||
from fetcher import BinanceDataFetcher
|
||||
|
||||
# Fetch data
|
||||
fetcher = BinanceDataFetcher()
|
||||
df = fetcher.fetch_ohlcv('BTC', '1h')
|
||||
|
||||
# Preprocess for classification
|
||||
preprocessor = CryptoPreprocessor(lookback=168, forecast_horizon=24)
|
||||
X, y, processed_df = preprocessor.preprocess_pipeline(df, task='classification')
|
||||
|
||||
print(f"\nFeatures shape: {X.shape}")
|
||||
print(f"Labels shape: {y.shape}")
|
||||
print(f"Label distribution: {np.bincount(y.astype(int))}")
|
||||
|
||||
# Split data
|
||||
(X_train, y_train), (X_val, y_val), (X_test, y_test) = split_data(X, y)
|
||||
Reference in new issue
Block a user