68 lines
2.0 KiB
Python
68 lines
2.0 KiB
Python
"""Migrate existing CSV cache data to SQLite database."""
|
||
|
||
import sys
|
||
import os
|
||
from pathlib import Path
|
||
import re
|
||
|
||
# Add project root to path
|
||
sys.path.append(str(Path(__file__).resolve().parent.parent))
|
||
|
||
import pandas as pd
|
||
from data.database import DatabaseManager
|
||
|
||
def migrate():
|
||
cache_dir = Path("./data/cache")
|
||
db_path = cache_dir / "market_data.db"
|
||
|
||
print(f"Initializing database at {db_path}...")
|
||
db_manager = DatabaseManager(str(db_path))
|
||
|
||
# Find all CSV files
|
||
csv_files = list(cache_dir.glob("*.csv"))
|
||
print(f"Found {len(csv_files)} CSV files to migrate.")
|
||
|
||
success_count = 0
|
||
fail_count = 0
|
||
|
||
for csv_file in csv_files:
|
||
try:
|
||
filename = csv_file.name
|
||
print(f"Processing {filename}...")
|
||
|
||
# Parse filename
|
||
parts = filename.split('_')
|
||
if len(parts) < 2:
|
||
print(f" ⚠ Skipping {filename}: Invalid naming format")
|
||
continue
|
||
|
||
symbol = parts[0]
|
||
timeframe = parts[1]
|
||
|
||
# Validate symbol and timeframe (basic check)
|
||
if timeframe not in ['1h', '15m']:
|
||
# Relaxed check, just warning
|
||
print(f" ℹ Note: Unfamiliar timeframe {timeframe} in {filename}")
|
||
|
||
# Read CSV
|
||
df = pd.read_csv(csv_file)
|
||
if 'timestamp' not in df.columns:
|
||
print(f" ⚠ Skipping {filename}: No timestamp column")
|
||
continue
|
||
|
||
# Save to DB
|
||
db_manager.save_ohlcv(df, symbol, timeframe)
|
||
print(f" ✓ Imported {len(df)} records for {symbol} {timeframe}")
|
||
success_count += 1
|
||
|
||
except Exception as e:
|
||
print(f" ❌ Error processing {csv_file.name}: {e}")
|
||
fail_count += 1
|
||
|
||
print("\nMigration complete!")
|
||
print(f"Successfully migrated: {success_count} files")
|
||
print(f"Failed: {fail_count} files")
|
||
|
||
if __name__ == "__main__":
|
||
migrate()
|