Spaces:
Sleeping
Sleeping
File size: 3,430 Bytes
27cb60a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
"""Final validation of complete 24-month LTA + Net Positions datasets."""
import polars as pl
from pathlib import Path
print("\n" + "=" * 80)
print("FINAL DATA COLLECTION VALIDATION")
print("=" * 80)
# =========================================================================
# LTA Dataset
# =========================================================================
lta_path = Path('data/raw/phase1_24month/jao_lta.parquet')
lta = pl.read_parquet(lta_path)
print("\n[1/2] LTA (Long Term Allocations)")
print("-" * 80)
print(f" Records: {len(lta):,}")
print(f" Columns: {len(lta.columns)} (1 timestamp + {len(lta.columns)-3} borders + 2 masking flags)")
print(f" File size: {lta_path.stat().st_size / (1024**2):.2f} MB")
print(f" Date range: {lta['mtu'].min()} to {lta['mtu'].max()}")
print(f" Unique timestamps: {lta['mtu'].n_unique():,}")
# Check October 2023
oct_2023 = lta.filter((pl.col('mtu').dt.year() == 2023) & (pl.col('mtu').dt.month() == 10))
days_2023 = sorted(oct_2023['mtu'].dt.day().unique().to_list())
masked_2023 = oct_2023.filter(pl.col('is_masked') == True)
print(f"\n October 2023:")
print(f" Days present: {days_2023}")
print(f" Total records: {len(oct_2023)}")
print(f" Masked records: {len(masked_2023)} ({len(masked_2023)/len(lta)*100:.3f}%)")
# Check October 2024
oct_2024 = lta.filter((pl.col('mtu').dt.year() == 2024) & (pl.col('mtu').dt.month() == 10))
days_2024 = sorted(oct_2024['mtu'].dt.day().unique().to_list())
print(f"\n October 2024:")
print(f" Days present: {days_2024}")
print(f" Total records: {len(oct_2024)}")
# =========================================================================
# Net Positions Dataset
# =========================================================================
np_path = Path('data/raw/phase1_24month/jao_net_positions.parquet')
np_df = pl.read_parquet(np_path)
print("\n[2/2] Net Positions (Domain Boundaries)")
print("-" * 80)
print(f" Records: {len(np_df):,}")
print(f" Columns: {len(np_df.columns)} (1 timestamp + 28 zones + 1 collection_date)")
print(f" File size: {np_path.stat().st_size / (1024**2):.2f} MB")
print(f" Date range: {np_df['mtu'].min()} to {np_df['mtu'].max()}")
print(f" Unique dates: {np_df['mtu'].dt.date().n_unique()}")
# Expected: Oct 1, 2023 to Sep 30, 2025 = 731 days
expected_days = 731
print(f" Expected days: {expected_days}")
print(f" Coverage: {np_df['mtu'].dt.date().n_unique() / expected_days * 100:.1f}%")
# =========================================================================
# Summary
# =========================================================================
print("\n" + "=" * 80)
print("COLLECTION STATUS")
print("=" * 80)
lta_complete = (days_2023 == list(range(1, 32))) and (days_2024 == list(range(1, 32)))
np_complete = (np_df['mtu'].dt.date().n_unique() >= expected_days - 1) # Allow 1 day variance
if lta_complete and np_complete:
print("[SUCCESS] Data collection complete!")
print(f" ✓ LTA: {len(lta):,} records with {len(masked_2023)} masked (Oct 27-31, 2023)")
print(f" ✓ Net Positions: {len(np_df):,} records covering {np_df['mtu'].dt.date().n_unique()} days")
else:
print("[WARNING] Data collection incomplete:")
if not lta_complete:
print(f" - LTA October coverage issue")
if not np_complete:
print(f" - Net Positions has {np_df['mtu'].dt.date().n_unique()}/{expected_days} expected days")
print("=" * 80)
print()
|