Tutorial 3: Exploratory Data Analysis & Distribution Validation

"""Tutorial 3: Validate the feature table through exploratory analysis"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# --- Load feature table ---

print("Loading feature table from Tutorial 2...")
features = pd.read_parquet('data/processed/features_powerball.parquet')
print(f"Loaded {len(features)} draws with {len(features.columns)} features")

# --- Compute summary statistics ---

print("\nSummary Statistics:")

# For the mean feature, we expect around 35
mean_of_means = features['mean'].mean()
std_of_means = features['mean'].std()
print(f"\nDraw Mean:")
print(f"  Average: {mean_of_means:.2f} (expect ~35)")
print(f"  Std Dev: {std_of_means:.2f}")

# For odd count, we expect around 2.5
mean_odd_count = features['odd_count'].mean()
std_odd_count = features['odd_count'].std()
print(f"\nOdd Count:")
print(f"  Average: {mean_odd_count:.2f} (expect ~2.5)")
print(f"  Std Dev: {std_odd_count:.2f}")

# For range, we expect varied values
mean_range = features['range'].mean()
std_range = features['range'].std()
print(f"\nRange:")
print(f"  Average: {mean_range:.2f}")
print(f"  Std Dev: {std_range:.2f}")

# --- Create distribution plots ---

print("\nCreating diagnostic plots...")

# Plot 1: Distribution of means
plt.figure(figsize=(10, 6))
plt.hist(features['mean'], bins=30, color='steelblue', edgecolor='black', alpha=0.7)
plt.axvline(x=35, color='red', linestyle='--', linewidth=2, label='Expected (35)')
plt.axvline(x=mean_of_means, color='green', linestyle='-', linewidth=2, label=f'Observed ({mean_of_means:.1f})')
plt.xlabel('Draw Mean', fontsize=12)
plt.ylabel('Frequency', fontsize=12)
plt.title('Distribution of Draw Means (Should Center Near 35)', fontsize=14)
plt.legend()
plt.grid(axis='y', alpha=0.3)
plt.tight_layout()
plt.savefig('outputs/tutorial3/mean_distribution.png', dpi=150)
plt.close()
print("  Saved: mean_distribution.png")

# Plot 2: Odd count distribution
odd_count_frequencies = features['odd_count'].value_counts().sort_index()

# Theoretical probabilities (from binomial distribution)
total_draws = len(features)
expected_frequencies = {
    0: 0.029 * total_draws,
    1: 0.153 * total_draws,
    2: 0.318 * total_draws,
    3: 0.330 * total_draws,
    4: 0.170 * total_draws,
    5: 0.034 * total_draws
}

plt.figure(figsize=(10, 6))

x_positions = np.arange(6)
bar_width = 0.35

# Plot observed vs expected
observed_values = [odd_count_frequencies.get(i, 0) for i in range(6)]
expected_values = [expected_frequencies[i] for i in range(6)]

plt.bar(x_positions - bar_width/2, observed_values, bar_width, 
        label='Observed', color='steelblue', edgecolor='black')
plt.bar(x_positions + bar_width/2, expected_values, bar_width, 
        label='Expected', color='coral', edgecolor='black', alpha=0.7)

plt.xlabel('Number of Odd Balls', fontsize=12)
plt.ylabel('Frequency', fontsize=12)
plt.title('Odd Count Distribution (Should Peak at 2-3)', fontsize=14)
plt.xticks(x_positions, x_positions)
plt.legend()
plt.grid(axis='y', alpha=0.3)
plt.tight_layout()
plt.savefig('outputs/tutorial3/oddcount_distribution.png', dpi=150)
plt.close()
print("  Saved: oddcount_distribution.png")

# Plot 3: Decade buckets
decade_columns = ['decade_00_09', 'decade_10_19', 'decade_20_29', 
                  'decade_30_39', 'decade_40_49', 'decade_50_59', 'decade_60_69']
decade_totals = [features[col].sum() for col in decade_columns]

plt.figure(figsize=(10, 6))
decades = ['0-9', '10-19', '20-29', '30-39', '40-49', '50-59', '60-69']
plt.bar(decades, decade_totals, color='steelblue', edgecolor='black')
plt.xlabel('Decade', fontsize=12)
plt.ylabel('Total Hits', fontsize=12)
plt.title('Decade Distribution (Should Be Roughly Balanced)', fontsize=14)
plt.grid(axis='y', alpha=0.3)
plt.tight_layout()
plt.savefig('outputs/tutorial3/decade_distribution.png', dpi=150)
plt.close()
print("  Saved: decade_distribution.png")

# Plot 4: Mean over time
features['draw_date'] = pd.to_datetime(features['draw_date'])
features_sorted = features.sort_values('draw_date')

plt.figure(figsize=(12, 6))
plt.scatter(features_sorted['draw_date'], features_sorted['mean'], 
            alpha=0.3, s=10, color='steelblue', label='Draw Mean')

# Add a rolling average to see if there's a trend
rolling_window = 50
rolling_mean = features_sorted['mean'].rolling(window=rolling_window, center=True).mean()
plt.plot(features_sorted['draw_date'], rolling_mean, 
         color='red', linewidth=2, label=f'Rolling Average ({rolling_window} draws)')

# Add expected value line
plt.axhline(y=35, color='green', linestyle='--', linewidth=1.5, label='Expected (35)')

plt.xlabel('Date', fontsize=12)
plt.ylabel('Draw Mean', fontsize=12)
plt.title('Draw Mean Over Time (Should Be Stable Around 35)', fontsize=14)
plt.legend()
plt.grid(alpha=0.3)
plt.tight_layout()
plt.savefig('outputs/tutorial3/mean_over_time.png', dpi=150)
plt.close()
print("  Saved: mean_over_time.png")

# --- Final summary ---

print("\nEDA Complete!")
print("\nAll plots saved to: outputs/tutorial3/")
print("\nWhat to look for:")
print("  - Mean distribution: should center near 35")
print("  - Odd count: should peak at 2-3")
print("  - Decades: should be roughly balanced")
print("  - Mean over time: should be stable (no upward/downward trends)")