# Install required packages
!pip install yfinance plotly scipy pandas numpy seaborn --quiet

# ===================
# IMPORTS
# ===================

# Standard Library
import warnings
import sys
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Tuple

# Numerical Computing
import numpy as np
import pandas as pd

# Statistics
from scipy import stats
from scipy.stats import norm, lognorm

# Visualization
import matplotlib.pyplot as plt
import matplotlib.ticker as mticker
import seaborn as sns
import plotly.graph_objects as go
import plotly.express as px
from plotly.subplots import make_subplots


# Configuration
warnings.filterwarnings("ignore")
pd.set_option("display.max_columns", 20)
pd.set_option("display.float_format", "{:,.2f}".format)
np.random.seed(42)

print("All imports successful.")
print(f"   pandas  : {pd.__version__}")
print(f"   numpy   : {np.__version__}")
print(f"   plotly  : {go.__version__ if hasattr(go, '__version__') else 'loaded'}")
print(f"   yfinance: {'not available — using synthetic data'}")

def load_or_generate_trade_data(
    ticker: str = "BTC-USD",
    n_trades: int = 5000,
    days_back: int = 30 # days_back is no longer used, kept for signature consistency
) -> pd.DataFrame:
    """
    Generates realistic synthetic trade data that mimics market behavior.

    The synthetic data includes a lognormal size distribution with embedded
    institutional-sized trades, tailored for either equity or crypto assets.

    Parameters
    ----------
    ticker : str
        Equity ticker symbol (e.g., 'SPY', 'AAPL') or crypto symbol (e.g., 'BTC-USD')
        to influence the characteristics of the synthetic data.
    n_trades : int
        Number of synthetic trades to generate.
    days_back : int
        This parameter is no longer used as live data fetching has been removed.
        It's kept for function signature consistency.

    Returns
    -------
    pd.DataFrame
        DataFrame with columns: timestamp, price, size, notional, source.
    """
    df: Optional[pd.DataFrame] = None

    # ------------------------------------------------------------------
    # Directly generate realistic synthetic trade data (yfinance removed)
    # ------------------------------------------------------------------
    print(f"Generating {n_trades:,} synthetic trades (realistic lognormal distribution)...")

    rng = np.random.default_rng(seed=42)

    # Time axis — simulate one trading day worth of minutes spread across 30 days
    base_time = pd.Timestamp("2024-01-02 09:30:00")
    timestamps = pd.date_range(base_time, periods=n_trades, freq="6s")  # ~6s between trades

    if ticker.upper() == "BTC-USD":
        # Crypto-specific synthetic data (BTC-like)
        prices = 60000.0 * np.exp(np.cumsum(rng.normal(loc=0.0, scale=0.0005, size=n_trades))) # Higher vol
        # Trade sizes in units of BTC (can be fractional)
        normal_sizes = rng.lognormal(mean=-3.0, sigma=1.5, size=n_trades) # Median ~0.05 BTC
        normal_sizes = np.clip(normal_sizes, 0.0001, 5.0) # Clip between 0.0001 and 5 BTC

        # Inject ~2% institutional block trades (20-500 BTC)
        n_blocks = int(n_trades * 0.02)
        block_indices = rng.choice(n_trades, size=n_blocks, replace=False)
        block_sizes = rng.uniform(20, 500, size=n_blocks) # 20-500 BTC
        normal_sizes[block_indices] = block_sizes
        print("  NOTE: This is SIMULATED crypto data for educational purposes only.")

    else:
        # Equity-specific synthetic data (SPY-like)
        prices = 450.0 * np.exp(np.cumsum(rng.normal(loc=0.0, scale=0.0002, size=n_trades)))
        # Trade sizes — lognormal with heavy tails (realistic equity market)
        # Median ~200 shares, long tail reaching 100k+
        normal_sizes = rng.lognormal(mean=5.2, sigma=1.4, size=n_trades)
        normal_sizes = np.round(normal_sizes / 100) * 100  # round to lot sizes
        normal_sizes = np.clip(normal_sizes, 100, 50_000)

        # Inject ~2% institutional block trades
        n_blocks = int(n_trades * 0.02)
        block_indices = rng.choice(n_trades, size=n_blocks, replace=False)
        block_sizes = rng.uniform(100_000, 2_000_000, size=n_blocks)  # 100k–2M shares
        normal_sizes[block_indices] = np.round(block_sizes / 1000) * 1000
        print("  NOTE: This is SIMULATED equity data for educational purposes only.")

    df = pd.DataFrame({
        "timestamp": timestamps,
        "price": prices,
        "size": normal_sizes,
        "source": "synthetic",
    })
    df["notional"] = df["price"] * df["size"]
    print(f"Synthetic data generated: {len(df):,} trades")

    df = df.reset_index(drop=True)
    return df

def validate_trade_data(df: pd.DataFrame) -> Dict[str, object]:
    """
    Validate schema, completeness, and sanity of trade data.

    Parameters
    ----------
    df : pd.DataFrame
        Trade data with expected columns: timestamp, price, size, notional.

    Returns
    -------
    Dict[str, object]
        Validation report with keys: is_valid, n_rows, issues, warnings.
    """
    required_columns = {"timestamp", "price", "size", "notional"}
    issues: List[str] = []
    warnings_list: List[str] = []

    # --- Schema check ---
    missing_cols = required_columns - set(df.columns)
    if missing_cols:
        issues.append(f"Missing required columns: {missing_cols}")

    if issues:
        return {"is_valid": False, "n_rows": len(df), "issues": issues, "warnings": warnings_list}

    # --- Null check ---
    null_counts = df[["price", "size", "notional"]].isnull().sum()
    for col, cnt in null_counts.items():
        if cnt > 0:
            issues.append(f"Column '{col}' has {cnt} null values.")

    # --- Negative values ---
    if (df["price"] <= 0).any():
        n_neg = (df["price"] <= 0).sum()
        issues.append(f"{n_neg} rows have non-positive price.")

    if (df["size"] <= 0).any():
        n_neg = (df["size"] <= 0).sum()
        issues.append(f"{n_neg} rows have non-positive size.")

    # --- Row count ---
    if len(df) < 10:
        issues.append(f"Insufficient data: only {len(df)} rows.")
    elif len(df) < 100:
        warnings_list.append(f"Low row count ({len(df)}): statistical results may be unstable.")

    # --- Duplicate timestamps ---
    dup_ts = df["timestamp"].duplicated().sum()
    if dup_ts > len(df) * 0.1:
        warnings_list.append(f"{dup_ts} duplicate timestamps (>{10}% of data) — expected for bar data.")

    is_valid = len(issues) == 0
    status = "PASSED" if is_valid else "❌ FAILED"

    print(f"\nData Validation Report — {status}")
    print(f"   Rows examined : {len(df):,}")
    print(f"   Columns       : {list(df.columns)}")
    if issues:
        print(f"   Issues ({len(issues)}):")
        for iss in issues:
            print(f"     ❌ {iss}")
    if warnings_list:
        print(f"   Warnings ({len(warnings_list)}):")
        for w in warnings_list:
            print(f"      {w}")
    if is_valid and not warnings_list:
        print("   All checks passed — data is clean.")

    return {
        "is_valid": is_valid,
        "n_rows": len(df),
        "issues": issues,
        "warnings": warnings_list,
    }

def preprocess_trade_data(df: pd.DataFrame) -> pd.DataFrame:
    """
    Clean, sort, and feature-engineer the trade DataFrame.

    Steps performed:
    1. Drop rows with null price/size/notional.
    2. Remove non-positive price or size.
    3. Sort by timestamp.
    4. Add log-transformed size/notional (stabilizes z-scores on right-skewed data).
    5. Add time-of-day features.
    6. Reset index.

    Parameters
    ----------
    df : pd.DataFrame
        Validated trade data.

    Returns
    -------
    pd.DataFrame
        Cleaned and enriched DataFrame.
    """
    print("Preprocessing trade data...")
    n_before = len(df)

    # Step 1: Drop nulls
    clean = df.dropna(subset=["price", "size", "notional"]).copy()

    # Step 2: Remove invalid rows
    clean = clean[(clean["price"] > 0) & (clean["size"] > 0)].copy()

    # Step 3: Sort by timestamp
    clean = clean.sort_values("timestamp").reset_index(drop=True)

    # Step 4: Log transforms — critical for statistical stability
    clean["log_size"] = np.log(clean["size"])
    clean["log_notional"] = np.log(clean["notional"])

    # Step 5: Time features
    clean["timestamp"] = pd.to_datetime(clean["timestamp"])
    clean["hour"] = clean["timestamp"].dt.hour
    clean["minute"] = clean["timestamp"].dt.minute

    # Step 6: Sequential trade index
    clean["trade_index"] = np.arange(len(clean))

    n_after = len(clean)
    n_dropped = n_before - n_after

    print(f"   Rows before cleaning : {n_before:,}")
    print(f"   Rows after cleaning  : {n_after:,}")
    print(f"   Rows dropped         : {n_dropped:,}")
    print(f"   Columns added        : log_size, log_notional, hour, minute, trade_index")
    print(f"   Time range           : {clean['timestamp'].min()} → {clean['timestamp'].max()}")

    return clean

def calculate_trade_statistics(df: pd.DataFrame) -> Dict[str, float]:
    """
    Compute a full statistical profile of the trade size distribution.

    Parameters
    ----------
    df : pd.DataFrame
        Preprocessed trade data containing 'size' and 'log_size' columns.

    Returns
    -------
    Dict[str, float]
        Statistical summary including moments, percentiles, and thresholds.
    """
    sizes = df["size"].values
    log_sizes = df["log_size"].values

    stat_dict: Dict[str, float] = {
        # Raw-space moments
        "mean":     float(np.mean(sizes)),
        "median":   float(np.median(sizes)),
        "std":      float(np.std(sizes)),
        "skewness": float(stats.skew(sizes)),
        "kurtosis": float(stats.kurtosis(sizes)),
        "min":      float(np.min(sizes)),
        "max":      float(np.max(sizes)),

        # Percentile thresholds
        "p75":  float(np.percentile(sizes, 75)),
        "p90":  float(np.percentile(sizes, 90)),
        "p95":  float(np.percentile(sizes, 95)),
        "p99":  float(np.percentile(sizes, 99)),
        "p999": float(np.percentile(sizes, 99.9)),

        # Log-space statistics (used for log z-score method)
        "log_mean": float(np.mean(log_sizes)),
        "log_std":  float(np.std(log_sizes)),

        # Volume multiple threshold (5× mean)
        "volume_multiple_threshold": float(5.0 * np.mean(sizes)),
        "volume_multiple_k": 5.0,

        # Total notional
        "total_volume":   float(np.sum(sizes)),
        "total_notional": float(df["notional"].sum()),
        "n_trades":       float(len(df)),
    }

    # Print formatted report
    print("\n" + "═" * 55)
    print("  TRADE SIZE STATISTICS")
    print("═" * 55)
    print(f"  Total trades       : {stat_dict['n_trades']:>12,.0f}")
    print(f"  Total volume       : {stat_dict['total_volume']:>12,.0f} shares")
    print(f"  Total notional     : ${stat_dict['total_notional']:>12,.0f}")
    print("─" * 55)
    print(f"  Mean size          : {stat_dict['mean']:>12,.1f} shares")
    print(f"  Median size        : {stat_dict['median']:>12,.1f} shares")
    print(f"  Std deviation      : {stat_dict['std']:>12,.1f} shares")
    print(f"  Skewness           : {stat_dict['skewness']:>12.2f}  (>0 = right-skewed)")
    print(f"  Excess kurtosis    : {stat_dict['kurtosis']:>12.2f}  (>3 = heavy tails)")
    print("─" * 55)
    print(f"  75th percentile    : {stat_dict['p75']:>12,.0f} shares")
    print(f"  90th percentile    : {stat_dict['p90']:>12,.0f} shares")
    print(f"  95th percentile    : {stat_dict['p95']:>12,.0f} shares")
    print(f"  99th percentile    : {stat_dict['p99']:>12,.0f} shares")
    print(f"  99.9th percentile  : {stat_dict['p999']:>12,.0f} shares")
    print("─" * 55)
    print(f"  5× Volume multiple : {stat_dict['volume_multiple_threshold']:>12,.0f} shares")
    print("═" * 55)

    return stat_dict

def detect_large_trades(
    df: pd.DataFrame,
    trade_stats: Dict[str, float],
    percentile_threshold: float = 95.0,
    zscore_threshold: float = 3.0,
    volume_multiple: float = 5.0,
) -> pd.DataFrame:
    """
    Apply three methods to detect and flag large/institutional trades.

    Method 1 — Percentile Threshold:
        Any trade with size >= percentile_threshold-th percentile is flagged.
        No distributional assumption required. Simple and robust.

    Method 2 — Log Z-Score:
        Compute z-score on log(size) to account for the lognormal distribution
        of trade sizes. Flag trades where log_z >= zscore_threshold.
        More sensitive to extreme outliers than raw z-score.

    Method 3 — Volume Multiple:
        Flag trades where size >= volume_multiple × mean_size.
        Intuitive rule used by many trading desks.

    Parameters
    ----------
    df : pd.DataFrame
        Preprocessed trade data with 'size' and 'log_size' columns.
    trade_stats : Dict[str, float]
        Statistical summary from calculate_trade_statistics().
    percentile_threshold : float
        Percentile cutoff (default 95.0 → top 5%).
    zscore_threshold : float
        Log-space z-score cutoff (default 3.0).
    volume_multiple : float
        Multiple of mean for rule-based detection (default 5.0).

    Returns
    -------
    pd.DataFrame
        Original DataFrame with detection columns appended.
    """
    flagged = df.copy()

    # ----------------------------------
    # Method 1: Percentile Threshold
    # ----------------------------------
    p_cutoff = np.percentile(flagged["size"], percentile_threshold)
    flagged["large_percentile"] = flagged["size"] >= p_cutoff

    # --------------------------
    # Method 2: Log Z-Score
    # --------------------------
    mu_log = trade_stats["log_mean"]
    sigma_log = trade_stats["log_std"]
    flagged["log_zscore"] = (flagged["log_size"] - mu_log) / sigma_log
    flagged["large_zscore"] = flagged["log_zscore"] >= zscore_threshold

    # Also compute raw z-score for reference
    mu_raw = trade_stats["mean"]
    sigma_raw = trade_stats["std"]
    flagged["zscore"] = (flagged["size"] - mu_raw) / (sigma_raw + 1e-9)

    # ---------------------------------
    # Method 3: Volume Multiple Rule
    # ---------------------------------
    vm_cutoff = volume_multiple * trade_stats["mean"]
    flagged["large_volume_multiple"] = flagged["size"] >= vm_cutoff

    # ------------------
    # Combined flags
    # ------------------
    flagged["large_any"] = (
        flagged["large_percentile"] |
        flagged["large_zscore"] |
        flagged["large_volume_multiple"]
    )
    flagged["large_consensus"] = (
        flagged["large_percentile"] &
        flagged["large_zscore"] &
        flagged["large_volume_multiple"]
    )

    # ----------
    # Summary
    # ----------
    n = len(flagged)
    n1 = flagged["large_percentile"].sum()
    n2 = flagged["large_zscore"].sum()
    n3 = flagged["large_volume_multiple"].sum()
    n_any = flagged["large_any"].sum()
    n_con = flagged["large_consensus"].sum()

    print("\n" + "═" * 60)
    print("  LARGE TRADE DETECTION RESULTS")
    print("═" * 60)
    print(f"  Total trades analyzed      : {n:>10,}")
    print(f"  Percentile threshold        : {p_cutoff:>10,.0f} shares  ({percentile_threshold}th pctile)")
    print(f"  Log Z-Score threshold       : {zscore_threshold:>10.1f}σ")
    print(f"  Volume Multiple threshold   : {vm_cutoff:>10,.0f} shares  ({volume_multiple}× mean)")
    print("─" * 60)
    print(f"  Method 1 — Percentile       : {n1:>6,} flagged  ({100*n1/n:.2f}%)")
    print(f"  Method 2 — Log Z-Score      : {n2:>6,} flagged  ({100*n2/n:.2f}%)")
    print(f"  Method 3 — Volume Multiple  : {n3:>6,} flagged  ({100*n3/n:.2f}%)")
    print("─" * 60)
    print(f"  Flagged by ANY method       : {n_any:>6,} trades  ({100*n_any/n:.2f}%)")
    print(f"  Flagged by ALL 3 (consensus): {n_con:>6,} trades  ({100*n_con/n:.2f}%)")
    print("═" * 60)

    # Volume concentration of flagged trades
    vol_large = flagged.loc[flagged["large_consensus"], "size"].sum()
    vol_total = flagged["size"].sum()
    if vol_total > 0:
        print(f"  Consensus trades account for {100*vol_large/vol_total:.1f}% of total volume")
    print("═" * 60)

    return flagged

def visualize_trade_distribution(
    df: pd.DataFrame,
    trade_stats: Dict[str, float]
) -> None:
    """
    Visualize the trade size distribution with histogram, KDE, and ECDF.

    Parameters
    ----------
    df : pd.DataFrame
        Preprocessed trade data with 'size' and 'log_size'.
    trade_stats : Dict[str, float]
        Statistical summary from calculate_trade_statistics().
    """
    log_sizes = df["log_size"].values
    sizes = df["size"].values

    fig = make_subplots(
        rows=1, cols=2,
        subplot_titles=[
            "Log(Trade Size) Distribution — Histogram + KDE",
            "Empirical CDF — Detection Thresholds"
        ]
    )

    # ── Panel 1: Histogram + KDE in log-space
    counts, bin_edges = np.histogram(log_sizes, bins=60)
    bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2

    fig.add_trace(
        go.Bar(
            x=bin_centers, y=counts,
            name="Frequency",
            marker_color="steelblue",
            opacity=0.65,
        ),
        row=1, col=1
    )

    # KDE overlay
    kde_x = np.linspace(log_sizes.min(), log_sizes.max(), 300)
    kde = stats.gaussian_kde(log_sizes)
    kde_y = kde(kde_x) * len(log_sizes) * (bin_edges[1] - bin_edges[0])  # scale to bar height
    fig.add_trace(
        go.Scatter(
            x=kde_x, y=kde_y,
            mode="lines", name="KDE",
            line=dict(color="crimson", width=2.5)
        ),
        row=1, col=1
    )

    # Threshold lines in log-space
    p95_log = np.log(trade_stats["p95"])
    p99_log = np.log(trade_stats["p99"])
    for val, label, color in [
        (p95_log, "P95", "orange"),
        (p99_log, "P99", "red"),
    ]:
        fig.add_vline(x=val, line_dash="dash", line_color=color,
                      annotation_text=label, annotation_position="top right",
                      row=1, col=1)

    # ── Panel 2: Empirical CDF
    sorted_sizes = np.sort(sizes)
    cdf_y = np.arange(1, len(sorted_sizes) + 1) / len(sorted_sizes)

    fig.add_trace(
        go.Scatter(
            x=np.log(sorted_sizes + 1), y=cdf_y,
            mode="lines", name="Empirical CDF",
            line=dict(color="steelblue", width=2)
        ),
        row=1, col=2
    )

    for pctile, label, color in [
        (trade_stats["p90"],  "P90", "gold"),
        (trade_stats["p95"],  "P95", "orange"),
        (trade_stats["p99"],  "P99", "red"),
        (trade_stats["p999"], "P99.9", "darkred"),
    ]:
        fig.add_vline(
            x=np.log(pctile + 1),
            line_dash="dot", line_color=color,
            annotation_text=label,
            annotation_position="top left",
            row=1, col=2
        )

    fig.update_layout(
        title_text="Trade Size Distribution Analysis",
        height=500,
        template="plotly_dark",
        showlegend=True,
        legend=dict(x=0.01, y=0.99),
    )
    fig.update_xaxes(title_text="Log(Trade Size)", row=1, col=1)
    fig.update_yaxes(title_text="Count", row=1, col=1)
    fig.update_xaxes(title_text="Log(Trade Size)", row=1, col=2)
    fig.update_yaxes(title_text="Cumulative Probability", row=1, col=2)

    fig.show()

def visualize_large_trades(
    df: pd.DataFrame,
    trade_stats: Dict[str, float],
    max_points: int = 2000
) -> None:
    """
    Time-series scatter plot highlighting large vs normal trades.

    Parameters
    ----------
    df : pd.DataFrame
        Flagged trade data from detect_large_trades().
    trade_stats : Dict[str, float]
        Statistical summary.
    max_points : int
        Maximum normal trades to plot (subsampled for performance).
    """
    normal = df[~df["large_any"]].copy()
    flagged_any = df[df["large_any"] & ~df["large_consensus"]].copy()
    flagged_con = df[df["large_consensus"]].copy()

    # Subsample normal trades for performance
    if len(normal) > max_points:
        normal = normal.sample(n=max_points, random_state=42)

    fig = make_subplots(
        rows=2, cols=1,
        shared_xaxes=True,
        subplot_titles=[
            "Trade Size Over Time — Large Trade Detection",
            "Trade Price Over Time"
        ],
        vertical_spacing=0.12,
        row_heights=[0.65, 0.35],
    )

    # ── Normal trades
    fig.add_trace(
        go.Scatter(
            x=normal["timestamp"], y=normal["size"],
            mode="markers",
            marker=dict(color="#4a9eff", size=3, opacity=0.4),
            name="Normal Trade",
            hovertemplate="%{x}<br>Size: %{y:,.0f}"
        ),
        row=1, col=1
    )

    # ── Trades flagged by at least one method
    if len(flagged_any) > 0:
        fig.add_trace(
            go.Scatter(
                x=flagged_any["timestamp"], y=flagged_any["size"],
                mode="markers",
                marker=dict(color="orange", size=8, symbol="diamond", opacity=0.8,
                            line=dict(color="darkorange", width=1)),
                name="Large (any method)",
                hovertemplate="%{x}<br>Size: %{y:,.0f}<br>Flagged: 1+ method"
            ),
            row=1, col=1
        )

    # ── Consensus large trades (all 3 methods)
    if len(flagged_con) > 0:
        fig.add_trace(
            go.Scatter(
                x=flagged_con["timestamp"], y=flagged_con["size"],
                mode="markers",
                marker=dict(color="red", size=14, symbol="star", opacity=1.0,
                            line=dict(color="darkred", width=1.5)),
                name="Consensus Large (all 3)",
                hovertemplate="%{x}<br>Size: %{y:,.0f}<br><b>CONSENSUS LARGE TRADE</b>"
            ),
            row=1, col=1
        )

    # ── Threshold lines
    for thresh, label, color in [
        (trade_stats["p95"], "P95", "rgba(255,165,0,0.6)"),
        (trade_stats["p99"], "P99", "rgba(255,0,0,0.6)"),
    ]:
        fig.add_hline(
            y=thresh, line_dash="dash", line_color=color,
            annotation_text=label, annotation_position="right",
            row=1, col=1
        )

    # ── Price panel
    all_trades_sorted = df.sort_values("timestamp")
    fig.add_trace(
        go.Scatter(
            x=all_trades_sorted["timestamp"],
            y=all_trades_sorted["price"],
            mode="lines",
            line=dict(color="#aaffaa", width=1),
            name="Price",
        ),
        row=2, col=1
    )

    fig.update_layout(
        title_text="🚨 Large Trade Detection — Time Series View",
        height=700,
        template="plotly_dark",
        showlegend=True,
        legend=dict(x=0.01, y=0.99),
        hovermode="x unified",
    )
    fig.update_yaxes(title_text="Trade Size (shares)", row=1, col=1)
    fig.update_yaxes(title_text="Price ($)", row=2, col=1)
    fig.update_xaxes(title_text="Timestamp", row=2, col=1)

    fig.show()

def visualize_method_comparison(df: pd.DataFrame) -> None:
    """
    Compare detection methods: trade count and volume concentration.

    Parameters
    ----------
    df : pd.DataFrame
        Flagged trade data from detect_large_trades().
    """
    methods = {
        "Percentile (P95)": "large_percentile",
        "Log Z-Score (z≥3)": "large_zscore",
        "Vol Multiple (5×)": "large_volume_multiple",
        "ANY Method": "large_any",
        "ALL Methods (Consensus)": "large_consensus",
    }

    total = len(df)
    total_vol = df["size"].sum()

    labels, trade_pcts, vol_pcts = [], [], []
    for label, col in methods.items():
        mask = df[col]
        trade_pcts.append(100 * mask.sum() / total)
        vol_pcts.append(100 * df.loc[mask, "size"].sum() / total_vol)
        labels.append(label)

    colors = ["#4a9eff", "#ff7043", "#ab47bc", "#26a69a", "#ef5350"]

    fig = make_subplots(
        rows=1, cols=2,
        subplot_titles=[
            "% of Trades Flagged by Each Method",
            "% of Total Volume in Flagged Trades",
        ]
    )

    fig.add_trace(
        go.Bar(x=labels, y=trade_pcts, marker_color=colors,
               name="% Trades", text=[f"{v:.2f}%" for v in trade_pcts],
               textposition="outside"),
        row=1, col=1
    )
    fig.add_trace(
        go.Bar(x=labels, y=vol_pcts, marker_color=colors,
               name="% Volume", text=[f"{v:.1f}%" for v in vol_pcts],
               textposition="outside"),
        row=1, col=2
    )

    fig.update_layout(
        title_text="Detection Method Comparison",
        height=480,
        template="plotly_dark",
        showlegend=False,
        bargap=0.3,
    )
    fig.update_yaxes(title_text="% of All Trades", row=1, col=1)
    fig.update_yaxes(title_text="% of Total Volume", row=1, col=2)

    fig.show()

    # Key insight
    con_vol_pct = vol_pcts[labels.index("ALL Methods (Consensus)")]
    con_trade_pct = trade_pcts[labels.index("ALL Methods (Consensus)")]
    print(f"\nKEY INSIGHT: Consensus large trades account for only {con_trade_pct:.2f}% "
          f"of all trades but {con_vol_pct:.1f}% of total volume.")

def visualize_zscore_analysis(df: pd.DataFrame) -> None:
    """
    Plot the log-space z-score distribution for all trades.

    Parameters
    ----------
    df : pd.DataFrame
        Flagged trade data containing 'log_zscore' and detection flags.
    """
    normal_z = df.loc[~df["large_consensus"], "log_zscore"].values
    large_z = df.loc[df["large_consensus"], "log_zscore"].values

    fig = go.Figure()

    # Normal trades histogram
    fig.add_trace(go.Histogram(
        x=normal_z,
        name="Normal Trades",
        nbinsx=80,
        marker_color="steelblue",
        opacity=0.7,
        histnorm="probability density",
    ))

    # Large trades histogram
    if len(large_z) > 0:
        fig.add_trace(go.Histogram(
            x=large_z,
            name="Consensus Large Trades",
            nbinsx=30,
            marker_color="crimson",
            opacity=0.85,
            histnorm="probability density",
        ))

    # Standard normal PDF
    x_range = np.linspace(-4, max(df["log_zscore"].max(), 5), 300)
    fig.add_trace(go.Scatter(
        x=x_range, y=norm.pdf(x_range),
        mode="lines", name="N(0,1) Reference",
        line=dict(color="gold", width=2, dash="dot")
    ))

    # Threshold lines
    for z_thresh, label, color in [(2, "z=2", "yellow"), (3, "z=3", "orange"), (4, "z=4", "red")]:
        fig.add_vline(x=z_thresh, line_dash="dash", line_color=color,
                      annotation_text=label, annotation_position="top right")

    fig.update_layout(
        title="Log Z-Score Distribution — Normal vs Large Trades",
        xaxis_title="Log-Space Z-Score",
        yaxis_title="Probability Density",
        template="plotly_dark",
        barmode="overlay",
        height=480,
        legend=dict(x=0.7, y=0.95),
    )

    fig.show()

def generate_summary_report(
    df: pd.DataFrame,
    trade_stats: Dict[str, float]
) -> Dict[str, object]:
    """
    Generate and display a comprehensive large-trade detection report.

    Parameters
    ----------
    df : pd.DataFrame
        Flagged trade data.
    trade_stats : Dict[str, float]
        Statistical summary.

    Returns
    -------
    Dict[str, object]
        Machine-readable report dictionary.
    """
    source = df["source"].iloc[0] if "source" in df.columns else "unknown"
    consensus = df[df["large_consensus"]].copy()
    any_flagged = df[df["large_any"]].copy()

    report = {
        "data_source": source,
        "total_trades": len(df),
        "flagged_any": len(any_flagged),
        "flagged_consensus": len(consensus),
        "pct_flagged_any": 100 * len(any_flagged) / len(df),
        "pct_flagged_consensus": 100 * len(consensus) / len(df),
        "consensus_volume_pct": 100 * consensus["size"].sum() / df["size"].sum() if len(consensus) > 0 else 0,
        "consensus_notional_pct": 100 * consensus["notional"].sum() / df["notional"].sum() if len(consensus) > 0 else 0,
        "max_trade_size": df["size"].max(),
        "max_log_zscore": df["log_zscore"].max(),
    }

    print("\n" + "═" * 65)
    print("  LARGE TRADE DETECTION — FINAL SUMMARY REPORT")
    print("═" * 65)
    print(f"  Data source               : {source.upper()}")
    print(f"  Analysis period           : {df['timestamp'].min().date()} → {df['timestamp'].max().date()}")
    print(f"  Total trades analyzed     : {report['total_trades']:>10,}")
    print("─" * 65)
    print(f"  Flagged (any method)      : {report['flagged_any']:>10,}  ({report['pct_flagged_any']:.2f}%)")
    print(f"  Flagged (consensus)       : {report['flagged_consensus']:>10,}  ({report['pct_flagged_consensus']:.2f}%)")
    print("─" * 65)
    print(f"  Consensus volume share    : {report['consensus_volume_pct']:>10.2f}% of total volume")
    print(f"  Consensus notional share  : {report['consensus_notional_pct']:>10.2f}% of total notional")
    print("─" * 65)
    print(f"  Largest trade observed    : {report['max_trade_size']:>10,.0f} shares")
    print(f"  Max log z-score           : {report['max_log_zscore']:>10.2f}σ")
    print("═" * 65)

    if len(consensus) > 0:
        print("\n  TOP 10 LARGEST CONSENSUS TRADES:")
        top10 = (
            consensus[["timestamp", "price", "size", "notional", "log_zscore"]]
            .sort_values("size", ascending=False)
            .head(10)
            .copy()
        )
        top10["size"] = top10["size"].map("{:,.0f}".format)
        top10["notional"] = top10["notional"].map("${:,.0f}".format)
        top10["log_zscore"] = top10["log_zscore"].map("{:.2f}σ".format)
        top10["price"] = top10["price"].map("${:.2f}".format)
        print(top10.to_string(index=False))

    return report

def main(
    ticker: str = "BTC-USDT",
    n_synthetic: int = 5000,
    percentile_threshold: float = 95.0,
    zscore_threshold: float = 3.0,
    volume_multiple: float = 5.0,
) -> Dict[str, object]:
    """
    Full large-trade detection pipeline.

    Executes:
    1. Data acquisition (live or synthetic)
    2. Data validation
    3. Preprocessing and feature engineering
    4. Statistical analysis
    5. Large-trade detection (3 methods)
    6. Visualizations
    7. Summary report

    Parameters
    ----------
    ticker : str
        Equity ticker (e.g., 'SPY', 'AAPL', 'QQQ').
    n_synthetic : int
        Number of synthetic trades to generate if live data fails.
    percentile_threshold : float
        Percentile cutoff for Method 1.
    zscore_threshold : float
        Log z-score cutoff for Method 2.
    volume_multiple : float
        Mean multiple for Method 3.

    Returns
    -------
    Dict[str, object]
        {'df': flagged_df, 'stats': trade_stats, 'report': final_report}
    """
    sep = "═" * 65
    print(sep)
    print(" LARGE TRADE DETECTION PIPELINE")
    print(f"  Ticker: {ticker}  |  Date: {datetime.now().strftime('%Y-%m-%d %H:%M')}")
    print(sep)

    # ─── STEP 1: Data Acquisition
    print("\n[STEP 1/7] Data Acquisition")
    raw_df = load_or_generate_trade_data(
        ticker=ticker, n_trades=n_synthetic, days_back=30
    )

    # ─── STEP 2: Validation
    print("\n[STEP 2/7] Data Validation")
    val = validate_trade_data(raw_df)
    if not val["is_valid"]:
        raise ValueError(f"Data validation failed: {val['issues']}")

    # ─── STEP 3: Preprocessing
    print("\n[STEP 3/7] Preprocessing")
    clean_df = preprocess_trade_data(raw_df)

    # ─── STEP 4: Statistics
    print("\n[STEP 4/7] Statistical Analysis")
    trade_stats = calculate_trade_statistics(clean_df)

    # ─── STEP 5: Detection
    print("\n[STEP 5/7] Large Trade Detection")
    flagged_df = detect_large_trades(
        df=clean_df,
        trade_stats=trade_stats,
        percentile_threshold=percentile_threshold,
        zscore_threshold=zscore_threshold,
        volume_multiple=volume_multiple,
    )

    # ─── STEP 6: Visualizations
    print("\n[STEP 6/7] Generating Visualizations")
    visualize_trade_distribution(flagged_df, trade_stats)
    visualize_large_trades(flagged_df, trade_stats)
    visualize_method_comparison(flagged_df)
    visualize_zscore_analysis(flagged_df)

    # ─── STEP 7: Report
    print("\n[STEP 7/7] Final Report")
    final_report = generate_summary_report(flagged_df, trade_stats)

    print(f"\n{sep}")
    print("  PIPELINE COMPLETE")
    print(sep)

    return {"df": flagged_df, "stats": trade_stats, "report": final_report}

if __name__ == "__main__":
    results = main(
        ticker="BTC-USD",
        n_synthetic=5000,
        percentile_threshold=95.0,
        zscore_threshold=3.0,
        volume_multiple=5.0,
    )

    # Make results accessible for further exploration
    flagged_df = results["df"]
    trade_stats = results["stats"]
    report = results["report"]

# ─── Quick exploration of detection results

# 1. Distribution of z-scores for consensus large trades
if "flagged_df" in dir():
    consensus = flagged_df[flagged_df["large_consensus"]]
    print("=== Z-score summary for CONSENSUS large trades ===")
    print(consensus["log_zscore"].describe().map("{:.2f}".format))

    print("\n=== Time of day distribution of large trades ===")
    if "hour" in consensus.columns:
        hour_counts = consensus["hour"].value_counts().sort_index()
        fig = px.bar(
            x=hour_counts.index, y=hour_counts.values,
            labels={"x": "Hour of Day", "y": "Number of Large Trades"},
            title="⏰ Intraday Distribution of Consensus Large Trades",
            template="plotly_dark",
            color=hour_counts.values,
            color_continuous_scale="Reds",
        )
        fig.update_layout(height=400, showlegend=False)
        fig.show()
    else:
        print("Hour data not available (single-day or non-intraday data).")

def sensitivity_analysis(
    df: pd.DataFrame,
    trade_stats: Dict[str, float]
) -> None:
    """
    Show how many trades are flagged at different threshold settings.
    Helps practitioners choose appropriate thresholds for their use case.
    """
    print("Threshold Sensitivity Analysis")
    print("─" * 50)

    # Percentile sensitivity
    print("\nPercentile thresholds:")
    print(f"  {'Percentile':>12} | {'Cutoff (shares)':>18} | {'# Flagged':>10} | {'% Flagged':>10}")
    print("  " + "-" * 56)
    for p in [90, 95, 97, 99, 99.5, 99.9]:
        cutoff = np.percentile(df["size"], p)
        n_flag = (df["size"] >= cutoff).sum()
        print(f"  {p:>12.1f} | {cutoff:>18,.0f} | {n_flag:>10,} | {100*n_flag/len(df):>9.2f}%")

    # Z-score sensitivity
    print("\nLog Z-Score thresholds:")
    print(f"  {'Z-Score':>10} | {'# Flagged':>10} | {'% Flagged':>10}")
    print("  " + "-" * 34)
    for z in [2.0, 2.5, 3.0, 3.5, 4.0, 5.0]:
        n_flag = (df["log_zscore"] >= z).sum()
        print(f"  {z:>10.1f} | {n_flag:>10,} | {100*n_flag/len(df):>9.3f}%")

    # Volume multiple sensitivity
    print("\nVolume Multiple thresholds:")
    print(f"  {'Multiple (k×)':>14} | {'Cutoff (shares)':>18} | {'# Flagged':>10} | {'% Flagged':>10}")
    print("  " + "-" * 60)
    for k in [2, 3, 5, 10, 20, 50]:
        cutoff = k * trade_stats["mean"]
        n_flag = (df["size"] >= cutoff).sum()
        print(f"  {k:>14}× | {cutoff:>18,.0f} | {n_flag:>10,} | {100*n_flag/len(df):>9.3f}%")


# Run sensitivity analysis
if "flagged_df" in dir() and "trade_stats" in dir():
    sensitivity_analysis(flagged_df, trade_stats)