!pip install yfinance pandas numpy matplotlib seaborn scipy --quiet

import yfinance as yf
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sns
from scipy import stats
import warnings

warnings.filterwarnings('ignore')
%matplotlib inline
plt.rcParams['figure.figsize'] = (14, 5)
plt.rcParams['axes.spines.top'] = False
plt.rcParams['axes.spines.right'] = False
sns.set_palette('muted')
print('Imports ready.')

START_DATE         = '2017-01-01'
ROLLING_WINDOWS    = [30, 60, 90]   # days for rolling correlation
GOLD_TICKER        = 'GC=F'
BTC_TICKER         = 'BTC-USD'
REGIME_CORR_THRESH = 0.3            # |corr| > this → meaningful relationship
USE_SYNTHETIC      = False          # set True to skip real data fetch

MACRO_EVENTS = [
    {'date': '2020-03-12', 'label': 'COVID Crash'},
    {'date': '2021-11-10', 'label': 'BTC ATH'},
    {'date': '2022-03-16', 'label': 'First 2022 Hike'},
    {'date': '2022-06-18', 'label': 'LUNA Collapse'},
    {'date': '2022-11-11', 'label': 'FTX Collapse'},
    {'date': '2023-03-10', 'label': 'SVB Failure'},
    {'date': '2024-01-10', 'label': 'BTC ETF Approval'},
]

def fetch_asset_pair(
    gold_ticker: str,
    btc_ticker: str,
    start: str
) -> pd.DataFrame:
    """
    Fetch daily close prices for Gold and BTC and align them.

    Parameters
    ----------
    gold_ticker : str
        Yahoo Finance ticker for Gold futures (e.g., 'GC=F').
    btc_ticker : str
        Yahoo Finance ticker for Bitcoin (e.g., 'BTC-USD').
    start : str
        Start date in 'YYYY-MM-DD' format.

    Returns
    -------
    pd.DataFrame
        Columns: gold, btc — daily close prices aligned on trading days.
        BTC trades 24/7; gold prices forward-filled on weekends.

    Notes
    -----
    Gold futures (GC=F) roll monthly — you may see small gaps around roll dates.
    For continuous Gold pricing, consider SPDR Gold Shares ETF ('GLD') as an
    alternative, though it has slight tracking error.
    """
    raw = yf.download([gold_ticker, btc_ticker], start=start, progress=False, auto_adjust=True)
    prices = raw['Close'].copy()
    prices.columns = ['gold', 'btc']
    prices.index = pd.to_datetime(prices.index)
    prices = prices.ffill().dropna()
    print(f'Fetched {len(prices)} days  ({prices.index[0].date()} → {prices.index[-1].date()})')
    return prices


def generate_synthetic_pair(start: str, n_days: int = 2500) -> pd.DataFrame:
    """
    Generate synthetic Gold and BTC price series with realistic statistical properties.

    Parameters
    ----------
    start : str
        Start date in 'YYYY-MM-DD' format.
    n_days : int
        Number of trading days to generate.

    Returns
    -------
    pd.DataFrame
        Columns: gold, btc — synthetic price series.
    """
    np.random.seed(42)
    dates = pd.date_range(start, periods=n_days, freq='B')

    # Regime-varying correlation: correlation shifts over time
    corr_series = np.concatenate([
        np.full(600, 0.55),   # 2017-early 2019: moderate positive
        np.full(400, 0.15),   # 2019-2020: low
        np.full(500, 0.60),   # 2020-2021: COVID inflation trade
        np.full(500, -0.20),  # 2022: BTC crashes harder
        np.full(500, 0.40),   # 2023-2024: recovering
    ])[:n_days]

    gold_rets, btc_rets = [], []
    for rho in corr_series:
        z = np.random.randn()
        e = np.random.randn()
        g = 0.0002 + 0.008 * z
        b = 0.0005 + 0.035 * (rho * z + np.sqrt(1 - rho**2) * e)
        gold_rets.append(g)
        btc_rets.append(b)

    gold_prices = 1800 * np.exp(np.cumsum(gold_rets))
    btc_prices  = 10000 * np.exp(np.cumsum(btc_rets))

    return pd.DataFrame({'gold': gold_prices, 'btc': btc_prices}, index=dates)


if USE_SYNTHETIC:
    prices = generate_synthetic_pair(START_DATE)
    print('Using synthetic data.')
else:
    try:
        prices = fetch_asset_pair(GOLD_TICKER, BTC_TICKER, START_DATE)
    except Exception as e:
        print(f'Live data failed ({e}). Falling back to synthetic.')
        prices = generate_synthetic_pair(START_DATE)

print(prices.tail(5))

def compute_rolling_correlation(
    prices: pd.DataFrame,
    windows: list
) -> pd.DataFrame:
    """
    Compute rolling Pearson correlations between log-returns of two assets.

    Parameters
    ----------
    prices : pd.DataFrame
        Columns: gold, btc — daily close prices.
    windows : list of int
        Rolling window sizes in days.

    Returns
    -------
    pd.DataFrame
        Rolling correlation at each window size, indexed by date.
        Columns: corr_30d, corr_60d, corr_90d (or similar).

    Notes
    -----
    Log-returns (not price levels) are used for correlation — correlating price
    levels can show spurious high correlation simply because both assets trend up.
    Spearman rank correlation is less sensitive to extreme moves and is also computed.
    """
    log_rets = np.log(prices / prices.shift(1)).dropna()
    corrs = pd.DataFrame(index=log_rets.index)

    for w in windows:
        corrs[f'corr_{w}d'] = log_rets['gold'].rolling(w).corr(log_rets['btc'])

    # Spearman at 60d
    spearman_vals = []
    window = 60
    for i in range(len(log_rets)):
        if i < window:
            spearman_vals.append(np.nan)
        else:
            g_slice = log_rets['gold'].iloc[i-window:i]
            b_slice = log_rets['btc'].iloc[i-window:i]
            rho, _ = stats.spearmanr(g_slice, b_slice)
            spearman_vals.append(rho)
    corrs['spearman_60d'] = spearman_vals

    return corrs


corrs = compute_rolling_correlation(prices, ROLLING_WINDOWS)
print(f'Correlation computed. Mean 60d corr: {corrs["corr_60d"].mean():.3f}')

def classify_correlation_regime(
    corrs: pd.DataFrame,
    window_col: str = 'corr_60d',
    high_thresh: float = 0.4,
    low_thresh: float = -0.2
) -> pd.Series:
    """
    Classify the Gold-BTC correlation regime into three states.

    Parameters
    ----------
    corrs : pd.DataFrame
        Rolling correlation DataFrame from compute_rolling_correlation().
    window_col : str
        Column to use for regime classification.
    high_thresh : float
        Correlation above this = 'coupled' regime (both move together).
    low_thresh : float
        Correlation below this = 'decoupled_negative' regime.

    Returns
    -------
    pd.Series
        Regime label per date: 'coupled', 'neutral', or 'decoupled_negative'.

    Notes
    -----
    'Coupled' regimes often coincide with macro-driven markets where gold and BTC
    both respond to USD strength / inflation narrative. 'Decoupled' regimes usually
    mean crypto is in its own idiosyncratic cycle (bull run or crash) uncorrelated
    with gold's store-of-value dynamics.
    """
    c = corrs[window_col]
    regime = pd.Series('neutral', index=c.index)
    regime[c >= high_thresh] = 'coupled'
    regime[c <= low_thresh]  = 'decoupled_negative'
    regime[c.isna()]          = 'unknown'
    return regime


corrs['regime'] = classify_correlation_regime(corrs)
regime_counts = corrs['regime'].value_counts()
print('Regime distribution:')
print(regime_counts)

def plot_rolling_correlation(
    corrs: pd.DataFrame,
    prices: pd.DataFrame,
    macro_events: list
) -> None:
    """
    Three-panel chart: price history, rolling correlations, and regime.

    Parameters
    ----------
    corrs : pd.DataFrame
        Output of compute_rolling_correlation().
    prices : pd.DataFrame
        Gold and BTC price series.
    macro_events : list of dict
        Each dict: {'date': 'YYYY-MM-DD', 'label': 'Event Name'}.
    """
    fig, axes = plt.subplots(3, 1, figsize=(15, 14), sharex=True)

    # Panel 1: Prices (normalized)
    normalized = prices / prices.iloc[0] * 100
    axes[0].plot(normalized.index, normalized['gold'], label='Gold (rebased)', color='goldenrod', linewidth=1.5)
    axes[0].plot(normalized.index, normalized['btc'],  label='BTC (rebased)',  color='orange',    linewidth=1.5)
    axes[0].set_yscale('log')
    axes[0].set_ylabel('Rebased Price (log, base=100)')
    axes[0].set_title('Gold vs BTC — Normalized Price Comparison')
    axes[0].legend()

    # Panel 2: Rolling correlations
    for col in ['corr_30d', 'corr_60d', 'corr_90d']:
        axes[1].plot(corrs.index, corrs[col], label=col, linewidth=1.2, alpha=0.8)
    axes[1].axhline(0, color='black', linewidth=0.8, linestyle='--')
    axes[1].axhline(0.4, color='green', linewidth=0.5, linestyle=':', alpha=0.7)
    axes[1].axhline(-0.2, color='red', linewidth=0.5, linestyle=':', alpha=0.7)
    axes[1].set_ylim(-1, 1)
    axes[1].set_ylabel('Pearson Correlation')
    axes[1].set_title('Rolling Gold-BTC Log-Return Correlation')
    axes[1].legend()

    # Panel 3: Regime background
    regime_colors = {'coupled': 'green', 'neutral': 'grey', 'decoupled_negative': 'red', 'unknown': 'white'}
    regime = corrs['regime']
    prev_date = corrs.index[0]
    prev_reg  = regime.iloc[0]
    for date, reg in regime.items():
        if reg != prev_reg:
            axes[2].axvspan(prev_date, date, alpha=0.3, color=regime_colors.get(prev_reg, 'white'), label=prev_reg)
            prev_date = date
            prev_reg  = reg
    axes[2].axvspan(prev_date, corrs.index[-1], alpha=0.3, color=regime_colors.get(prev_reg, 'white'))
    axes[2].plot(corrs.index, corrs['corr_60d'], color='black', linewidth=1.5)
    axes[2].axhline(0, color='black', linewidth=0.8, linestyle='--')
    axes[2].set_ylabel('Correlation (60d)')
    axes[2].set_title('Regime Classification — Green=Coupled, Red=Decoupled, Grey=Neutral')

    # Macro event lines on all panels
    for event in macro_events:
        edate = pd.to_datetime(event['date'])
        for ax in axes:
            ax.axvline(edate, color='navy', alpha=0.4, linewidth=1.0, linestyle='--')
        axes[0].text(edate, normalized['gold'].max() * 0.8, event['label'],
                     fontsize=7, rotation=85, va='top', ha='right', color='navy')

    plt.tight_layout()
    plt.show()


def plot_scatter_by_regime(
    prices: pd.DataFrame,
    corrs: pd.DataFrame
) -> None:
    """
    Scatter plot of daily Gold vs BTC returns, colored by regime.

    Parameters
    ----------
    prices : pd.DataFrame
        Gold and BTC close prices.
    corrs : pd.DataFrame
        Correlation DataFrame with 'regime' column.
    """
    log_rets = np.log(prices / prices.shift(1)).dropna()
    combined = log_rets.join(corrs['regime'], how='inner')
    combined = combined.dropna()

    fig, axes = plt.subplots(1, 3, figsize=(15, 5))
    palette = {'coupled': 'green', 'neutral': 'steelblue', 'decoupled_negative': 'red'}

    for ax, regime_name in zip(axes, ['coupled', 'neutral', 'decoupled_negative']):
        sub = combined[combined['regime'] == regime_name]
        ax.scatter(sub['gold'] * 100, sub['btc'] * 100,
                   alpha=0.3, s=8, color=palette[regime_name])
        if len(sub) > 5:
            m, b, r, _, _ = stats.linregress(sub['gold'], sub['btc'])
            x_line = np.linspace(sub['gold'].min(), sub['gold'].max(), 100)
            ax.plot(x_line * 100, (m * x_line + b) * 100, 'k--', linewidth=1)
            ax.set_title(f'{regime_name.upper()}\nSlope={m:.1f}, r={r:.2f}, N={len(sub)}')
        ax.set_xlabel('Gold Daily Return (%)')
        ax.set_ylabel('BTC Daily Return (%)')
        ax.axhline(0, color='black', linewidth=0.5)
        ax.axvline(0, color='black', linewidth=0.5)

    plt.suptitle('Gold vs BTC Return Scatter by Regime', fontsize=13)
    plt.tight_layout()
    plt.show()


plot_rolling_correlation(corrs, prices, MACRO_EVENTS)
plot_scatter_by_regime(prices, corrs)

def compute_correlation_signal(
    corrs: pd.DataFrame,
    window_col: str = 'corr_60d'
) -> pd.Series:
    """
    Generate a normalized correlation signal for use in strategy notebooks.

    Parameters
    ----------
    corrs : pd.DataFrame
        Rolling correlation DataFrame.
    window_col : str
        Column to normalize into a signal.

    Returns
    -------
    pd.Series
        Signal in [-1, 1] range. Positive = coupled regime (buy both).
        Negative = decoupled regime (trade them independently).
    """
    raw = corrs[window_col].dropna()
    # Smooth with 10d EMA to avoid regime flip noise
    signal = raw.ewm(span=10).mean()
    return signal.rename('gold_btc_corr_signal')


signal = compute_correlation_signal(corrs)
print(f'Correlation signal — latest value: {signal.iloc[-1]:.3f}')
print(f'Signal stats: mean={signal.mean():.3f}, std={signal.std():.3f}')

def export_correlation_data(
    prices: pd.DataFrame,
    corrs: pd.DataFrame,
    signal: pd.Series
) -> None:
    """
    Export prices, rolling correlations, and signal to CSV.

    Parameters
    ----------
    prices : pd.DataFrame
        Gold and BTC close prices.
    corrs : pd.DataFrame
        Rolling correlations and regime.
    signal : pd.Series
        Smoothed correlation signal.
    """
    combined = prices.join(corrs, how='left').join(signal, how='left')
    combined.to_csv('gold_btc_correlation.csv')
    print(f'Exported gold_btc_correlation.csv ({len(combined)} rows)')


export_correlation_data(prices, corrs, signal)