!pip install yfinance pandas-datareader pandas numpy matplotlib seaborn scikit-learn --quiet

import yfinance as yf
import pandas_datareader.data as web
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import MinMaxScaler
import warnings

warnings.filterwarnings('ignore')
%matplotlib inline
plt.rcParams['figure.figsize'] = (14, 5)
plt.rcParams['axes.spines.top']   = False
plt.rcParams['axes.spines.right'] = False
print('Imports ready.')

START_DATE = '2017-01-01'
FRED_API_KEY = ''  # leave blank for anonymous requests (rate limited)

# Signal weights — must sum to 1.0
SIGNAL_WEIGHTS = {
    'vix_score':        0.25,
    'yield_curve_score': 0.20,
    'dxy_score':        0.20,
    'btc_spx_corr_score': 0.15,
    'gold_btc_ratio_score': 0.20,
}
assert abs(sum(SIGNAL_WEIGHTS.values()) - 1.0) < 1e-9, 'Weights must sum to 1.0'

# Regime thresholds (composite score in [0, 1])
RISK_ON_THRESHOLD  = 0.60  # score > 0.60 → risk-on
RISK_OFF_THRESHOLD = 0.40  # score < 0.40 → risk-off

USE_SYNTHETIC = False
print('Config ready. Weights:', SIGNAL_WEIGHTS)

def fetch_regime_data(start: str) -> pd.DataFrame:
    """
    Fetch all input signals needed for regime classification.

    Fetches: VIX, DXY, BTC, SPX, Gold from Yahoo Finance;
    and 2Y/10Y Treasury yields from FRED.

    Parameters
    ----------
    start : str
        Start date in 'YYYY-MM-DD' format.

    Returns
    -------
    pd.DataFrame
        Columns: vix, dxy, btc, spx, gold — plus dgs2, dgs10 if FRED available.

    Notes
    -----
    If FRED yields are unavailable, the yield curve component is skipped and
    remaining weights are renormalized automatically.
    Yield data is only available on business days; forward-filled for crypto days.
    """
    tickers = {'^VIX': 'vix', 'DX-Y.NYB': 'dxy', 'BTC-USD': 'btc', '^GSPC': 'spx', 'GC=F': 'gold'}
    raw = yf.download(list(tickers.keys()), start=start, progress=False, auto_adjust=True)
    prices = raw['Close'].rename(columns=tickers)
    prices.index = pd.to_datetime(prices.index)
    prices = prices.ffill()

    # Try FRED for yield curve
    try:
        dgs2  = web.DataReader('DGS2',  'fred', start).rename(columns={'DGS2':  'dgs2'})
        dgs10 = web.DataReader('DGS10', 'fred', start).rename(columns={'DGS10': 'dgs10'})
        yields = dgs2.join(dgs10, how='outer').ffill()
        yields.index = pd.to_datetime(yields.index)
        prices = prices.join(yields, how='left').ffill()
        prices['yield_spread'] = prices['dgs10'] - prices['dgs2']
        print('FRED yield data merged.')
    except Exception:
        print('FRED unavailable — yield curve signal disabled.')
        prices['dgs2'] = np.nan
        prices['dgs10'] = np.nan
        prices['yield_spread'] = np.nan

    prices = prices.dropna(subset=['vix', 'dxy', 'btc', 'spx', 'gold'])
    print(f'Regime data: {len(prices)} rows, {prices.columns.tolist()}')
    return prices


def generate_synthetic_regime_data(start: str, n_days: int = 2000) -> pd.DataFrame:
    """
    Synthetic multi-asset data for regime testing.

    Parameters
    ----------
    start : str
        Start date.
    n_days : int
        Number of business days.

    Returns
    -------
    pd.DataFrame
        Synthetic vix, dxy, btc, spx, gold, yield_spread columns.
    """
    np.random.seed(42)
    dates = pd.date_range(start, periods=n_days, freq='B')

    spx_rets  = 0.0003 + 0.01 * np.random.randn(n_days)
    vix       = np.abs(20 - 8 * np.cumsum(spx_rets) + 3 * np.random.randn(n_days)).clip(10, 80)
    dxy_rets  = 0.0001 - 0.3 * spx_rets + 0.003 * np.random.randn(n_days)
    btc_rets  = 0.0005 + 2.5 * spx_rets + 0.030 * np.random.randn(n_days)
    gold_rets = 0.0002 - 0.5 * spx_rets + 0.008 * np.random.randn(n_days)
    yield_spread = 1.5 + 0.3 * np.sin(np.linspace(0, 4 * np.pi, n_days)) + 0.1 * np.random.randn(n_days)

    return pd.DataFrame({
        'vix':          vix,
        'dxy':          95 * np.exp(np.cumsum(dxy_rets)),
        'btc':          10000 * np.exp(np.cumsum(btc_rets)),
        'spx':          2500 * np.exp(np.cumsum(spx_rets)),
        'gold':         1800 * np.exp(np.cumsum(gold_rets)),
        'yield_spread': yield_spread,
    }, index=dates)


if USE_SYNTHETIC:
    data = generate_synthetic_regime_data(START_DATE)
    print('Using synthetic data.')
else:
    try:
        data = fetch_regime_data(START_DATE)
    except Exception as e:
        print(f'Live fetch failed ({e}). Using synthetic.')
        data = generate_synthetic_regime_data(START_DATE)

print(data.tail(3))

def compute_signal_scores(data: pd.DataFrame, roll: int = 60) -> pd.DataFrame:
    """
    Transform raw data into normalized [0, 1] risk-on scores for each signal.

    Score = 1 (fully risk-on), Score = 0 (fully risk-off).

    Parameters
    ----------
    data : pd.DataFrame
        Raw macro data from fetch_regime_data().
    roll : int
        Rolling window for correlation signals.

    Returns
    -------
    pd.DataFrame
        One column per signal, normalized to [0, 1].

    Notes
    -----
    All signals are computed on a rolling percentile rank basis to remain adaptive
    across different market eras. A VIX of 25 might be normal in 2020 but extreme
    in 2017 — using rank-based scoring avoids hardcoding level thresholds.
    """
    scores = pd.DataFrame(index=data.index)
    roll_window = 252  # look-back for percentile rank

    # VIX score: low VIX = risk-on → score = 1 - percentile_rank(VIX)
    scores['vix_score'] = 1 - data['vix'].rolling(roll_window).rank(pct=True)

    # Yield curve score: steeper = risk-on (positive spread → higher score)
    if data['yield_spread'].notna().sum() > 100:
        scores['yield_curve_score'] = data['yield_spread'].rolling(roll_window).rank(pct=True)
    else:
        scores['yield_curve_score'] = 0.5  # neutral if no data

    # DXY score: falling DXY = risk-on → score = 1 - percentile_rank(DXY 20d change)
    dxy_roc = data['dxy'].pct_change(20)
    scores['dxy_score'] = 1 - dxy_roc.rolling(roll_window).rank(pct=True)

    # BTC-SPX correlation score: when correlation is moderate positive, risk-on bull market
    # Very high correlation (>0.7) during stress = risk-off panic
    # We want mid-range correlation as most risk-on
    log_rets = np.log(data[['btc', 'spx']] / data[['btc', 'spx']].shift(1))
    rolling_corr = log_rets['btc'].rolling(roll).corr(log_rets['spx'])
    # Score peaks at corr=0.5, falls as corr approaches 1.0 (panic) or -1 (crypto crash)
    scores['btc_spx_corr_score'] = np.where(
        rolling_corr < 0.5,
        rolling_corr.clip(-1, 0.5) + 1,  # rising corr from -1 to 0.5 → 0 to 1.5 → cap at 1
        1 - (rolling_corr - 0.5) * 2      # corr > 0.5 → score falls
    ).clip(0, 1)

    # Gold/BTC ratio score: falling ratio = BTC outperforming gold = risk-on
    gold_btc_ratio = data['gold'] / data['btc'] * 1000  # normalize to roughly same scale
    gold_btc_roc   = gold_btc_ratio.pct_change(20)
    scores['gold_btc_ratio_score'] = 1 - gold_btc_roc.rolling(roll_window).rank(pct=True)

    return scores


scores = compute_signal_scores(data)
print('Signal score ranges:')
print(scores.describe().round(3))

def compute_composite_regime(
    scores: pd.DataFrame,
    weights: dict,
    risk_on_threshold: float,
    risk_off_threshold: float,
    smooth_span: int = 5
) -> pd.DataFrame:
    """
    Combine individual signal scores into a weighted composite regime indicator.

    Parameters
    ----------
    scores : pd.DataFrame
        Signal scores in [0, 1] from compute_signal_scores().
    weights : dict
        Mapping of score column name to weight (must sum to 1.0).
    risk_on_threshold : float
        Composite score above this → 'risk_on'.
    risk_off_threshold : float
        Composite score below this → 'risk_off'.
    smooth_span : int
        EMA span for smoothing the composite score to reduce flip noise.

    Returns
    -------
    pd.DataFrame
        Columns: composite_score, regime.
        regime: 'risk_on', 'neutral', 'risk_off'.

    Notes
    -----
    Missing scores (NaN from rolling warm-up) are handled by renormalizing
    available weights on each row, so the indicator starts working earlier.
    The 5-day EMA smoothing prevents the regime from flipping on single-day anomalies.
    """
    composite = pd.Series(0.0, index=scores.index)
    total_weight = pd.Series(0.0, index=scores.index)

    for col, w in weights.items():
        if col in scores.columns:
            valid_mask = scores[col].notna()
            composite  += scores[col].fillna(0) * w
            total_weight += pd.Series(np.where(valid_mask, w, 0), index=scores.index)

    # Renormalize by actual available weight
    composite = composite / total_weight.replace(0, np.nan)
    composite = composite.ewm(span=smooth_span).mean()

    regime = pd.Series('neutral', index=composite.index)
    regime[composite >= risk_on_threshold]  = 'risk_on'
    regime[composite <= risk_off_threshold] = 'risk_off'

    result = pd.DataFrame({'composite_score': composite, 'regime': regime})
    print('Regime distribution:')
    print(result['regime'].value_counts())
    return result


regime_df = compute_composite_regime(
    scores, SIGNAL_WEIGHTS, RISK_ON_THRESHOLD, RISK_OFF_THRESHOLD
)
print(f'\nCurrent regime: {regime_df["regime"].iloc[-1]}')
print(f'Current score : {regime_df["composite_score"].iloc[-1]:.3f}')

def backtest_regime_strategy(
    data: pd.DataFrame,
    regime_df: pd.DataFrame
) -> pd.DataFrame:
    """
    Backtest a BTC position sizing strategy driven by the regime indicator.

    Position sizing:
    - risk_on:  100% BTC
    - neutral:   50% BTC
    - risk_off:   0% BTC (cash)

    Parameters
    ----------
    data : pd.DataFrame
        Must include 'btc' column.
    regime_df : pd.DataFrame
        Output of compute_composite_regime().

    Returns
    -------
    pd.DataFrame
        Equity curves and drawdowns for strategy vs buy-and-hold.
    """
    combined = data[['btc']].join(regime_df, how='inner').dropna()
    combined['btc_ret'] = combined['btc'].pct_change()

    pos_map = {'risk_on': 1.0, 'neutral': 0.5, 'risk_off': 0.0}
    combined['position']     = combined['regime'].map(pos_map).shift(1)
    combined['strategy_ret'] = combined['position'] * combined['btc_ret']

    combined['cum_btc']      = (1 + combined['btc_ret']).cumprod()
    combined['cum_strategy'] = (1 + combined['strategy_ret']).cumprod()

    for col in ['cum_btc', 'cum_strategy']:
        peak = combined[col].cummax()
        combined[f'dd_{col}'] = (combined[col] - peak) / peak * 100

    combined = combined.dropna()
    total_btc  = (combined['cum_btc'].iloc[-1] - 1) * 100
    total_strat = (combined['cum_strategy'].iloc[-1] - 1) * 100
    max_dd_btc  = combined['dd_cum_btc'].min()
    max_dd_strat = combined['dd_cum_strategy'].min()

    print(f'Buy-and-hold BTC:  {total_btc:.1f}%  (max DD: {max_dd_btc:.1f}%)')
    print(f'Regime strategy:   {total_strat:.1f}%  (max DD: {max_dd_strat:.1f}%)')
    return combined


bt = backtest_regime_strategy(data, regime_df)

def plot_regime_dashboard(
    data: pd.DataFrame,
    scores: pd.DataFrame,
    regime_df: pd.DataFrame,
    bt: pd.DataFrame
) -> None:
    """
    Five-panel regime dashboard showing all signals and strategy performance.

    Parameters
    ----------
    data : pd.DataFrame
        Raw market data.
    scores : pd.DataFrame
        Individual signal scores.
    regime_df : pd.DataFrame
        Composite score and regime classification.
    bt : pd.DataFrame
        Backtest results.
    """
    fig, axes = plt.subplots(4, 1, figsize=(15, 18), sharex=True)

    # Panel 1: BTC with regime shading
    axes[0].plot(data.index, data['btc'], color='orange', linewidth=1.5)
    axes[0].set_yscale('log')
    color_map = {'risk_on': 'green', 'neutral': 'grey', 'risk_off': 'red'}
    prev_date = regime_df.index[0]
    prev_reg  = regime_df['regime'].iloc[0]
    for date, reg in regime_df['regime'].items():
        if reg != prev_reg:
            axes[0].axvspan(prev_date, date, alpha=0.15, color=color_map.get(prev_reg, 'white'))
            prev_date = date
            prev_reg  = reg
    axes[0].set_title('BTC Price — Green=Risk-On, Red=Risk-Off, Grey=Neutral')
    axes[0].set_ylabel('BTC (log)')

    # Panel 2: Individual signal scores
    for col, color in zip(scores.columns, ['red', 'blue', 'navy', 'purple', 'goldenrod']):
        axes[1].plot(scores.index, scores[col], label=col, linewidth=0.8, alpha=0.8, color=color)
    axes[1].axhline(0.5, color='black', linewidth=0.5, linestyle='--')
    axes[1].set_ylim(0, 1)
    axes[1].set_ylabel('Signal Score [0=risk-off, 1=risk-on]')
    axes[1].set_title('Individual Risk-On/Off Signal Scores')
    axes[1].legend(fontsize=8, loc='upper left')

    # Panel 3: Composite score
    composite = regime_df['composite_score']
    axes[2].plot(composite.index, composite, color='black', linewidth=1.5)
    axes[2].fill_between(composite.index, composite, RISK_ON_THRESHOLD,
                          where=(composite >= RISK_ON_THRESHOLD), alpha=0.3, color='green')
    axes[2].fill_between(composite.index, composite, RISK_OFF_THRESHOLD,
                          where=(composite <= RISK_OFF_THRESHOLD), alpha=0.3, color='red')
    axes[2].axhline(RISK_ON_THRESHOLD,  color='green', linewidth=0.8, linestyle='--')
    axes[2].axhline(RISK_OFF_THRESHOLD, color='red',   linewidth=0.8, linestyle='--')
    axes[2].set_ylim(0, 1)
    axes[2].set_ylabel('Composite Score')
    axes[2].set_title(f'Composite Risk-On/Off Score (risk-on>{RISK_ON_THRESHOLD}, risk-off<{RISK_OFF_THRESHOLD})')

    # Panel 4: Strategy performance
    axes[3].plot(bt.index, bt['cum_btc']      * 100, color='orange', linewidth=1.5, label='Buy & Hold BTC')
    axes[3].plot(bt.index, bt['cum_strategy'] * 100, color='green',  linewidth=1.5, label='Regime Strategy')
    axes[3].set_yscale('log')
    axes[3].set_ylabel('Portfolio (log, base=100)')
    axes[3].set_title('Regime Strategy vs Buy-and-Hold')
    axes[3].legend()

    plt.tight_layout()
    plt.show()


plot_regime_dashboard(data, scores, regime_df, bt)

def export_regime_data(scores, regime_df, bt):
    """
    Export signal scores, regime classification, and backtest results.

    Parameters
    ----------
    scores : pd.DataFrame
        Individual signal scores.
    regime_df : pd.DataFrame
        Composite regime classification.
    bt : pd.DataFrame
        Backtest equity curves.
    """
    scores.to_csv('regime_signal_scores.csv')
    regime_df.to_csv('macro_regime.csv')
    bt[['btc', 'regime', 'position', 'cum_btc', 'cum_strategy']].to_csv('regime_backtest.csv')
    print('Exported: regime_signal_scores.csv')
    print('Exported: macro_regime.csv')
    print('Exported: regime_backtest.csv')


export_regime_data(scores, regime_df, bt)