!pip install yfinance pandas-datareader pandas numpy matplotlib seaborn requests --quiet

import yfinance as yf
import pandas_datareader.data as web
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sns
import requests
from datetime import datetime, date
import warnings

warnings.filterwarnings('ignore')
%matplotlib inline
plt.rcParams['figure.figsize'] = (14, 5)
plt.rcParams['axes.spines.top'] = False
plt.rcParams['axes.spines.right'] = False
sns.set_palette('husl')
print('Imports ready.')

# ── CONFIGURATION ─────────────────────────────────────────────────────────────
START_DATE    = '2019-01-01'
END_DATE      = datetime.today().strftime('%Y-%m-%d')
FRED_API_KEY  = 'YOUR_FRED_API_KEY'   # optional — free at fred.stlouisfed.org

# FRED series IDs for macroeconomic indicators
FRED_SERIES = {
    'cpi_yoy':        'CPIAUCSL',     # Consumer Price Index (all items)
    'fed_rate':       'FEDFUNDS',     # Effective Fed Funds Rate
    'gdp_growth':     'A191RL1Q225SBEA',  # Real GDP growth rate QoQ
    'unemployment':   'UNRATE',       # Unemployment rate
    'treasury_2y':    'DGS2',         # 2-Year Treasury yield
    'treasury_10y':   'DGS10',        # 10-Year Treasury yield
    'm2_money':       'M2SL',         # M2 Money Supply
}

# Yahoo Finance tickers
YF_TICKERS = {
    'dxy':   'DX-Y.NYB',   # US Dollar Index
    'vix':   '^VIX',       # CBOE Volatility Index
    'sp500': '^GSPC',      # S&P 500
    'gold':  'GC=F',       # Gold futures
    'btc':   'BTC-USD',    # Bitcoin price
}
# ─────────────────────────────────────────────────────────────────────────────

def fetch_fred_series(
    series_ids: dict,
    start: str,
    end: str,
    api_key: str = None
) -> pd.DataFrame:
    """
    Fetch multiple economic time series from the FRED database.

    Parameters
    ----------
    series_ids : dict
        {column_name: FRED_series_id}. Example: {'cpi': 'CPIAUCSL'}.
    start : str
        Start date in 'YYYY-MM-DD' format.
    end : str
        End date in 'YYYY-MM-DD' format.
    api_key : str, optional
        FRED API key for higher rate limits. Uses pandas-datareader without key.

    Returns
    -------
    pd.DataFrame
        DataFrame with each series as a column. FRED series are released on
        irregular schedules (monthly, quarterly) — raw values, not forward-filled.

    Notes
    -----
    Some FRED series are released with a lag. CPI data, for example, is published
    ~2 weeks after month-end. Always be aware of release lags when combining
    FRED data with real-time price data to avoid look-ahead bias.
    """
    frames = {}
    for col_name, series_id in series_ids.items():
        try:
            if api_key and api_key != 'YOUR_FRED_API_KEY':
                url = (f'https://api.stlouisfed.org/fred/series/observations'
                       f'?series_id={series_id}&observation_start={start}'
                       f'&observation_end={end}&api_key={api_key}&file_type=json')
                resp = requests.get(url, timeout=10).json()
                obs  = resp.get('observations', [])
                s = pd.Series(
                    {o['date']: float(o['value']) for o in obs if o['value'] != '.'},
                    name=col_name
                )
                s.index = pd.to_datetime(s.index)
            else:
                s = web.DataReader(series_id, 'fred', start, end)[series_id]
                s.name = col_name
            frames[col_name] = s
            print(f'  {col_name} ({series_id}): {len(s)} observations')
        except Exception as e:
            print(f'  WARNING: Failed to fetch {series_id}: {e}')

    df = pd.DataFrame(frames)
    df.index = pd.to_datetime(df.index)
    print(f'FRED fetch complete: {len(df.columns)} series, {len(df)} raw rows')
    return df


def compute_cpi_yoy_change(fred_df: pd.DataFrame, cpi_col: str = 'cpi_yoy') -> pd.DataFrame:
    """
    Compute the year-over-year percentage change in CPI (the actual inflation rate).

    Parameters
    ----------
    fred_df : pd.DataFrame
        FRED DataFrame containing the CPI level series.
    cpi_col : str
        Column name of the CPI level series (default CPIAUCSL is a level, not % change).

    Returns
    -------
    pd.DataFrame
        fred_df with an added 'cpi_inflation_pct' column (YoY %).

    Notes
    -----
    CPIAUCSL from FRED is a price level index (not a rate). The commonly cited
    'inflation rate' is the YoY percentage change of this index.
    """
    if cpi_col in fred_df.columns:
        fred_df['cpi_inflation_pct'] = fred_df[cpi_col].pct_change(periods=12) * 100
    return fred_df


def compute_yield_curve_spread(fred_df: pd.DataFrame) -> pd.DataFrame:
    """
    Compute the 10Y-2Y Treasury yield spread (yield curve slope indicator).

    Parameters
    ----------
    fred_df : pd.DataFrame
        FRED DataFrame with 'treasury_10y' and 'treasury_2y' columns.

    Returns
    -------
    pd.DataFrame
        fred_df with added 'yield_spread_10y_2y' column.
        Negative spread = inverted yield curve = recession warning.

    Notes
    -----
    The 2s10s spread is the most watched recession indicator. Every US recession
    since 1950 has been preceded by a yield curve inversion (negative spread).
    For crypto, an inversion signals risk-off environment — historically bearish.
    """
    if 'treasury_10y' in fred_df.columns and 'treasury_2y' in fred_df.columns:
        fred_df['yield_spread_10y_2y'] = fred_df['treasury_10y'] - fred_df['treasury_2y']
    return fred_df


# ── Fetch FRED data ───────────────────────────────────────────────────────────
print('Fetching FRED data...')
fred_df = fetch_fred_series(FRED_SERIES, START_DATE, END_DATE, FRED_API_KEY)
fred_df = compute_cpi_yoy_change(fred_df)
fred_df = compute_yield_curve_spread(fred_df)
print(fred_df.tail())

def fetch_yahoo_finance_data(
    tickers: dict,
    start: str,
    end: str
) -> pd.DataFrame:
    """
    Fetch daily close prices for multiple tickers from Yahoo Finance.

    Parameters
    ----------
    tickers : dict
        {column_name: yahoo_ticker}. Example: {'btc': 'BTC-USD'}.
    start : str
        Start date 'YYYY-MM-DD'.
    end : str
        End date 'YYYY-MM-DD'.

    Returns
    -------
    pd.DataFrame
        Daily close price DataFrame. Each column is one ticker.
        Missing days (weekends, holidays) are kept as NaN — do not forward-fill
        until you are ready to run analysis to avoid obscuring data gaps.
    """
    frames = {}
    for col_name, ticker in tickers.items():
        try:
            data = yf.download(ticker, start=start, end=end, progress=False, auto_adjust=True)
            frames[col_name] = data['Close'].squeeze().rename(col_name)
            print(f'  {col_name} ({ticker}): {len(data)} rows')
        except Exception as e:
            print(f'  WARNING: {ticker} failed: {e}')

    df = pd.DataFrame(frames)
    df.index = pd.to_datetime(df.index)
    df.index.name = 'date'
    return df


print('Fetching Yahoo Finance data...')
yf_df = fetch_yahoo_finance_data(YF_TICKERS, START_DATE, END_DATE)
print(yf_df.tail())

def align_macro_data(
    fred_df: pd.DataFrame,
    yf_df: pd.DataFrame,
    freq: str = 'D'
) -> pd.DataFrame:
    """
    Merge FRED and Yahoo Finance data to a unified daily DataFrame.

    Parameters
    ----------
    fred_df : pd.DataFrame
        FRED data (monthly, quarterly, or daily).
    yf_df : pd.DataFrame
        Yahoo Finance data (daily).
    freq : str
        Target resampling frequency. 'D' = daily.

    Returns
    -------
    pd.DataFrame
        Unified DataFrame at daily frequency. Low-frequency FRED series
        are forward-filled (last known value carried forward).

    Notes
    -----
    Forward-filling is appropriate here because macro indicators like the
    Fed funds rate are fixed between meeting dates — there is no new value
    until the next FOMC meeting, so carrying the last reading forward is
    economically correct, not a data manipulation.
    """
    # Build a full daily date range covering both sources
    start = min(fred_df.index.min(), yf_df.index.min() if not yf_df.empty else fred_df.index.min())
    end   = max(fred_df.index.max(), yf_df.index.max() if not yf_df.empty else fred_df.index.max())
    daily_idx = pd.date_range(start, end, freq=freq)

    fred_daily = fred_df.reindex(daily_idx).ffill()
    yf_daily   = yf_df.reindex(daily_idx)

    macro_df = fred_daily.join(yf_daily, how='outer')
    macro_df = macro_df.dropna(how='all')  # drop rows with no data at all

    print(f'Unified macro DataFrame: {len(macro_df)} rows, {len(macro_df.columns)} columns')
    print(f'Date range: {macro_df.index[0].date()} to {macro_df.index[-1].date()}')
    return macro_df


macro_df = align_macro_data(fred_df, yf_df)
display_cols = ['cpi_inflation_pct', 'fed_rate', 'yield_spread_10y_2y', 'dxy', 'vix', 'btc']
existing_cols = [col for col in display_cols if col in macro_df.columns]
if existing_cols:
    print(macro_df[existing_cols].tail(10))
else:
    print("None of the requested display columns are available in macro_df.")

def plot_macro_dashboard(macro_df: pd.DataFrame) -> None:
    """
    Multi-panel dashboard of key macro indicators alongside BTC price.

    Parameters
    ----------
    macro_df : pd.DataFrame
        Unified macro DataFrame from align_macro_data().

    Notes
    -----
    Uses a twin y-axis for BTC price alongside each indicator to visually
    compare directional relationships. Pay attention to periods where BTC
    and an indicator move together (correlation) vs in opposite directions.
    """
    indicators = [
        ('cpi_inflation_pct', 'CPI Inflation YoY (%)', 'red'),
        ('fed_rate',          'Fed Funds Rate (%)',    'purple'),
        ('yield_spread_10y_2y', '10Y-2Y Yield Spread', 'orange'),
        ('dxy',               'DXY (Dollar Index)',    'green'),
        ('vix',               'VIX (Fear Index)',      'darkred'),
    ]

    available = [(col, label, color) for col, label, color in indicators if col in macro_df.columns]
    n = len(available)
    fig, axes = plt.subplots(n, 1, figsize=(14, 4 * n), sharex=True)
    if n == 1:
        axes = [axes]

    for ax, (col, label, color) in zip(axes, available):
        series = macro_df[col].dropna()
        ax.plot(series.index, series, color=color, linewidth=1.5, label=label)
        ax.set_ylabel(label, color=color, fontsize=9)
        ax.tick_params(axis='y', labelcolor=color)

        if 'btc' in macro_df.columns:
            ax2 = ax.twinx()
            btc = macro_df['btc'].dropna()
            ax2.plot(btc.index, btc, color='gold', linewidth=1, alpha=0.5, label='BTC')
            ax2.set_ylabel('BTC Price (USD)', color='goldenrod', fontsize=9)
            ax2.tick_params(axis='y', labelcolor='goldenrod')

        if col == 'yield_spread_10y_2y':
            ax.axhline(0, color='black', linewidth=0.8, linestyle='--', alpha=0.6)
            ax.fill_between(series.index, 0, series, where=series < 0,
                            alpha=0.2, color='red', label='Inverted')

        ax.set_title(f'{label} vs BTC Price')

    plt.tight_layout()
    plt.show()


def plot_correlation_heatmap(macro_df: pd.DataFrame) -> None:
    """
    Correlation heatmap between all macro indicators and BTC price.

    Parameters
    ----------
    macro_df : pd.DataFrame
        Unified macro DataFrame.

    Notes
    -----
    Uses daily returns (pct_change) rather than levels for correlation.
    Level-based correlation is spurious for trending series — two series
    that both trend upward will show high correlation even if they are
    economically unrelated.
    """
    # Define all desired columns for correlation
    correlation_cols = [
        'cpi_inflation_pct', 'fed_rate', 'yield_spread_10y_2y',
        'dxy', 'vix', 'sp500', 'gold', 'btc'
    ]

    # Filter to include only columns that actually exist in macro_df
    existing_correlation_cols = [col for col in correlation_cols if col in macro_df.columns]

    if not existing_correlation_cols or len(existing_correlation_cols) < 2:
        print("Not enough columns available in macro_df to compute correlation heatmap. Need at least two.")
        return

    returns = macro_df[existing_correlation_cols].dropna(how='all')

    if returns.empty or len(returns.columns) < 2:
        print("Not enough valid data or columns to compute correlation heatmap after dropping NaNs.")
        return

    returns = returns.pct_change().dropna()

    if returns.empty or len(returns.columns) < 2:
        print("Not enough valid data or columns to compute correlation heatmap after calculating returns and dropping NaNs.")
        return

    corr = returns.corr()

    fig, ax = plt.subplots(figsize=(10, 8))
    mask = np.triu(np.ones_like(corr, dtype=bool))
    sns.heatmap(corr, annot=True, fmt='.2f', cmap='RdYlGn',
                center=0, vmin=-1, vmax=1, mask=mask, ax=ax,
                linewidths=0.5, annot_kws={'size': 9})
    ax.set_title('Daily Return Correlations: Macro Indicators vs Crypto')
    plt.tight_layout()
    plt.show()


plot_macro_dashboard(macro_df)
plot_correlation_heatmap(macro_df)

def export_macro_data(macro_df: pd.DataFrame, filename: str = 'macro_indicators.csv') -> None:
    """
    Export the unified macro DataFrame to CSV.

    Parameters
    ----------
    macro_df : pd.DataFrame
        Unified daily macro DataFrame from align_macro_data().
    filename : str
        Output CSV filename.

    Notes
    -----
    This CSV is the primary input for Notebooks 126-132 (correlation analysis,
    risk-on/off regime detection, macro event strategies, etc.).
    """
    macro_df.to_csv(filename)
    print(f'Exported: {filename}  ({len(macro_df)} rows x {len(macro_df.columns)} columns)')
    print(f'Columns: {list(macro_df.columns)}')


export_macro_data(macro_df, 'macro_indicators.csv')