!pip install numpy pandas matplotlib seaborn requests --quiet

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import requests
import warnings

warnings.filterwarnings('ignore')
%matplotlib inline
plt.rcParams['figure.figsize'] = (14, 5)
plt.rcParams['axes.spines.top']   = False
plt.rcParams['axes.spines.right'] = False
sns.set_palette('husl')
print('Imports ready.')

SYMBOL              = 'BTCUSDT'      # Binance/Bybit symbol
FUNDING_INTERVAL_H  = 8             # hours between funding payments
EXTREME_FUNDING_PCT = 0.10          # per-period funding rate (%) that triggers extreme signal
BASIS_EXTREME_PCT   = 0.50          # basis > this % of spot = overheated signal
USE_SYNTHETIC       = True          # set False to fetch from Binance public API
N_PERIODS           = 500           # number of 8-hour periods to simulate

def fetch_binance_funding_history(
    symbol: str,
    limit: int = 500
) -> pd.DataFrame:
    """
    Fetch historical funding rate data from Binance public API.

    Parameters
    ----------
    symbol : str
        Trading pair symbol (e.g., 'BTCUSDT').
    limit : int
        Number of funding rate periods to fetch (max 1000).

    Returns
    -------
    pd.DataFrame
        Columns: timestamp, funding_rate, mark_price.

    Notes
    -----
    Binance perpetual funding rates are settled every 8 hours at 00:00, 08:00, 16:00 UTC.
    Bybit uses the same schedule. OKX uses 8h but with different rate formula.
    The funding rate is applied to the notional value of the open position.
    A rate of 0.01% per 8h = 0.03% daily = ~11% annualized carry cost for longs.
    """
    url = f'https://fapi.binance.com/fapi/v1/fundingRate'
    params = {'symbol': symbol, 'limit': limit}
    try:
        resp = requests.get(url, params=params, timeout=10)
        resp.raise_for_status()
        data = resp.json()
        df = pd.DataFrame(data)
        df['timestamp']    = pd.to_datetime(df['fundingTime'], unit='ms')
        df['funding_rate'] = df['fundingRate'].astype(float) * 100  # convert to percentage
        df['mark_price']   = df['markPrice'].astype(float)
        return df[['timestamp', 'funding_rate', 'mark_price']].sort_values('timestamp').reset_index(drop=True)
    except Exception as e:
        raise RuntimeError(f'Binance API request failed: {e}')


def generate_synthetic_perp_data(
    n_periods: int,
    start_price: float = 50_000
) -> pd.DataFrame:
    """
    Generate synthetic perpetual futures data with realistic basis and funding dynamics.

    Embeds three market phases:
    - Bull run: positive funding, high basis, OI increasing
    - Neutral: funding near zero, small basis
    - Bear/liquidation: negative funding spike, negative basis

    Parameters
    ----------
    n_periods : int
        Number of 8-hour funding periods.
    start_price : float
        Initial spot price.

    Returns
    -------
    pd.DataFrame
        Columns: timestamp, spot_price, perp_price, basis_usd, basis_pct,
                 funding_rate_pct, open_interest, market_phase.
    """
    np.random.seed(42)
    timestamps = pd.date_range('2023-01-01', periods=n_periods, freq='8h')

    # Phase assignments
    phases = np.full(n_periods, 'neutral')
    phases[:n_periods//4] = 'bull_run'
    phases[3*n_periods//4:] = 'bear'

    # Spot price simulation with phase-dependent drift
    spot_rets = np.zeros(n_periods)
    for i, phase in enumerate(phases):
        drift = {'bull_run': 0.004, 'neutral': 0.0, 'bear': -0.003}.get(phase, 0.0)
        spot_rets[i] = drift + np.random.normal(0, 0.015)

    spot_prices = start_price * np.exp(np.cumsum(spot_rets))

    # Basis as % of spot — mean-reverts around phase-specific level
    basis_mean = {'bull_run': 0.4, 'neutral': 0.05, 'bear': -0.2}
    basis_pct  = np.zeros(n_periods)
    b = 0.05
    for i, phase in enumerate(phases):
        target = basis_mean.get(phase, 0.05)
        b = b + 0.15 * (target - b) + np.random.normal(0, 0.05)
        b = np.clip(b, -1.0, 2.0)
        basis_pct[i] = b

    perp_prices  = spot_prices * (1 + basis_pct / 100)
    basis_usd    = perp_prices - spot_prices
    funding_rate = basis_pct / 3 + np.random.normal(0, 0.01, n_periods)  # funding ≈ basis / 3 per period

    # Open interest: rises in bull, falls in bear
    oi_base = 500_000  # BTC
    oi_multiplier = {'bull_run': 1.5, 'neutral': 1.0, 'bear': 0.7}
    oi = np.array([oi_base * oi_multiplier.get(p, 1.0) * (1 + np.random.normal(0, 0.02)) for p in phases])

    return pd.DataFrame({
        'timestamp':       timestamps,
        'spot_price':      np.round(spot_prices, 2),
        'perp_price':      np.round(perp_prices, 2),
        'basis_usd':       np.round(basis_usd, 2),
        'basis_pct':       np.round(basis_pct, 4),
        'funding_rate_pct': np.round(funding_rate, 4),
        'open_interest':   np.round(oi, 0),
        'market_phase':    phases,
    })


if USE_SYNTHETIC:
    data = generate_synthetic_perp_data(N_PERIODS)
    print('Using synthetic perpetual data.')
else:
    try:
        funding_data = fetch_binance_funding_history(SYMBOL)
        print(f'Fetched {len(funding_data)} funding periods from Binance.')
        data = funding_data
    except Exception as e:
        print(f'Live fetch failed ({e}). Using synthetic.')
        data = generate_synthetic_perp_data(N_PERIODS)

print(data.tail(5))

def compute_basis_signals(
    data: pd.DataFrame,
    extreme_funding_pct: float,
    basis_extreme_pct: float,
    roll: int = 21  # periods (~7 days at 8h intervals)
) -> pd.DataFrame:
    """
    Compute derived basis signals: rolling stats, extreme flags, sentiment score.

    Parameters
    ----------
    data : pd.DataFrame
        Perpetual data with 'basis_pct', 'funding_rate_pct', 'open_interest'.
    extreme_funding_pct : float
        Funding rate (per period, %) that triggers extreme signal.
    basis_extreme_pct : float
        Basis (% of spot) that triggers extreme premium signal.
    roll : int
        Rolling window for percentile-rank signals.

    Returns
    -------
    pd.DataFrame
        Extended DataFrame with signal columns.

    Notes
    -----
    The 'basis_sentiment' composite score (0=extreme bearish, 1=extreme bullish)
    combines funding rate, basis percentage, and OI trend.
    It is a CONTRARIAN signal in excess: extreme bullish reading (>0.8) suggests
    overextension and is bearish for SPOT price. Extreme bearish (<0.2) suggests
    over-leveraged shorts and is bullish for spot (squeeze risk).
    """
    out = data.copy()

    # Rolling funding stats
    out['funding_rolling_mean'] = out['funding_rate_pct'].rolling(roll).mean()
    out['funding_cumulative']   = out['funding_rate_pct'].cumsum()  # total carry earned

    # Extreme event flags
    out['extreme_positive_funding'] = (out['funding_rate_pct'] >= extreme_funding_pct).astype(int)
    out['extreme_negative_funding'] = (out['funding_rate_pct'] <= -extreme_funding_pct / 2).astype(int)
    out['extreme_positive_basis']   = (out['basis_pct'] >= basis_extreme_pct).astype(int)
    out['extreme_negative_basis']   = (out['basis_pct'] <= -basis_extreme_pct / 3).astype(int)

    # OI trend: rising OI + positive funding = leveraged longs (bearish for spot)
    out['oi_change_pct'] = out['open_interest'].pct_change() * 100 if 'open_interest' in out.columns else 0.0

    # Composite sentiment: higher = more bullish leverage (CONTRARIAN bearish signal)
    funding_score = out['funding_rate_pct'].rolling(roll).rank(pct=True)
    basis_score   = out['basis_pct'].rolling(roll).rank(pct=True)
    out['basis_sentiment'] = (0.6 * funding_score.fillna(0.5) + 0.4 * basis_score.fillna(0.5)).clip(0, 1)

    out['sentiment_signal'] = pd.cut(
        out['basis_sentiment'].fillna(0.5),
        bins=[-0.01, 0.25, 0.75, 1.01],
        labels=['contrarian_buy', 'neutral', 'contrarian_sell']
    )

    return out


signals = compute_basis_signals(data, EXTREME_FUNDING_PCT, BASIS_EXTREME_PCT)

print('Basis signal summary:')
print(f'  Mean funding rate:       {signals["funding_rate_pct"].mean():.4f}%')
print(f'  Extreme positive funding: {signals["extreme_positive_funding"].sum()} periods')
print(f'  Extreme negative funding: {signals["extreme_negative_funding"].sum()} periods')
print(f'  Current sentiment:        {signals["sentiment_signal"].iloc[-1]}')
print(f'  Cumulative funding earned: {signals["funding_cumulative"].iloc[-1]:.2f}% (as basis capture)')

def backtest_cash_and_carry(
    signals: pd.DataFrame,
    enter_funding_thresh: float = 0.05,
    exit_funding_thresh: float = 0.01,
    position_size_usdt: float = 10_000
) -> pd.DataFrame:
    """
    Backtest a cash-and-carry (delta-neutral basis capture) strategy.

    Enter trade when funding rate exceeds enter_funding_thresh:
    - Long spot BTC
    - Short equal notional value of BTC perpetual
    Net position: delta-neutral, earns funding payments from short perp.

    Parameters
    ----------
    signals : pd.DataFrame
        Output of compute_basis_signals().
    enter_funding_thresh : float
        Funding rate (%) to enter the carry trade.
    exit_funding_thresh : float
        Funding rate (%) to exit (rate has dropped, less attractive).
    position_size_usdt : float
        Notional position size in USDT.

    Returns
    -------
    pd.DataFrame
        Backtest results with entry/exit flags, cumulative PnL.

    Notes
    -----
    This is a simplified backtest — it ignores:
    - Exchange margin requirements (short perp requires collateral)
    - Borrow costs for spot (if using borrowed BTC)
    - Liquidation risk on the short perp during extreme moves
    - Transaction costs (~0.05% per trade side)
    In practice, annualized yields of 10-30% were achievable during bull markets.
    """
    bt = signals[['timestamp', 'spot_price', 'perp_price', 'funding_rate_pct', 'basis_pct']].copy()
    in_trade      = False
    cumulative_pnl = 0.0
    entry_price   = None
    pnl_per_period = []
    in_trade_flag  = []

    for _, row in bt.iterrows():
        fr = row['funding_rate_pct']

        if not in_trade and fr >= enter_funding_thresh:
            in_trade   = True
            entry_price = row['spot_price']

        period_pnl = 0.0
        if in_trade:
            # Funding received from short perp position
            notional    = position_size_usdt
            period_pnl  = notional * (fr / 100)  # received each period
            cumulative_pnl += period_pnl

            if fr <= exit_funding_thresh:
                in_trade   = False
                entry_price = None

        pnl_per_period.append(period_pnl)
        in_trade_flag.append(int(in_trade))

    bt['pnl_per_period']  = pnl_per_period
    bt['cumulative_pnl']  = np.cumsum(pnl_per_period)
    bt['in_carry_trade']  = in_trade_flag
    bt['return_on_notional_pct'] = bt['cumulative_pnl'] / position_size_usdt * 100

    total_return = bt['return_on_notional_pct'].iloc[-1]
    n_periods_active = sum(in_trade_flag)
    daily_periods    = 3
    ann_return       = (total_return / len(bt)) * daily_periods * 365

    print(f'Cash-and-Carry Backtest Results:')
    print(f'  Total return on notional: {total_return:.2f}%')
    print(f'  Annualized return:        {ann_return:.1f}%')
    print(f'  Periods in trade:         {n_periods_active} / {len(bt)} ({n_periods_active/len(bt)*100:.1f}%)')
    print(f'  Total funding earned:     ${bt["cumulative_pnl"].iloc[-1]:.0f} on ${position_size_usdt:.0f} notional')
    return bt


carry_bt = backtest_cash_and_carry(signals)

def plot_perp_basis_dashboard(
    signals: pd.DataFrame,
    carry_bt: pd.DataFrame
) -> None:
    """
    Five-panel perpetual basis dashboard.

    Parameters
    ----------
    signals : pd.DataFrame
        Computed signals from compute_basis_signals().
    carry_bt : pd.DataFrame
        Cash-and-carry backtest results.
    """
    fig, axes = plt.subplots(4, 1, figsize=(15, 18), sharex=True)

    # Panel 1: Spot and perp prices
    if 'spot_price' in signals.columns:
        axes[0].plot(signals['timestamp'], signals['spot_price'], label='Spot',  color='steelblue', linewidth=1.5)
        axes[0].plot(signals['timestamp'], signals['perp_price'], label='Perp',  color='orange',   linewidth=1.5, linestyle='--')
    elif 'mark_price' in signals.columns:
        axes[0].plot(signals['timestamp'], signals['mark_price'], label='Mark Price', color='steelblue', linewidth=1.5)
    axes[0].set_ylabel('Price (USD)')
    axes[0].set_title('Spot vs Perpetual Price')
    axes[0].legend()

    # Panel 2: Basis %
    if 'basis_pct' in signals.columns:
        axes[1].bar(signals['timestamp'], signals['basis_pct'],
                    color=np.where(signals['basis_pct'] >= 0, 'green', 'red'), alpha=0.7, width=0.3)
        axes[1].axhline(0, color='black', linewidth=0.8)
        axes[1].axhline( BASIS_EXTREME_PCT,   color='red',  linewidth=0.8, linestyle='--', alpha=0.7)
        axes[1].axhline(-BASIS_EXTREME_PCT/3, color='blue', linewidth=0.8, linestyle='--', alpha=0.7)
        axes[1].set_ylabel('Basis (%)')
        axes[1].set_title('Perpetual Basis (% of Spot) — Red Line = Extreme Premium')

    # Panel 3: Funding rate
    axes[2].bar(signals['timestamp'], signals['funding_rate_pct'],
                color=np.where(signals['funding_rate_pct'] >= 0, 'orange', 'purple'), alpha=0.7, width=0.3)
    axes[2].axhline(0, color='black', linewidth=0.8)
    axes[2].axhline( EXTREME_FUNDING_PCT,   color='red',  linewidth=0.8, linestyle=':', label='Extreme positive')
    axes[2].axhline(-EXTREME_FUNDING_PCT/2, color='blue', linewidth=0.8, linestyle=':', label='Extreme negative')
    axes[2].set_ylabel('Funding Rate (%)')
    axes[2].set_title('8h Funding Rate — Orange=Positive (Longs Pay), Purple=Negative (Shorts Pay)')
    axes[2].legend()

    # Panel 4: Carry trade PnL
    axes[3].plot(carry_bt['timestamp'], carry_bt['return_on_notional_pct'], color='gold', linewidth=1.5)
    axes[3].fill_between(carry_bt['timestamp'], carry_bt['return_on_notional_pct'], 0,
                          where=(carry_bt['in_carry_trade'] == 1), alpha=0.3, color='green', label='Active carry trade')
    axes[3].axhline(0, color='black', linewidth=0.8, linestyle='--')
    axes[3].set_ylabel('Return on Notional (%)')
    axes[3].set_title('Cash-and-Carry Strategy Cumulative Return')
    axes[3].set_xlabel('Date')
    axes[3].legend()

    plt.tight_layout()
    plt.show()


plot_perp_basis_dashboard(signals, carry_bt)

def summarize_basis_signals(signals: pd.DataFrame) -> pd.DataFrame:
    """
    Generate a concise summary of current and recent basis signal conditions.

    Parameters
    ----------
    signals : pd.DataFrame
        Output of compute_basis_signals().

    Returns
    -------
    pd.DataFrame
        Summary table of current readings and historical context.
    """
    latest = signals.iloc[-1]
    roll_7d = signals.tail(21)  # ~7 days at 8h intervals

    summary = {
        'Current Funding Rate (%)':          round(float(latest['funding_rate_pct']), 4),
        'Current Basis (%)':                 round(float(latest.get('basis_pct', 0)), 4),
        '7d Avg Funding Rate (%)':           round(float(roll_7d['funding_rate_pct'].mean()), 4),
        'Extreme Funding Events (7d)':       int(roll_7d['extreme_positive_funding'].sum()),
        'Sentiment Signal':                  str(latest.get('sentiment_signal', 'N/A')),
        'Cumulative Funding Earned (7d, %)': round(float(roll_7d['funding_rate_pct'].sum()), 3),
        'Annualized Funding Rate (%)':       round(float(roll_7d['funding_rate_pct'].mean()) * 3 * 365, 1),
    }

    df = pd.DataFrame([summary]).T
    df.columns = ['Value']
    print('Basis Signal Summary:')
    print(df.to_string())
    return df


signal_summary = summarize_basis_signals(signals)

def export_perp_basis_data(
    signals: pd.DataFrame,
    carry_bt: pd.DataFrame,
    signal_summary: pd.DataFrame
) -> None:
    """
    Export perp basis signals, carry backtest, and summary.

    Parameters
    ----------
    signals : pd.DataFrame
        Full signal DataFrame.
    carry_bt : pd.DataFrame
        Cash-and-carry backtest results.
    signal_summary : pd.DataFrame
        Current signal summary table.
    """
    signals.to_csv('perp_basis_signals.csv', index=False)
    carry_bt.to_csv('carry_trade_backtest.csv', index=False)
    signal_summary.to_csv('basis_signal_summary.csv')
    print('Exported: perp_basis_signals.csv')
    print('Exported: carry_trade_backtest.csv')
    print('Exported: basis_signal_summary.csv')


export_perp_basis_data(signals, carry_bt, signal_summary)