!pip install pandas numpy matplotlib seaborn scipy --quiet

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sns
from scipy.stats import zscore
from datetime import datetime, timedelta, timezone
import warnings

warnings.filterwarnings('ignore')
%matplotlib inline
plt.rcParams['figure.figsize'] = (14, 5)
plt.rcParams['axes.spines.top'] = False
plt.rcParams['axes.spines.right'] = False
sns.set_palette('husl')
print('Imports ready.')

# ── CONFIGURATION ─────────────────────────────────────────────────────────────
SOURCE_WEIGHTS = {
    'reddit':    0.20,   # VADER compound scores from reddit_crypto_scraper
    'telegram':  0.25,   # VADER scores from telegram_channel_monitor
    'youtube':   0.10,   # VADER scores from youtube_sentiment_analysis
    'finbert':   0.30,   # FinBERT sentiment from finbert_crypto_sentiment
    'topic':     0.15,   # Normalized signal from topic_modeling_crypto_news
}

EMA_SPAN_FAST  = 3    # fast EMA in days (reactive)
EMA_SPAN_SLOW  = 7    # slow EMA in days (trend)
BUY_THRESHOLD  = 0.15  # composite score above this → BUY
SELL_THRESHOLD = -0.15 # composite score below this → SELL

# File paths — update these to match your exported CSV locations
DATA_FILES = {
    'reddit':   'reddit_daily_sentiment.csv',
    'telegram': 'telegram_daily_sentiment.csv',
    'youtube':  'youtube_daily_sentiment.csv',
    'finbert':  'finbert_daily_sentiment.csv',
    'topic':    'topic_modeling_crypto_news_signals.csv',
}
# ─────────────────────────────────────────────────────────────────────────────


def generate_synthetic_sentiment_data(n_days: int = 90, seed: int = 42) -> dict:
    """
    Generate synthetic daily sentiment data for pipeline testing.

    Parameters
    ----------
    n_days : int
        Number of days to generate.
    seed : int
        Random seed for reproducibility.

    Returns
    -------
    dict
        {source_name: pd.DataFrame} where each DataFrame has
        columns ['date', 'sentiment_score'] with values in [-1, +1].

    Notes
    -----
    Synthetic data contains a shared latent trend (simulating real market
    sentiment) plus source-specific noise. This mimics how different platforms
    reflect the same underlying mood with different signal-to-noise ratios.
    """
    np.random.seed(seed)
    dates = pd.date_range(end=datetime.now(tz=timezone.utc).date(), periods=n_days, freq='D')

    # Shared latent trend (sine wave + random walk)
    t = np.linspace(0, 4 * np.pi, n_days)
    latent = 0.3 * np.sin(t) + np.cumsum(np.random.randn(n_days) * 0.02)
    latent = np.clip(latent, -1, 1)

    noise_levels = {'reddit': 0.25, 'telegram': 0.20, 'youtube': 0.35, 'finbert': 0.15, 'topic': 0.30}
    sources = {}
    for src, noise in noise_levels.items():
        scores = np.clip(latent + np.random.randn(n_days) * noise, -1, 1)
        sources[src] = pd.DataFrame({'date': dates, 'sentiment_score': scores})

    print(f'Generated {n_days} days of synthetic sentiment data for {len(sources)} sources.')
    return sources


def load_sentiment_source(filepath: str, date_col: str = 'date', score_col: str = 'sentiment_score') -> pd.DataFrame:
    """
    Load a daily sentiment CSV exported by upstream notebooks.

    Parameters
    ----------
    filepath : str
        Path to the CSV file.
    date_col : str
        Name of the date column in the CSV.
    score_col : str
        Name of the sentiment score column.

    Returns
    -------
    pd.DataFrame
        Cleaned DataFrame with columns ['date', 'sentiment_score'].
        The 'date' column is cast to datetime.date for alignment.
    """
    df = pd.read_csv(filepath, parse_dates=[date_col])
    df = df.rename(columns={date_col: 'date', score_col: 'sentiment_score'})
    df['date'] = pd.to_datetime(df['date']).dt.date
    df = df[['date', 'sentiment_score']].dropna().sort_values('date')
    return df


# ── Load or generate data ──────────────────────────────────────────────────────
USE_SYNTHETIC = True  # Set False to load real CSV files

if USE_SYNTHETIC:
    source_data = generate_synthetic_sentiment_data(n_days=90)
else:
    source_data = {}
    for src, path in DATA_FILES.items():
        try:
            source_data[src] = load_sentiment_source(path)
            print(f'Loaded {src}: {len(source_data[src])} rows')
        except FileNotFoundError:
            print(f'WARNING: {path} not found. Skipping {src}.')

def normalize_to_unit_range(series: pd.Series, percentile_clip: float = 1.0) -> pd.Series:
    """
    Normalize a pandas Series to the range [−1, +1] with outlier clipping.

    Parameters
    ----------
    series : pd.Series
        Raw sentiment score series (any numeric scale).
    percentile_clip : float
        Percentile for winsorization (clips top/bottom X% before scaling).
        Use 1.0 to clip the top and bottom 1% — handles extreme outliers.

    Returns
    -------
    pd.Series
        Normalized series in [-1, +1]. NaN values are forward-filled.
    """
    lo = np.percentile(series.dropna(), percentile_clip)
    hi = np.percentile(series.dropna(), 100 - percentile_clip)
    clipped = series.clip(lo, hi)
    span = hi - lo
    if span == 0:
        return pd.Series(0.0, index=series.index)
    normalized = 2 * (clipped - lo) / span - 1
    return normalized.ffill()


def align_and_normalize_sources(source_data: dict) -> pd.DataFrame:
    """
    Align all sentiment sources to a common daily date index and normalize scores.

    Parameters
    ----------
    source_data : dict
        {source_name: pd.DataFrame} where each DataFrame has ['date', 'sentiment_score'].

    Returns
    -------
    pd.DataFrame
        Wide DataFrame indexed by date. Each column is one normalized source.
        Missing days are forward-filled (last known sentiment carried forward).

    Notes
    -----
    The alignment uses a full outer join so that no data point is discarded.
    Sources with sparse data (e.g., weekly YouTube uploads) are forward-filled
    to maintain alignment without introducing look-ahead bias.
    """
    frames = {}
    for src, df in source_data.items():
        s = df.set_index('date')['sentiment_score'].rename(src)
        frames[src] = s

    aligned = pd.DataFrame(frames).sort_index()
    # Forward fill missing values (no look-ahead bias — uses past data only)
    aligned = aligned.ffill().bfill()

    # Normalize each column independently
    for col in aligned.columns:
        aligned[col] = normalize_to_unit_range(aligned[col])

    print(f'Aligned data: {len(aligned)} days, {len(aligned.columns)} sources')
    print(f'Date range: {aligned.index[0]} → {aligned.index[-1]}')
    return aligned


aligned_df = align_and_normalize_sources(source_data)
print(aligned_df.tail())
aligned_df.plot(title='Normalized Sentiment by Source', alpha=0.7, figsize=(14, 5))
plt.axhline(0, color='black', linewidth=0.8, linestyle='--')
plt.ylabel('Normalized Score [-1, +1]')
plt.tight_layout()
plt.show()

def compute_weighted_composite(
    aligned_df: pd.DataFrame,
    weights: dict
) -> pd.Series:
    """
    Compute a weighted composite sentiment signal from normalized source signals.

    Parameters
    ----------
    aligned_df : pd.DataFrame
        Normalized source signals from align_and_normalize_sources().
        Columns correspond to source names.
    weights : dict
        {source_name: weight (float)}. Weights do not need to sum to 1 —
        they are re-normalized internally based on available sources.

    Returns
    -------
    pd.Series
        Daily composite score in [-1, +1]. Index is the same date index
        as aligned_df.

    Notes
    -----
    If a source is missing from aligned_df, its weight is redistributed
    proportionally among available sources. This makes the function robust
    to partial data availability.
    """
    available = {k: v for k, v in weights.items() if k in aligned_df.columns}
    total_weight = sum(available.values())
    if total_weight == 0:
        raise ValueError('No matching sources found in aligned DataFrame.')

    composite = pd.Series(0.0, index=aligned_df.index)
    for src, w in available.items():
        composite += aligned_df[src] * (w / total_weight)

    print(f'Composite signal computed from: {list(available.keys())}')
    print(f'Effective weights: { {k: round(v/total_weight, 3) for k, v in available.items()} }')
    return composite.rename('composite_raw')


def apply_ema_smoothing(
    composite: pd.Series,
    span_fast: int = 3,
    span_slow: int = 7
) -> pd.DataFrame:
    """
    Apply fast and slow exponential moving averages to the composite signal.

    Parameters
    ----------
    composite : pd.Series
        Raw composite sentiment series.
    span_fast : int
        EMA span for the fast (reactive) signal in days.
    span_slow : int
        EMA span for the slow (trend) signal in days.

    Returns
    -------
    pd.DataFrame
        Columns: composite_raw, ema_fast, ema_slow, ema_crossover.
        ema_crossover = ema_fast - ema_slow (positive = bullish momentum).

    Notes
    -----
    The crossover between fast and slow EMA mirrors the classic MACD concept
    applied to sentiment: when short-term sentiment rises faster than the
    longer-term baseline, bullish momentum is building.
    """
    df = composite.to_frame()
    df['ema_fast']     = composite.ewm(span=span_fast, adjust=False).mean()
    df['ema_slow']     = composite.ewm(span=span_slow, adjust=False).mean()
    df['ema_crossover'] = df['ema_fast'] - df['ema_slow']
    return df


composite_raw = compute_weighted_composite(aligned_df, SOURCE_WEIGHTS)
signal_df     = apply_ema_smoothing(composite_raw, EMA_SPAN_FAST, EMA_SPAN_SLOW)

# Plot composite vs smoothed
fig, ax = plt.subplots(figsize=(14, 5))
ax.plot(signal_df.index, signal_df['composite_raw'], alpha=0.35, color='grey', label='Raw Composite')
ax.plot(signal_df.index, signal_df['ema_fast'], linewidth=2, color='steelblue', label=f'EMA-{EMA_SPAN_FAST}')
ax.plot(signal_df.index, signal_df['ema_slow'], linewidth=2, color='tomato',    label=f'EMA-{EMA_SPAN_SLOW}')
ax.axhline(0, color='black', linewidth=0.8, linestyle='--')
ax.axhline( BUY_THRESHOLD,  color='green', linewidth=0.8, linestyle=':', alpha=0.8, label='Buy threshold')
ax.axhline(SELL_THRESHOLD, color='red',   linewidth=0.8, linestyle=':', alpha=0.8, label='Sell threshold')
ax.set_title('Composite Sentiment Signal with EMA Smoothing')
ax.set_ylabel('Score [-1, +1]')
ax.legend(fontsize=9)
plt.tight_layout()
plt.show()

def generate_discrete_signals(
    signal_df: pd.DataFrame,
    buy_threshold: float = 0.15,
    sell_threshold: float = -0.15,
    use_ema: bool = True
) -> pd.DataFrame:
    """
    Convert the continuous composite score into discrete BUY / SELL / HOLD labels.

    Parameters
    ----------
    signal_df : pd.DataFrame
        Output of apply_ema_smoothing(). Must contain 'ema_fast' and 'composite_raw'.
    buy_threshold : float
        Score above this value generates a BUY signal.
    sell_threshold : float
        Score below this value generates a SELL signal.
    use_ema : bool
        If True, uses the fast EMA for signal generation (less noisy).
        If False, uses the raw composite score.

    Returns
    -------
    pd.DataFrame
        Original signal_df with added columns:
        - signal: 'BUY', 'SELL', or 'HOLD'
        - confidence: float in [0, 1] — how strongly the score exceeds the threshold
        - signal_numeric: +1 (BUY), −1 (SELL), 0 (HOLD) for easy arithmetic
    """
    score_col = 'ema_fast' if use_ema else 'composite_raw'
    score = signal_df[score_col]
    result = signal_df.copy()

    conditions = [
        score >  buy_threshold,
        score <  sell_threshold,
    ]
    result['signal'] = np.select(conditions, ['BUY', 'SELL'], default='HOLD')
    result['signal_numeric'] = np.select(conditions, [1, -1], default=0)

    # Confidence: how far above/below the threshold, scaled to [0, 1]
    max_distance = 1 - abs(buy_threshold)
    result['confidence'] = (
        (score.abs() - abs(buy_threshold)).clip(lower=0) / max_distance
    ).clip(0, 1)

    counts = result['signal'].value_counts()
    print(f'Signals: BUY={counts.get("BUY", 0)}, SELL={counts.get("SELL", 0)}, HOLD={counts.get("HOLD", 0)}')
    return result


def plot_discrete_signals(signal_df: pd.DataFrame) -> None:
    """
    Visualize BUY/SELL/HOLD signals overlaid on the composite sentiment score.

    Parameters
    ----------
    signal_df : pd.DataFrame
        Output of generate_discrete_signals().
    """
    fig, axes = plt.subplots(2, 1, figsize=(14, 8), sharex=True)

    ax = axes[0]
    ax.plot(signal_df.index, signal_df['ema_fast'], color='steelblue', linewidth=2, label='EMA Fast')
    ax.axhline( BUY_THRESHOLD,  color='green', linewidth=0.8, linestyle='--')
    ax.axhline(SELL_THRESHOLD, color='red',   linewidth=0.8, linestyle='--')
    ax.axhline(0, color='black', linewidth=0.5)

    buys  = signal_df[signal_df['signal'] == 'BUY']
    sells = signal_df[signal_df['signal'] == 'SELL']
    ax.scatter(buys.index,  buys['ema_fast'],  color='green', marker='^', s=80, zorder=5, label='BUY')
    ax.scatter(sells.index, sells['ema_fast'], color='red',   marker='v', s=80, zorder=5, label='SELL')
    ax.set_title('Composite Sentiment Signal with BUY / SELL Markers')
    ax.set_ylabel('Score')
    ax.legend(fontsize=9)

    ax = axes[1]
    colors = signal_df['signal'].map({'BUY': 'green', 'SELL': 'red', 'HOLD': 'grey'})
    ax.bar(signal_df.index, signal_df['confidence'], color=colors, alpha=0.7)
    ax.set_title('Signal Confidence')
    ax.set_ylabel('Confidence [0, 1]')

    plt.tight_layout()
    plt.show()


signal_df = generate_discrete_signals(signal_df, BUY_THRESHOLD, SELL_THRESHOLD)
plot_discrete_signals(signal_df)

def generate_synthetic_ohlcv(dates: pd.DatetimeIndex, seed: int = 42) -> pd.DataFrame:
    """
    Generate synthetic BTC-like OHLCV data aligned to the signal dates.

    Parameters
    ----------
    dates : pd.DatetimeIndex
        Date index from the signal DataFrame.
    seed : int
        Random seed.

    Returns
    -------
    pd.DataFrame
        OHLCV DataFrame with columns: open, high, low, close, volume.
    """
    np.random.seed(seed)
    n = len(dates)
    log_returns = np.random.randn(n) * 0.025
    close = 40000 * np.exp(np.cumsum(log_returns))
    high   = close * (1 + np.abs(np.random.randn(n) * 0.008))
    low    = close * (1 - np.abs(np.random.randn(n) * 0.008))
    open_  = close * (1 + np.random.randn(n) * 0.005)
    volume = np.random.exponential(1e9, n)
    return pd.DataFrame({'open': open_, 'high': high, 'low': low, 'close': close, 'volume': volume}, index=dates)


def backtest_sentiment_signal(
    signal_df: pd.DataFrame,
    ohlcv_df: pd.DataFrame
) -> pd.DataFrame:
    """
    Backtest the sentiment signal against daily OHLCV price data.

    Parameters
    ----------
    signal_df : pd.DataFrame
        Signal DataFrame with 'signal_numeric' column (+1, 0, -1).
    ohlcv_df : pd.DataFrame
        OHLCV DataFrame aligned to the same date index.

    Returns
    -------
    pd.DataFrame
        Merged DataFrame with columns: signal_numeric, close, daily_return,
        strategy_return, cumulative_market, cumulative_strategy.

    Notes
    -----
    Strategy return = signal_numeric(t-1) * close_return(t)
    The signal is shifted by one day to avoid look-ahead bias (today's signal
    is acted on at tomorrow's open, proxied by tomorrow's close return).
    """
    bt = ohlcv_df[['close']].copy()
    bt['daily_return']    = bt['close'].pct_change()
    bt['signal_numeric']  = signal_df['signal_numeric'].reindex(bt.index).ffill()
    # Shift signal: today's signal applied to tomorrow's return
    bt['strategy_return'] = bt['signal_numeric'].shift(1) * bt['daily_return']
    bt['cumulative_market']   = (1 + bt['daily_return'].fillna(0)).cumprod()
    bt['cumulative_strategy'] = (1 + bt['strategy_return'].fillna(0)).cumprod()

    total_return   = bt['cumulative_strategy'].iloc[-1] - 1
    market_return  = bt['cumulative_market'].iloc[-1] - 1
    sharpe = (bt['strategy_return'].mean() / bt['strategy_return'].std()) * np.sqrt(252)
    print(f'Strategy return: {total_return:.1%}  |  Market return: {market_return:.1%}  |  Sharpe: {sharpe:.2f}')
    return bt


def plot_backtest_results(bt: pd.DataFrame) -> None:
    """
    Plot equity curves for strategy vs buy-and-hold.

    Parameters
    ----------
    bt : pd.DataFrame
        Output of backtest_sentiment_signal().
    """
    fig, ax = plt.subplots(figsize=(14, 5))
    ax.plot(bt.index, bt['cumulative_strategy'], linewidth=2, color='steelblue', label='Sentiment Strategy')
    ax.plot(bt.index, bt['cumulative_market'],   linewidth=2, color='grey',      label='Buy & Hold',      alpha=0.7)
    ax.set_title('Sentiment Signal Strategy vs Buy-and-Hold')
    ax.set_ylabel('Portfolio Value (starting at 1.0)')
    ax.legend()
    plt.tight_layout()
    plt.show()


ohlcv_df = generate_synthetic_ohlcv(signal_df.index.to_series().apply(pd.Timestamp))
bt_df    = backtest_sentiment_signal(signal_df, ohlcv_df)
plot_backtest_results(bt_df)

def export_signal(
    signal_df: pd.DataFrame,
    filename: str = 'sentiment_signal.csv'
) -> None:
    """
    Export the complete signal DataFrame to CSV.

    Parameters
    ----------
    signal_df : pd.DataFrame
        Full signal DataFrame from generate_discrete_signals().
    filename : str
        Output CSV filename.

    Outputs
    -------
    CSV with columns: date, composite_raw, ema_fast, ema_slow,
    ema_crossover, signal, confidence, signal_numeric.
    """
    out = signal_df.reset_index().rename(columns={'index': 'date'})
    out.to_csv(filename, index=False)
    print(f'Signal exported to {filename}  ({len(out)} rows)')
    print(out[['date', 'ema_fast', 'signal', 'confidence']].tail(10).to_string(index=False))


export_signal(signal_df, 'sentiment_signal.csv')