!pip install transformers vaderSentiment torch pandas matplotlib seaborn tqdm requests feedparser --quiet

import torch
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from transformers import BertTokenizer, BertForSequenceClassification, pipeline
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
from tqdm.auto import tqdm
import requests
import feedparser
import re
import time
from datetime import datetime, timezone
from dateutil import parser as dateparser

%matplotlib inline
plt.rcParams['figure.figsize'] = (12, 5)
plt.rcParams['axes.spines.top'] = False
plt.rcParams['axes.spines.right'] = False

DEVICE = 0 if torch.cuda.is_available() else -1
print(f'Device: {"GPU — " + torch.cuda.get_device_name(0) if DEVICE == 0 else "CPU"}')

FINBERT_MODEL = 'ProsusAI/finbert'
BATCH_SIZE    = 16
MAX_LENGTH    = 128
CONFIDENCE_THRESHOLD = 0.60   # Minimum confidence to trust a prediction


def load_finbert_pipeline(model_name: str, device: int = -1):
    """
    Load FinBERT as a HuggingFace text-classification pipeline.

    Parameters
    ----------
    model_name : HuggingFace model ID (default: ProsusAI/finbert)
    device     : 0 for GPU, -1 for CPU

    Returns
    -------
    HuggingFace pipeline object
    """
    print(f'Loading {model_name}...')
    t0 = time.time()
    nlp = pipeline(
        'text-classification',
        model=model_name,
        tokenizer=model_name,
        device=device,
        truncation=True,
        max_length=MAX_LENGTH,
        return_all_scores=True
    )
    print(f'Loaded in {time.time() - t0:.1f}s')
    return nlp


finbert = load_finbert_pipeline(FINBERT_MODEL, DEVICE)

CRYPTO_RSS_FEEDS = {
    'CoinDesk':       'https://www.coindesk.com/arc/outboundfeeds/rss/',
    'Cointelegraph':  'https://cointelegraph.com/rss',
    'Decrypt':        'https://decrypt.co/feed',
    'BitcoinMagazine': 'https://bitcoinmagazine.com/.rss/full/'
}


def fetch_rss_headlines(feeds: dict, max_per_source: int = 50) -> pd.DataFrame:
    """
    Fetch recent crypto news headlines from RSS feeds.

    Parameters
    ----------
    feeds           : Dictionary of {source_name: rss_url}
    max_per_source  : Maximum articles to fetch per source

    Returns
    -------
    DataFrame with columns: source, title, summary, published, link
    """
    all_articles = []
    print('Fetching RSS feeds...')

    for source, url in feeds.items():
        try:
            feed = feedparser.parse(url)
            count = 0
            for entry in feed.entries[:max_per_source]:
                published = None
                if hasattr(entry, 'published'):
                    try:
                        published = dateparser.parse(entry.published)
                    except Exception:
                        pass

                all_articles.append({
                    'source':    source,
                    'title':     entry.get('title', ''),
                    'summary':   re.sub('<[^>]+>', '', entry.get('summary', ''))[:300],
                    'published': published,
                    'link':      entry.get('link', '')
                })
                count += 1
            print(f'  {source}: {count} articles')
        except Exception as e:
            print(f'  WARNING: {source} failed — {e}')

    df = pd.DataFrame(all_articles)
    df.dropna(subset=['title'], inplace=True)
    df = df[df['title'].str.len() > 10].reset_index(drop=True)
    print(f'\nTotal articles: {len(df)}')
    return df


news_df = fetch_rss_headlines(CRYPTO_RSS_FEEDS)
news_df[['source', 'title', 'published']].head(5)

def run_finbert_inference(
    texts: list[str],
    nlp_pipeline,
    batch_size: int = 16,
    confidence_threshold: float = 0.0
) -> pd.DataFrame:
    """
    Run FinBERT inference on a list of texts.
    """
    cleaned = [re.sub(r'\s+', ' ', str(t)).strip()[:512] for t in texts]

    all_results = []
    for i in tqdm(range(0, len(cleaned), batch_size), desc='FinBERT'):
        batch = cleaned[i: i + batch_size]
        outputs = nlp_pipeline(batch, batch_size=len(batch))
        all_results.extend(outputs)

    rows = []
    for result in all_results:
        if isinstance(result, dict):
            result = [result]

        score_map = {item['label'].lower(): item['score'] for item in result}
        top = max(result, key=lambda x: x['score'])
        label_raw = top['label'].lower()
        confidence = top['score']
        final_label = label_raw if confidence >= confidence_threshold else 'uncertain'

        rows.append({
            'finbert_positive':   score_map.get('positive', 0.0),
            'finbert_neutral':    score_map.get('neutral', 0.0),
            'finbert_negative':   score_map.get('negative', 0.0),
            'finbert_label':      final_label,
            'finbert_confidence': confidence,
            'finbert_score':      score_map.get('positive', 0.0) - score_map.get('negative', 0.0)
        })

    return pd.DataFrame(rows)

# Score titles and summaries
title_scores   = run_finbert_inference(news_df['title'].tolist(), finbert, BATCH_SIZE, CONFIDENCE_THRESHOLD)
summary_scores = run_finbert_inference(news_df['summary'].tolist(), finbert, BATCH_SIZE, CONFIDENCE_THRESHOLD)

# Rename summary columns - using prefix to avoid the KeyError mismatch
summary_scores = summary_scores.add_prefix('summary_')

# Merge data
news_scored = pd.concat([news_df, title_scores, summary_scores], axis=1)

# Composite score calculation using the correct prefixes
news_scored['composite_score'] = (news_scored['finbert_score'] + news_scored['summary_finbert_score']) / 2
news_scored['composite_label'] = news_scored['composite_score'].apply(
    lambda s: 'positive' if s >= 0.1 else ('negative' if s <= -0.1 else 'neutral')
)

print(f'Scored {len(news_scored)} articles')
news_scored[['source', 'title', 'finbert_label', 'finbert_confidence', 'composite_score']].head(6)

FINANCIAL_TEST_CASES = [
    'Bitcoin ETF inflows hit record $500M as institutional demand surges',
    'SEC delays decision on spot Ethereum ETF amid regulatory uncertainty',
    'Crypto exchange reports record quarterly revenue beating analyst estimates',
    'DeFi protocol suffers $50M exploit due to smart contract vulnerability',
    'Federal Reserve signals potential rate cuts boosting risk asset sentiment',
    'Stablecoin issuer faces liquidity concerns amid market stress conditions',
    'Layer 2 adoption accelerates as transaction costs fall below $0.01',
    'Venture capital funding in Web3 declined 68% year over year in Q3',
    'Bitcoin mining difficulty reached all-time high following hash rate surge',
    'Regulatory crackdown forces crypto firms to exit several emerging markets'
]


def compare_models_on_financial_text(texts: list[str], finbert_pipeline) -> pd.DataFrame:
    """
    Score financial text snippets with FinBERT and VADER for comparison.

    Parameters
    ----------
    texts            : List of financial text strings
    finbert_pipeline : Loaded FinBERT pipeline

    Returns
    -------
    Comparison DataFrame
    """
    vader_analyzer = SentimentIntensityAnalyzer()

    finbert_scores = run_finbert_inference(texts, finbert_pipeline, batch_size=8)

    rows = []
    for i, text in enumerate(texts):
        vc = vader_analyzer.polarity_scores(text)['compound']
        rows.append({
            'text':              text[:80] + '...' if len(text) > 80 else text,
            'finbert_label':     finbert_scores.iloc[i]['finbert_label'],
            'finbert_score':     round(finbert_scores.iloc[i]['finbert_score'], 3),
            'vader_label':       'positive' if vc >= 0.05 else ('negative' if vc <= -0.05 else 'neutral'),
            'vader_compound':    round(vc, 3),
            'models_agree':      finbert_scores.iloc[i]['finbert_label'] == ('positive' if vc >= 0.05 else ('negative' if vc <= -0.05 else 'neutral'))
        })

    df = pd.DataFrame(rows)
    agree_pct = df['models_agree'].mean() * 100
    print(f'Agreement rate: {agree_pct:.1f}%')
    return df


comparison_df = compare_models_on_financial_text(FINANCIAL_TEST_CASES, finbert)
comparison_df.style.applymap(
    lambda v: 'background-color: #d4edda' if v == 'positive'
    else ('background-color: #f8d7da' if v == 'negative' else ''),
    subset=['finbert_label', 'vader_label']
)

SOURCE_CREDIBILITY = {
    'CoinDesk':        1.5,
    'Cointelegraph':   1.3,
    'Decrypt':         1.2,
    'BitcoinMagazine': 1.1
}


def build_daily_news_signal(df: pd.DataFrame, credibility_map: dict) -> pd.DataFrame:
    """
    Aggregate article-level FinBERT scores into a daily sentiment signal.

    Each article is weighted by source credibility. The resulting
    daily score reflects the credibility-weighted news sentiment.

    Parameters
    ----------
    df               : Scored news DataFrame with 'published' and 'composite_score'
    credibility_map  : Dictionary of {source_name: weight_multiplier}

    Returns
    -------
    Daily signal DataFrame
    """
    df = df.copy()
    df['published'] = pd.to_datetime(df['published'], utc=True, errors='coerce')
    df.dropna(subset=['published'], inplace=True)
    df['date']   = df['published'].dt.date
    df['weight'] = df['source'].map(credibility_map).fillna(1.0)

    daily = df.groupby('date').apply(
        lambda g: pd.Series({
            'weighted_sentiment': (g['composite_score'] * g['weight']).sum() / g['weight'].sum(),
            'mean_sentiment':     g['composite_score'].mean(),
            'article_count':      len(g),
            'pct_positive':       (g['composite_label'] == 'positive').mean() * 100,
            'pct_negative':       (g['composite_label'] == 'negative').mean() * 100,
            'sources':            ', '.join(g['source'].unique())
        })
    ).reset_index()

    daily['date'] = pd.to_datetime(daily['date'])
    daily.sort_values('date', inplace=True)
    return daily


daily_news = build_daily_news_signal(news_scored, SOURCE_CREDIBILITY)
daily_news.tail(5)

def plot_finbert_news_results(news_df: pd.DataFrame, daily_df: pd.DataFrame) -> None:
    fig, axes = plt.subplots(2, 2, figsize=(14, 9))

    # Top-left: label distribution
    counts = news_df['finbert_label'].value_counts()
    axes[0, 0].bar(counts.index, counts.values,
                   color=['#2ecc71' if l == 'positive' else '#e74c3c' if l == 'negative' else '#95a5a6' for l in counts.index],
                   edgecolor='white')
    axes[0, 0].set_title('FinBERT Label Distribution', fontweight='bold')
    axes[0, 0].set_ylabel('Article Count')

    # Top-right: confidence distribution
    news_df['finbert_confidence'].plot.hist(bins=20, ax=axes[0, 1], color='steelblue', edgecolor='white')
    axes[0, 1].axvline(0.6, color='red', linestyle='--', linewidth=1.5, label='Confidence threshold (0.6)')
    axes[0, 1].set_title('FinBERT Confidence Distribution', fontweight='bold')
    axes[0, 1].set_xlabel('Confidence Score')
    axes[0, 1].legend()

    # Bottom-left: sentiment by source
    by_source = news_df.groupby('source')['composite_score'].mean().sort_values()
    bar_colors = ['#e74c3c' if v < 0 else '#2ecc71' for v in by_source.values]
    by_source.plot(kind='barh', ax=axes[1, 0], color=bar_colors, edgecolor='white')
    axes[1, 0].axvline(0, color='black', linewidth=0.8)
    axes[1, 0].set_title('Avg Sentiment Score by Source', fontweight='bold')

    # Bottom-right: daily signal
    if len(daily_df) > 1:
        axes[1, 1].fill_between(daily_df['date'], daily_df['weighted_sentiment'], 0,
                                where=daily_df['weighted_sentiment'] >= 0, alpha=0.3, color='green')
        axes[1, 1].fill_between(daily_df['date'], daily_df['weighted_sentiment'], 0,
                                where=daily_df['weighted_sentiment'] < 0,  alpha=0.3, color='red')
        axes[1, 1].plot(daily_df['date'], daily_df['weighted_sentiment'], color='steelblue', linewidth=2)
        axes[1, 1].axhline(0, color='gray', linestyle='--', linewidth=0.8)
        axes[1, 1].set_title('Daily Weighted News Sentiment', fontweight='bold')
        plt.setp(axes[1, 1].xaxis.get_majorticklabels(), rotation=30)

    plt.suptitle('FinBERT Crypto News Sentiment Analysis', fontsize=14)
    plt.tight_layout()
    plt.show()


plot_finbert_news_results(news_scored, daily_news)

news_cols = ['source', 'title', 'published', 'finbert_label', 'finbert_score',
             'finbert_confidence', 'composite_score', 'composite_label', 'link']
news_scored[news_cols].to_csv('finbert_news_sentiment.csv', index=False)
daily_news.to_csv('finbert_daily_signal.csv', index=False)
print(f'Exported: {len(news_scored)} articles, {len(daily_news)} daily rows')
print('\nTop 5 most positive headlines:')
print(news_scored.nlargest(5, 'composite_score')[['source', 'title', 'composite_score']].to_string(index=False))
print('\nTop 5 most negative headlines:')
print(news_scored.nsmallest(5, 'composite_score')[['source', 'title', 'composite_score']].to_string(index=False))