!pip install spacy feedparser pandas matplotlib seaborn tqdm transformers torch --quiet
!python -m spacy download en_core_web_sm --quiet

import spacy
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sns
import feedparser
import re
from datetime import datetime, timezone
from dateutil import parser as dateparser
from collections import Counter
from tqdm.auto import tqdm

%matplotlib inline
plt.rcParams['figure.figsize'] = (12, 5)
plt.rcParams['axes.spines.top'] = False
plt.rcParams['axes.spines.right'] = False

nlp = spacy.load('en_core_web_sm')
print(f'spaCy loaded. Pipeline: {nlp.pipe_names}')

EVENT_TAXONOMY = {
    'hack':           {'keywords': ['hack', 'exploit', 'breach', 'stolen', 'vulnerability', 'attack', 'drained'],          'impact': 'bearish'},
    'regulatory':     {'keywords': ['sec', 'cftc', 'fine', 'lawsuit', 'ban', 'regulation', 'compliance', 'subpoena'],       'impact': 'bearish'},
    'adoption':       {'keywords': ['etf', 'institutional', 'approve', 'partnership', 'integration', 'launch', 'accept'],    'impact': 'bullish'},
    'listing':        {'keywords': ['listed', 'listing', 'added to', 'now trading', 'pairs added'],                         'impact': 'bullish'},
    'delisting':      {'keywords': ['delist', 'delisting', 'removed from', 'suspended'],                                    'impact': 'bearish'},
    'upgrade':        {'keywords': ['upgrade', 'hard fork', 'soft fork', 'mainnet', 'testnet', 'v2', 'migration'],          'impact': 'bullish'},
    'macro':          {'keywords': ['fed', 'inflation', 'rate hike', 'rate cut', 'cpi', 'gdp', 'recession', 'fomc'],        'impact': 'neutral'},
    'liquidation':    {'keywords': ['liquidat', 'margin call', 'forced sell', 'wipeout'],                                   'impact': 'bearish'},
    'fundraising':    {'keywords': ['raise', 'funding round', 'series a', 'series b', 'venture', 'investment'],             'impact': 'bullish'},
    'whale_activity': {'keywords': ['whale', 'large transfer', 'moved', 'wallet', 'dormant'],                               'impact': 'neutral'}
}

KNOWN_COINS = {
    'bitcoin': 'BTC', 'btc': 'BTC',
    'ethereum': 'ETH', 'eth': 'ETH',
    'solana': 'SOL', 'sol': 'SOL',
    'binance coin': 'BNB', 'bnb': 'BNB',
    'cardano': 'ADA', 'ada': 'ADA',
    'ripple': 'XRP', 'xrp': 'XRP',
    'dogecoin': 'DOGE', 'doge': 'DOGE',
    'avalanche': 'AVAX', 'avax': 'AVAX',
    'polygon': 'MATIC', 'matic': 'MATIC',
    'chainlink': 'LINK', 'link': 'LINK',
    'polkadot': 'DOT', 'dot': 'DOT',
    'uniswap': 'UNI', 'uni': 'UNI',
    'tether': 'USDT', 'usdt': 'USDT',
    'usdc': 'USDC', 'dai': 'DAI'
}

KNOWN_EXCHANGES = [
    'binance', 'coinbase', 'kraken', 'bybit', 'okx', 'bitfinex',
    'gemini', 'kucoin', 'huobi', 'ftx', 'bitget', 'mexc'
]

AMOUNT_PATTERN = re.compile(
    r'\$([\d,]+(?:\.\d+)?\s?(?:million|billion|m|b|k)?)',
    re.IGNORECASE
)

print(f'Taxonomy: {len(EVENT_TAXONOMY)} event types')
print(f'Known coins: {len(KNOWN_COINS)} entries')

CRYPTO_RSS_FEEDS = {
    'CoinDesk':      'https://www.coindesk.com/arc/outboundfeeds/rss/',
    'Cointelegraph': 'https://cointelegraph.com/rss',
    'Decrypt':       'https://decrypt.co/feed'
}


def fetch_news_articles(feeds: dict, max_per_source: int = 50) -> pd.DataFrame:
    articles = []
    for source, url in feeds.items():
        try:
            feed = feedparser.parse(url)
            for entry in feed.entries[:max_per_source]:
                title   = entry.get('title', '')
                summary = re.sub('<[^>]+>', '', entry.get('summary', ''))[:500]
                published = None
                if hasattr(entry, 'published'):
                    try:
                        published = dateparser.parse(entry.published)
                    except Exception:
                        pass
                articles.append({
                    'source': source,
                    'title': title,
                    'summary': summary,
                    'full_text': f'{title}. {summary}',
                    'published': published,
                    'link': entry.get('link', '')
                })
        except Exception as e:
            print(f'WARNING: {source} — {e}')

    df = pd.DataFrame(articles)
    df = df[df['title'].str.len() > 5].reset_index(drop=True)
    print(f'Fetched {len(df)} articles')
    return df


news_df = fetch_news_articles(CRYPTO_RSS_FEEDS)
news_df[['source', 'title', 'published']].head(5)

def extract_coins_from_text(text: str, coin_dict: dict) -> list[str]:
    """
    Extract cryptocurrency ticker symbols from text using a lookup dictionary.

    Looks for both full names ('bitcoin') and tickers ('BTC') in the text.
    Uses word boundary matching to avoid partial matches.

    Parameters
    ----------
    text      : Text string to search
    coin_dict : Dictionary mapping {name/ticker_lowercase: TICKER}

    Returns
    -------
    Sorted list of unique ticker symbols found
    """
    text_lower = text.lower()
    found = set()
    for name, ticker in coin_dict.items():
        pattern = r'\b' + re.escape(name) + r'\b'
        if re.search(pattern, text_lower):
            found.add(ticker)
    return sorted(found)


def extract_exchanges_from_text(text: str, exchanges: list[str]) -> list[str]:
    """
    Extract exchange names mentioned in text.

    Parameters
    ----------
    text      : Text to search
    exchanges : List of exchange name strings (lowercase)

    Returns
    -------
    List of matched exchange names
    """
    text_lower = text.lower()
    return [ex for ex in exchanges if re.search(r'\b' + re.escape(ex) + r'\b', text_lower)]


def extract_amounts(text: str) -> list[str]:
    """
    Extract dollar amounts from text using regex.

    Parameters
    ----------
    text : Text to search

    Returns
    -------
    List of matched amount strings (e.g., ['50 million', '300'])
    """
    return AMOUNT_PATTERN.findall(text)


def run_ner_on_text(text: str, spacy_model) -> dict:
    """
    Run spaCy NER and custom extraction on a single text.

    Extracts: organizations, monetary amounts, dates, persons,
    crypto coins (custom), and exchanges (custom).

    Parameters
    ----------
    text        : Text to analyze
    spacy_model : Loaded spaCy language model

    Returns
    -------
    Dictionary of extracted entity lists
    """
    doc = spacy_model(text[:1000])

    orgs    = list(set(ent.text for ent in doc.ents if ent.label_ == 'ORG'))
    money   = list(set(ent.text for ent in doc.ents if ent.label_ == 'MONEY'))
    dates   = list(set(ent.text for ent in doc.ents if ent.label_ == 'DATE'))
    persons = list(set(ent.text for ent in doc.ents if ent.label_ == 'PERSON'))

    coins     = extract_coins_from_text(text, KNOWN_COINS)
    exchanges = extract_exchanges_from_text(text, KNOWN_EXCHANGES)
    amounts   = extract_amounts(text)

    return {
        'orgs':      orgs,
        'money':     money,
        'dates':     dates,
        'persons':   persons,
        'coins':     coins,
        'exchanges': exchanges,
        'amounts':   amounts
    }


# Test NER on a sample headline
test_text = 'Binance fined $4.3 billion by US DOJ and FinCEN; CEO CZ resigns from exchange'
test_ner = run_ner_on_text(test_text, nlp)
print('NER test results:')
for key, values in test_ner.items():
    if values:
        print(f'  {key:12s}: {values}')

def classify_event_type(text: str, taxonomy: dict) -> tuple[str, str]:
    """
    Classify text into an event type based on keyword matching.

    Scores each event type by counting keyword matches.
    Returns the best-matching type and its default market impact.

    Parameters
    ----------
    text     : Text to classify (title + summary)
    taxonomy : Event taxonomy dictionary

    Returns
    -------
    Tuple of (event_type_string, market_impact_string)
    """
    text_lower = text.lower()
    scores = {}

    for event_type, config in taxonomy.items():
        score = sum(1 for kw in config['keywords'] if kw in text_lower)
        if score > 0:
            scores[event_type] = score

    if not scores:
        return 'general', 'neutral'

    best_type = max(scores, key=scores.get)
    impact    = taxonomy[best_type]['impact']
    return best_type, impact


def extract_events_from_dataframe(
    df: pd.DataFrame,
    spacy_model,
    taxonomy: dict
) -> pd.DataFrame:
    """
    Run full event extraction pipeline on all articles in a DataFrame.

    For each article: runs NER, classifies event type, and assembles
    a structured event record.

    Parameters
    ----------
    df          : News DataFrame with 'full_text' and 'title' columns
    spacy_model : Loaded spaCy model
    taxonomy    : Event taxonomy dictionary

    Returns
    -------
    Structured events DataFrame
    """
    results = []

    for _, row in tqdm(df.iterrows(), total=len(df), desc='Event extraction'):
        text = row.get('full_text', row.get('title', ''))
        ner  = run_ner_on_text(text, spacy_model)
        event_type, default_impact = classify_event_type(text, taxonomy)

        results.append({
            'source':         row.get('source', ''),
            'title':          row.get('title', ''),
            'published':      row.get('published'),
            'link':           row.get('link', ''),
            'event_type':     event_type,
            'default_impact': default_impact,
            'coins':          ner['coins'],
            'exchanges':      ner['exchanges'],
            'orgs':           ner['orgs'][:5],
            'amounts':        ner['amounts'][:3],
            'persons':        ner['persons'][:3],
            'has_amount':     len(ner['amounts']) > 0,
            'coin_count':     len(ner['coins']),
            'is_multi_coin':  len(ner['coins']) > 1
        })

    return pd.DataFrame(results)


events_df = extract_events_from_dataframe(news_df, nlp, EVENT_TAXONOMY)
print(f'Extracted {len(events_df)} events')
events_df[['title', 'event_type', 'default_impact', 'coins', 'amounts']].head(6)

def build_event_timeline(events_df: pd.DataFrame) -> pd.DataFrame:
    """
    Build a chronological event timeline with daily summaries.

    Groups events by day and computes: event counts by type,
    bullish/bearish ratio, most affected coins, and a risk score.

    The risk score is:
      (hack_count * 3 + regulatory_count * 2 + liquidation_count * 2) / total_events

    Parameters
    ----------
    events_df : Output from extract_events_from_dataframe()

    Returns
    -------
    Daily timeline DataFrame
    """
    df = events_df.copy()
    df['published'] = pd.to_datetime(df['published'], utc=True, errors='coerce')
    df.dropna(subset=['published'], inplace=True)
    df['date'] = df['published'].dt.date

    def daily_summary(g):
        all_coins  = [c for coins in g['coins'] for c in coins]
        top_coins  = [coin for coin, _ in Counter(all_coins).most_common(5)]
        event_type_counts = g['event_type'].value_counts().to_dict()
        bullish = (g['default_impact'] == 'bullish').sum()
        bearish = (g['default_impact'] == 'bearish').sum()

        risk_score = (
            event_type_counts.get('hack', 0) * 3 +
            event_type_counts.get('regulatory', 0) * 2 +
            event_type_counts.get('liquidation', 0) * 2
        ) / max(len(g), 1)

        return pd.Series({
            'total_events':   len(g),
            'bullish_events': int(bullish),
            'bearish_events': int(bearish),
            'net_sentiment':  (bullish - bearish) / max(len(g), 1),
            'risk_score':     round(risk_score, 3),
            'top_coins':      ', '.join(top_coins) if top_coins else 'general',
            'event_types':    str(event_type_counts),
            'hack_events':    event_type_counts.get('hack', 0),
            'regulatory_events': event_type_counts.get('regulatory', 0),
            'adoption_events':   event_type_counts.get('adoption', 0)
        })

    timeline = df.groupby('date').apply(daily_summary).reset_index()
    timeline['date'] = pd.to_datetime(timeline['date'])
    timeline.sort_values('date', inplace=True)
    print(f'Event timeline: {len(timeline)} days, {events_df["event_type"].value_counts().to_dict()}')
    return timeline


timeline_df = build_event_timeline(events_df)
timeline_df.tail(5)

def plot_event_analysis(events_df: pd.DataFrame, timeline_df: pd.DataFrame) -> None:
    """
    Four-panel visualization of extracted events.

    Panels: event type distribution, most affected coins,
    daily event volume, daily risk score timeline.

    Parameters
    ----------
    events_df  : Article-level events DataFrame
    timeline_df: Daily timeline DataFrame
    """
    fig, axes = plt.subplots(2, 2, figsize=(15, 9))

    # Top-left: event type counts
    event_counts = events_df['event_type'].value_counts().head(8)
    impact_colors = [EVENT_TAXONOMY.get(et, {}).get('impact', 'neutral') for et in event_counts.index]
    bar_colors = ['#e74c3c' if c == 'bearish' else '#2ecc71' if c == 'bullish' else '#95a5a6' for c in impact_colors]
    axes[0, 0].barh(event_counts.index[::-1], event_counts.values[::-1], color=bar_colors[::-1], edgecolor='white')
    axes[0, 0].set_title('Event Type Distribution', fontweight='bold')
    axes[0, 0].set_xlabel('Article Count')

    # Top-right: most affected coins
    all_coins = [c for coins_list in events_df['coins'] for c in coins_list]
    if all_coins:
        coin_counts = pd.Series(Counter(all_coins)).nlargest(10)
        axes[0, 1].bar(coin_counts.index, coin_counts.values, color='steelblue', edgecolor='white')
        axes[0, 1].set_title('Most Mentioned Coins', fontweight='bold')
        axes[0, 1].set_ylabel('Mention Count')
        axes[0, 1].tick_params(axis='x', rotation=30)

    # Bottom-left: daily event volume by impact
    if len(timeline_df) > 1:
        axes[1, 0].bar(timeline_df['date'], timeline_df['bullish_events'],
                       label='Bullish', color='#2ecc71', alpha=0.7)
        axes[1, 0].bar(timeline_df['date'], -timeline_df['bearish_events'],
                       label='Bearish', color='#e74c3c', alpha=0.7)
        axes[1, 0].axhline(0, color='black', linewidth=0.8)
        axes[1, 0].set_title('Daily Bullish vs Bearish Events', fontweight='bold')
        axes[1, 0].legend()
        plt.setp(axes[1, 0].xaxis.get_majorticklabels(), rotation=30)

    # Bottom-right: daily risk score
    if len(timeline_df) > 1:
        axes[1, 1].fill_between(timeline_df['date'], timeline_df['risk_score'],
                                alpha=0.4, color='#e74c3c')
        axes[1, 1].plot(timeline_df['date'], timeline_df['risk_score'],
                        color='#c0392b', linewidth=2)
        axes[1, 1].set_title('Daily Risk Score\n(hack × 3 + regulatory × 2 + liquidation × 2) / total',
                             fontsize=10, fontweight='bold')
        axes[1, 1].set_ylabel('Risk Score')
        plt.setp(axes[1, 1].xaxis.get_majorticklabels(), rotation=30)

    plt.suptitle('Crypto News Event Extraction Analysis', fontsize=14)
    plt.tight_layout()
    plt.show()


plot_event_analysis(events_df, timeline_df)

def flag_high_impact_events(
    events_df: pd.DataFrame,
    high_impact_types: list = None
) -> pd.DataFrame:
    """
    Filter events to return only high-impact ones worth alerting on.

    High impact criteria:
    - Event type is in high_impact_types list, OR
    - Article mentions a dollar amount (implies significant scale)

    Parameters
    ----------
    events_df          : Full events DataFrame
    high_impact_types  : Event types to always flag as high impact

    Returns
    -------
    Filtered DataFrame of high-impact events
    """
    if high_impact_types is None:
        high_impact_types = ['hack', 'regulatory', 'adoption', 'listing']

    mask = (
        events_df['event_type'].isin(high_impact_types) |
        events_df['has_amount']
    )
    high_impact = events_df[mask].copy()
    high_impact['alert_priority'] = high_impact['event_type'].map(
        {'hack': 1, 'regulatory': 2, 'liquidation': 3, 'adoption': 4, 'listing': 5}
    ).fillna(6).astype(int)
    high_impact.sort_values(['alert_priority', 'published'], inplace=True)
    print(f'High-impact events: {len(high_impact)} / {len(events_df)} total')
    return high_impact.reset_index(drop=True)


high_impact_df = flag_high_impact_events(events_df)
high_impact_df[['event_type', 'default_impact', 'coins', 'amounts', 'title']].head(10)

def export_event_results(
    events_df: pd.DataFrame,
    timeline_df: pd.DataFrame,
    high_impact_df: pd.DataFrame
) -> None:
    """
    Export event extraction results to CSV files.

    Three files:
    1. Full event database (all articles with extracted events)
    2. Daily timeline with risk scores
    3. High-impact events only (for alerting systems)

    Parameters
    ----------
    events_df      : Full events DataFrame
    timeline_df    : Daily timeline
    high_impact_df : Filtered high-impact events
    """
    str_events = events_df.copy()
    for col in ['coins', 'exchanges', 'orgs', 'amounts', 'persons']:
        if col in str_events.columns:
            str_events[col] = str_events[col].apply(lambda x: '|'.join(x) if isinstance(x, list) else str(x))

    str_events.to_csv('crypto_news_events.csv', index=False)
    timeline_df.to_csv('crypto_event_timeline.csv', index=False)

    str_high = high_impact_df.copy()
    for col in ['coins', 'exchanges', 'orgs', 'amounts', 'persons']:
        if col in str_high.columns:
            str_high[col] = str_high[col].apply(lambda x: '|'.join(x) if isinstance(x, list) else str(x))
    str_high.to_csv('high_impact_events.csv', index=False)

    print(f'Exported:')
    print(f'  crypto_news_events.csv     — {len(events_df)} rows')
    print(f'  crypto_event_timeline.csv  — {len(timeline_df)} rows')
    print(f'  high_impact_events.csv     — {len(high_impact_df)} rows')


export_event_results(events_df, timeline_df, high_impact_df)