!pip install gensim pyLDAvis feedparser bertopic sentence-transformers umap-learn hdbscan nltk requests tqdm --quiet
!python -m nltk.downloader stopwords wordnet punkt -q

import gensim
import gensim.corpora as corpora
from gensim.models import LdaMulticore
from gensim.models.coherencemodel import CoherenceModel
import pyLDAvis
import pyLDAvis.gensim_models as gensimvis
import feedparser
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import requests
import re
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from datetime import datetime, timezone
from dateutil import parser as dateparser
from tqdm.auto import tqdm
import warnings
import logging

warnings.filterwarnings('ignore')
logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.WARNING)
%matplotlib inline
plt.rcParams['figure.figsize'] = (13, 5)
plt.rcParams['axes.spines.top'] = False
plt.rcParams['axes.spines.right'] = False
sns.set_palette('husl')

LEMMATIZER = WordNetLemmatizer()
STOP_WORDS = set(stopwords.words('english'))
CRYPTO_STOPWORDS = {
    'crypto', 'cryptocurrency', 'market', 'price', 'trading',
    'blockchain', 'token', 'coin', 'exchange', 'said', 'says',
    'also', 'would', 'could', 'new', 'one', 'week', 'day'
}
STOP_WORDS |= CRYPTO_STOPWORDS

print('All imports successful.')

import pandas as pd
# ── CONFIGURATION ─────────────────────────────────────────────────────────────
NEWS_API_KEY = 'e7bbbd6a37a74ed7bdcd783dbfc7c67f'   # Replace with your NewsAPI key.
N_TOPICS = 10                               # number of LDA topics to discover
RANDOM_SEED = 42

RSS_FEEDS = [
    'https://cointelegraph.com/rss',
    'https://coindesk.com/arc/outboundfeeds/rss/',
    'https://decrypt.co/feed',
    'https://bitcoinmagazine.com/.rss/full/',
]
# ─────────────────────────────────────────────────────────────────────────────


def fetch_rss_news(feed_urls: list, max_per_feed: int = 100) -> pd.DataFrame:
    """
    Fetch news articles from a list of RSS feed URLs.

    Parameters
    ----------
    feed_urls : list of str
        RSS feed URLs to fetch articles from.
    max_per_feed : int
        Maximum number of articles to retrieve per feed.

    Returns
    -------
    pd.DataFrame
        DataFrame with columns: title, summary, published, source, text.
        'text' is the concatenation of title and summary for modeling.
    """
    records = []
    for url in feed_urls:
        try:
            feed = feedparser.parse(url)
            source_name = feed.feed.get('title', url)
            for entry in feed.entries[:max_per_feed]:
                published = None
                if hasattr(entry, 'published'):
                    try:
                        published = dateparser.parse(entry.published).replace(tzinfo=timezone.utc)
                    except Exception:
                        pass
                records.append({
                    'title':   entry.get('title', ''),
                    'summary': entry.get('summary', ''),
                    'published': published,
                    'source':  source_name,
                })
        except Exception as e:
            print(f'Feed failed ({url[:40]}): {e}')

    df = pd.DataFrame(records).dropna(subset=['title'])
    df['text'] = df['title'] + ' ' + df['summary'].fillna('')
    print(f'Fetched {len(df)} articles from {len(feed_urls)} RSS feeds.')
    return df


def fetch_news_api(api_key: str, pages: int = 5) -> pd.DataFrame:
    """
    Fetch structured crypto news from NewsAPI.

    Parameters
    ----------
    api_key : str
        API key for NewsAPI.
    pages : int
        Number of pages/results to retrieve (max 100 articles per page).

    Returns
    -------
    pd.DataFrame
        DataFrame with columns: title, summary, published, source, text.
        Returns an empty DataFrame if no API key is provided or fetching fails.
    """
    if not api_key or api_key == 'YOUR_API_KEY':
        print("No NewsAPI key provided. Skipping fetching from NewsAPI.")
        return pd.DataFrame(columns=['title', 'summary', 'published', 'source', 'text'])

    base_url = 'https://newsapi.org/v2/everything'
    records = []
    for page in range(1, pages + 1):
        params = {'apiKey': api_key, 'q': 'cryptocurrency', 'language': 'en', 'pageSize': 100, 'page': page}
        try:
            resp = requests.get(base_url, params=params, timeout=10)
            resp.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
            for article in resp.json().get('articles', []):
                published = None
                if article.get('publishedAt'):
                    try:
                        published = dateparser.parse(article.get('publishedAt')).replace(tzinfo=timezone.utc)
                    except Exception:
                        pass
                records.append({
                    'title':   article.get('title', ''),
                    'summary': article.get('description', ''),
                    'published': published,
                    'source':  article.get('source', {}).get('name', 'NewsAPI'),
                    'text':    article.get('title', '') + ' ' + (article.get('description', '') or ''),
                })
        except requests.exceptions.RequestException as e:
            print(f'NewsAPI page {page} failed: {e}')
            break
        except Exception as e:
            print(f'Error processing NewsAPI response for page {page}: {e}')
            break

    df = pd.DataFrame(records).dropna(subset=['title'])
    print(f'Fetched {len(df)} articles from NewsAPI.')
    return df


def combine_news_sources(*dfs: pd.DataFrame) -> pd.DataFrame:
    """
    Merge multiple news DataFrames and deduplicate by title.

    Parameters
    ----------
    *dfs : pd.DataFrame
        Any number of news DataFrames sharing the same column schema.

    Returns
    -------
    pd.DataFrame
        Combined, deduplicated DataFrame sorted by published date descending.
    """
    combined = pd.concat(dfs, ignore_index=True)
    combined['_key'] = combined['title'].str.lower().str.strip()
    combined = combined.drop_duplicates(subset='_key').drop(columns='_key')
    combined = combined.sort_values('published', ascending=False).reset_index(drop=True)
    print(f'Combined dataset: {len(combined)} unique articles.')
    return combined


# ── Run collection ─────────────────────────────────────────────────────────────
news_df = fetch_rss_news(RSS_FEEDS, max_per_feed=100)

alt_df = fetch_news_api(NEWS_API_KEY, pages=1)
news_df = combine_news_sources(news_df, alt_df)

print(news_df[['title', 'published', 'source']].head())

def clean_text(text: str) -> str:
    """
    Strip HTML, URLs, numbers, and punctuation from raw text.

    Parameters
    ----------
    text : str
        Raw article text that may contain HTML markup, hyperlinks, etc.

    Returns
    -------
    str
        Lowercased plain text containing only alphabetic characters and spaces.
    """
    text = re.sub(r'<[^>]+>', ' ', text)        # strip HTML tags
    text = re.sub(r'http\S+', ' ', text)         # remove URLs
    text = re.sub(r'[^a-zA-Z\s]', ' ', text)    # keep only letters
    text = re.sub(r'\s+', ' ', text).strip()
    return text.lower()


def tokenize_and_lemmatize(text: str, stop_words: set, min_token_len: int = 3) -> list:
    """
    Clean, tokenize, and lemmatize a single document string.

    Parameters
    ----------
    text : str
        Raw document text.
    stop_words : set of str
        Words to remove (English stopwords + domain-specific terms).
    min_token_len : int
        Discard tokens shorter than this length (removes noise like 'bt', 'th').

    Returns
    -------
    list of str
        Cleaned token list ready to feed into gensim corpus building.
    """
    cleaned = clean_text(text)
    tokens = [
        LEMMATIZER.lemmatize(tok)
        for tok in cleaned.split()
        if tok not in stop_words and len(tok) >= min_token_len
    ]
    return tokens


def build_gensim_corpus(
    tokenized_docs: list,
    no_below: int = 2,
    no_above: float = 0.90
) -> tuple:
    """
    Build a gensim Dictionary and Bag-of-Words corpus from tokenized documents.

    Parameters
    ----------
    tokenized_docs : list of list of str
        Each element is the token list for one document.
    no_below : int
        Remove tokens appearing in fewer than this many documents.
        Filters hapax legomena and rare misspellings.
    no_above : float
        Remove tokens appearing in more than this fraction of all documents.
        Filters de-facto stopwords not caught earlier.

    Returns
    -------
    tuple : (id2word, corpus)
        id2word : gensim.corpora.Dictionary  — word-to-integer mapping
        corpus  : list of list of (int, int) — BoW representation of each doc
    """
    id2word = corpora.Dictionary(tokenized_docs)
    id2word.filter_extremes(no_below=no_below, no_above=no_above)
    corpus = [id2word.doc2bow(doc) for doc in tokenized_docs]
    print(f'Vocabulary: {len(id2word):,} unique tokens')
    print(f'Corpus:     {len(corpus):,} documents')
    return id2word, corpus


# ── Run preprocessing ─────────────────────────────────────────────────────────
print('Tokenizing and lemmatizing...')
tokenized_docs = [
    tokenize_and_lemmatize(text, STOP_WORDS)
    for text in tqdm(news_df['text'].fillna(''))
]
id2word, corpus = build_gensim_corpus(tokenized_docs, no_below=2, no_above=0.90)

print(f'\nSample tokens (article 0): {tokenized_docs[0][:15]}')

def train_lda_model(
    corpus: list,
    id2word,
    n_topics: int = 10,
    passes: int = 15,
    workers: int = 2
) -> LdaMulticore:
    """
    Train a parallelized LDA model using gensim's LdaMulticore.

    Parameters
    ----------
    corpus : list
        BoW corpus from build_gensim_corpus().
    id2word : gensim.corpora.Dictionary
        Word-integer mapping.
    n_topics : int
        Number of latent topics. Start with 10 for general crypto news.
    passes : int
        Training iterations over the full corpus. More passes → better convergence.
    workers : int
        Parallel CPU workers. Set to (number of cores - 1).

    Returns
    -------
    LdaMulticore
        Trained LDA model ready for inference and visualization.
    """
    np.random.seed(RANDOM_SEED)
    model = LdaMulticore(
        corpus=corpus,
        id2word=id2word,
        num_topics=n_topics,
        passes=passes,
        workers=workers,
        random_state=RANDOM_SEED,
        per_word_topics=True,
    )
    print(f'LDA trained: {n_topics} topics, {passes} passes.')
    return model


def compute_topic_coherence(lda_model, tokenized_docs: list, id2word, corpus: list) -> float:
    """
    Compute the c_v coherence score — the standard LDA quality metric.

    Parameters
    ----------
    lda_model : LdaMulticore
        Trained LDA model to evaluate.
    tokenized_docs : list of list of str
        Original tokenized documents (not BoW).
    id2word : gensim.corpora.Dictionary
    corpus : list
        BoW corpus.

    Returns
    -------
    float
        Coherence score. Scores of 0.45–0.65 are typical for news data.
        Higher is better — topics whose top words co-occur frequently score higher.
    """
    cm = CoherenceModel(
        model=lda_model, texts=tokenized_docs,
        dictionary=id2word, coherence='c_v'
    )
    score = cm.get_coherence()
    print(f'Coherence (c_v): {score:.4f}')
    return score


def extract_topic_keywords(lda_model, n_words: int = 12) -> dict:
    """
    Extract the top keywords and their probabilities for each topic.

    Parameters
    ----------
    lda_model : LdaMulticore
        Trained LDA model.
    n_words : int
        Number of top words to return per topic.

    Returns
    -------
    dict
        {topic_id (int): [(word, probability), ...]} for all topics.
    """
    return {
        tid: lda_model.show_topic(tid, topn=n_words)
        for tid in range(lda_model.num_topics)
    }


def assign_dominant_topic(lda_model, corpus: list, news_df: pd.DataFrame) -> pd.DataFrame:
    """
    Append dominant topic ID and probability to each article in the DataFrame.

    Parameters
    ----------
    lda_model : LdaMulticore
        Trained LDA model.
    corpus : list
        BoW corpus aligned with news_df rows.
    news_df : pd.DataFrame
        Original news DataFrame to annotate.

    Returns
    -------
    pd.DataFrame
        news_df with added columns: dominant_topic (int), topic_prob (float).
    """
    rows = []
    for doc_bow in corpus:
        dist = lda_model.get_document_topics(doc_bow, minimum_probability=0)
        top = max(dist, key=lambda x: x[1])
        rows.append({'dominant_topic': top[0], 'topic_prob': round(top[1], 4)})
    result = news_df.copy()
    result['dominant_topic'] = [r['dominant_topic'] for r in rows]
    result['topic_prob']     = [r['topic_prob']     for r in rows]
    return result


def print_topic_summary(topic_keywords: dict, topic_labels: dict = None):
    """
    Print a human-readable summary of discovered topics with top keywords.

    Parameters
    ----------
    topic_keywords : dict
        Output of extract_topic_keywords().
    topic_labels : dict, optional
        {topic_id: 'Human-readable label'} assigned after inspecting keywords.
    """
    print('=' * 65)
    print('DISCOVERED TOPICS')
    print('=' * 65)
    for tid, words in topic_keywords.items():
        label = topic_labels.get(tid, f'Topic {tid}') if topic_labels else f'Topic {tid}'
        top_words = ', '.join([w for w, _ in words[:8]])
        print(f'[{label:<22}] {top_words}')
    print('=' * 65)


# ── Train LDA ─────────────────────────────────────────────────────────────────
lda_model  = train_lda_model(corpus, id2word, n_topics=N_TOPICS, passes=15)
coherence  = compute_topic_coherence(lda_model, tokenized_docs, id2word, corpus)
topic_kws  = extract_topic_keywords(lda_model, n_words=12)
print_topic_summary(topic_kws)

doc_df = assign_dominant_topic(lda_model, corpus, news_df)

# ── Assign human-readable labels after reviewing keywords above ───────────────
TOPIC_LABELS = {
    0: 'Regulatory & Legal',
    1: 'DeFi & Protocols',
    2: 'Macro & Fed Policy',
    3: 'Exchange & Liquidity',
    4: 'Bitcoin & Halvings',
    5: 'NFT & Gaming',
    6: 'Ethereum & Upgrades',
    7: 'Stablecoins & Risk',
    8: 'Institutional Adoption',
    9: 'On-Chain Metrics',
}
doc_df['topic_label'] = doc_df['dominant_topic'].map(TOPIC_LABELS)
print(doc_df[['title', 'dominant_topic', 'topic_prob', 'topic_label']].head(8))

def visualize_lda_interactive(lda_model, corpus: list, id2word):
    """
    Render an interactive pyLDAvis intertopic distance map in Colab.

    Parameters
    ----------
    lda_model : LdaMulticore
        Trained LDA model.
    corpus : list
        BoW corpus.
    id2word : gensim.corpora.Dictionary
        Word dictionary.

    Returns
    -------
    pyLDAvis PreparedData
        Interactive HTML visualization rendered inline in the notebook.

    Notes
    -----
    Use `mds='mmds'` (multi-dimensional scaling) for layout — stable and fast.
    Click a circle to inspect its top terms. Adjust the relevance slider (lambda)
    to weight term frequency vs topic exclusivity.
    """
    pyLDAvis.enable_notebook()
    vis = gensimvis.prepare(lda_model, corpus, id2word, mds='mmds', sort_topics=False)
    return pyLDAvis.display(vis)


def plot_topic_distribution(doc_df: pd.DataFrame, topic_labels: dict) -> None:
    """
    Bar chart of article count per discovered topic.

    Parameters
    ----------
    doc_df : pd.DataFrame
        DataFrame with 'dominant_topic' column from assign_dominant_topic().
    topic_labels : dict
        {topic_id: 'Label'} for axis labeling.
    """
    counts = doc_df['dominant_topic'].value_counts().sort_index()
    labels = [topic_labels.get(i, f'Topic {i}') for i in counts.index]
    palette = sns.color_palette('husl', len(labels))

    fig, ax = plt.subplots(figsize=(12, 5))
    bars = ax.bar(labels, counts.values, color=palette)
    ax.set_xlabel('Topic')
    ax.set_ylabel('Article Count')
    ax.set_title('Article Distribution Across Topics')
    plt.xticks(rotation=35, ha='right')
    for bar, count in zip(bars, counts.values):
        ax.text(bar.get_x() + bar.get_width() / 2,
                bar.get_height() + 0.5, str(count),
                ha='center', va='bottom', fontsize=9)
    plt.tight_layout()
    plt.show()


def plot_topic_trend_over_time(doc_df: pd.DataFrame, topic_labels: dict, top_n: int = 5) -> None:
    """
    Line chart showing how topic prevalence shifts over time.

    Parameters
    ----------
    doc_df : pd.DataFrame
        Must have 'published' (datetime) and 'dominant_topic' columns.
    topic_labels : dict
        {topic_id: 'Label'}.
    top_n : int
        Plot only the top N topics by total article count.

    Notes
    -----
    Spikes in specific topics can act as early signals — e.g., a surge in
    'Regulatory & Legal' articles often precedes increased volatility.
    """
    df = doc_df.dropna(subset=['published']).copy()
    df['date'] = pd.to_datetime(df['published']).dt.date
    top_ids = df['dominant_topic'].value_counts().head(top_n).index
    pivot = (
        df[df['dominant_topic'].isin(top_ids)]
        .groupby(['date', 'dominant_topic'])
        .size()
        .unstack(fill_value=0)
    )
    pivot.columns = [topic_labels.get(c, f'Topic {c}') for c in pivot.columns]

    fig, ax = plt.subplots(figsize=(14, 5))
    pivot.plot(ax=ax, linewidth=2, marker='o', markersize=4)
    ax.set_title('Topic Prevalence Over Time')
    ax.set_xlabel('Date')
    ax.set_ylabel('Article Count')
    ax.legend(loc='upper left', fontsize=9)
    plt.tight_layout()
    plt.show()


# ── Visualize ─────────────────────────────────────────────────────────────────
visualize_lda_interactive(lda_model, corpus, id2word)
plot_topic_distribution(doc_df, TOPIC_LABELS)
plot_topic_trend_over_time(doc_df, TOPIC_LABELS, top_n=5)

def train_bertopic_model(texts: list, min_topic_size: int = 8, n_topics=None) -> tuple:
    """
    Train a BERTopic model on raw (un-tokenized) text documents.

    Parameters
    ----------
    texts : list of str
        Raw article texts. BERTopic handles preprocessing internally.
    min_topic_size : int
        Minimum cluster size for HDBSCAN. Larger values → fewer, broader topics.
        For 500 articles try 8–15; for 5000+ articles try 20–50.
    n_topics : int or None
        Force a specific number of topics after clustering (merges small topics).
        Use None for automatic detection.

    Returns
    -------
    tuple : (topic_model, topics, probs)
        topic_model : BERTopic — trained model with built-in visualization methods
        topics      : list of int — topic ID per document (-1 = outlier)
        probs       : np.ndarray — topic probability per document

    Notes
    -----
    Topic -1 contains outlier documents that did not fit any cluster.
    High outlier counts suggest min_topic_size is too large.
    """
    from bertopic import BERTopic
    from sentence_transformers import SentenceTransformer

    embedding_model = SentenceTransformer('all-MiniLM-L6-v2')
    topic_model = BERTopic(
        embedding_model=embedding_model,
        nr_topics=n_topics,
        min_topic_size=min_topic_size,
        verbose=True,
    )
    topics, probs = topic_model.fit_transform(texts)
    n_discovered = len(set(topics)) - (1 if -1 in topics else 0)
    n_outliers   = topics.count(-1) if isinstance(topics, list) else (topics == -1).sum()
    print(f'BERTopic: {n_discovered} topics | {n_outliers} outliers (-1)')
    return topic_model, topics, probs


def get_bertopic_topic_summary(topic_model) -> pd.DataFrame:
    """
    Return a summary DataFrame of all BERTopic topics with representative terms.

    Parameters
    ----------
    topic_model : BERTopic
        Trained BERTopic model.

    Returns
    -------
    pd.DataFrame
        Columns: Topic, Count, Name, Representation.
        Sorted by document count descending. Outlier topic (-1) excluded.
    """
    info = topic_model.get_topic_info()
    info = info[info['Topic'] != -1].sort_values('Count', ascending=False)
    print(info[['Topic', 'Count', 'Name']].head(15).to_string(index=False))
    return info


# ── Run BERTopic (GPU recommended) ────────────────────────────────────────────
raw_texts   = news_df['text'].fillna('').tolist()
bert_model, bert_topics, bert_probs = train_bertopic_model(raw_texts, min_topic_size=8)
bert_summary = get_bertopic_topic_summary(bert_model)

# Built-in BERTopic visualizations
bert_model.visualize_topics()       # intertopic distance map
bert_model.visualize_barchart(top_n_topics=10)  # top words per topic

BULLISH_TOPICS = {'Institutional Adoption', 'Bitcoin & Halvings', 'Ethereum & Upgrades'}
BEARISH_TOPICS = {'Regulatory & Legal', 'Stablecoins & Risk', 'Exchange & Liquidity'}


def compute_daily_topic_signal(
    doc_df: pd.DataFrame,
    bullish_topics: set,
    bearish_topics: set
) -> pd.DataFrame:
    """
    Compute a daily net topic signal from article topic labels.

    Parameters
    ----------
    doc_df : pd.DataFrame
        Must have 'published' and 'topic_label' columns.
    bullish_topics : set of str
        Topic labels associated with positive market conditions.
    bearish_topics : set of str
        Topic labels associated with negative market conditions.

    Returns
    -------
    pd.DataFrame
        Daily rows with columns: date, bullish, bearish, neutral, net_signal,
        normalized_signal (range -1 to +1).
    """
    df = doc_df.dropna(subset=['published', 'topic_label']).copy()
    df['date'] = pd.to_datetime(df['published']).dt.date
    df['signal_type'] = df['topic_label'].apply(
        lambda t: 'bullish' if t in bullish_topics
        else ('bearish' if t in bearish_topics else 'neutral')
    )
    daily = (
        df.groupby(['date', 'signal_type'])
        .size()
        .unstack(fill_value=0)
        .reindex(columns=['bullish', 'bearish', 'neutral'], fill_value=0)
    )
    daily['net_signal']         = daily['bullish'] - daily['bearish']
    total                       = daily['bullish'] + daily['bearish'] + 1
    daily['normalized_signal']  = daily['net_signal'] / total
    return daily.reset_index()


def detect_topic_surge(
    doc_df: pd.DataFrame,
    topic_label: str,
    window_hours: int = 24,
    z_threshold: float = 1.5
) -> pd.DataFrame:
    """
    Detect statistically significant surges in a specific topic's article volume.

    Parameters
    ----------
    doc_df : pd.DataFrame
        Must have 'published' and 'topic_label' columns.
    topic_label : str
        The topic name to monitor (e.g., 'Regulatory & Legal').
    window_hours : int
        Rolling window in hours for baseline computation.
    z_threshold : float
        Z-score threshold above which an hour is flagged as a surge.
        A value of 1.5 catches 93rd-percentile spikes.

    Returns
    -------
    pd.DataFrame
        Hourly article counts with z_score and boolean surge flag.
    """
    df = doc_df[doc_df['topic_label'] == topic_label].dropna(subset=['published']).copy()
    df = df.set_index(pd.to_datetime(df['published'])).sort_index()
    hourly = df.resample('1H').size().rename('count').reset_index()
    hourly.columns = ['datetime', 'count']

    hourly['rolling_mean'] = hourly['count'].rolling(window_hours, min_periods=1).mean()
    hourly['rolling_std']  = hourly['count'].rolling(window_hours, min_periods=1).std().fillna(1).clip(lower=0.1)
    hourly['z_score']      = (hourly['count'] - hourly['rolling_mean']) / hourly['rolling_std']
    hourly['surge']        = hourly['z_score'] > z_threshold

    print(f'[{topic_label}]: {hourly["surge"].sum()} surge events (z > {z_threshold})')
    return hourly


def plot_topic_signal(daily_signal: pd.DataFrame) -> None:
    """
    Visualize the daily net topic signal alongside bullish/bearish article volumes.

    Parameters
    ----------
    daily_signal : pd.DataFrame
        Output from compute_daily_topic_signal().
    """
    fig, axes = plt.subplots(2, 1, figsize=(14, 8), sharex=True)

    ax = axes[0]
    colors = ['green' if v >= 0 else 'red' for v in daily_signal['net_signal']]
    ax.bar(daily_signal['date'], daily_signal['net_signal'], color=colors, alpha=0.75)
    ax.axhline(0, color='black', linewidth=0.8)
    ax.set_title('Daily Net Topic Signal  (Bullish Topics − Bearish Topics)')
    ax.set_ylabel('Net Count')

    ax = axes[1]
    ax.bar(daily_signal['date'],  daily_signal['bullish'],  label='Bullish', color='green', alpha=0.65)
    ax.bar(daily_signal['date'], -daily_signal['bearish'],  label='Bearish', color='red',   alpha=0.65)
    ax.axhline(0, color='black', linewidth=0.8)
    ax.set_title('Bullish vs Bearish Article Volume')
    ax.set_ylabel('Article Count')
    ax.legend()

    plt.tight_layout()
    plt.show()


# ── Generate signals ──────────────────────────────────────────────────────────
daily_signal = compute_daily_topic_signal(doc_df, BULLISH_TOPICS, BEARISH_TOPICS)
plot_topic_signal(daily_signal)

surge_df = detect_topic_surge(doc_df, 'Regulatory & Legal', window_hours=24, z_threshold=1.5)
print(surge_df[surge_df['surge']].head())

def export_topic_results(
    doc_df: pd.DataFrame,
    daily_signal: pd.DataFrame,
    prefix: str = 'topic_modeling_crypto_news'
) -> None:
    """
    Export article topic assignments and daily signal DataFrames to CSV.

    Parameters
    ----------
    doc_df : pd.DataFrame
        Article-level DataFrame with topic columns from assign_dominant_topic().
    daily_signal : pd.DataFrame
        Daily aggregated signal from compute_daily_topic_signal().
    prefix : str
        Filename prefix for both output CSVs.

    Outputs
    -------
    {prefix}_articles.csv  — article-level topic labels
    {prefix}_signals.csv   — daily bullish/bearish/net signal
    """
    article_cols = ['title', 'published', 'source', 'dominant_topic', 'topic_prob', 'topic_label']
    articles_out = f'{prefix}_articles.csv'
    signals_out  = f'{prefix}_signals.csv'

    doc_df[article_cols].to_csv(articles_out, index=False)
    daily_signal.to_csv(signals_out, index=False)

    print(f'Exported: {articles_out}  ({len(doc_df)} articles)')
    print(f'Exported: {signals_out}   ({len(daily_signal)} daily rows)')


export_topic_results(doc_df, daily_signal)