!pip install anthropic openai feedparser pandas matplotlib requests tqdm groq --quiet

import anthropic
import openai
import groq
import feedparser
import pandas as pd
import matplotlib.pyplot as plt
import requests
import json
import re
import time
from datetime import datetime, timezone
from dateutil import parser as dateparser
from tqdm.auto import tqdm

%matplotlib inline
plt.rcParams['figure.figsize'] = (12, 5)
plt.rcParams['axes.spines.top'] = False
plt.rcParams['axes.spines.right'] = False

print('Imports successful.')

# ─── CONFIGURATION ──────────────────────────────────────────────────────────
ANTHROPIC_API_KEY = 'YOUR_ANTHROPIC_API_KEY'  # or use Colab Secrets
OPENAI_API_KEY    = 'YOUR_OPENAI_API_KEY'
GROQ_API_KEY      = 'YOUR_GROQ_API_KEY'       # or use Colab Secrets

# Which LLM provider to use: 'anthropic', 'openai', or 'groq'
LLM_PROVIDER = 'groq'

# Model IDs
CLAUDE_MODEL = 'claude-haiku-4-5-20251001'  # Fastest + cheapest Claude model
GPT_MODEL    = 'gpt-4o-mini'
GROQ_MODEL   = 'llama-3.3-70b-versatile'             # A fast open-source model from Groq

# How many articles to send to the LLM (cost control)
MAX_ARTICLES_FOR_LLM = 20

# Rate limit: pause between API calls (seconds)
API_CALL_DELAY = 0.5
# ────────────────────────────────────────────────────────────────────────────

def create_llm_client(provider: str, anthropic_key: str, openai_key: str, groq_key: str):
    """
    Initialize and return an LLM API client.

    Parameters
    ----------
    provider     : 'anthropic', 'openai', or 'groq'
    anthropic_key: Anthropic API key
    openai_key   : OpenAI API key
    groq_key     : Groq API key

    Returns
    -------
    Tuple of (client, provider_name)
    """
    if provider == 'anthropic':
        client = anthropic.Anthropic(api_key=anthropic_key)
        print(f'Anthropic client ready. Model: {CLAUDE_MODEL}')
        return client, 'anthropic'
    elif provider == 'openai':
        client = openai.OpenAI(api_key=openai_key)
        print(f'OpenAI client ready. Model: {GPT_MODEL}')
        return client, 'openai'
    elif provider == 'groq':
        client = groq.Groq(api_key=groq_key)
        print(f'Groq client ready. Model: {GROQ_MODEL}')
        return client, 'groq'
    else:
        raise ValueError(f'Unknown provider: {provider}. Use "anthropic", "openai", or "groq".')


llm_client, llm_provider = create_llm_client(LLM_PROVIDER, ANTHROPIC_API_KEY, OPENAI_API_KEY, GROQ_API_KEY)

CRYPTO_RSS_FEEDS = {
    'CoinDesk':      'https://www.coindesk.com/arc/outboundfeeds/rss/',
    'Cointelegraph': 'https://cointelegraph.com/rss',
    'Decrypt':       'https://decrypt.co/feed'
}


def fetch_news_for_llm(feeds: dict, max_per_source: int = 15) -> pd.DataFrame:
    """
    Fetch news articles from RSS feeds, combining title and summary
    into a single text field for LLM analysis.

    Parameters
    ----------
    feeds          : Dictionary of {source_name: rss_url}
    max_per_source : Articles per source

    Returns
    -------
    DataFrame with 'full_text' column (title + summary)
    """
    articles = []
    for source, url in feeds.items():
        try:
            feed = feedparser.parse(url)
            for entry in feed.entries[:max_per_source]:
                title   = entry.get('title', '')
                summary = re.sub('<[^>]+>', '', entry.get('summary', ''))[:500]
                published = None
                if hasattr(entry, 'published'):
                    try:
                        published = dateparser.parse(entry.published)
                    except Exception:
                        pass
                articles.append({
                    'source':    source,
                    'title':     title,
                    'summary':   summary,
                    'full_text': f'{title}. {summary}'.strip(),
                    'published': published,
                    'link':      entry.get('link', '')
                })
        except Exception as e:
            print(f'WARNING: {source} — {e}')

    df = pd.DataFrame(articles)
    df = df[df['full_text'].str.len() > 20].reset_index(drop=True)
    print(f'Fetched {len(df)} articles from {len(feeds)} sources')
    return df


news_df = fetch_news_for_llm(CRYPTO_RSS_FEEDS)
news_df[['source', 'title', 'published']].head(5)

EXTRACTION_SYSTEM_PROMPT = """You are a crypto trading analyst. Analyze news articles and return ONLY valid JSON with these exact fields:
{
  "sentiment": one of ["bullish", "bearish", "neutral"],
  "sentiment_score": float from -1.0 (very bearish) to 1.0 (very bullish),
  "affected_coins": list of ticker symbols mentioned (e.g. ["BTC", "ETH"]),
  "event_type": one of ["regulatory", "adoption", "hack", "macro", "technical", "earnings", "other"],
  "impact_level": one of ["high", "medium", "low"],
  "time_horizon": one of ["intraday", "swing", "long-term"],
  "one_line_summary": string (max 100 chars),
  "trading_implication": string (max 150 chars)
}
Return ONLY the JSON object. No explanation, no markdown, no extra text."""


def call_llm(
    client,
    provider: str,
    text: str,
    system_prompt: str
) -> str:
    """
    Call an LLM API (Anthropic, OpenAI, or Groq) and return the response text.

    Parameters
    ----------
    client        : LLM API client
    provider      : 'anthropic', 'openai', or 'groq'
    text          : User message (the news article text)
    system_prompt : System instruction for the model

    Returns
    -------
    Raw response string from the LLM
    """
    if provider == 'anthropic':
        response = client.messages.create(
            model=CLAUDE_MODEL,
            max_tokens=256,
            system=system_prompt,
            messages=[{'role': 'user', 'content': text}]
        )
        return response.content[0].text

    elif provider == 'openai':
        response = client.chat.completions.create(
            model=GPT_MODEL,
            max_tokens=256,
            messages=[
                {'role': 'system', 'content': system_prompt},
                {'role': 'user',   'content': text}
            ]
        )
        return response.choices[0].message.content

    elif provider == 'groq':
        response = client.chat.completions.create(
            model=GROQ_MODEL,
            max_tokens=256,
            messages=[
                {'role': 'system', 'content': system_prompt},
                {'role': 'user', 'content': text}
            ]
        )
        return response.choices[0].message.content


def parse_llm_json(raw_response: str) -> dict:
    """
    Parse JSON from an LLM response, handling common formatting issues.

    LLMs sometimes wrap JSON in markdown code blocks or add trailing text.
    This function extracts and parses the JSON robustly.

    Parameters
    ----------
    raw_response : Raw string from LLM API

    Returns
    -------
    Parsed dictionary, or empty dict on failure
    """
    json_match = re.search(r'\{.*\}', raw_response, re.DOTALL)
    if not json_match:
        return {}
    try:
        return json.loads(json_match.group(0))
    except json.JSONDecodeError:
        return {}


def extract_article_insights(
    df: pd.DataFrame,
    client,
    provider: str,
    max_articles: int = 20,
    delay: float = 0.5
) -> pd.DataFrame:
    """
    Run LLM structured extraction on news articles.

    Processes `max_articles` from the DataFrame, calls the LLM for each,
    and merges the structured output back into the DataFrame.

    Parameters
    ----------
    df           : News DataFrame with 'full_text' column
    client       : LLM API client
    provider     : 'anthropic', 'openai', or 'groq'
    max_articles : How many articles to process (cost limit)
    delay        : Seconds to wait between API calls

    Returns
    -------
    DataFrame with LLM-extracted fields added
    """
    subset = df.head(max_articles).copy()
    results = []

    for _, row in tqdm(subset.iterrows(), total=len(subset), desc='LLM extraction'):
        raw = call_llm(client, provider, row['full_text'][:800], EXTRACTION_SYSTEM_PROMPT)
        parsed = parse_llm_json(raw)
        results.append(parsed)
        time.sleep(delay)

    extracted = pd.DataFrame(results)
    result = pd.concat([subset.reset_index(drop=True), extracted], axis=1)
    print(f'Extracted insights for {len(result)} articles')
    return result


extracted_df = extract_article_insights(news_df, llm_client, llm_provider, MAX_ARTICLES_FOR_LLM, API_CALL_DELAY)
extracted_df[['title', 'sentiment', 'sentiment_score', 'affected_coins', 'event_type', 'impact_level']].head(6)

DIGEST_SYSTEM_PROMPT = """You are a senior crypto analyst writing a daily trading briefing. Given a list of news headlines and their extracted sentiments, write a concise daily digest in this exact format:

**Overall Sentiment:** [BULLISH/BEARISH/NEUTRAL]
**Key Events:** [2-3 bullet points of the most important news]
**Most Affected Coins:** [comma-separated list]
**Dominant Theme:** [single sentence describing the day's narrative]
**Trader Takeaway:** [1-2 sentence action-oriented summary]

Be concise and actionable. Focus on what matters for price."""


def generate_daily_digest(
    extracted_df: pd.DataFrame,
    client,
    provider: str,
    top_n: int = 10
) -> str:
    """
    Generate a daily trading digest from extracted article insights.

    Selects the top_n highest-impact articles and sends their summaries
    to the LLM for synthesis into a single daily briefing.

    Parameters
    ----------
    extracted_df : Output from extract_article_insights()
    client       : LLM API client
    provider     : 'anthropic' or 'openai'
    top_n        : Number of articles to include in digest

    Returns
    -------
    Digest string from the LLM
    """
    # Prioritize high-impact articles
    priority_order = {'high': 0, 'medium': 1, 'low': 2}
    df = extracted_df.copy()
    if 'impact_level' in df.columns:
        df['_priority'] = df['impact_level'].map(priority_order).fillna(1)
        df = df.sort_values('_priority').head(top_n)

    article_summaries = []
    for _, row in df.iterrows():
        coins = row.get('affected_coins', [])
        coins_str = ', '.join(coins) if isinstance(coins, list) else str(coins)
        summary_line = f"- [{row.get('sentiment', 'neutral').upper()}] {row['title']}. Coins: {coins_str}."
        article_summaries.append(summary_line)

    input_text = '\n'.join(article_summaries)
    digest = call_llm(client, provider, input_text, DIGEST_SYSTEM_PROMPT)
    return digest


if 'sentiment' in extracted_df.columns:
    daily_digest = generate_daily_digest(extracted_df, llm_client, llm_provider)
    print('=== DAILY CRYPTO TRADING DIGEST ===')
    print(f'Date: {datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")}')
    print(daily_digest)

from transformers import pipeline as hf_pipeline


def summarize_with_bart(
    texts: list[str],
    max_summary_length: int = 80,
    min_summary_length: int = 20
) -> list[str]:
    """
    Summarize a list of texts using BART (no API key required).

    BART is a seq2seq model trained for abstractive summarization.
    It generates new sentences rather than extracting existing ones.

    Parameters
    ----------
    texts              : List of text strings to summarize
    max_summary_length : Maximum tokens in each summary
    min_summary_length : Minimum tokens in each summary

    Returns
    -------
    List of summary strings
    """
    print('Loading BART summarizer (downloads ~1.6GB on first run)...')
    summarizer = hf_pipeline(
        'summarization',
        model='facebook/bart-large-cnn',
        device=0 if torch.cuda.is_available() else -1
    )

    summaries = []
    for text in tqdm(texts, desc='BART summarization'):
        text = text[:1024]  # BART max input length
        if len(text.split()) < 20:
            summaries.append(text)
            continue
        result = summarizer(text, max_length=max_summary_length,
                            min_length=min_summary_length, do_sample=False)
        summaries.append(result[0]['summary_text'])

    return summaries


# Uncomment to use BART instead of LLM API:
# import torch
# bart_summaries = summarize_with_bart(news_df['full_text'].head(10).tolist())
# news_df.loc[:9, 'bart_summary'] = bart_summaries
# news_df[['title', 'bart_summary']].head(5)
print('BART summarizer ready. Uncomment the lines above to run without an LLM API key.')

def build_llm_daily_signal(df: pd.DataFrame) -> pd.DataFrame:
    """
    Aggregate LLM-extracted sentiment scores into a daily signal.

    Impact level is converted to a numeric weight:
    high=3, medium=2, low=1

    Parameters
    ----------
    df : Output from extract_article_insights() with 'sentiment_score' column

    Returns
    -------
    Daily signal DataFrame
    """
    if 'sentiment_score' not in df.columns:
        print('No sentiment_score column — run LLM extraction first.')
        return pd.DataFrame()

    df = df.copy()
    df['published'] = pd.to_datetime(df['published'], utc=True, errors='coerce')
    df.dropna(subset=['published', 'sentiment_score'], inplace=True)
    df['date']           = df['published'].dt.date
    df['sentiment_score'] = pd.to_numeric(df['sentiment_score'], errors='coerce').fillna(0)
    df['impact_weight']  = df['impact_level'].map({'high': 3, 'medium': 2, 'low': 1}).fillna(1)

    daily = df.groupby('date').apply(
        lambda g: pd.Series({
            'llm_weighted_sentiment': (g['sentiment_score'] * g['impact_weight']).sum() / g['impact_weight'].sum(),
            'mean_sentiment_score':   g['sentiment_score'].mean(),
            'article_count':          len(g),
            'high_impact_count':      (g['impact_level'] == 'high').sum(),
            'bullish_count':          (g['sentiment'] == 'bullish').sum(),
            'bearish_count':          (g['sentiment'] == 'bearish').sum(),
            'top_coins':              ', '.join(set([c for coins in g['affected_coins'].dropna()
                                                     for c in (coins if isinstance(coins, list) else [])]))
        })
    ).reset_index()

    daily['date'] = pd.to_datetime(daily['date'])
    return daily.sort_values('date')


daily_signal = build_llm_daily_signal(extracted_df)
if not daily_signal.empty:
    print(f'Daily signal: {len(daily_signal)} days')
    daily_signal.tail()

def plot_llm_results(extracted_df: pd.DataFrame) -> None:
    if 'sentiment' not in extracted_df.columns:
        print('No LLM results to plot.')
        return

    fig, axes = plt.subplots(1, 3, figsize=(16, 5))

    # Sentiment distribution
    counts = extracted_df['sentiment'].value_counts()
    axes[0].bar(counts.index, counts.values,
                color=['#2ecc71' if l == 'bullish' else '#e74c3c' if l == 'bearish' else '#95a5a6'
                       for l in counts.index], edgecolor='white')
    axes[0].set_title('LLM Sentiment Labels', fontweight='bold')

    # Event type distribution
    event_counts = extracted_df['event_type'].value_counts().head(6)
    axes[1].barh(event_counts.index, event_counts.values, color='steelblue', edgecolor='white')
    axes[1].set_title('Event Type Distribution', fontweight='bold')

    # Impact level
    impact_counts = extracted_df['impact_level'].value_counts()
    axes[2].pie(impact_counts.values, labels=impact_counts.index, autopct='%1.0f%%',
                colors=['#e74c3c', '#f39c12', '#2ecc71'])
    axes[2].set_title('Impact Level Distribution', fontweight='bold')

    plt.suptitle('LLM-Extracted News Analysis', fontsize=14)
    plt.tight_layout()
    plt.show()


plot_llm_results(extracted_df)

# Export
export_cols = ['source', 'title', 'published', 'sentiment', 'sentiment_score',
               'affected_coins', 'event_type', 'impact_level', 'time_horizon',
               'one_line_summary', 'trading_implication', 'link']
available_cols = [c for c in export_cols if c in extracted_df.columns]
extracted_df[available_cols].to_csv('llm_news_analysis.csv', index=False)

if not daily_signal.empty:
    daily_signal.to_csv('llm_daily_signal.csv', index=False)

print('Export complete.')