# Install required libraries (Colab resets on each session)
!pip install praw vaderSentiment textblob wordcloud --quiet

import praw
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sns
from wordcloud import WordCloud
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
from textblob import TextBlob
import re
import string
from datetime import datetime, timezone
from collections import Counter

# Notebook display settings
%matplotlib inline
plt.rcParams['figure.figsize'] = (12, 5)
plt.rcParams['axes.spines.top'] = False
plt.rcParams['axes.spines.right'] = False
sns.set_palette('husl')

print('All imports successful.')

import random
from datetime import datetime, timedelta, timezone

# ─── CONFIGURATION ────────────────────────────────────────────────────────────
# Toggle this to True to use synthetic data instead of calling the Reddit API
USE_DUMMY_DATA = True

# Replace these with your actual credentials (ignored if USE_DUMMY_DATA is True)
REDDIT_CLIENT_ID     = 'YOUR_CLIENT_ID'
REDDIT_CLIENT_SECRET = 'YOUR_CLIENT_SECRET'
REDDIT_USER_AGENT    = 'crypto_scraper/1.0 by YourRedditUsername'

# Subreddits to scrape
TARGET_SUBREDDITS = ['CryptoCurrency', 'Bitcoin', 'ethereum', 'CryptoMarkets', 'altcoin']

# How many posts to fetch per subreddit
POSTS_LIMIT = 100

# Post sort method: 'hot', 'new', 'top', 'rising'
SORT_METHOD = 'hot'
# ──────────────────────────────────────────────────────────────────────────────

def create_reddit_client(client_id: str, client_secret: str, user_agent: str) -> praw.Reddit:
    if USE_DUMMY_DATA:
        print('Mode: Dummy Data Enabled. Skipping Reddit Authentication.')
        return None

    reddit = praw.Reddit(
        client_id=client_id,
        client_secret=client_secret,
        user_agent=user_agent,
        read_only=True
    )
    print(f'Reddit client created. Read-only: {reddit.read_only}')
    return reddit

reddit = create_reddit_client(REDDIT_CLIENT_ID, REDDIT_CLIENT_SECRET, REDDIT_USER_AGENT)

def generate_dummy_data(subreddit_name: str, limit: int) -> list[dict]:
    """Generates synthetic Reddit-like data for testing."""
    templates = [
        ("Bitcoin is going to the moon! 🚀", "I just bought the dip and feeling great. HODL!"),
        ("Is Ethereum a good investment?", "Thinking about moving my bags from BTC to ETH. Thoughts?"),
        ("Market crash is coming...", "Seeing a lot of red candles. Might be time to exit for a while. Very bearish."),
        ("New altcoin gem discovered!", "Check out this new project. Great fundamentals and community."),
        ("Regulations are killing crypto", "Government is making it harder for us. This is bad news.")
    ]

    dummy_posts = []
    for i in range(limit):
        title, body = random.choice(templates)
        # Randomize dates over the last 7 days
        random_days = random.randint(0, 7)
        created_date = datetime.now(timezone.utc) - timedelta(days=random_days)

        dummy_posts.append({
            'post_id':       f'dummy_{subreddit_name}_{i}',
            'subreddit':     subreddit_name,
            'title':         f"[{i}] {title}",
            'selftext':      body,
            'score':         random.randint(1, 2000),
            'upvote_ratio':  random.uniform(0.5, 0.99),
            'num_comments':  random.randint(5, 500),
            'created_utc':   created_date,
            'url':           'https://www.reddit.com',
            'author':        'crypto_enthusiast',
            'is_self':       True,
            'flair':         'Discussion'
        })
    return dummy_posts

def scrape_subreddit_posts(reddit_client, subreddit_name, limit=100, sort='hot'):
    if USE_DUMMY_DATA:
        posts = generate_dummy_data(subreddit_name, limit)
        print(f'  r/{subreddit_name}: {len(posts)} dummy posts generated')
        return posts

    subreddit = reddit_client.subreddit(subreddit_name)
    feed = {'hot': subreddit.hot, 'new': subreddit.new, 'top': subreddit.top, 'rising': subreddit.rising}.get(sort, subreddit.hot)
    posts = []
    for submission in feed(limit=limit):
        posts.append({
            'post_id':       submission.id,
            'subreddit':     subreddit_name,
            'title':         submission.title,
            'selftext':      submission.selftext,
            'score':         submission.score,
            'upvote_ratio':  submission.upvote_ratio,
            'num_comments':  submission.num_comments,
            'created_utc':   datetime.fromtimestamp(submission.created_utc, tz=timezone.utc),
            'url':           submission.url,
            'author':        str(submission.author),
            'is_self':       submission.is_self,
            'flair':         submission.link_flair_text
        })
    print(f'  r/{subreddit_name}: {len(posts)} posts fetched')
    return posts

def scrape_multiple_subreddits(reddit_client, subreddit_list, limit=100, sort='hot'):
    all_posts = []
    mode_str = "DUMMY MODE" if USE_DUMMY_DATA else "API MODE"
    print(f'Scraping {len(subreddit_list)} subreddits ({mode_str}, {sort}, limit={limit})...')

    for sub in subreddit_list:
        try:
            posts = scrape_subreddit_posts(reddit_client, sub, limit, sort)
            all_posts.extend(posts)
        except Exception as e:
            print(f'  WARNING: Could not scrape r/{sub} — {e}')

    df = pd.DataFrame(all_posts)
    if not df.empty:
        df.drop_duplicates(subset='post_id', inplace=True)
        df.sort_values('created_utc', ascending=False, inplace=True)
        df.reset_index(drop=True, inplace=True)

    print(f'\nTotal posts collected: {len(df)}')
    return df

raw_df = scrape_multiple_subreddits(reddit, TARGET_SUBREDDITS, POSTS_LIMIT, SORT_METHOD)
display(raw_df.head(3))

# Common crypto stop words to exclude from word clouds (not from sentiment analysis)
WORDCLOUD_STOP_WORDS = {
    'crypto', 'bitcoin', 'btc', 'eth', 'ethereum', 'coin', 'coins',
    'market', 'price', 'just', 'like', 'get', 'one', 'will', 'think',
    'know', 'people', 'time', 'new', 'good', 'going', 'much', 'also',
    'still', 'even', 'want', 'see', 'make', 'use', 'year', 'really',
    'many', 'right', 'said', 'say', 'back', 'way', 'something', 'every',
    'subreddit', 'post', 'comment', 'reddit', 'https', 'com', 'www'
}


def clean_text(text: str) -> str:
    """
    Clean raw Reddit text for sentiment analysis.

    Removes URLs, markdown formatting, special characters, and extra whitespace.
    Preserves natural language structure needed by VADER and TextBlob.

    Parameters
    ----------
    text : Raw text string

    Returns
    -------
    Cleaned text string
    """
    if not isinstance(text, str):
        return ''

    text = re.sub(r'http\S+|www\.\S+', '', text)    # remove URLs
    text = re.sub(r'>.*?\n', ' ', text)              # remove Reddit quotes
    text = re.sub(r'[*_~`#]', '', text)              # remove markdown symbols
    text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', text)  # keep link text
    text = re.sub(r'\s+', ' ', text)                 # normalize whitespace
    return text.strip()


def combine_title_body(title: str, body: str, title_weight: int = 2) -> str:
    """
    Combine post title and body into a single analysis string.

    The title is repeated `title_weight` times to reflect its higher
    importance — it's what users read first and influences upvote behavior.

    Parameters
    ----------
    title        : Post title text
    body         : Post body (selftext)
    title_weight : How many times to repeat the title (default=2)

    Returns
    -------
    Combined, cleaned text string
    """
    clean_title = clean_text(title)
    clean_body  = clean_text(body)
    repeated_title = ' '.join([clean_title] * title_weight)
    combined = f'{repeated_title} {clean_body}'.strip()
    return combined


def preprocess_dataframe(df: pd.DataFrame) -> pd.DataFrame:
    """
    Apply all preprocessing steps to the raw scraped DataFrame.

    Adds columns: 'clean_title', 'clean_body', 'combined_text'.

    Parameters
    ----------
    df : Raw DataFrame from scrape_multiple_subreddits()

    Returns
    -------
    DataFrame with new text columns added
    """
    df = df.copy()
    df['clean_title']    = df['title'].apply(clean_text)
    df['clean_body']     = df['selftext'].apply(clean_text)
    df['combined_text']  = df.apply(
        lambda row: combine_title_body(row['title'], row['selftext']), axis=1
    )
    df['text_length']    = df['combined_text'].apply(len)
    print(f'Preprocessing complete. Avg text length: {df["text_length"].mean():.0f} chars')
    return df


processed_df = preprocess_dataframe(raw_df)
processed_df[['title', 'clean_title', 'combined_text', 'text_length']].head(3)

vader = SentimentIntensityAnalyzer()


def get_vader_scores(text: str) -> dict:
    """
    Compute VADER sentiment scores for a text string.

    Parameters
    ----------
    text : Cleaned text to analyze

    Returns
    -------
    Dictionary with keys: vader_neg, vader_neu, vader_pos, vader_compound
    """
    if not text or len(text) < 3:
        return {'vader_neg': 0.0, 'vader_neu': 1.0, 'vader_pos': 0.0, 'vader_compound': 0.0}

    scores = vader.polarity_scores(text)
    return {
        'vader_neg':      scores['neg'],
        'vader_neu':      scores['neu'],
        'vader_pos':      scores['pos'],
        'vader_compound': scores['compound']
    }


def get_textblob_scores(text: str) -> dict:
    """
    Compute TextBlob sentiment scores for a text string.

    Parameters
    ----------
    text : Cleaned text to analyze

    Returns
    -------
    Dictionary with keys: tb_polarity, tb_subjectivity
    """
    if not text or len(text) < 3:
        return {'tb_polarity': 0.0, 'tb_subjectivity': 0.0}

    blob = TextBlob(text)
    return {
        'tb_polarity':     blob.sentiment.polarity,
        'tb_subjectivity': blob.sentiment.subjectivity
    }


def label_sentiment(compound_score: float, positive_threshold: float = 0.05, negative_threshold: float = -0.05) -> str:
    """
    Convert a VADER compound score into a categorical label.

    Parameters
    ----------
    compound_score      : VADER compound value in range [-1, 1]
    positive_threshold  : Minimum compound score for POSITIVE label
    negative_threshold  : Maximum compound score for NEGATIVE label

    Returns
    -------
    'POSITIVE', 'NEGATIVE', or 'NEUTRAL'
    """
    if compound_score >= positive_threshold:
        return 'POSITIVE'
    elif compound_score <= negative_threshold:
        return 'NEGATIVE'
    return 'NEUTRAL'


def run_sentiment_analysis(df: pd.DataFrame) -> pd.DataFrame:
    """
    Apply VADER and TextBlob sentiment analysis to all posts in the DataFrame.

    Adds columns: vader_neg, vader_neu, vader_pos, vader_compound,
                  tb_polarity, tb_subjectivity, sentiment_label,
                  combined_polarity (average of VADER compound and TB polarity)

    Parameters
    ----------
    df : Preprocessed DataFrame with 'combined_text' column

    Returns
    -------
    DataFrame with sentiment score columns added
    """
    df = df.copy()

    # Apply VADER
    vader_scores = df['combined_text'].apply(get_vader_scores)
    df = pd.concat([df, pd.DataFrame(list(vader_scores))], axis=1)

    # Apply TextBlob
    tb_scores = df['combined_text'].apply(get_textblob_scores)
    df = pd.concat([df, pd.DataFrame(list(tb_scores))], axis=1)

    # Categorical label based on VADER compound
    df['sentiment_label'] = df['vader_compound'].apply(label_sentiment)

    # Ensemble polarity: average of VADER compound and TextBlob polarity
    df['combined_polarity'] = (df['vader_compound'] + df['tb_polarity']) / 2

    label_counts = df['sentiment_label'].value_counts()
    print('Sentiment distribution:')
    for label, count in label_counts.items():
        pct = count / len(df) * 100
        print(f'  {label:10s}: {count:4d}  ({pct:.1f}%)')

    return df


sentiment_df = run_sentiment_analysis(processed_df)
sentiment_df[['title', 'vader_compound', 'tb_polarity', 'tb_subjectivity', 'sentiment_label']].head(5)

def plot_sentiment_distribution(df: pd.DataFrame) -> None:
    """
    Plot a bar chart of sentiment label counts and a KDE of VADER compound scores.

    Parameters
    ----------
    df : DataFrame with 'sentiment_label' and 'vader_compound' columns
    """
    fig, axes = plt.subplots(1, 2, figsize=(14, 5))

    # Left: bar chart of label counts
    label_counts = df['sentiment_label'].value_counts().reindex(['POSITIVE', 'NEUTRAL', 'NEGATIVE'])
    colors = ['#2ecc71', '#95a5a6', '#e74c3c']
    bars = axes[0].bar(label_counts.index, label_counts.values, color=colors, width=0.5, edgecolor='white')
    axes[0].set_title('Sentiment Label Distribution', fontsize=13, fontweight='bold')
    axes[0].set_ylabel('Number of Posts')
    for bar, val in zip(bars, label_counts.values):
        axes[0].text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 1,
                     str(val), ha='center', va='bottom', fontsize=11)

    # Right: KDE of VADER compound score
    axes[1].axvline(x=0, color='gray', linestyle='--', linewidth=1, alpha=0.7)
    axes[1].axvspan(-1, -0.05, alpha=0.08, color='red', label='Bearish zone')
    axes[1].axvspan(0.05,  1,  alpha=0.08, color='green', label='Bullish zone')
    df['vader_compound'].plot.kde(ax=axes[1], color='steelblue', linewidth=2)
    axes[1].set_title('VADER Compound Score Distribution', fontsize=13, fontweight='bold')
    axes[1].set_xlabel('Compound Score  (−1 = very negative, +1 = very positive)')
    axes[1].set_xlim(-1, 1)
    axes[1].legend()

    plt.suptitle('Overall Crypto Reddit Sentiment', fontsize=15, y=1.02)
    plt.tight_layout()
    plt.show()


plot_sentiment_distribution(sentiment_df)

def plot_sentiment_by_subreddit(df: pd.DataFrame) -> None:
    """
    Stacked bar chart showing sentiment breakdown per subreddit.

    Parameters
    ----------
    df : DataFrame with 'subreddit', 'sentiment_label', and 'vader_compound' columns
    """
    fig, axes = plt.subplots(1, 2, figsize=(15, 5))

    # Left: stacked bar of sentiment labels
    pivot = df.groupby(['subreddit', 'sentiment_label']).size().unstack(fill_value=0)
    for col in ['POSITIVE', 'NEUTRAL', 'NEGATIVE']:
        if col not in pivot.columns:
            pivot[col] = 0
    pivot = pivot[['POSITIVE', 'NEUTRAL', 'NEGATIVE']]
    pivot_pct = pivot.div(pivot.sum(axis=1), axis=0) * 100

    pivot_pct.plot(kind='bar', stacked=True, ax=axes[0],
                   color=['#2ecc71', '#95a5a6', '#e74c3c'], edgecolor='white')
    axes[0].set_title('Sentiment Breakdown by Subreddit (%)', fontsize=12, fontweight='bold')
    axes[0].set_ylabel('Percentage of Posts')
    axes[0].set_xlabel('')
    axes[0].tick_params(axis='x', rotation=30)
    axes[0].legend(loc='upper right')

    # Right: mean compound score per subreddit
    mean_compound = df.groupby('subreddit')['vader_compound'].mean().sort_values(ascending=True)
    bar_colors = ['#e74c3c' if v < 0 else '#2ecc71' for v in mean_compound.values]
    mean_compound.plot(kind='barh', ax=axes[1], color=bar_colors, edgecolor='white')
    axes[1].axvline(x=0, color='black', linewidth=0.8)
    axes[1].set_title('Mean VADER Compound Score by Subreddit', fontsize=12, fontweight='bold')
    axes[1].set_xlabel('Mean Compound Score')

    plt.tight_layout()
    plt.show()


plot_sentiment_by_subreddit(sentiment_df)

def generate_sentiment_wordclouds(df: pd.DataFrame, stop_words: set = None) -> None:
    """
    Generate side-by-side word clouds for positive and negative posts.

    Word size reflects frequency — larger words appear more often in posts
    with that sentiment label.

    Parameters
    ----------
    df         : DataFrame with 'sentiment_label' and 'combined_text' columns
    stop_words : Optional set of words to exclude from clouds
    """
    if stop_words is None:
        stop_words = WORDCLOUD_STOP_WORDS

    positive_text = ' '.join(df[df['sentiment_label'] == 'POSITIVE']['combined_text'].tolist())
    negative_text = ' '.join(df[df['sentiment_label'] == 'NEGATIVE']['combined_text'].tolist())

    fig, axes = plt.subplots(1, 2, figsize=(16, 6))

    for ax, text, title, bg, colormap in [
        (axes[0], positive_text, 'Positive Posts — Top Keywords', '#f0fff0', 'Greens'),
        (axes[1], negative_text, 'Negative Posts — Top Keywords', '#fff0f0', 'Reds')
    ]:
        if text.strip():
            wc = WordCloud(
                width=700, height=400,
                background_color=bg,
                colormap=colormap,
                stopwords=stop_words,
                max_words=80,
                collocations=False
            ).generate(text)
            ax.imshow(wc, interpolation='bilinear')
        else:
            ax.text(0.5, 0.5, 'No data', ha='center', va='center', transform=ax.transAxes)
        ax.set_title(title, fontsize=12, fontweight='bold')
        ax.axis('off')

    plt.suptitle('Most Frequent Words by Sentiment', fontsize=14)
    plt.tight_layout()
    plt.show()


generate_sentiment_wordclouds(sentiment_df)

def compute_daily_sentiment(
    df: pd.DataFrame,
    date_col: str = 'created_utc',
    score_col: str = 'vader_compound',
    weight_col: str = 'score'
) -> pd.DataFrame:
    """
    Aggregate post-level sentiment into a daily weighted sentiment time series.

    Parameters
    ----------
    df         : DataFrame with sentiment scores and engagement data
    date_col   : Column containing datetime values
    score_col  : Sentiment column to aggregate (default: 'vader_compound')
    weight_col : Engagement column for weighting (default: 'score')

    Returns
    -------
    Daily DataFrame with columns: date, mean_sentiment, weighted_sentiment,
                                   post_count, total_engagement
    """
    df = df.copy()
    df['date'] = pd.to_datetime(df[date_col]).dt.date

    # Clip negative scores to 1 to avoid negative weights distorting the average
    df['_weight'] = df[weight_col].clip(lower=1)

    daily = df.groupby('date').apply(
        lambda g: pd.Series({
            'mean_sentiment':     g[score_col].mean(),
            'weighted_sentiment': (g[score_col] * g['_weight']).sum() / g['_weight'].sum(),
            'post_count':         len(g),
            'total_engagement':   g[weight_col].sum(),
            'pct_positive':       (g['sentiment_label'] == 'POSITIVE').mean() * 100,
            'pct_negative':       (g['sentiment_label'] == 'NEGATIVE').mean() * 100
        })
    ).reset_index()

    daily['date'] = pd.to_datetime(daily['date'])
    daily.sort_values('date', inplace=True)
    return daily


daily_df = compute_daily_sentiment(sentiment_df)
print(f'Daily sentiment data: {len(daily_df)} days')
daily_df.tail(5)

def plot_daily_sentiment_trend(daily_df: pd.DataFrame) -> None:
    """
    Plot the daily weighted sentiment signal with post volume overlay.

    Parameters
    ----------
    daily_df : Output from compute_daily_sentiment()
    """
    if len(daily_df) < 2:
        print('Not enough days for a trend chart. Try scraping more posts with SORT_METHOD="new".')
        return

    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(14, 7), sharex=True)

    # Top: sentiment line
    ax1.axhline(y=0, color='gray', linestyle='--', linewidth=1, alpha=0.6)
    ax1.fill_between(daily_df['date'], daily_df['weighted_sentiment'], 0,
                     where=daily_df['weighted_sentiment'] >= 0,
                     alpha=0.3, color='#2ecc71', label='Bullish')
    ax1.fill_between(daily_df['date'], daily_df['weighted_sentiment'], 0,
                     where=daily_df['weighted_sentiment'] < 0,
                     alpha=0.3, color='#e74c3c', label='Bearish')
    ax1.plot(daily_df['date'], daily_df['weighted_sentiment'],
             color='steelblue', linewidth=2, label='Weighted Sentiment')
    ax1.set_ylabel('Weighted VADER Compound')
    ax1.set_title('Daily Crypto Reddit Sentiment Signal', fontsize=13, fontweight='bold')
    ax1.legend(loc='upper left')
    ax1.set_ylim(-1, 1)

    # Bottom: post volume
    ax2.bar(daily_df['date'], daily_df['post_count'], color='steelblue', alpha=0.6, width=0.8)
    ax2.set_ylabel('Number of Posts')
    ax2.set_xlabel('Date')
    ax2.set_title('Daily Post Volume', fontsize=11)
    ax2.xaxis.set_major_formatter(mdates.DateFormatter('%b %d'))
    plt.xticks(rotation=30)

    plt.tight_layout()
    plt.show()


plot_daily_sentiment_trend(daily_df)

def export_results(
    posts_df: pd.DataFrame,
    daily_df: pd.DataFrame,
    posts_filename: str = 'reddit_crypto_posts_sentiment.csv',
    daily_filename: str = 'reddit_crypto_daily_signal.csv'
) -> None:
    """
    Export post-level and daily-aggregated DataFrames to CSV files.

    Files are saved in the current working directory (Colab: /content/).
    Download them via the Files panel (folder icon in the left sidebar).

    Parameters
    ----------
    posts_df       : Full post DataFrame with sentiment scores
    daily_df       : Daily aggregated sentiment DataFrame
    posts_filename : Output filename for post-level data
    daily_filename : Output filename for daily signal
    """
    # Post-level export — keep key columns
    post_cols = [
        'post_id', 'subreddit', 'created_utc', 'title', 'score',
        'upvote_ratio', 'num_comments', 'vader_compound', 'vader_pos',
        'vader_neg', 'tb_polarity', 'tb_subjectivity', 'combined_polarity',
        'sentiment_label', 'text_length', 'url'
    ]
    posts_df[post_cols].to_csv(posts_filename, index=False)
    print(f'Post-level data saved: {posts_filename}  ({len(posts_df)} rows)')

    # Daily signal export
    daily_df.to_csv(daily_filename, index=False)
    print(f'Daily signal saved:    {daily_filename}  ({len(daily_df)} rows)')


export_results(sentiment_df, daily_df)