# Install all necessary dependencies
!pip install vaderSentiment textblob wordcloud emoji yt-dlp --quiet

import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sns
from wordcloud import WordCloud
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
from textblob import TextBlob
import re
import html
from datetime import datetime, timezone
from dateutil import parser as dateparser
from yt_dlp import YoutubeDL # Added for yt-dlp functionality

%matplotlib inline
plt.rcParams['figure.figsize'] = (13, 5)
plt.rcParams['axes.spines.top'] = False
plt.rcParams['axes.spines.right'] = False
sns.set_palette('husl')

print('All imports successful.')

# Search queries — will run each query independently
SEARCH_QUERIES = ['bitcoin analysis', 'ethereum price prediction', 'crypto market 2024', 'crypto', 'cryptocurrency']

# Videos per search query
VIDEOS_PER_QUERY = 5

# Comments to fetch per video
COMMENTS_PER_VIDEO = 100

# yt-dlp doesn't have a direct 'relevance' sort for comments, so this will be unused
# COMMENT_ORDER = 'relevance'
# ────────────────────────────────────────────────────────────────────────────

# This cell was a duplicate of configuration settings and has been removed.

def get_video_info_and_comments_ytdlp(video_id: str, max_comments: int = 10):
    """
    Fetches video metadata and comments for a single YouTube video ID using yt-dlp.
    Returns video info and a list of comment dictionaries.
    """
    comments_list = []
    video_info = {}
    video_url = f"https://www.youtube.com/watch?v={video_id}"

    try:
        # First, extract detailed video info and comments
        ydl_opts_comments = {
            'getcomments': True,
            'skip_download': True,
            'quiet': True,
            'no_warnings': True,
            'extract_flat': False, # Extract full metadata and comments
            'force_generic_extractor': True,
        }
        with YoutubeDL(ydl_opts_comments) as ydl:
            detailed_info_dict = ydl.extract_info(video_url, download=False)

        if detailed_info_dict:
            # Populate video_info dictionary
            video_info = {
                'video_id': detailed_info_dict.get('id'),
                'title': detailed_info_dict.get('title'),
                'channel_title': detailed_info_dict.get('channel'),
                'published_at': pd.to_datetime(detailed_info_dict.get('upload_date'), format='%Y%m%d', errors='coerce').replace(tzinfo=timezone.utc),
                'description': detailed_info_dict.get('description', '')[:300],
                'view_count': detailed_info_dict.get('view_count', 0),
                'like_count': detailed_info_dict.get('like_count', 0),
                'comment_count': detailed_info_dict.get('comment_count', 0)
            }

            # Populate comments_list
            raw_comments = detailed_info_dict.get('comments', [])
            for count, c in enumerate(raw_comments):
                if count >= max_comments:
                    break
                comments_list.append({
                    'video_id': detailed_info_dict.get('id'),
                    'comment_id': c.get('id'),
                    'text': c.get('text'),
                    'like_count': c.get('like_count', 0),
                    'reply_count': c.get('reply_count', 0),
                    'published_at': pd.to_datetime(c.get('timestamp'), unit='s', errors='coerce').replace(tzinfo=timezone.utc)
                })

    except Exception as e:
        print(f"Error extracting data for {video_url}: {e}")

    return video_info, comments_list

def fetch_all_data_with_ytdlp(queries: list[str], videos_per_query: int, comments_per_video: int):
    """
    Searches YouTube for videos based on queries using yt-dlp, then fetches their metadata and comments.
    Returns two DataFrames: videos_df and comments_df.
    """
    all_video_info = []
    all_comments = []
    processed_video_ids = set()

    print(f'Searching {len(queries)} queries using yt-dlp...')

    for query in queries:
        try:
            # Use yt-dlp's search functionality (ytsearchN:) to get video IDs
            ydl_opts_search = {
                'quiet': True,
                'no_warnings': True,
                'extract_flat': True, # Only extract basic info for search results
                'force_generic_extractor': True,
            }
            with YoutubeDL(ydl_opts_search) as ydl:
                search_results = ydl.extract_info(f'ytsearch{videos_per_query}:{query}', download=False)
                entries = search_results.get('entries', [])

                for entry in entries:
                    video_id = entry.get('id')

                    if video_id and video_id not in processed_video_ids:
                        print(f'  Fetching detailed data for video: {video_id} (Query: "{query}")')
                        video_data, comments = get_video_info_and_comments_ytdlp(video_id, comments_per_video)

                        if video_data.get('video_id'): # Check if video_info was successfully populated
                            video_data['query'] = query # Add the query that found it
                            all_video_info.append(video_data)
                            all_comments.extend(comments)
                            processed_video_ids.add(video_id)

        except Exception as e:
            print(f'  WARNING: yt-dlp search for "{query}" failed — {e}')

    videos_df = pd.DataFrame(all_video_info)
    if not videos_df.empty:
        # Ensure 'view_count' is numeric for sorting
        videos_df['view_count'] = pd.to_numeric(videos_df['view_count'], errors='coerce').fillna(0).astype(int)
        videos_df.drop_duplicates(subset='video_id', inplace=True)
        videos_df.sort_values('view_count', ascending=False, inplace=True)
        videos_df.reset_index(drop=True, inplace=True)
    print(f'\nTotal unique videos found: {len(videos_df)}')

    comments_df = pd.DataFrame(all_comments)
    if not comments_df.empty:
        comments_df.drop_duplicates(subset=['comment_id', 'video_id'], inplace=True)
    print(f'Total comments fetched: {len(comments_df)}')

    return videos_df, comments_df

# --- EXECUTE THE NEW DATA FETCHING ---
videos_df, comments_df = fetch_all_data_with_ytdlp(SEARCH_QUERIES, VIDEOS_PER_QUERY, COMMENTS_PER_VIDEO)
videos_df[['title', 'channel_title', 'view_count', 'comment_count', 'published_at']].head(5)

# This cell contained redundant `yt-dlp` function definitions and their execution, which are now correctly placed and executed within cell 'CbssxN2hkIcp'.

YT_STOP_WORDS = {
    'video', 'youtube', 'channel', 'subscribe', 'like', 'comment', 'watch',
    'crypto', 'bitcoin', 'btc', 'eth', 'coin', 'market', 'price', 'just',
    'will', 'one', 'get', 'now', 'good', 'great', 'new', 'also', 'really'
}


def clean_youtube_comment(text: str) -> str:
    """
    Clean a raw YouTube comment for sentiment analysis.

    Preserves capitalization and punctuation intensity (important for VADER).
    Removes URLs, timestamps, and excess repetition.

    Parameters
    ----------
    text : Raw comment string (already HTML-unescaped)

    Returns
    -------
    Cleaned comment string
    """
    if not isinstance(text, str):
        return ''

    text = re.sub(r'http\S+|www\.\S+', '', text)          # URLs
    text = re.sub(r'\b\d{1,2}:\d{2}\b', '', text)         # timestamps like 4:32
    text = re.sub(r'(.)\1{3,}', r'\1\1', text)            # cap repeated chars (!!!!! → !!)
    text = re.sub(r'\n+', ' ', text)                       # newlines to space
    text = re.sub(r'\s+', ' ', text)
    return text.strip()


def is_spam(text: str, min_length: int = 8) -> bool:
    """
    Simple heuristic to detect spam comments.

    Flags comments that are too short, all-caps, or contain
    common promotional keywords.

    Parameters
    ----------
    text       : Cleaned comment text
    min_length : Minimum character length to be considered valid

    Returns
    -------
    True if comment is likely spam
    """
    if len(text) < min_length:
        return True
    spam_keywords = ['dm me', 'whatsapp', 'telegram me', 'recovery', 'hack', 'giveaway']
    lower = text.lower()
    return any(kw in lower for kw in spam_keywords)


def preprocess_comments_df(df: pd.DataFrame) -> pd.DataFrame:
    """
    Clean all comments and remove spam.

    Adds 'clean_text', 'text_length', and 'is_spam' columns.

    Parameters
    ----------
    df : Raw comments DataFrame

    Returns
    -------
    Filtered DataFrame with cleaned text
    """
    df = df.copy()
    df['clean_text']  = df['text'].apply(clean_youtube_comment)
    df['text_length'] = df['clean_text'].apply(len)
    df['is_spam']     = df['clean_text'].apply(is_spam)

    before = len(df)
    df = df[~df['is_spam']].reset_index(drop=True)
    print(f'Removed {before - len(df)} spam/short comments. Remaining: {len(df)}')
    return df


clean_comments = preprocess_comments_df(comments_df)
clean_comments[['clean_text', 'like_count', 'text_length']].head(3)

vader = SentimentIntensityAnalyzer()


def score_comment(text: str) -> dict:
    """
    Score a single comment with VADER and TextBlob.

    Parameters
    ----------
    text : Cleaned comment string

    Returns
    -------
    Dictionary with vader_compound, tb_polarity, tb_subjectivity
    """
    if not text or len(text) < 3:
        return {'vader_compound': 0.0, 'tb_polarity': 0.0, 'tb_subjectivity': 0.0}
    vs   = vader.polarity_scores(text)
    blob = TextBlob(text)
    return {
        'vader_compound':  vs['compound'],
        'vader_pos':       vs['pos'],
        'vader_neg':       vs['neg'],
        'tb_polarity':     blob.sentiment.polarity,
        'tb_subjectivity': blob.sentiment.subjectivity
    }


def run_comment_sentiment(df: pd.DataFrame) -> pd.DataFrame:
    """
    Apply sentiment scoring to all comments.

    Adds sentiment columns and a 'like_weight' for aggregation.
    Like weight = like_count + 1 (to avoid zero weights).

    Parameters
    ----------
    df : Preprocessed comments DataFrame

    Returns
    -------
    DataFrame with sentiment scores
    """
    df = df.copy()
    scores = df['clean_text'].apply(score_comment)
    df = pd.concat([df, pd.DataFrame(list(scores))], axis=1)

    df['sentiment_label'] = df['vader_compound'].apply(
        lambda c: 'POSITIVE' if c >= 0.05 else ('NEGATIVE' if c <= -0.05 else 'NEUTRAL')
    )
    df['like_weight'] = (df['like_count'] + 1).clip(lower=1)

    dist = df['sentiment_label'].value_counts()
    print('Comment sentiment:')
    for label, count in dist.items():
        print(f'  {label:10s}: {count:5d}  ({count/len(df)*100:.1f}%)')
    return df


scored_comments = run_comment_sentiment(clean_comments)
scored_comments[['clean_text', 'vader_compound', 'tb_polarity', 'sentiment_label', 'like_count']].head(5)

def aggregate_video_sentiment(
    comments_df: pd.DataFrame,
    videos_df: pd.DataFrame
) -> pd.DataFrame:
    """
    Aggregate comment-level sentiment to video level and join with metadata.

    Computes both simple mean and like-weighted mean sentiment per video.
    Joins with videos_df to include view_count and published_at.

    Parameters
    ----------
    comments_df : Sentiment-scored comments DataFrame
    videos_df   : Video metadata DataFrame

    Returns
    -------
    Video-level DataFrame with sentiment scores
    """
    video_sentiment = comments_df.groupby('video_id').apply(
        lambda g: pd.Series({
            'mean_vader':        g['vader_compound'].mean(),
            'weighted_vader':    (g['vader_compound'] * g['like_weight']).sum()
                                  / g['like_weight'].sum(),
            'mean_tb_polarity':  g['tb_polarity'].mean(),
            'mean_subjectivity': g['tb_subjectivity'].mean(),
            'comment_count':     len(g),
            'pct_positive':      (g['sentiment_label'] == 'POSITIVE').mean() * 100,
            'pct_negative':      (g['sentiment_label'] == 'NEGATIVE').mean() * 100,
        })
    ).reset_index()

    merged = video_sentiment.merge(
        videos_df[['video_id', 'title', 'channel_title', 'published_at', 'view_count', 'like_count']],
        on='video_id', how='left'
    )
    merged.sort_values('view_count', ascending=False, inplace=True)
    merged.reset_index(drop=True, inplace=True)
    print(f'Video-level sentiment: {len(merged)} videos')
    return merged


video_sentiment_df = aggregate_video_sentiment(scored_comments, videos_df)
video_sentiment_df[['title', 'mean_vader', 'pct_positive', 'pct_negative', 'view_count']].head(5)

def plot_video_sentiment_scatter(df: pd.DataFrame) -> None:
    """
    Scatter plot: sentiment score vs view count, sized by comment count.

    High-view videos on the right. Bullish ones above 0, bearish below.

    Parameters
    ----------
    df : Video-level sentiment DataFrame
    """
    fig, ax = plt.subplots(figsize=(12, 6))

    scatter = ax.scatter(
        df['view_count'] / 1e6,
        df['mean_vader'],
        s=df['comment_count'].clip(upper=500) * 2,
        c=df['mean_vader'],
        cmap='RdYlGn',
        vmin=-0.5, vmax=0.5,
        alpha=0.75, edgecolors='white', linewidths=0.5
    )
    ax.axhline(0, color='gray', linestyle='--', linewidth=1)
    ax.axhline(0.05, color='green', linestyle=':', linewidth=0.8, alpha=0.5)
    ax.axhline(-0.05, color='red', linestyle=':', linewidth=0.8, alpha=0.5)
    plt.colorbar(scatter, ax=ax, label='VADER Compound Score')
    ax.set_xlabel('Video Views (millions)')
    ax.set_ylabel('Mean VADER Compound')
    ax.set_title('Comment Sentiment vs Video Views\n(bubble size = comment count)', fontsize=13, fontweight='bold')
    plt.tight_layout()
    plt.show()


def plot_comment_sentiment_distribution(df: pd.DataFrame) -> None:
    """
    Bar chart of comment label counts and KDE of vader_compound.

    Parameters
    ----------
    df : Scored comments DataFrame
    """
    fig, axes = plt.subplots(1, 2, figsize=(14, 5))

    counts = df['sentiment_label'].value_counts().reindex(['POSITIVE', 'NEUTRAL', 'NEGATIVE'])
    axes[0].bar(counts.index, counts.values,
                color=['#2ecc71', '#95a5a6', '#e74c3c'], width=0.5, edgecolor='white')
    axes[0].set_title('Comment Sentiment Labels', fontsize=12, fontweight='bold')
    axes[0].set_ylabel('Number of Comments')
    for i, (_, v) in enumerate(counts.items()):
        axes[0].text(i, v + 10, str(v), ha='center', fontsize=11)

    df['vader_compound'].plot.kde(ax=axes[1], color='steelblue', linewidth=2)
    axes[1].axvline(0, color='gray', linestyle='--', linewidth=1)
    axes[1].axvspan(-1, -0.05, alpha=0.08, color='red')
    axes[1].axvspan(0.05, 1,   alpha=0.08, color='green')
    axes[1].set_xlim(-1, 1)
    axes[1].set_title('VADER Compound Distribution', fontsize=12, fontweight='bold')

    plt.suptitle('YouTube Crypto Comment Sentiment', fontsize=14)
    plt.tight_layout()
    plt.show()


plot_video_sentiment_scatter(video_sentiment_df)
plot_comment_sentiment_distribution(scored_comments)

def plot_yt_wordclouds(df: pd.DataFrame) -> None:
    pos_text = ' '.join(df[df['sentiment_label'] == 'POSITIVE']['clean_text'])
    neg_text = ' '.join(df[df['sentiment_label'] == 'NEGATIVE']['clean_text'])
    fig, axes = plt.subplots(1, 2, figsize=(16, 6))
    for ax, text, title, bg, cmap in [
        (axes[0], pos_text, 'Bullish Comments — Top Keywords', '#f0fff0', 'Greens'),
        (axes[1], neg_text, 'Bearish Comments — Top Keywords', '#fff0f0', 'Reds')
    ]:
        if text.strip():
            wc = WordCloud(width=700, height=400, background_color=bg,
                           colormap=cmap, stopwords=YT_STOP_WORDS,
                           max_words=80, collocations=False).generate(text)
            ax.imshow(wc, interpolation='bilinear')
        ax.set_title(title, fontsize=12, fontweight='bold')
        ax.axis('off')
    plt.tight_layout()
    plt.show()


plot_yt_wordclouds(scored_comments)

def build_daily_yt_signal(video_df: pd.DataFrame) -> pd.DataFrame:
    """
    Aggregate video-level sentiment to daily signals weighted by view count.

    Videos with more views are weighted higher — they reach more retail investors.

    Parameters
    ----------
    video_df : Video-level sentiment DataFrame with 'published_at'

    Returns
    -------
    Daily signal DataFrame
    """
    df = video_df.copy()
    df['date'] = pd.to_datetime(df['published_at']).dt.date
    df['view_weight'] = (df['view_count'] + 1).clip(lower=1)

    daily = df.groupby('date').apply(
        lambda g: pd.Series({
            'view_weighted_sentiment': (g['mean_vader'] * g['view_weight']).sum() / g['view_weight'].sum(),
            'mean_sentiment':          g['mean_vader'].mean(),
            'video_count':             len(g),
            'total_views':             g['view_count'].sum()
        })
    ).reset_index()
    daily['date'] = pd.to_datetime(daily['date'])
    daily.sort_values('date', inplace=True)
    return daily


def export_yt_results(comments_df, video_df, daily_df):
    comments_df.to_csv('youtube_comments_sentiment.csv', index=False)
    video_df.to_csv('youtube_video_sentiment.csv', index=False)
    daily_df.to_csv('youtube_daily_signal.csv', index=False)
    print(f'Exported: {len(comments_df)} comments, {len(video_df)} videos, {len(daily_df)} daily rows')


daily_yt = build_daily_yt_signal(video_sentiment_df)
export_yt_results(scored_comments, video_sentiment_df, daily_yt)
daily_yt.tail(5)