!pip install telethon vaderSentiment textblob wordcloud nest_asyncio --quiet

import asyncio
import nest_asyncio
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sns
from wordcloud import WordCloud
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
from textblob import TextBlob
from telethon import TelegramClient
from telethon.tl.types import Channel, Chat
import re
from datetime import datetime, timezone, timedelta

# Allow asyncio to run inside Colab's existing event loop
nest_asyncio.apply()

%matplotlib inline
plt.rcParams['figure.figsize'] = (13, 5)
plt.rcParams['axes.spines.top'] = False
plt.rcParams['axes.spines.right'] = False
sns.set_palette('husl')

print('All imports successful.')

# ─── CONFIGURATION ─────────────────────────────────────────────────────────
API_ID   = 12345678           # Replace with your api_id (integer)
API_HASH = 'YOUR_API_HASH'   # Replace with your api_hash (string)
SESSION_NAME = 'crypto_monitor'

# SET TO True TO TEST WITHOUT TELEGRAM CREDENTIALS
USE_DUMMY_DATA = True

# Channels to monitor
TARGET_CHANNELS = [
    'CryptoNewsAlerts',
    'Bitcoin',
    'ethereum',
    'CryptoSignals'
]

MESSAGES_LIMIT = 200
DAYS_BACK = 7
# ───────────────────────────────────────────────────────────────────────────

def create_telegram_client(api_id: int, api_hash: str, session_name: str) -> TelegramClient:
    client = TelegramClient(session_name, api_id, api_hash)
    print(f'Telegram client created. Session: {session_name}')
    return client

client = None
if not USE_DUMMY_DATA:
    client = create_telegram_client(API_ID, API_HASH, SESSION_NAME)

import pandas as pd
import numpy as np
from datetime import datetime, timedelta, timezone

def generate_dummy_telegram_data(n_messages=100):
    channels = ['CryptoNewsAlerts', 'Bitcoin', 'ethereum', 'CryptoSignals']
    sample_texts = [
        "Bitcoin is looking incredibly bullish today! to the moon!",
        "Market is crashing, everyone sell now! Panic at the exchanges!",
        "Ethereum 2.0 update scheduled for next week. Great news for the ecosystem.",
        "Neutral price action observed in the last 4 hours for BTC.",
        "Total scam alert! Avoid this new token at all costs.",
        "Huge whale move detected: 50,000 BTC moved to cold storage.",
        "The regulatory news from the US is causing some uncertainty in the market.",
        "Just bought the dip! Long term holding strategy for #ETH."
    ]

    data = []
    now = datetime.now(timezone.utc)

    for i in range(n_messages):
        ch = np.random.choice(channels)
        text = np.random.choice(sample_texts) + f" (msg {i})"
        date = now - timedelta(hours=np.random.randint(0, 168)) # Last 7 days

        data.append({
            'channel': ch,
            'message_id': 1000 + i,
            'text': text,
            'date': date,
            'views': np.random.randint(100, 50000),
            'forwards': np.random.randint(0, 500),
            'replies': np.random.randint(0, 100),
            'sender_id': str(np.random.randint(100000, 999999))
        })

    df = pd.DataFrame(data)
    df.sort_values('date', ascending=False, inplace=True)
    return df.reset_index(drop=True)

# Replace the real scraper output with dummy data
raw_df = generate_dummy_telegram_data(250)
print(f"Generated {len(raw_df)} dummy messages across {raw_df['channel'].nunique()} channels.")
display(raw_df.head())

async def fetch_channel_messages(tg_client, channel, limit=200, days_back=None):
    messages = []
    offset_date = None
    if days_back:
        offset_date = datetime.now(timezone.utc) - timedelta(days=days_back)

    async for msg in tg_client.iter_messages(channel, limit=limit, offset_date=offset_date):
        if not msg.text: continue
        messages.append({
            'channel': channel,
            'message_id': msg.id,
            'text': msg.text,
            'date': msg.date.replace(tzinfo=timezone.utc) if msg.date.tzinfo is None else msg.date,
            'views': getattr(msg, 'views', 0) or 0,
            'forwards': getattr(msg, 'forwards', 0) or 0,
            'replies': msg.replies.replies if msg.replies else 0,
            'sender_id': str(msg.sender_id)
        })
    return messages

async def fetch_all_channels(tg_client, channels, limit=200, days_back=None):
    all_messages = []
    for ch in channels:
        try:
            msgs = await fetch_channel_messages(tg_client, ch, limit, days_back)
            all_messages.extend(msgs)
        except Exception as e: print(f'Warning: {e}')
    return pd.DataFrame(all_messages)

async def run_scraper():
    if USE_DUMMY_DATA:
        print("MODE: Using Dummy Data (Bypassing Telegram Login)")
        return generate_dummy_telegram_data(250)
    else:
        print("MODE: Fetching from Live Telegram API")
        async with client:
            return await fetch_all_channels(client, TARGET_CHANNELS, MESSAGES_LIMIT, DAYS_BACK)

# Execute based on the flag
raw_df = asyncio.run(run_scraper())
display(raw_df.head(3))

WORDCLOUD_STOP_WORDS = {
    'crypto', 'bitcoin', 'btc', 'eth', 'telegram', 'channel', 'forwarded',
    'market', 'price', 'just', 'will', 'new', 'one', 'get', 'like', 'now',
    'today', 'coin', 'tokens', 'trading', 'trade', 'buy', 'sell', 'good'
}


def clean_telegram_text(text: str) -> str:
    """
    Clean raw Telegram message text for sentiment analysis.

    Removes URLs, formatting markdown, forward headers, and extra whitespace.
    Preserves emojis (VADER handles common ones) and hashtag words.

    Parameters
    ----------
    text : Raw Telegram message string

    Returns
    -------
    Cleaned text string
    """
    if not isinstance(text, str):
        return ''

    text = re.sub(r'Forwarded from.*?\n', '', text)      # remove forward headers
    text = re.sub(r'http\S+|www\.\S+|t\.me/\S+', '', text)  # remove URLs & t.me links
    text = re.sub(r'@\w+', '', text)                      # remove @mentions
    text = re.sub(r'#(\w+)', r'\1', text)                # keep hashtag words
    text = re.sub(r'[*_`~]', '', text)                   # remove markdown
    text = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', text) # inline links
    text = re.sub(r'\s+', ' ', text)
    return text.strip()


def preprocess_telegram_df(df: pd.DataFrame) -> pd.DataFrame:
    """
    Apply preprocessing to the raw Telegram messages DataFrame.

    Adds 'clean_text' and 'text_length' columns.
    Drops messages that become empty after cleaning.

    Parameters
    ----------
    df : Raw DataFrame from fetch_all_channels()

    Returns
    -------
    Preprocessed DataFrame
    """
    df = df.copy()
    df['clean_text']  = df['text'].apply(clean_telegram_text)
    df['text_length'] = df['clean_text'].apply(len)
    df = df[df['text_length'] > 5].reset_index(drop=True)
    print(f'After cleaning: {len(df)} messages remain (avg {df["text_length"].mean():.0f} chars)')
    return df


processed_df = preprocess_telegram_df(raw_df)
processed_df[['channel', 'date', 'clean_text', 'views']].head(4)

vader = SentimentIntensityAnalyzer()


def score_message_sentiment(text: str) -> dict:
    """
    Compute VADER and TextBlob sentiment scores for a single message.

    Parameters
    ----------
    text : Cleaned message text

    Returns
    -------
    Dictionary with vader_compound, tb_polarity, tb_subjectivity
    """
    if not text or len(text) < 3:
        return {'vader_compound': 0.0, 'tb_polarity': 0.0, 'tb_subjectivity': 0.0}

    vs = vader.polarity_scores(text)
    blob = TextBlob(text)
    return {
        'vader_compound':  vs['compound'],
        'vader_pos':       vs['pos'],
        'vader_neg':       vs['neg'],
        'tb_polarity':     blob.sentiment.polarity,
        'tb_subjectivity': blob.sentiment.subjectivity
    }


def label_sentiment(compound: float) -> str:
    if compound >= 0.05:  return 'POSITIVE'
    if compound <= -0.05: return 'NEGATIVE'
    return 'NEUTRAL'


def run_sentiment_pipeline(df: pd.DataFrame) -> pd.DataFrame:
    """
    Apply sentiment scoring to all messages in the DataFrame.

    Adds sentiment score columns and a categorical label.
    Computes an engagement weight = views + (forwards * 3) to
    up-weight forwarded messages as they have wider reach.

    Parameters
    ----------
    df : Preprocessed DataFrame with 'clean_text' column

    Returns
    -------
    DataFrame with sentiment columns added
    """
    df = df.copy()

    scores = df['clean_text'].apply(score_message_sentiment)
    df = pd.concat([df, pd.DataFrame(list(scores))], axis=1)

    df['sentiment_label']    = df['vader_compound'].apply(label_sentiment)
    df['engagement_weight']  = df['views'] + (df['forwards'] * 3)
    df['engagement_weight']  = df['engagement_weight'].clip(lower=1)

    dist = df['sentiment_label'].value_counts()
    print('Sentiment distribution:')
    for label, count in dist.items():
        print(f'  {label:10s}: {count:4d}  ({count/len(df)*100:.1f}%)')
    return df


sentiment_df = run_sentiment_pipeline(processed_df)
sentiment_df[['channel', 'clean_text', 'vader_compound', 'tb_subjectivity', 'sentiment_label', 'views']].head(5)

def plot_channel_sentiment_breakdown(df: pd.DataFrame) -> None:
    """
    Stacked bar chart: sentiment label proportions per channel.

    Parameters
    ----------
    df : DataFrame with 'channel' and 'sentiment_label' columns
    """
    pivot = df.groupby(['channel', 'sentiment_label']).size().unstack(fill_value=0)
    for col in ['POSITIVE', 'NEUTRAL', 'NEGATIVE']:
        if col not in pivot.columns:
            pivot[col] = 0
    pivot_pct = pivot[['POSITIVE', 'NEUTRAL', 'NEGATIVE']].div(pivot.sum(axis=1), axis=0) * 100

    ax = pivot_pct.plot(kind='bar', stacked=True,
                        color=['#2ecc71', '#95a5a6', '#e74c3c'],
                        figsize=(12, 5), edgecolor='white')
    ax.set_title('Sentiment Breakdown by Telegram Channel', fontsize=13, fontweight='bold')
    ax.set_ylabel('% of Messages')
    ax.set_xlabel('')
    ax.tick_params(axis='x', rotation=30)
    ax.legend(loc='upper right')
    plt.tight_layout()
    plt.show()


plot_channel_sentiment_breakdown(sentiment_df)

def plot_sentiment_over_time(df: pd.DataFrame, resample_freq: str = '4H') -> None:
    """
    Line chart of engagement-weighted sentiment resampled over time.

    Parameters
    ----------
    df             : DataFrame with 'date', 'vader_compound', 'engagement_weight'
    resample_freq  : Pandas offset alias for resampling (default '4H' = 4-hour buckets)
    """
    df = df.copy()
    df = df.set_index('date').sort_index()
    if df.index.tz is None:
        df.index = df.index.tz_localize('UTC')

    resampled = df['vader_compound'].resample(resample_freq).mean()
    volume    = df['message_id'].resample(resample_freq).count()

    fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(14, 7), sharex=True)

    ax1.axhline(0, color='gray', linestyle='--', linewidth=1, alpha=0.6)
    ax1.fill_between(resampled.index, resampled, 0,
                     where=resampled >= 0, alpha=0.3, color='#2ecc71')
    ax1.fill_between(resampled.index, resampled, 0,
                     where=resampled < 0,  alpha=0.3, color='#e74c3c')
    ax1.plot(resampled.index, resampled, color='steelblue', linewidth=2)
    ax1.set_ylabel('Avg VADER Compound')
    ax1.set_title(f'Telegram Sentiment Signal ({resample_freq} buckets)', fontsize=13, fontweight='bold')
    ax1.set_ylim(-1, 1)

    ax2.bar(volume.index, volume.values, color='steelblue', alpha=0.6, width=pd.Timedelta(resample_freq) * 0.8)
    ax2.set_ylabel('Message Count')
    ax2.set_xlabel('Time (UTC)')
    ax2.xaxis.set_major_formatter(mdates.DateFormatter('%b %d %H:%M'))
    plt.xticks(rotation=30)

    plt.tight_layout()
    plt.show()


plot_sentiment_over_time(sentiment_df)

def generate_telegram_wordclouds(df: pd.DataFrame) -> None:
    """
    Side-by-side word clouds for positive and negative Telegram messages.

    Parameters
    ----------
    df : DataFrame with 'sentiment_label' and 'clean_text' columns
    """
    pos_text = ' '.join(df[df['sentiment_label'] == 'POSITIVE']['clean_text'])
    neg_text = ' '.join(df[df['sentiment_label'] == 'NEGATIVE']['clean_text'])

    fig, axes = plt.subplots(1, 2, figsize=(16, 6))
    for ax, text, title, bg, cmap in [
        (axes[0], pos_text, 'Bullish Messages — Top Words', '#f0fff0', 'Greens'),
        (axes[1], neg_text, 'Bearish Messages — Top Words', '#fff0f0', 'Reds')
    ]:
        if text.strip():
            wc = WordCloud(width=700, height=400, background_color=bg,
                           colormap=cmap, stopwords=WORDCLOUD_STOP_WORDS,
                           max_words=80, collocations=False).generate(text)
            ax.imshow(wc, interpolation='bilinear')
        else:
            ax.text(0.5, 0.5, 'No data', ha='center', va='center', transform=ax.transAxes)
        ax.set_title(title, fontsize=12, fontweight='bold')
        ax.axis('off')

    plt.tight_layout()
    plt.show()


generate_telegram_wordclouds(sentiment_df)

def compute_hourly_signal(df: pd.DataFrame) -> pd.DataFrame:
    """
    Aggregate message sentiment into an hourly signal DataFrame.

    Uses engagement_weight (views + 3*forwards) to compute a
    weighted average sentiment score per hour.

    Parameters
    ----------
    df : Sentiment-scored DataFrame with 'date' and 'engagement_weight'

    Returns
    -------
    Hourly DataFrame with columns: hour, mean_sentiment, weighted_sentiment,
                                    message_count, total_engagement
    """
    df = df.copy()
    df['hour'] = pd.to_datetime(df['date']).dt.floor('H')

    hourly = df.groupby('hour').apply(
        lambda g: pd.Series({
            'mean_sentiment':     g['vader_compound'].mean(),
            'weighted_sentiment': (g['vader_compound'] * g['engagement_weight']).sum()
                                  / g['engagement_weight'].sum(),
            'message_count':      len(g),
            'total_views':        g['views'].sum(),
            'total_forwards':     g['forwards'].sum(),
            'pct_positive':       (g['sentiment_label'] == 'POSITIVE').mean() * 100,
            'pct_negative':       (g['sentiment_label'] == 'NEGATIVE').mean() * 100
        })
    ).reset_index()

    print(f'Hourly signal: {len(hourly)} hours of data')
    return hourly


hourly_signal = compute_hourly_signal(sentiment_df)
hourly_signal.tail(5)

def export_telegram_results(
    messages_df: pd.DataFrame,
    hourly_df: pd.DataFrame,
    messages_file: str = 'telegram_messages_sentiment.csv',
    hourly_file: str   = 'telegram_hourly_signal.csv'
) -> None:
    """
    Export message-level and hourly signal DataFrames to CSV.

    Parameters
    ----------
    messages_df   : Full messages DataFrame with sentiment scores
    hourly_df     : Hourly aggregated signal DataFrame
    messages_file : Output filename for message data
    hourly_file   : Output filename for hourly signal
    """
    msg_cols = ['channel', 'message_id', 'date', 'clean_text',
                'vader_compound', 'vader_pos', 'vader_neg',
                'tb_polarity', 'tb_subjectivity',
                'sentiment_label', 'views', 'forwards', 'engagement_weight']
    messages_df[msg_cols].to_csv(messages_file, index=False)
    print(f'Messages saved: {messages_file}  ({len(messages_df)} rows)')

    hourly_df.to_csv(hourly_file, index=False)
    print(f'Hourly signal: {hourly_file}    ({len(hourly_df)} rows)')


export_telegram_results(sentiment_df, hourly_signal)

# Final Verification: Run dummy data through the entire pipeline now that all functions are defined
try:
    # 1. Preprocess
    processed_dummy_df = preprocess_telegram_df(raw_df)

    # 2. Run sentiment analysis
    sentiment_dummy_df = run_sentiment_pipeline(processed_dummy_df)

    # 3. Generate hourly signals
    hourly_dummy_signal = compute_hourly_signal(sentiment_dummy_df)

    # 4. Test visualization functions
    plot_channel_sentiment_breakdown(sentiment_dummy_df)
    plot_sentiment_over_time(sentiment_dummy_df)
    generate_telegram_wordclouds(sentiment_dummy_df)

    print("\n--- Verification Results ---")
    display(sentiment_dummy_df[['channel', 'clean_text', 'sentiment_label', 'engagement_weight']].head())
except NameError as e:
    print(f"Error: {e}. Please ensure you have executed all cells above this one first.")