import sys

# Install core libraries
!{sys.executable} -m pip install pandas matplotlib seaborn tqdm nltk snscrape requests

# Download NLTK data for sentiment analysis (moved to import cell for consolidation)

import sys
import datetime
import time
import random
import logging
from collections import deque

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
# import snscrape.modules.twitter as sntwitter # Conceptual for scraping, currently unreliable
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from tqdm.notebook import tqdm
import requests # For general web interaction and exponential backoff concept

import nltk

# Download NLTK data for sentiment analysis
try:
    nltk.data.find('sentiment/vader_lexicon.zip')
except LookupError:
    nltk.download('vader_lexicon')

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

def create_scraper_state(query: str, since: str, until: str, max_tweets: int = 1000) -> dict:
    """
    Initializes the state dictionary for the Twitter/X scraping process.

    Parameters
    ----------
    query : str
        The search query to use for scraping (e.g., '#bitcoin').
    since : str
        The start date for scraping in 'YYYY-MM-DD' format.
    until : str
        The end date for scraping in 'YYYY-MM-DD' format.
    max_tweets : int, optional
        The maximum number of tweets to retrieve. Defaults to 1000.

    Returns
    -------
    dict
        An initialized state dictionary containing scraping parameters and an empty list for `scraped_data`.

    Examples
    --------
    >>> state = create_scraper_state(query='#ethereum', since='2023-01-01', until='2023-01-07', max_tweets=500)
    >>> assert 'query' in state
    >>> assert state['max_tweets'] == 500
    """
    logger.info(f"Initializing scraper state for query: '{query}' from {since} to {until}")
    state = {
        'query': query,
        'since': since,
        'until': until,
        'max_tweets': max_tweets,
        'scraped_data': []
    }
    logger.debug("Scraper state initialized successfully.")
    return state

def exponential_backoff_retry(func: callable, exceptions: tuple, max_retries: int = 5, initial_delay: float = 1.0):
    """
    Implements an exponential backoff strategy for retrying a function.

    Parameters
    ----------
    func : callable
        The function to execute.
    exceptions : tuple
        A tuple of exception types to catch and retry on.
    max_retries : int, optional
        The maximum number of retry attempts. Defaults to 5.
    initial_delay : float, optional
        The initial delay in seconds before the first retry. Defaults to 1.0.

    Returns
    -------
    Any
        The result of the `func` if successful.

    Raises
    ------
    Exception
        Re-raises the last exception if all retries fail.

    Examples
    --------
    >>> def flaky_api_call():
    ...     if random.random() < 0.7: # Simulate 70% failure rate
    ...         raise requests.exceptions.RequestException("API error")
    ...     return "Data"
    >>> # result = exponential_backoff_retry(flaky_api_call, (requests.exceptions.RequestException,)) # This will now work as intended as logger is defined.
    >>> # assert result == "Data" # This might still fail sometimes, for demonstration.
    """
    delay = initial_delay
    for i in range(max_retries):
        try:
            return func()
        except exceptions as e:
            logger.warning(f"Attempt {i + 1}/{max_retries} failed with {type(e).__name__}: {e}")
            if i == max_retries - 1:
                logger.error(f"Max retries reached. Failing permanently.")
                raise
            jitter = random.uniform(0.5 * delay, 1.5 * delay)
            sleep_time = min(delay + jitter, 60.0) # Cap sleep time for practicality
            logger.info(f"Retrying in {sleep_time:.2f} seconds...")
            time.sleep(sleep_time)
            delay *= 2

def scrape_tweets(state: dict) -> dict:
    """
    Simulates scraping tweets based on the provided state.
    **Note: Due to recent Twitter/X API changes, direct scraping via snscrape may be unreliable.**
    This function demonstrates the structure, assuming a functional scraping method.

    Parameters
    ----------
    state : dict
        The current scraper state dictionary containing 'query', 'since', 'until', and 'max_tweets'.

    Returns
    -------
    dict
        The updated state dictionary with `scraped_data` populated.

    Examples
    --------
    >>> # state = create_scraper_state(query='#dogecoin', since='2023-01-01', until='2023-01-02', max_tweets=10)
    >>> # updated_state = scrape_tweets(state)
    >>> # assert len(updated_state['scraped_data']) <= 10
    """
    query = state['query']
    since = state['since']
    until = state['until']
    max_tweets = state['max_tweets']
    scraped_tweets_list = []

    search_string = f"{query} since:{since} until:{until}"
    logger.info(f"Starting conceptual tweet scraping for: '{search_string}' (max_tweets={max_tweets})")

    # Simulate scraping due to snscrape limitations for actual X data
    # In a real scenario, this would iterate through sntwitter.TwitterSearchScraper(search_string).get_items()
    # and perform error handling with exponential_backoff_retry for each API call.

    # Generate dummy data for demonstration purposes
    for i in tqdm(range(min(max_tweets, 500)), desc="Simulating Tweet Scraping"):
        tweet_date = datetime.datetime.strptime(since, '%Y-%m-%d') + datetime.timedelta(days=random.randint(0, (datetime.datetime.strptime(until, '%Y-%m-%d') - datetime.datetime.strptime(since, '%Y-%m-%d')).days))
        tweet_time = tweet_date + datetime.timedelta(hours=random.randint(0,23), minutes=random.randint(0,59))
        scraped_tweets_list.append({
            'date': tweet_time.isoformat(),
            'id': f'{random.randint(1000000000000000000, 9999999999999999999)}',
            'content': f"This is a sample tweet about {query}. Sentiment: {'positive' if random.random() > 0.5 else 'negative' if random.random() < 0.2 else 'neutral'}. crypto trading analysis!",
            'username': f"user_{random.randint(1, 100)}",
            'reply_count': random.randint(0, 100),
            'retweet_count': random.randint(0, 500),
            'like_count': random.randint(0, 2000)
        })

    # Actual snscrape usage (commented out due to likely breaking changes on Twitter/X)
    # try:
    #     for i, tweet in enumerate(sntwitter.TwitterSearchScraper(search_string).get_items()):
    #         if i >= max_tweets: break
    #         scraped_tweets_list.append({
    #             'date': tweet.date.isoformat(),
    #             'id': tweet.id,
    #             'content': tweet.rawContent,
    #             'username': tweet.user.username,
    #             'reply_count': tweet.replyCount,
    #             'retweet_count': tweet.retweetCount,
    #             'like_count': tweet.likeCount
    #         })
    # except Exception as e:
    #     logger.error(f"Error during snscrape operation: {e}. Falling back to dummy data.")
    #     # Placeholder for dummy data generation if snscrape fails

    state['scraped_data'] = scraped_tweets_list
    logger.info(f"Finished conceptual tweet scraping. Collected {len(state['scraped_data'])} tweets.")
    return state

def process_tweets(state: dict) -> dict:
    """
    Processes raw scraped tweet data into a pandas DataFrame and performs basic cleaning.

    Parameters
    ----------
    state : dict
        The current scraper state dictionary containing 'scraped_data'.

    Returns
    -------
    dict
        The updated state dictionary with a new key 'processed_df' containing the pandas DataFrame.

    Examples
    --------
    >>> state_with_data = {'scraped_data': [{'date': '2023-01-01T12:00:00', 'content': 'Test', 'reply_count': 1}]}
    >>> updated_state = process_tweets(state_with_data)
    >>> assert 'processed_df' in updated_state
    >>> assert isinstance(updated_state['processed_df'], pd.DataFrame)
    """
    logger.info("Starting tweet data processing.")
    if not state['scraped_data']:
        logger.warning("No scraped data found. Returning empty DataFrame.")
        state['processed_df'] = pd.DataFrame()
        return state

    df = pd.DataFrame(state['scraped_data'])

    # Convert 'date' column to datetime objects
    df['date'] = pd.to_datetime(df['date'])

    # Ensure numerical columns are correctly typed, filling NaNs with 0 before conversion
    numerical_cols = ['reply_count', 'retweet_count', 'like_count']
    for col in numerical_cols:
        if col in df.columns:
            df[col] = df[col].fillna(0).astype(int)

    state['processed_df'] = df
    logger.info(f"Tweet data processed. DataFrame shape: {df.shape}")
    logger.debug(f"Processed DataFrame columns: {df.columns.tolist()}")
    return state

def analyze_sentiment(state: dict) -> dict:
    """
    Performs sentiment analysis on tweet content using NLTK VADER.

    Parameters
    ----------
    state : dict
        The current scraper state dictionary containing 'processed_df'.

    Returns
    -------
    dict
        The updated state dictionary with 'processed_df' including a 'sentiment_score' column.

    Examples
    --------
    >>> df_example = pd.DataFrame({'content': ['This is great!', 'I hate this.', 'It\'s okay.'], 'date': pd.to_datetime(['2023-01-01', '2023-01-01', '2023-01-01'])})
    >>> state_with_df = {'processed_df': df_example}
    >>> updated_state = analyze_sentiment(state_with_df)
    >>> assert 'sentiment_score' in updated_state['processed_df'].columns
    """
    logger.info("Starting sentiment analysis on tweets.")
    if state['processed_df'].empty:
        logger.warning("Processed DataFrame is empty. Skipping sentiment analysis.")
        return state

    sid = SentimentIntensityAnalyzer()

    def get_vader_sentiment(text: str) -> float:
        return sid.polarity_scores(text)['compound']

    state['processed_df']['sentiment_score'] = state['processed_df']['content'].apply(get_vader_sentiment)

    logger.info("Sentiment analysis completed. 'sentiment_score' column added.")
    logger.debug(f"Sentiment scores head:\n{state['processed_df']['sentiment_score'].head()}")
    return state

def aggregate_daily_metrics(state: dict) -> dict:
    """
    Aggregates tweet data to daily metrics, including tweet count, average sentiment,
    and total engagement (likes, retweets, replies).

    Parameters
    ----------
    state : dict
        The current scraper state dictionary containing 'processed_df'.

    Returns
    -------
    dict
        The updated state dictionary with a new key 'daily_metrics_df' containing the aggregated daily DataFrame.

    Examples
    --------
    >>> df_example = pd.DataFrame({
    ...     'date': pd.to_datetime(['2023-01-01 10:00', '2023-01-01 14:00', '2023-01-02 09:00']),
    ...     'sentiment_score': [0.5, -0.1, 0.8],
    ...     'like_count': [10, 5, 20],
    ...     'retweet_count': [2, 1, 5],
    ...     'reply_count': [1, 0, 2]
    ... })
    >>> state_with_df = {'processed_df': df_example}
    >>> updated_state = aggregate_daily_metrics(state_with_df)
    >>> assert 'daily_metrics_df' in updated_state
    >>> assert updated_state['daily_metrics_df'].shape[0] == 2 # Two unique dates
    """
    logger.info("Aggregating tweet data to daily metrics.")
    if state['processed_df'].empty:
        logger.warning("Processed DataFrame is empty. Cannot aggregate daily metrics.")
        state['daily_metrics_df'] = pd.DataFrame()
        return state

    df = state['processed_df'].copy()
    df['day'] = df['date'].dt.date

    daily_metrics_df = df.groupby('day').agg(
        tweet_count=('id', 'count'),
        avg_sentiment=('sentiment_score', 'mean'),
        total_likes=('like_count', 'sum'),
        total_retweets=('retweet_count', 'sum'),
        total_replies=('reply_count', 'sum')
    ).reset_index()

    daily_metrics_df['day'] = pd.to_datetime(daily_metrics_df['day'])
    state['daily_metrics_df'] = daily_metrics_df

    logger.info(f"Daily metrics aggregated. DataFrame shape: {daily_metrics_df.shape}")
    logger.debug(f"Daily metrics head:\n{daily_metrics_df.head()}")
    return state

def save_data(state: dict, output_filepath: str) -> dict:
    """
    Saves the processed tweet data (DataFrame) to a CSV file.

    Parameters
    ----------
    state : dict
        The current scraper state dictionary containing 'processed_df'.
    output_filepath : str
        The path to the CSV file where the data will be saved.

    Returns
    -------
    dict
        The unchanged state dictionary.

    Examples
    --------
    >>> df_to_save = pd.DataFrame({'col1': [1,2], 'col2': ['A','B']})
    >>> state_to_save = {'processed_df': df_to_save}
    >>> # save_data(state_to_save, 'test_output.csv') # This would create a file.
    """
    logger.info(f"Attempting to save data to '{output_filepath}'.")
    if 'processed_df' in state and not state['processed_df'].empty:
        try:
            state['processed_df'].to_csv(output_filepath, index=False)
            logger.info(f"Data successfully saved to '{output_filepath}'.")
        except Exception as e:
            logger.error(f"Error saving data to '{output_filepath}': {e}")
    else:
        logger.warning("No processed DataFrame found or it is empty. No data saved.")
    return state

# Define scraping parameters
CRYPTO_QUERY = '#bitcoin'
START_DATE = '2023-01-01'
END_DATE = '2023-01-08' # A week of data
MAX_TWEETS_TO_SCRAPE = 1000

# Create the initial state
scraper_state = create_scraper_state(
    query=CRYPTO_QUERY,
    since=START_DATE,
    until=END_DATE,
    max_tweets=MAX_TWEETS_TO_SCRAPE
)

print(f"Initial scraper state:\n{scraper_state}")

# Perform conceptual scraping
scraper_state = scrape_tweets(scraper_state)

print(f"\nNumber of (simulated) tweets scraped: {len(scraper_state['scraped_data'])}")
if scraper_state['scraped_data']:
    print(f"First 3 scraped tweets:\n{scraper_state['scraped_data'][:3]}")

# Process the scraped data into a DataFrame
scraper_state = process_tweets(scraper_state)

print(f"\nProcessed DataFrame info:")
scraper_state['processed_df'].info()
print(f"\nFirst 5 rows of processed DataFrame:\n{scraper_state['processed_df'].head()}")

# Perform sentiment analysis
scraper_state = analyze_sentiment(scraper_state)

print(f"\nDataFrame head with sentiment scores:\n{scraper_state['processed_df'][['content', 'sentiment_score']].head()}")
print(f"\nSentiment score descriptive statistics:\n{scraper_state['processed_df']['sentiment_score'].describe()}")

# Aggregate daily metrics
scraper_state = aggregate_daily_metrics(scraper_state)

print(f"\nDaily metrics DataFrame head:\n{scraper_state['daily_metrics_df'].head()}")

if not scraper_state['daily_metrics_df'].empty:
    plt.figure(figsize=(14, 6))

    # Plot 1: Daily Tweet Count
    plt.subplot(1, 2, 1)
    sns.lineplot(x='day', y='tweet_count', data=scraper_state['daily_metrics_df'], marker='o')
    plt.title(f'Daily Tweet Volume for {CRYPTO_QUERY}')
    plt.xlabel('Date')
    plt.ylabel('Number of Tweets')
    plt.grid(True)
    plt.xticks(rotation=45)

    # Plot 2: Daily Average Sentiment
    plt.subplot(1, 2, 2)
    sns.lineplot(x='day', y='avg_sentiment', data=scraper_state['daily_metrics_df'], marker='o', color='green')
    plt.axhline(y=0, color='r', linestyle='--', label='Neutral Sentiment')
    plt.title(f'Daily Average Sentiment for {CRYPTO_QUERY}')
    plt.xlabel('Date')
    plt.ylabel('Average Sentiment Score')
    plt.grid(True)
    plt.xticks(rotation=45)
    plt.legend()

    plt.tight_layout()
    plt.show()
else:
    logger.warning("Daily metrics DataFrame is empty, skipping daily volume and sentiment plots.")

if 'processed_df' in scraper_state and not scraper_state['processed_df'].empty and 'sentiment_score' in scraper_state['processed_df'].columns:
    plt.figure(figsize=(8, 6))
    sns.histplot(scraper_state['processed_df']['sentiment_score'], bins=30, kde=True)
    plt.title(f'Distribution of Sentiment Scores for {CRYPTO_QUERY} Tweets')
    plt.xlabel('Sentiment Score (VADER Compound)')
    plt.ylabel('Frequency')
    plt.grid(True)
    plt.show()
else:
    logger.warning("Processed DataFrame or sentiment_score column missing/empty, skipping sentiment distribution plot.")

OUTPUT_FILENAME = 'crypto_tweets_processed.csv'
scraper_state = save_data(scraper_state, OUTPUT_FILENAME)

# Display first few rows of the saved (or hypothetical saved) data
if 'processed_df' in scraper_state and not scraper_state['processed_df'].empty:
    print(f"\nSample of data that would be saved to '{OUTPUT_FILENAME}':\n")
    display(scraper_state['processed_df'].head())