// src/economic/update.rs
use super::{scraper::*, storage::*, helpers::*, types::*};
use crate::check_shutdown;
use crate::{config::Config, scraper::webdriver::{ScrapeTask, ChromeDriverPool}, util::directories::DataPaths, util::logger};
use chrono::{Local};
use std::sync::{Arc, atomic::{AtomicBool}};
use std::collections::HashMap;

/// Runs the full update for economic data using streaming to minimize memory usage
pub async fn run_full_update(config: &Config, pool: &Arc<ChromeDriverPool>, shutdown_flag: &Arc<AtomicBool>) -> anyhow::Result<()> {
    let paths = DataPaths::new(".")?;

    logger::log_info("Economic Update: Initializing...").await;

    let today_str = chrono::Local::now().date_naive().format("%Y-%m-%d").to_string();
    let end_date = config.target_end_date();

    logger::log_info("=== Economic Update ===").await;

    check_shutdown!(shutdown_flag);

    // Step 1: Build lightweight index instead of loading all events
    logger::log_info("Step 1: Building event index...").await;
    let chunks = scan_existing_chunks(&paths).await?;
    let event_index = build_event_index(&chunks).await?;
    logger::log_info(&format!("  Economic Update: Indexed {} events from {} chunks", 
        event_index.len(), chunks.len())).await;

    check_shutdown!(shutdown_flag);

    // Step 2: Determine start date
    let start_date = if event_index.is_empty() {
        logger::log_warn("Step 2: No existing events found, starting from config date").await;
        config.economic_start_date.clone()
    } else {
        // Find the latest date in the index
        let max_date = event_index.iter()
            .map(|e| &e.date)
            .max()
            .cloned()
            .unwrap_or(today_str.clone());
        
        if max_date >= today_str {
            logger::log_info("  Events exist for today, starting from today").await;
            today_str.clone()
        } else {
            let next = chrono::NaiveDate::parse_from_str(&max_date, "%Y-%m-%d")
                .ok()
                .and_then(|d| d.succ_opt())
                .map(|d| d.format("%Y-%m-%d").to_string())
                .unwrap_or(today_str.clone());
            logger::log_info(&format!("  Resuming from: {}", next)).await;
            next
        }
    };

    check_shutdown!(shutdown_flag);

    // Step 3: Scrape new events in batches
    logger::log_info(&format!("Step 3: Scraping events from {} → {}", start_date, end_date)).await;
    let new_events = scrape_all_economic_events(&start_date, &end_date, pool).await?;
    logger::log_info(&format!("  Scraped {} new events", new_events.len())).await;

    check_shutdown!(shutdown_flag);

    // Step 4: Process events in streaming fashion
    logger::log_info(&format!("Step 4: Detecting changes")).await;
    let (changes, updated_events) = process_events_streaming(&chunks, &new_events, &today_str).await?;
    logger::log_info(&format!("  Detected {} changes", changes.len())).await;
    if !changes.is_empty() {
        logger::log_info(&format!("  Saving {} changes to log", changes.len())).await;
        save_changes(&paths, &changes).await?;
        logger::log_info("  Changes saved successfully").await;
    }

    check_shutdown!(shutdown_flag);

    // Step 5: Save consolidated events
    logger::log_info(&format!("Step 5: Saving {} total events to chunks", updated_events.len())).await;
    save_optimized_chunks(&paths, updated_events).await?;
    logger::log_info(&format!("  ✓ Economic update complete — {} changes detected", changes.len())).await;

    Ok(())
}

/// Process events using streaming to minimize memory usage
async fn process_events_streaming(
    chunks: &[ChunkInfo],
    new_events: &[EconomicEvent],
    today: &str,
) -> anyhow::Result<(Vec<EventChange>, Vec<EconomicEvent>)> {
    let mut all_changes = Vec::new();
    let mut final_events: HashMap<String, EconomicEvent> = HashMap::new();
    
    // Step 1: Load existing events in batches
    logger::log_info("Processing existing events in batches...").await;
    
    for chunk in chunks {
        logger::log_info(&format!("Loading chunk: {:?}", chunk.path.file_name())).await;
        
        let content = tokio::fs::read_to_string(&chunk.path).await?;
        let events: Vec<EconomicEvent> = serde_json::from_str(&content)?;
        
        // Add to final events map
        for e in events {
            final_events.insert(event_key(&e), e);
        }
        
        // Clear memory periodically
        if final_events.len() > 10000 {
            logger::log_info(&format!("Loaded {} events so far...", final_events.len())).await;
        }
    }
    
    logger::log_info(&format!("Loaded {} existing events total", final_events.len())).await;
    
    // Step 2: Process new events in batches
    logger::log_info("Processing new events...").await;
    
    for (idx, batch) in new_events.chunks(500).enumerate() {
        logger::log_info(&format!("Processing batch {} ({} events)", idx + 1, batch.len())).await;
        
        let batch_result = process_batch(batch, &mut final_events, today);
        all_changes.extend(batch_result.changes);
        
        // Yield to prevent blocking
        tokio::task::yield_now().await;
    }
    
    logger::log_info(&format!("Processing complete. Total events: {}", final_events.len())).await;
    
    // Convert HashMap to Vec for saving
    let events_vec: Vec<EconomicEvent> = final_events.into_values().collect();
    
    Ok((all_changes, events_vec))
}

/// Scrapes all economic events from start to end date
pub async fn scrape_all_economic_events(
    start: &str, 
    end: &str, 
    pool: &Arc<ChromeDriverPool>
) -> anyhow::Result<Vec<EconomicEvent>> {
    let url = "https://www.finanzen.net/termine/wirtschaftsdaten/".to_string();
    let start_clone = start.to_string();
    let end_clone = end.to_string();

    let task = ScrapeTask::new(url, move |client| async move {
        goto_and_prepare(&client).await?;
        let mut all_events = Vec::new();
        let mut current = start_clone;

        while current <= end_clone {
            set_date_range(&client, &current, &end_clone).await?;
            tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
            let new_events = extract_events(&client).await?;
            
            if new_events.is_empty() { 
                break; 
            }
            
            all_events.extend(new_events.clone());

            // Prevent memory buildup - process in chunks if too large
            if all_events.len() > 5000 {
                logger::log_info(&format!("Scraped {} events so far, continuing...", all_events.len())).await;
            }

            let next = new_events.iter()
                .filter_map(|e| chrono::NaiveDate::parse_from_str(&e.date, "%Y-%m-%d").ok())
                .max()
                .and_then(|d| d.succ_opt())
                .map(|d| d.format("%Y-%m-%d").to_string())
                .unwrap_or(end_clone.clone());

            if next > end_clone { break; }
            current = next;
        }
        
        Ok(all_events)
    });

    task.execute_with_pool(pool).await
}

/// Process a batch of events and detect changes
pub fn process_batch(
    new_events: &[EconomicEvent],
    existing: &mut HashMap<String, EconomicEvent>,
    today: &str,
) -> ScrapeResult {
    let mut changes = Vec::new();
    let mut removed = std::collections::HashSet::new();

    let identity_map = build_identity_lookup(existing);
    let date_map = build_date_event_lookup(existing);

    for new in new_events {
        let key = event_key(new);

        if let Some(old) = existing.get(&key) {
            changes.extend(detect_changes(old, new, today));
            existing.insert(key, new.clone());
            continue;
        }

        let date_key = format!("{}|{}|{}", new.country, new.event, new.date);
        if let Some(occurrences) = date_map.get(&date_key) {
            if let Some((old_key, old_event)) = occurrences.iter().find(|(k, _)| *k != key) {
                if new.date.as_str() > today {
                    changes.push(EventChange {
                        date: new.date.clone(),
                        event: new.event.clone(),
                        country: new.country.clone(),
                        field_changed: "time".to_string(),
                        old_value: old_event.time.clone(),
                        new_value: new.time.clone(),
                        detected_at: Local::now().format("%Y-%m-%d %H:%M:%S").to_string(),
                    });
                }
                removed.insert(old_key.clone());
                existing.remove(old_key);
            }
        }

        existing.insert(key, new.clone());
    }

    ScrapeResult { changes, removed_keys: removed }
}