// src/corporate/scraper.rs
use super::{types::*, helpers::*, openfigi::*};
//use crate::corporate::openfigi::OpenFigiClient;
use crate::{scraper::webdriver::*};
use fantoccini::{Client, Locator};
use scraper::{Html, Selector};
use chrono::{DateTime, Duration, NaiveDate, Utc};
use tokio::{time::{Duration as TokioDuration, sleep}};
use reqwest::Client as HttpClient;
use serde_json::{json, Value};
use zip::ZipArchive;
use std::{collections::HashMap, sync::Arc};
use std::io::{Read};
use anyhow::{anyhow, Result};

const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";

/// Discover all exchanges where this ISIN trades by querying Yahoo Finance and enriching with OpenFIGI API calls.
/// 
/// # Arguments
/// * `isin` - The ISIN to search for.
/// * `known_ticker` - A known ticker symbol for fallback or initial check.
/// 
/// # Returns
/// A vector of FigiInfo structs containing enriched data from API calls.
/// 
/// # Errors
/// Returns an error if HTTP requests fail, JSON parsing fails, or OpenFIGI API responds with an error.
pub async fn discover_available_exchanges(isin: &str, known_ticker: &str) -> anyhow::Result<Vec<FigiInfo>> {
    println!("    Discovering exchanges for ISIN {}", isin);
    
    let mut potential: Vec<(String, PrimaryInfo)> = Vec::new();
    
    // Try the primary ticker first
    if let Ok(info) = check_ticker_exists(known_ticker).await {
        potential.push((known_ticker.to_string(), info));
    }
    
    // Search for ISIN directly on Yahoo to find other listings
    let search_url = format!(
        "https://query2.finance.yahoo.com/v1/finance/search?q={}&quotesCount=20&newsCount=0",
        isin
    );
    
    let resp = HttpClient::new()
        .get(&search_url)
        .header("User-Agent", USER_AGENT)
        .send()
        .await?;
    
    let json = resp.json::<Value>().await?;
    
    if let Some(quotes) = json["quotes"].as_array() {
        for quote in quotes {
            // First: filter by quoteType directly from search results (faster rejection)
            let quote_type = quote["quoteType"].as_str().unwrap_or("");
            if quote_type.to_uppercase() != "EQUITY" {
                continue; // Skip bonds, ETFs, mutual funds, options, etc.
            }

            if let Some(symbol) = quote["symbol"].as_str() {
                // Avoid duplicates
                if potential.iter().any(|(s, _)| s == symbol) {
                    continue;
                }

                // Double-check with full quote data (some search results are misleading)
                if let Ok(info) = check_ticker_exists(symbol).await {
                    potential.push((symbol.to_string(), info));
                }
            }
        }
    }

    if potential.is_empty() {
        return Ok(vec![]);
    }

    // Enrich with OpenFIGI API
    let client = OpenFigiClient::new()?;

    let mut discovered_figis = Vec::new();

    if !client.has_key() {
        // Fallback without API key - create FigiInfo with default/empty fields
        for (symbol, info) in potential {
            println!("      Found equity listing: {} on {} ({}) - no FIGI (fallback mode)", symbol, info.exchange_mic, info.currency);
            let figi_info = FigiInfo {
                isin: info.isin,
                figi: String::new(),
                name: info.name,
                ticker: symbol,
                mic_code: info.exchange_mic,
                currency: info.currency,
                compositeFIGI: String::new(),
                securityType: String::new(),
                marketSector: String::new(),
                shareClassFIGI: String::new(),
                securityType2: String::new(),
                securityDescription: String::new(),
            };
            discovered_figis.push(figi_info);
        }
        return Ok(discovered_figis);
    }

    // With API key, batch the mapping requests
    let chunk_size = 100;
    for chunk in potential.chunks(chunk_size) {
        let mut jobs = vec![];
        for (symbol, info) in chunk {
            jobs.push(json!({
                "idType": "TICKER",
                "idValue": symbol,
                "micCode": info.exchange_mic,
                "marketSecDes": "Equity",
            }));
        }

        let resp = client.get_figi_client()
            .post("https://api.openfigi.com/v3/mapping")
            .header("Content-Type", "application/json")
            .json(&jobs)
            .send()
            .await?;

        if !resp.status().is_success() {
            return Err(anyhow::anyhow!("OpenFIGI mapping failed with status: {}", resp.status()));
        }

        let parsed: Vec<Value> = resp.json().await?;

        for (i, item) in parsed.iter().enumerate() {
            let (symbol, info) = &chunk[i];
            if let Some(data) = item["data"].as_array() {
                if let Some(entry) = data.first() {
                    let market_sec = entry["marketSector"].as_str().unwrap_or("");
                    if market_sec != "Equity" {
                        continue;
                    }
                    println!("      Found equity listing: {} on {} ({}) - FIGI: {}", symbol, info.exchange_mic, info.currency, entry["figi"]);
                    let figi_info = FigiInfo {
                        isin: info.isin.clone(),
                        figi: entry["figi"].as_str().unwrap_or("").to_string(),
                        name: entry["name"].as_str().unwrap_or(&info.name).to_string(),
                        ticker: symbol.clone(),
                        mic_code: info.exchange_mic.clone(),
                        currency: info.currency.clone(),
                        compositeFIGI: entry["compositeFIGI"].as_str().unwrap_or("").to_string(),
                        securityType: entry["securityType"].as_str().unwrap_or("").to_string(),
                        marketSector: market_sec.to_string(),
                        shareClassFIGI: entry["shareClassFIGI"].as_str().unwrap_or("").to_string(),
                        securityType2: entry["securityType2"].as_str().unwrap_or("").to_string(),
                        securityDescription: entry["securityDescription"].as_str().unwrap_or("").to_string(),
                    };
                    discovered_figis.push(figi_info);
                } else {
                    println!("      No data returned for ticker {} on MIC {}", symbol, info.exchange_mic);
                }
            } else if let Some(error) = item["error"].as_str() {
                println!("      OpenFIGI error for ticker {}: {}", symbol, error);
            }
        }

        // Respect rate limit (6 seconds between requests with key)
        sleep(TokioDuration::from_secs(6)).await;
    }

    Ok(discovered_figis)
}

/// Check if a ticker exists on Yahoo Finance and return core metadata.
///
/// This function calls the public Yahoo Finance quoteSummary endpoint and extracts:
/// - ISIN (when available)
/// - Company name
/// - Exchange MIC code
/// - Trading currency
///
/// It strictly filters to only accept **equity** securities.
///
/// # Arguments
/// * `ticker` - The ticker symbol to validate (e.g., "AAPL", "7203.T", "BMW.DE")
///
/// # Returns
/// `Ok(PrimaryInfo)` on success, `Err` if ticker doesn't exist, is not equity, or data is malformed.
///
/// # Errors
/// - Ticker not found
/// - Not an equity (ETF, bond, etc.)
/// - Missing critical fields
/// - Network or JSON parsing errors
pub async fn check_ticker_exists(ticker: &str) -> anyhow::Result<PrimaryInfo> {
    let url = format!(
        "https://query1.finance.yahoo.com/v10/finance/quoteSummary/{}?modules=price%2CassetProfile",
        ticker
    );

    let resp = match HttpClient::new()
        .get(&url)
        .header("User-Agent", USER_AGENT)
        .send()
        .await
    {
        Ok(resp) => resp,
        Err(err) => {
            return Err(anyhow::anyhow!(
                "Failed to reach Yahoo Finance for ticker {}: {}",
                ticker,
                err
            ));
        }
    };

    if !resp.status().is_success() {
        return Err(anyhow::anyhow!("Yahoo returned HTTP {} for ticker {}", resp.status(), ticker));
    }

    let json: Value = match resp
        .json()
        .await {
            Ok(resp) => resp,
            Err(err) => {
                return Err(anyhow::anyhow!(
                    "Failed to parse JSON response from Yahoo Finance {}: {}",
                    ticker,
                    err
                ));
            }
        };

    let result_array = json["quoteSummary"]["result"]
        .as_array()
        .ok_or_else(|| anyhow::anyhow!("Missing 'quoteSummary.result' in response"))?;

    if result_array.is_empty() || result_array[0].is_null() {
        return Err(anyhow::anyhow!("No quote data returned for ticker {}", ticker));
    }

    let quote = &result_array[0]["price"];
    let profile = &result_array[0]["assetProfile"];

    // === 1. Must be EQUITY ===
    let quote_type = quote["quoteType"]
        .as_str()
        .unwrap_or("")
        .to_ascii_uppercase();

    if quote_type != "EQUITY" {
        println!("      → Skipping {} (quoteType: {})", ticker, quote_type);
        return Err(anyhow::anyhow!("Not an equity security: {}", quote_type));
    }

    // === 2. Extract basic info ===
    let long_name = quote["longName"]
        .as_str()
        .or_else(|| quote["shortName"].as_str())
        .unwrap_or(ticker)
        .trim()
        .to_string();

    let currency = quote["currency"]
        .as_str()
        .unwrap_or("USD")
        .to_string();

    let exchange_mic = quote["exchange"]
        .as_str()
        .unwrap_or("")
        .to_string();

    if exchange_mic.is_empty() {
        return Err(anyhow::anyhow!("Missing exchange MIC for ticker {}", ticker));
    }

    // === 3. Extract ISIN (from assetProfile if available) ===
    let isin = profile["isin"]
        .as_str()
        .and_then(|s| if s.len() == 12 && s.chars().all(|c| c.is_ascii_alphanumeric()) { Some(s) } else { None })
        .unwrap_or("")
        .to_ascii_uppercase();

    // === 4. Final sanity check: reject obvious debt securities ===
    let name_upper = long_name.to_ascii_uppercase();
    if name_upper.contains(" BOND") ||
       name_upper.contains(" NOTE") ||
       name_upper.contains(" DEBENTURE") ||
       name_upper.contains(" PREFERRED") && !name_upper.contains(" STOCK") {
        return Err(anyhow::anyhow!("Security name suggests debt instrument: {}", long_name));
    }

    println!(
        "      → Valid equity: {} | {} | {} | ISIN: {}",
        ticker,
        long_name,
        exchange_mic,
        if isin.is_empty() { "N/A" } else { &isin }
    );

    Ok(PrimaryInfo {
        isin,
        name: long_name,
        exchange_mic,
        currency,
    })
}

/// Convert Yahoo's exchange name to MIC code (best effort)
fn exchange_name_to_mic(name: &str) -> String {
    match name {
        "NMS" | "NasdaqGS" | "NASDAQ" => "XNAS",
        "NYQ" | "NYSE" => "XNYS",
        "LSE" | "London" => "XLON",
        "FRA" | "Frankfurt" | "GER" | "XETRA" => "XFRA",
        "PAR" | "Paris" => "XPAR",
        "AMS" | "Amsterdam" => "XAMS",
        "MIL" | "Milan" => "XMIL",
        "JPX" | "Tokyo" => "XJPX",
        "HKG" | "Hong Kong" => "XHKG",
        "SHH" | "Shanghai" => "XSHG",
        "SHZ" | "Shenzhen" => "XSHE",
        "TOR" | "Toronto" => "XTSE",
        "ASX" | "Australia" => "XASX",
        "SAU" | "Saudi" => "XSAU",
        "SWX" | "Switzerland" => "XSWX",
        "BSE" | "Bombay" => "XBSE",
        "NSE" | "NSI" => "XNSE",
        "TAI" | "Taiwan" => "XTAI",
        "SAO" | "Sao Paulo" => "BVMF",
        "MCE" | "Madrid" => "XMAD",
        _ => name, // Fallback to name itself
    }.to_string()
}

/// Fetches earnings events for a ticker using a dedicated ScrapeTask.
///
/// This function creates and executes a ScrapeTask to navigate to the Yahoo Finance earnings calendar,
/// reject cookies, and extract the events.
///
/// # Arguments
/// * `ticker` - The stock ticker symbol.
///
/// # Returns
/// A vector of CompanyEvent structs on success.
///
/// # Errors
/// Returns an error if the task execution fails, e.g., chromedriver spawn or navigation issues.
pub async fn fetch_earnings_with_pool(
    ticker: &str,
    pool: &Arc<ChromeDriverPool>,
) -> anyhow::Result<Vec<CompanyEvent>> {
    let ticker = ticker.to_string();
    let url = format!("https://finance.yahoo.com/calendar/earnings?symbol={}", ticker);

    let ticker_cloned = ticker.clone();

    pool.execute(url, move |client| {
        let ticker = ticker_cloned.clone();
        Box::pin(async move {
            reject_yahoo_cookies(&client).await?;
            extract_earnings_events(&client, &ticker).await
        })
    }).await
}

/// Extracts earnings events from the currently loaded Yahoo Finance earnings calendar page.
///
/// This function assumes the client is already navigated to the correct URL (e.g., 
/// https://finance.yahoo.com/calendar/earnings?symbol={ticker}) and cookies are handled.
///
/// It waits for the earnings table, extracts rows, parses cells into CompanyEvent structs,
/// and handles date parsing, float parsing, and optional fields.
///
/// # Arguments
/// * `client` - The fantoccini Client with the page loaded.
/// * `ticker` - The stock ticker symbol for the events.
///
/// # Returns
/// A vector of CompanyEvent on success.
///
/// # Errors
/// Returns an error if:
/// - Table or elements not found.
/// - Date or float parsing fails.
/// - WebDriver operations fail.
///
/// # Examples
///
/// ```no_run
/// use fantoccini::Client;
/// use crate::corporate::scraper::extract_earnings;
///
/// #[tokio::main]
/// async fn main() -> Result<()> {
///     // Assume client is set up and navigated
///     let events = extract_earnings(&client, "AAPL").await?;
///     Ok(())
/// }
/// ```
pub async fn extract_earnings_events(client: &Client, ticker: &str) -> Result<Vec<CompanyEvent>> {
    // Wait for the table to load
    let table = client
        .wait()
        .for_element(Locator::Css(r#"table[data-test="cal-table"]"#))
        .await
        .map_err(|e| anyhow!("Failed to find earnings table: {}", e))?;

    // Find all rows in tbody
    let rows = table
        .find_all(Locator::Css("tbody tr"))
        .await
        .map_err(|e| anyhow!("Failed to find table rows: {}", e))?;

    let mut events = Vec::with_capacity(rows.len());

    for row in rows {
        let cells = row
            .find_all(Locator::Css("td"))
            .await
            .map_err(|e| anyhow!("Failed to find cells in row: {}", e))?;

        if cells.len() < 5 {
            continue; // Skip incomplete rows
        }

        // Extract and parse date
        let date_str = cells[0]
            .text()
            .await
            .map_err(|e| anyhow!("Failed to get date text: {}", e))?;
        let date = parse_yahoo_date(&date_str)
            .map_err(|e| anyhow!("Failed to parse date '{}': {}", date_str, e))?
            .format("%Y-%m-%d")
            .to_string();

        // Extract time, replace "Time Not Supplied" with empty
        let time = cells[1]
            .text()
            .await
            .map_err(|e| anyhow!("Failed to get time text: {}", e))?
            .replace("Time Not Supplied", "");

        // Extract period
        let period = cells[2]
            .text()
            .await
            .map_err(|e| anyhow!("Failed to get period text: {}", e))?;

        // Parse EPS forecast
        let eps_forecast_str = cells[3]
            .text()
            .await
            .map_err(|e| anyhow!("Failed to get EPS forecast text: {}", e))?;
        let eps_forecast = parse_float(&eps_forecast_str);

        // Parse EPS actual
        let eps_actual_str = cells[4]
            .text()
            .await
            .map_err(|e| anyhow!("Failed to get EPS actual text: {}", e))?;
        let eps_actual = parse_float(&eps_actual_str);

        // Parse surprise % if available
        let surprise_pct = if cells.len() > 5 {
            let surprise_str = cells[5]
                .text()
                .await
                .map_err(|e| anyhow!("Failed to get surprise text: {}", e))?;
            parse_float(&surprise_str)
        } else {
            None
        };

        events.push(CompanyEvent {
            ticker: ticker.to_string(),
            date,
            time,
            period,
            eps_forecast,
            eps_actual,
            revenue_forecast: None,
            revenue_actual: None,
            surprise_pct,
            source: "Yahoo".to_string(),
        });
    }

    if events.is_empty() {
        eprintln!("Warning: No earnings events extracted for ticker {}", ticker);
    } else {
        println!("Extracted {} earnings events for {}", events.len(), ticker);
    }

    Ok(events)
}

fn parse_price(v: Option<&Value>) -> f64 {
    v.and_then(|x| x.as_str())
        .and_then(|s| s.replace('$', "").replace(',', "").parse::<f64>().ok())
        .or_else(|| v.and_then(|x| x.as_f64()))
        .unwrap_or(0.0)
}

fn parse_volume(v: Option<&Value>) -> u64 {
    v.and_then(|x| x.as_str())
        .and_then(|s| s.replace(',', "").parse::<u64>().ok())
        .or_else(|| v.and_then(|x| x.as_u64()))
        .unwrap_or(0)
}

pub async fn fetch_daily_price_history(
    ticker: &str,
    start_str: &str,
    end_str: &str,
) -> anyhow::Result<Vec<CompanyPrice>> {
    let start = NaiveDate::parse_from_str(start_str, "%Y-%m-%d")?;
    let end   = NaiveDate::parse_from_str(end_str, "%Y-%m-%d")? + Duration::days(1);

    let mut all_prices = Vec::new();
    let mut current = start;

    while current < end {
        let chunk_end = current + Duration::days(730);
        let actual_end = chunk_end.min(end);

        let period1 = current.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp();
        let period2 = actual_end.and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp();

        println!("    Fetching {ticker} {} → {}", current, actual_end - Duration::days(1));

        let url = format!(
            "https://query1.finance.yahoo.com/v8/finance/chart/{ticker}?period1={period1}&period2={period2}&interval=1d&includeAdjustedClose=true"
        );

        let json: Value = HttpClient::new()
            .get(&url)
            .header("User-Agent", USER_AGENT)
            .send()
            .await?
            .json()
            .await?;

        let result = &json["chart"]["result"][0];
        let timestamps = result["timestamp"].as_array().ok_or_else(|| anyhow::anyhow!("No timestamps"))?;
        let quote = &result["indicators"]["quote"][0];
        let meta = &result["meta"];
        let currency = meta["currency"].as_str().unwrap_or("USD").to_string();

        let opens = quote["open"].as_array();
        let highs = quote["high"].as_array();
        let lows  = quote["low"].as_array();
        let closes = quote["close"].as_array();
        let adj_closes = result["indicators"]["adjclose"][0]["adjclose"].as_array()
            .or_else(|| closes);
        let volumes = quote["volume"].as_array();

        for (i, ts_val) in timestamps.iter().enumerate() {
            let ts = ts_val.as_i64().unwrap_or(0);
            let dt: DateTime<Utc> = DateTime::from_timestamp(ts, 0).unwrap_or_default();
            let date_str = dt.format("%Y-%m-%d").to_string();

            if date_str < start_str.to_string() || date_str > end_str.to_string() {
                continue;
            }

            let open = parse_price(opens.and_then(|a| a.get(i)));
            let high = parse_price(highs.and_then(|a| a.get(i)));
            let low  = parse_price(lows.and_then(|a| a.get(i)));
            let close = parse_price(closes.and_then(|a| a.get(i)));
            let adj_close = parse_price(adj_closes.and_then(|a| a.get(i)));
            let volume = parse_volume(volumes.and_then(|a| a.get(i)));

            all_prices.push(CompanyPrice {
                ticker: ticker.to_string(),
                date: date_str,
                time: "".to_string(),
                open,
                high,
                low,
                close,
                adj_close,
                volume,
                currency: currency.clone(),
            });
        }

        sleep(TokioDuration::from_millis(200)).await;
        current = actual_end;
    }

    all_prices.sort_by_key(|p| (p.date.clone(), p.time.clone()));
    all_prices.dedup_by(|a, b| a.date == b.date && a.time == b.time);

    println!("    Got {} daily bars for {ticker}", all_prices.len());
    Ok(all_prices)
}

pub async fn fetch_price_history_5min(
    ticker: &str,
    _start: &str,
    _end: &str,
) -> anyhow::Result<Vec<CompanyPrice>> {
    let now = Utc::now().timestamp();
    let period1 = now - 5184000;
    let period2 = now;

    let url = format!(
        "https://query1.finance.yahoo.com/v8/finance/chart/{ticker}?period1={period1}&period2={period2}&interval=5m&includeAdjustedClose=true"
    );

    let json: Value = HttpClient::new()
        .get(&url)
        .header("User-Agent", USER_AGENT)
        .send()
        .await?
        .json()
        .await?;

    let result = &json["chart"]["result"][0];
    let timestamps = result["timestamp"].as_array().ok_or_else(|| anyhow::anyhow!("No timestamps"))?;
    let quote = &result["indicators"]["quote"][0];
    let meta = &result["meta"];
    let currency = meta["currency"].as_str().unwrap_or("USD").to_string();

    let mut prices = Vec::new();

    for (i, ts_val) in timestamps.iter().enumerate() {
        let ts = ts_val.as_i64().unwrap_or(0);
        let dt: DateTime<Utc> = DateTime::from_timestamp(ts, 0).unwrap_or_default();
        let date_str = dt.format("%Y-%m-%d").to_string();
        let time_str = dt.format("%H:%M:%S").to_string();

        let open = parse_price(quote["open"].as_array().and_then(|a| a.get(i)));
        let high = parse_price(quote["high"].as_array().and_then(|a| a.get(i)));
        let low  = parse_price(quote["low"].as_array().and_then(|a| a.get(i)));
        let close = parse_price(quote["close"].as_array().and_then(|a| a.get(i)));
        let volume = parse_volume(quote["volume"].as_array().and_then(|a| a.get(i)));

        prices.push(CompanyPrice {
            ticker: ticker.to_string(),
            date: date_str,
            time: time_str,
            open,
            high,
            low,
            close,
            adj_close: close,
            volume,
            currency: currency.clone(),
        });
    }

    prices.sort_by_key(|p| (p.date.clone(), p.time.clone()));
    Ok(prices)
}

/// Fetch the URL of the latest ISIN↔LEI mapping CSV from GLEIF
/// Overengineered; we could just use the static URL, but this shows how to scrape if needed
pub async fn _fetch_latest_gleif_isin_lei_mapping_url(client: &Client) -> anyhow::Result<String> {
    let url = format!("https://www.gleif.org/de/lei-data/lei-mapping/download-isin-to-lei-relationship-files");
    client.goto(&url).await?;

    let html = client.source().await?;
    let _document = Html::parse_document(&html);
    let _row_sel = Selector::parse("table tbody tr").unwrap();
    let isin_lei = "".to_string();

    Ok(isin_lei)
}

pub async fn download_isin_lei_csv() -> anyhow::Result<Option<String>> {
    let url = "https://mapping.gleif.org/api/v2/isin-lei/9315e3e3-305a-4e71-b062-46714740fa8d/download";
    let zip_path = "data/isin_lei.zip";
    let csv_path = "data/isin_lei.csv";

    if let Err(e) = std::fs::create_dir_all("data") {
        println!("Failed to create data directory: {e}");
        return Ok(None);
    }

    // Download ZIP
    let bytes = match reqwest::Client::builder()
        .user_agent(USER_AGENT)
        .timeout(std::time::Duration::from_secs(30))
        .build()
        .and_then(|c| Ok(c))
    {
        Ok(client) => match client.get(url).send().await {
            Ok(resp) if resp.status().is_success() => match resp.bytes().await {
                Ok(b) => b,
                Err(e) => {
                    println!("Failed to read ZIP bytes: {e}");
                    return Ok(None);
                }
            },
            Ok(resp) => {
                println!("Server returned HTTP {}", resp.status());
                return Ok(None);
            }
            Err(e) => {
                println!("Failed to download ISIN/LEI ZIP: {e}");
                return Ok(None);
            }
        },
        Err(e) => {
            println!("Failed to create HTTP client: {e}");
            return Ok(None);
        }
    };

    if let Err(e) = tokio::fs::write(zip_path, &bytes).await {
        println!("Failed to write ZIP file: {e}");
        return Ok(None);
    }

    // Extract CSV
    let archive = match std::fs::File::open(zip_path)
        .map(ZipArchive::new)
    {
        Ok(Ok(a)) => a,
        Ok(Err(e)) => {
            println!("Invalid ZIP: {e}");
            return Ok(None);
        }
        Err(e) => {
            println!("Cannot open ZIP file: {e}");
            return Ok(None);
        }
    };

    let mut archive = archive;

    let idx = match (0..archive.len()).find(|&i| {
        archive.by_index(i)
            .map(|f| f.name().ends_with(".csv"))
            .unwrap_or(false)
    }) {
        Some(i) => i,
        None => {
            println!("ZIP did not contain a CSV file");
            return Ok(None);
        }
    };

    let mut csv_file = match archive.by_index(idx) {
        Ok(f) => f,
        Err(e) => {
            println!("Failed to read CSV entry: {e}");
            return Ok(None);
        }
    };

    let mut csv_bytes = Vec::new();
    if let Err(e) = csv_file.read_to_end(&mut csv_bytes) {
        println!("Failed to extract CSV: {e}");
        return Ok(None);
    }

    if let Err(e) = tokio::fs::write(csv_path, &csv_bytes).await {
        println!("Failed to save CSV file: {e}");
        return Ok(None);
    }

    Ok(Some(csv_path.to_string()))
}


pub async fn load_isin_lei_csv() -> anyhow::Result<HashMap<String, Vec<String>>> {
    // 1. Download + extract the CSV (this is now async)
    let csv_path = match download_isin_lei_csv().await? {
        Some(p) => p,
        None => {
            println!("ISIN/LEI download failed; continuing with empty map");
            return Ok(HashMap::new());
        }
    };

    // 2. Open and parse the CSV synchronously (fast enough, ~8M lines is fine)
    let file = match std::fs::File::open(&csv_path) {
        Ok(f) => f,
        Err(e) => {
            println!("Cannot open CSV '{}': {}", csv_path, e);
            return Ok(HashMap::new());
        }
    };

    let mut rdr = csv::ReaderBuilder::new()
        .has_headers(false)
        .from_reader(std::io::BufReader::new(file));

    let mut map: HashMap<String, Vec<String>> = HashMap::new();

    for result in rdr.records() {
        let record = match result {
            Ok(r) => r,
            Err(e) => {
                println!("CSV parse error: {}", e);
                continue;
            }
        };

        if record.len() < 2 { continue; }

        let lei = record[0].to_string();
        let isin = record[1].to_string();
        map.entry(lei).or_default().push(isin);
    }

    println!("Loaded ISIN↔LEI map with {} LEIs and {} total ISINs", 
        map.len(), 
        map.values().map(|v| v.len()).sum::<usize>()
    );

    Ok(map)
}

pub async fn reject_yahoo_cookies(client: &Client) -> anyhow::Result<()> {
    for _ in 0..10 {
        let clicked: bool = client
            .execute(
                r#"(() => {
                    const btn = document.querySelector('#consent-page .reject-all');
                    if (btn) {
                        btn.click();
                        return true;
                    }
                    return false;
                })()"#,
                vec![],
            )
            .await?
            .as_bool()
            .unwrap_or(false);

        if clicked { break; }
        sleep(TokioDuration::from_millis(500)).await;
    }

    println!("Rejected Yahoo cookies if button existed");
    Ok(())
}