use crate::util::directories::DataPaths;
use crate::util::logger;

// src/corporate/openfigi.rs
use super::{types::*};
use reqwest::Client as HttpClient;
use reqwest::header::{HeaderMap, HeaderValue};
use serde_json::{json, Value};
use csv::{ReaderBuilder, StringRecord, WriterBuilder};
use chrono::NaiveDate;
use std::collections::{HashMap, HashSet};
use std::path::{Path};
use std::time::Instant;
use tokio::time::{sleep, Duration};
use tokio::fs as tokio_fs;
use tokio::io::AsyncWriteExt;
use anyhow::{Context, anyhow};

#[derive(Clone)]
pub struct OpenFigiClient {
    client: HttpClient,
    has_key: bool,
}

impl OpenFigiClient {
    /// Creates a new OpenFIGI client, optionally with an API key.
    ///
    /// Loads the API key from the `OPENFIGI_API_KEY` environment variable if present.
    ///
    /// # Errors
    ///
    /// Returns an error if the HTTP client cannot be built or if the API key header is invalid.
    pub async fn new() -> anyhow::Result<Self> {
        let api_key = dotenvy::var("OPENFIGI_API_KEY").ok();
        let has_key = api_key.is_some();

        let mut builder = HttpClient::builder()
            .user_agent("Mozilla/5.0 (compatible; OpenFIGI-Rust/1.0)")
            .timeout(Duration::from_secs(30));

        if let Some(key) = &api_key {
            let mut headers = HeaderMap::new();
            headers.insert("X-OPENFIGI-APIKEY", HeaderValue::from_str(key)?);
            builder = builder.default_headers(headers);
        }

        let client = builder.build().context("Failed to build HTTP client")?;

        let msg = format!(
            "OpenFIGI client initialized: {}",
            if has_key { "with API key" } else { "no key (limited mode)" }
        );
        logger::log_info(&msg).await;

        Ok(Self { client, has_key })
    }

    /// Maps a batch of ISINs to FigiInfo structs, filtering for equities only.
    ///
    /// Batches requests according to rate limits (100 jobs/req with key, 5 without).
    /// Optimizes inter-request delays to approach the rate limit without exceeding it:
    /// - With key: ~240ms sleep per request (to sustain ~4 req/sec or 250 req/min).
    /// - Without key: 2.4s sleep (to sustain 25 req/min).
    /// Handles 429 rate limits with header-based backoff.
    /// Collects detailed FigiInfo from responses, using `exchCode` as proxy for `mic_code`.
    ///
    /// # Arguments
    ///
    /// * `isins` - Slice of ISIN strings to map (deduplicated internally if needed).
    ///
    /// # Returns
    ///
    /// A vector of `FigiInfo` structs for equity instruments.
    ///
    /// # Errors
    ///
    /// Returns an error on HTTP failures, JSON parsing issues, invalid API keys,
    /// or repeated rate limit violations after backoff.
    ///
    /// # Examples
    ///
    /// ```no_run
    /// # use anyhow::Result;
    /// # async fn example(client: &OpenFigiClient) -> Result<()> {
    /// let isins = vec!["US0378331005".to_string(), "US5949181045".to_string()];
    /// let figis = client.map_isins_to_figi_infos(&isins).await?;
    /// # Ok(())
    /// # }
    /// ```
    pub async fn map_isins_to_figi_infos(&self, isins: &[String]) -> anyhow::Result<Vec<FigiInfo>> {
        if isins.is_empty() {
            return Ok(vec![]);
        }

        let mut all_figi_infos = Vec::new();
        let chunk_size = if self.has_key { 100 } else { 5 };
        let inter_sleep = if self.has_key {
            Duration::from_millis(240) // ~4.16 req/sec (250/min)
        } else {
            Duration::from_millis(2400) // 25/min
        };

        let start_time = Instant::now();
        let mut req_count = 0;

        for chunk in isins.chunks(chunk_size) {
            let jobs: Vec<Value> = chunk.iter()
                .map(|isin| json!({
                    "idType": "ID_ISIN",
                    "idValue": isin,
                    //"marketSecDes": "Equity",
                }))
                .collect();

            // Retry logic with exponential backoff for transient failures
            let mut retry_count = 0;
            let max_retries = 5;
            let mut backoff_ms = 1000u64;
            
            loop {
                let resp_result = self.client
                    .post("https://api.openfigi.com/v3/mapping")
                    .header("Content-Type", "application/json")
                    .json(&jobs)
                    .send()
                    .await;

                let resp = match resp_result {
                    Ok(r) => r,
                    Err(e) => {
                        retry_count += 1;
                        if retry_count >= max_retries {
                            let err_msg = format!("Failed to send mapping request after {} retries: {}", max_retries, e);
                            logger::log_error(&err_msg).await;
                            return Err(anyhow!(err_msg));
                        }
                        let warn_msg = format!("Transient error sending mapping request (attempt {}/{}): {}", retry_count, max_retries, e);
                        eprintln!("{}", warn_msg);
                        logger::log_warn(&warn_msg).await;
                        let retry_msg = format!("  Retrying in {}ms...", backoff_ms);
                        println!("{}", retry_msg);
                        logger::log_info(&retry_msg).await;
                        sleep(Duration::from_millis(backoff_ms)).await;
                        backoff_ms = (backoff_ms * 2).min(60000); // Cap at 60s
                        continue;
                    }
                };

                let status = resp.status();
                let headers = resp.headers().clone();
                let body = resp.text().await.context("Failed to read response body")?;

                if status.is_client_error() || status.is_server_error() {
                    if status == 429 {
                        let reset_sec = headers
                            .get("ratelimit-reset")
                            .and_then(|v| v.to_str().ok())
                            .and_then(|s| s.parse::<u64>().ok())
                            .unwrap_or(10);
                        let rate_msg = format!("Rate limited—backing off {}s", reset_sec);
                        println!("{}", rate_msg);
                        logger::log_warn(&rate_msg).await;
                        sleep(Duration::from_secs(reset_sec.max(10))).await;
                        continue; // Retry the same chunk
                    } else if status == 401 {
                        return Err(anyhow!("Invalid OpenFIGI API key: {}", body));
                    } else if status == 413 {
                        return Err(anyhow!("Payload too large—reduce chunk size: {}", body));
                    } else if status.is_server_error() {
                        // Transient server error, retry with backoff
                        retry_count += 1;
                        if retry_count >= max_retries {
                            let err_msg = format!("OpenFIGI server error {} after {} retries: {}", status, max_retries, body);
                            logger::log_error(&err_msg).await;
                            return Err(anyhow!(err_msg));
                        }
                        let warn_msg = format!("Server error {} (attempt {}/{}), retrying in {}ms...", status, retry_count, max_retries, backoff_ms);
                        eprintln!("{}", warn_msg);
                        logger::log_warn(&warn_msg).await;
                        sleep(Duration::from_millis(backoff_ms)).await;
                        backoff_ms = (backoff_ms * 2).min(60000);
                        continue;
                    }
                    return Err(anyhow!("OpenFIGI error {}: {}", status, body));
                }

                let results: Vec<Value> = serde_json::from_str(&body)
                    .context("Failed to parse response JSON")?;

                for (isin, result) in chunk.iter().zip(results) {
                    if let Some(data) = result["data"].as_array() {
                        for item in data {
                            let sec_type = item["securityType"].as_str().unwrap_or("");
                            let market_sec = item["marketSector"].as_str().unwrap_or("");
                            
                            // Capture all security types, let caller filter by market sector if needed
                            let figi = match item["figi"].as_str() {
                                Some(f) => f.to_string(),
                                None => continue,
                            };

                            let figi_info = FigiInfo {
                                isin: isin.clone(),
                                figi,
                                name: item["name"].as_str().unwrap_or("").to_string(),
                                ticker: item["ticker"].as_str().unwrap_or("").to_string(),
                                exch_code: item["exchCode"].as_str().unwrap_or("").to_string(),
                                composite_figi: item["compositeFIGI"].as_str().unwrap_or("").to_string(),
                                security_type: sec_type.to_string(),
                                market_sector: market_sec.to_string(),
                                share_class_figi: item["shareClassFIGI"].as_str().unwrap_or("").to_string(),
                                security_type2: item["securityType2"].as_str().unwrap_or("").to_string(),
                                security_description: item["securityDescription"].as_str().unwrap_or("").to_string(),
                            };

                            all_figi_infos.push(figi_info);
                        }
                    }
                }
                
                // Successfully processed this chunk, break out of retry loop
                break;
            }

            req_count += 1;
            if req_count % 25 == 0 {
                // Optional: Enforce 6-sec window for bursts
                let elapsed = start_time.elapsed();
                if self.has_key {
                    if elapsed < Duration::from_secs(6) {
                        sleep(Duration::from_secs(6) - elapsed).await;
                    }
                } else {
                    if elapsed < Duration::from_secs(6) {
                        sleep(Duration::from_secs(60) - elapsed).await;
                    }
                }
            } else {
                sleep(inter_sleep).await;
            }
        }

        Ok(all_figi_infos)
    }
}

/// Extracts the date from a GLEIF CSV filename in the format "isin-lei-DDMMYYYY.csv".
/// 
/// # Arguments
/// 
/// * `filename` - The GLEIF CSV filename (e.g., "isin-lei-24112025.csv")
/// 
/// # Returns
/// 
/// A string in the format "DDMMYYYY" (e.g., "24112025") if successfully parsed, otherwise the original filename.
fn extract_gleif_date_from_filename(filename: &str) -> String {
    // Pattern: isin-lei-DDMMYYYY.csv
    if let Some(start_idx) = filename.find("isin-lei-") {
        let rest = &filename[start_idx + 9..]; // Skip "isin-lei-"
        if rest.len() >= 8 && rest[0..8].chars().all(|c| c.is_numeric()) {
            return rest[0..8].to_string();
        }
    }
    filename.to_string()
}

/// Loads the list of market sectors from cache/openfigi/marketSecDes.json
/// 
/// # Returns
/// 
/// Vec of market sector strings (e.g., ["Comdty", "Corp", "Curncy", "Equity", ...])
/// If the file doesn't exist or can't be parsed, returns a sensible default list.
async fn load_market_sectors() -> anyhow::Result<Vec<String>> {
    let dir = DataPaths::new(".")?;
    let cache_file = dir.cache_openfigi_dir().join("marketSecDes.json");
    
    if !cache_file.exists() {
        // Return default if file doesn't exist
        let warn_msg = format!("Warning: {} not found, using default sectors", cache_file.display());
        eprintln!("{}", warn_msg);
        logger::log_warn(&warn_msg).await;
        return Ok(vec![
            "Comdty".to_string(),
            "Corp".to_string(),
            "Curncy".to_string(),
            "Equity".to_string(),
            "Govt".to_string(),
            "Index".to_string(),
            "M-Mkt".to_string(),
            "Mtge".to_string(),
            "Muni".to_string(),
            "Pfd".to_string(),
        ]);
    }
    
    let content = tokio_fs::read_to_string(&cache_file).await
        .context("Failed to read marketSecDes.json")?;
    
    let json: Value = serde_json::from_str(&content)
        .context("Failed to parse marketSecDes.json")?;
    
    let sectors: Vec<String> = json["values"]
        .as_array()
        .ok_or_else(|| anyhow!("'values' field not found in marketSecDes.json"))?
        .iter()
        .filter_map(|v| v.as_str().map(|s| s.to_string()))
        .collect();
    
    if sectors.is_empty() {
        return Err(anyhow!("No sectors found in marketSecDes.json"));
    }
    
    let msg = format!("Loaded {} market sectors from cache", sectors.len());
    logger::log_info(&msg).await;
    Ok(sectors)
}

/// Finds the most recent GLEIF CSV file in the cache/gleif directory.
/// 
/// Returns the extracted date in format "DDMMYYYY" from the filename.
/// If no GLEIF file is found, returns None.
async fn find_most_recent_gleif_date(gleif_cache_dir: &Path) -> anyhow::Result<Option<String>> {
    // First check for subdirectories named as DDMMYYYY and pick the most recent date
    let mut dir_entries = tokio_fs::read_dir(gleif_cache_dir)
        .await
        .context("Failed to read gleif cache directory")?;

    let mut found_dates: Vec<NaiveDate> = Vec::new();

    while let Some(entry) = dir_entries.next_entry().await? {
        let path = entry.path();
        if path.is_dir() {
            if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
                // Expect folder name in DDMMYYYY
                if name.len() == 8 && name.chars().all(|c| c.is_numeric()) {
                    if let Ok(nd) = NaiveDate::parse_from_str(name, "%d%m%Y") {
                        found_dates.push(nd);
                    }
                }
            }
        }
    }

    if !found_dates.is_empty() {
        found_dates.sort();
        if let Some(most_recent) = found_dates.last() {
            let date_str = most_recent.format("%d%m%Y").to_string();
            let msg = format!("  Found GLEIF data dated (from subdirs): {}", date_str);
            logger::log_info(&msg).await;
            return Ok(Some(date_str));
        }
    }

    // Fallback: look for CSV files in the directory as before
    let mut entries = tokio_fs::read_dir(gleif_cache_dir)
        .await
        .context("Failed to read gleif cache directory")?;
    let mut csv_files = Vec::new();

    while let Some(entry) = entries.next_entry().await? {
        let path = entry.path();
        if let Some(filename) = path.file_name() {
            let filename_str = filename.to_string_lossy();
            if filename_str.ends_with(".csv") && filename_str.contains("isin-lei-") {
                csv_files.push(filename_str.to_string());
            }
        }
    }

    if csv_files.is_empty() {
        return Ok(None);
    }

    // Sort files in reverse order (most recent first) based on date in filename
    csv_files.sort();
    csv_files.reverse();

    let most_recent = &csv_files[0];
    let date = extract_gleif_date_from_filename(most_recent);

    let msg = format!("  Found GLEIF data dated: {}", date);

    logger::log_info(&msg).await;
    Ok(Some(date))
}

/// Builds a LEI-to-FigiInfo map with automatic retry on transient failures.
///
/// This is a wrapper around build_lei_to_figi_infos_internal that handles transient errors
/// by automatically retrying after a delay if the mapping process fails. The mapping can
/// resume from where it left off since already-processed LEIs are saved incrementally.
///
/// # Arguments
///
/// * `lei_to_isins` - HashMap of LEI to Vec<ISIN> (used for fetching missing entries).
/// * `gleif_date` - Optional date in format "DDMMYYYY". If None, uses the most recent GLEIF file.
///
/// # Returns
///
/// The complete (or partial if interrupted) HashMap<LEI, Vec<FigiInfo>>.
///
/// # Errors
///
/// Returns an error only on fatal errors (file I/O, invalid API key, etc.).
/// Transient errors are retried automatically.
pub async fn build_lei_to_figi_infos(lei_to_isins: &HashMap<String, Vec<String>>, gleif_date: Option<&str>) -> anyhow::Result<HashMap<String, Vec<FigiInfo>>> {
    let mut retry_count = 0;
    let max_retries = 3;
    
    loop {
        match build_lei_to_figi_infos_internal(lei_to_isins, gleif_date).await {
            Ok(map) => {
                if !map.is_empty() {
                    let msg = format!("✓ LEI→FIGI mapping completed successfully with {} entries", map.len());
                    
                    logger::log_info(&msg).await;
                }
                return Ok(map);
            }
            Err(e) => {
                let error_msg = e.to_string();
                
                // Check if this is a fatal error or transient
                let is_fatal = error_msg.contains("Invalid OpenFIGI API key") 
                    || error_msg.contains("No GLEIF CSV file found")
                    || error_msg.contains("Failed to create");
                
                if is_fatal {
                    let err = format!("Fatal error in LEI→FIGI mapping: {}", e);
                    eprintln!("{}", err);
                    logger::log_error(&err).await;
                    return Err(e);
                }
                
                retry_count += 1;
                if retry_count >= max_retries {
                    let err = format!("LEI→FIGI mapping failed after {} retries: {}", max_retries, e);
                    eprintln!("{}", err);
                    logger::log_error(&err).await;
                    return Err(e);
                }
                
                let wait_secs = 60 * retry_count;
                let warn_msg = format!("Transient error in LEI→FIGI mapping (attempt {}/{}): {}", retry_count, max_retries, e);
                eprintln!("{}", warn_msg);
                logger::log_warn(&warn_msg).await;
                let retry_msg = format!("Retrying mapping in {}s...", wait_secs);
                println!("{}", retry_msg);
                logger::log_info(&retry_msg).await;
                sleep(Duration::from_secs(wait_secs as u64)).await;
            }
        }
    }
}

/// Internal implementation of LEI-to-FigiInfo mapping.
///
/// This is the actual worker function that performs the mapping. It handles already-processed
/// LEIs gracefully but will fail on transient errors, which are caught and retried by the
/// wrapper function build_lei_to_figi_infos.
///
/// Tracks three outcomes:
/// 1. Hit with marketSector: saved to sector-specific folder
/// 2. Hit without marketSector: saved to "uncategorized" folder
/// 3. No_hit (empty results): LEI marked for removal from GLEIF CSV
async fn build_lei_to_figi_infos_internal(lei_to_isins: &HashMap<String, Vec<String>>, gleif_date: Option<&str>) -> anyhow::Result<HashMap<String, Vec<FigiInfo>>> {
    let dir = DataPaths::new(".")?;
    let gleif_cache_dir = dir.cache_gleif_dir();
    let map_cache_dir = dir.cache_gleif_openfigi_map_dir();

    // Determine the GLEIF date to use
    let date = if let Some(d) = gleif_date {
        let msg = format!("Using provided GLEIF date: {}", d);
        logger::log_info(&msg).await;
        d.to_string()
    } else {
        // Find the most recent GLEIF file
        logger::log_info("Searching for most recent GLEIF file...").await;
        match find_most_recent_gleif_date(&gleif_cache_dir).await? {
            Some(d) => d,
            None => {
                let err = "No GLEIF CSV file found in cache/gleif directory";
                logger::log_error(err).await;
                return Err(anyhow!(err));
            },
        }
    };

    // Creat date-based subdirectory in the gleif cache
    let gleif_date_dir = gleif_cache_dir.join(&date);

    // Create date-based subdirectory in the mapping cache
    let msg = format!("Creating date directory for: {}", date);
    logger::log_info(&msg).await;
    let date_dir = map_cache_dir.join(&date);
    tokio_fs::create_dir_all(&date_dir).await.context("Failed to create date directory")?;
    
    // Load market sectors dynamically from cache
    logger::log_info("Loading market sectors...").await;
    let sector_dirs = load_market_sectors().await?;
    let mut sector_maps: HashMap<String, HashMap<String, Vec<FigiInfo>>> = HashMap::new();
    
    // Create uncategorized folder
    let msg = format!("Creating {} sector directories...", sector_dirs.len());
    logger::log_info(&msg).await;
    let uncategorized_dir = date_dir.join("uncategorized");
    tokio_fs::create_dir_all(&uncategorized_dir).await.context("Failed to create uncategorized directory")?;
    let uncategorized_path = uncategorized_dir.join("lei_to_figi.jsonl");
    let uncategorized_map = load_lei_to_figi_jsonl(&uncategorized_path).await?;
    sector_maps.insert("uncategorized".to_string(), uncategorized_map);
    
    for sector in &sector_dirs {
        let sector_dir = date_dir.join(sector);
        tokio_fs::create_dir_all(&sector_dir).await.context("Failed to create sector directory")?;
        
        // Load existing mappings for this sector
        let path = sector_dir.join("lei_to_figi.jsonl");
        let lei_map = load_lei_to_figi_jsonl(&path).await?;
        sector_maps.insert(sector.clone(), lei_map);
    }

    let client = OpenFigiClient::new().await?;
    if !client.has_key {
        let total_entries: usize = sector_maps.values().map(|m| m.len()).sum();
        let msg = format!("No API key—using partial LEI→FIGI maps with {} total entries", total_entries);
        
        logger::log_warn(&msg).await;
        return Ok(sector_maps.get("Equity").cloned().unwrap_or_default());
    }

    // Sort LEIs for deterministic processing order
    logger::log_info("Starting LEI→FIGI mapping process...").await;
    let mut leis: Vec<_> = lei_to_isins.keys().cloned().collect();
    leis.sort();

    let mut processed = sector_maps.values().map(|m| m.len()).sum::<usize>();
    let total = leis.len();
    let mut no_hit_leis = Vec::new(); // Track LEIs with no data found (no_hit)
    let mut leis_to_delete_batch = Vec::new(); // Batch delete every 100 LEIs
    
    let msg = format!("Total LEIs to process: {}, already processed: {}", total, processed);
    
    logger::log_info(&msg).await;

    for lei in leis {
        // Check if LEI is already processed in any sector (including uncategorized)
        let mut already_processed = false;
        for sector_map in sector_maps.values() {
            if sector_map.contains_key(&lei) {
                already_processed = true;
                break;
            }
        }
        
        if already_processed {
            continue;
        }

        let isins = match lei_to_isins.get(&lei) {
            Some(i) => i,
            None => continue,
        };

        let unique_isins: Vec<_> = isins.iter().cloned().collect::<HashSet<_>>().into_iter().collect();
        let debug_msg = format!("Processing LEI {} with {} ISINs...", lei, unique_isins.len());
        logger::log_info(&debug_msg).await;
        
        let all_figi_infos = client.map_isins_to_figi_infos(&unique_isins).await?;

        // Case 1: no_hit - API succeeded but returned no data
        if all_figi_infos.is_empty() {
            let no_hit_msg = format!("  no_hit: LEI {} returned no FIGIs", lei);
            logger::log_warn(&no_hit_msg).await;
            no_hit_leis.push(lei.clone());
            leis_to_delete_batch.push(lei.clone());
            
            // Delete every 100 no_hit LEIs to prevent progress loss on interrupt
            if leis_to_delete_batch.len() >= 100 {
                let batch_msg = format!("Batch deleting {} LEIs from GLEIF CSV...", leis_to_delete_batch.len());
                logger::log_info(&batch_msg).await;
                if let Err(e) = remove_leis_batch_from_gleif_csv(&gleif_date_dir, &leis_to_delete_batch).await {
                    let warn_msg = format!("Warning: Failed to batch remove LEIs from GLEIF CSV: {}", e);
                    eprintln!("{}", warn_msg);
                    logger::log_warn(&warn_msg).await;
                }
                leis_to_delete_batch.clear();
            }
            
            continue;
        }
        
        let hit_msg = format!("  hit: LEI {} found {} FIGIs", lei, all_figi_infos.len());
        logger::log_info(&hit_msg).await;

        // Organize results by marketSector
        let mut figis_by_sector: HashMap<String, Vec<FigiInfo>> = HashMap::new();
        let mut uncategorized_figis = Vec::new();
        
        for figi_info in all_figi_infos {
            let sector = figi_info.market_sector.clone();
            
            if sector.is_empty() {
                // Case 2: Hit but no marketSecDes - save to uncategorized
                uncategorized_figis.push(figi_info);
            } else {
                // Case 1: Hit with marketSector - organize by sector
                figis_by_sector.entry(sector).or_insert_with(Vec::new).push(figi_info);
            }
        }

        // Save uncategorized FIGIs if any
        if !uncategorized_figis.is_empty() {
            uncategorized_figis.sort_by_key(|f| f.figi.clone());
            uncategorized_figis.dedup_by_key(|f| f.figi.clone());
            
            append_lei_to_figi_jsonl(&uncategorized_path, &lei, &uncategorized_figis).await
                .context("Failed to append to uncategorized JSONL")?;
            
            if let Some(uncategorized_map) = sector_maps.get_mut("uncategorized") {
                uncategorized_map.insert(lei.clone(), uncategorized_figis);
            }
        }

        // Save to appropriate sector files
        for (sector, mut figis) in figis_by_sector {
            if !figis.is_empty() {
                figis.sort_by_key(|f| f.figi.clone());
                figis.dedup_by_key(|f| f.figi.clone());
                
                // Save to sector's JSONL file
                let sector_dir = date_dir.join(&sector);
                let path = sector_dir.join("lei_to_figi.jsonl");
                append_lei_to_figi_jsonl(&path, &lei, &figis).await.context("Failed to append to JSONL")?;
                
                // Update in-memory sector map
                if let Some(sector_map) = sector_maps.get_mut(&sector) {
                    sector_map.insert(lei.clone(), figis);
                }
            }
        }

        processed += 1;
        if processed % 100 == 0 {
            let totals: Vec<String> = sector_dirs.iter().map(|s| {
                let count = sector_maps.get(s).map(|m| m.len()).unwrap_or(0);
                format!("{}:{}", s, count)
            }).collect();
            let progress_msg = format!("Processed {}/{} LEIs → [{}] no_hit: {}", processed, total, totals.join(", "), no_hit_leis.len());
            println!("{}", progress_msg);
            logger::log_info(&progress_msg).await;
        }

        tokio::time::sleep(Duration::from_millis(100)).await;
    }
    
    // Delete any remaining LEIs in the batch
    if !leis_to_delete_batch.is_empty() {
        let batch_msg = format!("Final batch: Deleting {} LEIs from GLEIF CSV...", leis_to_delete_batch.len());
        logger::log_info(&batch_msg).await;
        if let Err(e) = remove_leis_batch_from_gleif_csv(gleif_cache_dir, &leis_to_delete_batch).await {
            let warn_msg = format!("Warning: Failed to delete final batch from GLEIF CSV: {}", e);
            eprintln!("{}", warn_msg);
            logger::log_warn(&warn_msg).await;
        }
    }
    
    // Log final summary for no_hit LEIs (they've already been removed incrementally)
    if !no_hit_leis.is_empty() {
        let no_hit_summary = format!("no_hit (removed in batches from GLEIF): {} LEIs", no_hit_leis.len());
        println!("{}", no_hit_summary);
        logger::log_info(&no_hit_summary).await;
    }
    
    // Return Equity sector as the main result
    Ok(sector_maps.get("Equity").cloned().unwrap_or_default())
}

/// Loads LEI-to-FigiInfo map from a JSON Lines file.
///
/// Each line is expected to be a JSON object: {"lei": "ABC", "figis": [FigiInfo...]}
///
/// # Arguments
///
/// * `path` - Path to the .jsonl file.
///
/// # Returns
///
/// The loaded HashMap<LEI, Vec<FigiInfo>>.
///
/// # Errors
///
/// Returns an error if the file cannot be opened or if any line fails to parse as JSON.
async fn load_lei_to_figi_jsonl(path: &Path) -> anyhow::Result<HashMap<String, Vec<FigiInfo>>> {
    let mut map = HashMap::new();

    if !path.exists() {
        return Ok(map);
    }

    let content = tokio_fs::read_to_string(path).await.context("Failed to read JSONL file")?;

    for (line_num, line) in content.lines().enumerate() {
        if line.trim().is_empty() {
            continue;
        }

        let entry: Value = serde_json::from_str(&line).context(format!("Failed to parse JSON on line {}", line_num + 1))?;
        let lei = entry["lei"].as_str().context("Missing 'lei' field")?.to_string();
        let figis: Vec<FigiInfo> = serde_json::from_value(entry["figis"].clone()).context("Invalid 'figis' field")?;

        map.insert(lei, figis);
    }

    let msg = format!("Loaded LEI→FIGI map with {} entries from {}", map.len(), path.display());
    
    logger::log_info(&msg).await;
    Ok(map)
}

/// Appends a single LEI entry to the JSON Lines file.
///
/// # Arguments
///
/// * `path` - Path to the .jsonl file.
/// * `lei` - The LEI key.
/// * `figis` - The Vec<FigiInfo> for this LEI.
///
/// # Errors
///
/// Returns an error if the file cannot be opened for append or if serialization fails.
async fn append_lei_to_figi_jsonl(path: &Path, lei: &str, figis: &[FigiInfo]) -> anyhow::Result<()> {
    let entry = json!({
        "lei": lei,
        "figis": figis,
    });

    let line = serde_json::to_string(&entry).context("Failed to serialize entry")? + "\n";
    
    let mut file = tokio_fs::OpenOptions::new()
        .create(true)
        .append(true)
        .open(path)
        .await
        .context("Failed to open JSONL file for append")?;
    
    file.write_all(line.as_bytes())
        .await
        .context("Failed to write to JSONL file")?;

    Ok(())
}

/// Removes multiple invalid LEIs from the GLEIF CSV file in a single batch operation.
///
/// This function is more efficient than removing LEIs one at a time.
/// It reads the GLEIF CSV once, filters out all specified LEIs, and overwrites the file once.
///
/// # Arguments
///
/// * `gleif_cache_dir` - Path to the cache/gleif directory
/// * `leis_to_remove` - Vec of LEI strings to remove
///
/// # Returns
/// Ok(()) if successful, Err if file operations fail.
async fn remove_leis_batch_from_gleif_csv(gleif_cache_dir: &Path, leis_to_remove: &[String]) -> anyhow::Result<()> {
    if leis_to_remove.is_empty() {
        return Ok(());
    }
    
    // Find the most recent GLEIF CSV file
    let mut entries = tokio_fs::read_dir(gleif_cache_dir)
        .await
        .context("Failed to read gleif cache directory")?;
    
    let mut csv_files = Vec::new();

    while let Some(entry) = entries.next_entry().await? {
        let path = entry.path();
        if let Some(filename) = path.file_name() {
            let filename_str = filename.to_string_lossy();
            if filename_str.ends_with(".csv") && filename_str.contains("isin-lei-") {
                csv_files.push(path);
            }
        }
    }
    
    if csv_files.is_empty() {
        logger::log_warn("No GLEIF CSV files found for batch removal operation").await;
        return Ok(());
    }
    
    // Prefer an original (non-_clean) GLEIF CSV if available; otherwise use the most recent file.
    csv_files.sort();
    csv_files.reverse();

    let mut gleif_file: &std::path::PathBuf = &csv_files[0];
    // Try to find the most recent filename that does NOT end with "_clean.csv"
    if let Some(non_clean) = csv_files.iter().find(|p| {
        p.file_name()
            .and_then(|n| n.to_str())
            .map(|s| !s.to_lowercase().ends_with("_clean.csv"))
            .unwrap_or(false)
    }) {
        gleif_file = non_clean;
    }
    
    // Prepare clean file path: insert "_clean" before extension
    let orig_path = gleif_file;
    let file_name = orig_path.file_name().and_then(|n| n.to_str()).unwrap_or("gleif.csv");
    let mut stem = orig_path.file_stem().and_then(|s| s.to_str()).unwrap_or("isin-lei").to_string();
    let parent = orig_path.parent().unwrap_or_else(|| Path::new("."));
    // Avoid creating a double "_clean_clean.csv". If stem already ends with "_clean", keep it.
    if stem.to_lowercase().ends_with("_clean") {
        // stem is already clean; keep same filename (no double suffix)
        // e.g., stem="isin-lei-24112025_clean" -> clean_name="isin-lei-24112025_clean.csv"
    } else {
        stem = format!("{}_clean", stem);
    }

    let clean_name = format!("{}.csv", stem);
    let clean_path = parent.join(&clean_name);

    // If a clean file already exists, operate on it; otherwise read original and write clean file
    let source_path = if clean_path.exists() { &clean_path } else { orig_path };

    let debug_msg = format!("Reading GLEIF source for batch removal: {} (writing to {})", source_path.display(), clean_path.display());
    logger::log_info(&debug_msg).await;

    // Cleanup any accidental double-clean files in the same directory: if a file ends with
    // "_clean_clean.csv" replace it with single "_clean.csv" or remove it if target exists.
    if let Ok(mut dir_entries) = tokio_fs::read_dir(parent).await {
        while let Ok(Some(entry)) = dir_entries.next_entry().await {
            if let Some(name) = entry.file_name().to_str().map(|s| s.to_string()) {
                if name.to_lowercase().ends_with("_clean_clean.csv") {
                    let offending = entry.path();
                    let candidate = offending.file_name().and_then(|n| n.to_str()).unwrap_or("");
                    let target_name = candidate.replacen("_clean_clean.csv", "_clean.csv", 1);
                    let target_path = parent.join(target_name);

                    if !target_path.exists() {
                        // Rename offending -> target
                        let _ = tokio_fs::rename(&offending, &target_path).await;
                        let msg = format!("Renamed {} -> {}", offending.display(), target_path.display());
                        logger::log_info(&msg).await;
                    } else {
                        // Target exists already; remove offending duplicate
                        let _ = tokio_fs::remove_file(&offending).await;
                        let msg = format!("Removed duplicate {}", offending.display());
                        logger::log_info(&msg).await;
                    }
                }
            }
        }
    }

    // Read file into memory and parse with csv crate for robust handling of quoted fields
    let content = tokio_fs::read_to_string(source_path)
        .await
        .context("Failed to read GLEIF CSV source")?;

    // Convert LEIs to remove into a HashSet (normalized)
    let remove_set: std::collections::HashSet<String> = leis_to_remove
        .iter()
        .map(|s| s.trim().trim_matches('"').to_uppercase())
        .collect();

    // Build CSV reader: try with headers first; allow flexible records
    let mut reader = ReaderBuilder::new()
        .has_headers(true)
        .flexible(true)
        .from_reader(content.as_bytes());

    // Remember headers (if present) and then iterate records.
    let headers_record = match reader.headers() {
        Ok(h) => Some(h.clone()),
        Err(_) => None,
    };

    // We'll collect kept records and count original rows
    let mut kept_records: Vec<StringRecord> = Vec::new();
    let mut original_count: usize = 0;
    let mut removed_count: usize = 0;

    // For robustness, search all columns for a matching LEI instead of relying on a single column index.
    for result in reader.records() {
        let record = result.context("Failed to parse CSV record")?;
        original_count += 1;

        // Check every field for a match in the remove set
        let mut matched = false;
        for field in record.iter() {
            let norm = field.trim().trim_matches('"').to_uppercase();
            if remove_set.contains(&norm) {
                matched = true;
                break;
            }
        }

        if matched {
            removed_count += 1;
        } else {
            kept_records.push(record.clone());
        }
    }

    let new_count = kept_records.len();

    // Write back using csv writer to preserve quoting/format into clean file
    let mut wtr = WriterBuilder::new().has_headers(true).from_writer(vec![]);
    // If original had headers, write them back
    if let Some(headers) = headers_record {
        wtr.write_record(headers.iter())?;
    }

    for rec in &kept_records {
        wtr.write_record(rec.iter())?;
    }

    let out_bytes = wtr.into_inner().context("Failed to finalize CSV writer")?;
    let out_str = String::from_utf8(out_bytes).context("CSV output not valid UTF-8")?;

    tokio_fs::write(&clean_path, out_str)
        .await
        .context("Failed to write filtered GLEIF CSV clean file")?;

    let success_msg = format!(
        "✓ Batch attempted to remove {} LEIs from GLEIF CSV (was {} records, now {} records, removed {} rows) -> {}",
        leis_to_remove.len(), original_count, new_count, removed_count, clean_path.display()
    );
    println!("{}", success_msg);
    logger::log_info(&success_msg).await;

    Ok(())
}

/// Loads or builds HashMaps for companies, warrants, and options.
///
/// This function:
/// 1. Attempts to load existing data from cache
/// 2. Processes new FIGI data and classifies by securityType:
///    - "Common Stock" → companies HashMap (grouped by ISIN)
///    - "Equity WRT" → warrants HashMap (parsed from name)
///    - "Equity Option" → options HashMap (parsed from name)
/// 3. Updates/extends existing entries
/// 4. Saves results to separate JSON files
///
/// # Arguments
/// * `figi_to_lei` - HashMap mapping LEI to Vec<FigiInfo>.
///
/// # Returns
/// A tuple of (companies, warrants, options) HashMaps.
///
/// # Errors
/// Returns an error if file I/O fails or JSON serialization fails.
pub async fn load_or_build_all_securities(
    figi_to_lei: &HashMap<String, Vec<FigiInfo>>
) -> anyhow::Result<(
    HashMap<String, CompanyInfo>,
    HashMap<String, HashMap<String, WarrantInfo>>,
    HashMap<String, HashMap<String, OptionInfo>>
)> {
    // Load existing data
    let mut commons = load_from_cache("data/corporate/by_name/common_stocks.json").await?
        .unwrap_or_else(HashMap::new);
    let mut warrants = load_from_cache("data/corporate/by_name/warrants.json").await?
        .unwrap_or_else(HashMap::new);
    let mut options = load_from_cache("data/corporate/by_name/options.json").await?
        .unwrap_or_else(HashMap::new);
    /*let mut preferred = load_from_cache("data/corporate/by_name/preferred.json").await?
        .unwrap_or_else(HashMap::new);*/
    
    
    println!("Loaded existing data:");
    println!("  - Companies: {}", commons.len());
    println!("  - Warrants: {}", warrants.len());
    println!("  - Options: {}", options.len());
    
    let mut stats = ProcessingStats::new(commons.len(), warrants.len(), options.len());
    
    println!("Processing {} LEI entries from FIGI data...", figi_to_lei.len());
    
    for (_lei, figi_infos) in figi_to_lei.iter() {
        if figi_infos.is_empty() {
            continue;
        }
        
        // Group FigiInfos by security type
        let mut common_stocks = Vec::new();
        let mut warrant_securities = Vec::new();
        let mut option_securities = Vec::new();
        
        for figi_info in figi_infos {
            match figi_info.security_type.as_str() {
                "Common Stock" => common_stocks.push(figi_info.clone()),
                "Equity WRT" => warrant_securities.push(figi_info.clone()),
                "Equity Option" => option_securities.push(figi_info.clone()),
                _ => {} // Ignore other types
            }
        }
        
        // Process common stocks -> companies
        if !common_stocks.is_empty() {
            process_common_stocks(&mut commons, &common_stocks, &mut stats);
        }
        
        // Process warrants
        if !warrant_securities.is_empty() {
            process_warrants(&mut warrants, &warrant_securities, &mut stats);
        }
        
        // Process options
        if !option_securities.is_empty() {
            process_options(&mut options, &option_securities, &mut stats);
        }
    }
    
    stats.print_summary(commons.len(), warrants.len(), options.len());
    
    // Save all three HashMaps
    save_to_cache("data/corporate/by_name/common_stocks.json", &commons).await?;
    save_to_cache("data/corporate/by_name/warrants.json", &warrants).await?;
    save_to_cache("data/corporate/by_name/options.json", &options).await?;
    
    Ok((commons, warrants, options))
}

/// Statistics tracker for processing
#[derive(Debug)]
struct ProcessingStats {
    initial_companies: usize,
    initial_warrants: usize,
    initial_options: usize,
    companies_added: usize,
    companies_updated: usize,
    warrants_added: usize,
    warrants_updated: usize,
    options_added: usize,
    options_updated: usize,
}

impl ProcessingStats {
    fn new(companies: usize, warrants: usize, options: usize) -> Self {
        Self {
            initial_companies: companies,
            initial_warrants: warrants,
            initial_options: options,
            companies_added: 0,
            companies_updated: 0,
            warrants_added: 0,
            warrants_updated: 0,
            options_added: 0,
            options_updated: 0,
        }
    }
    
    fn print_summary(&self, final_companies: usize, final_warrants: usize, final_options: usize) {
        println!("\n=== Processing Statistics ===");
        println!("Companies:");
        println!("  - Initial: {}", self.initial_companies);
        println!("  - Added: {}", self.companies_added);
        println!("  - Updated: {}", self.companies_updated);
        println!("  - Total: {}", final_companies);
        println!("Warrants:");
        println!("  - Initial: {}", self.initial_warrants);
        println!("  - Added: {}", self.warrants_added);
        println!("  - Updated: {}", self.warrants_updated);
        println!("  - Total: {}", final_warrants);
        println!("Options:");
        println!("  - Initial: {}", self.initial_options);
        println!("  - Added: {}", self.options_added);
        println!("  - Updated: {}", self.options_updated);
        println!("  - Total: {}", final_options);
    }
}

/// Process common stocks into companies HashMap
fn process_common_stocks(
    companies: &mut HashMap<String, CompanyInfo>,
    figi_infos: &[FigiInfo],
    stats: &mut ProcessingStats,
) {
    let name = figi_infos[0].name.clone();
    if name.is_empty() {
        return;
    }
    
    // Group by ISIN
    let grouped_by_isin = group_by_isin(figi_infos);
    
    if let Some(existing) = companies.get_mut(&name) {
        // Update existing company
        let mut updated = false;
        for (isin, new_figis) in grouped_by_isin {
            if let Some(existing_figis) = existing.securities.get_mut(&isin) {
                let merged = merge_figi_list(existing_figis, &new_figis);
                if merged.len() > existing_figis.len() {
                    *existing_figis = merged;
                    updated = true;
                }
            } else {
                existing.securities.insert(isin.clone(), new_figis);
                updated = true;
            }
        }
        
        // Update primary ISIN if needed
        if existing.primary_isin.is_empty() || !existing.securities.contains_key(&existing.primary_isin) {
            if let Some(first_isin) = existing.securities.keys().next() {
                existing.primary_isin = first_isin.clone();
            }
        }
        
        if updated {
            stats.companies_updated += 1;
        }
    } else {
        // Add new company
        let primary_isin = grouped_by_isin.keys().next().cloned().unwrap_or_default();
        
        companies.insert(name.clone(), CompanyInfo {
            name,
            primary_isin,
            securities: grouped_by_isin,
        });
        
        stats.companies_added += 1;
    }
}

/// Process warrants into warrants HashMap
fn process_warrants(
    warrants: &mut HashMap<String, HashMap<String, WarrantInfo>>,
    warrant_securities: &[FigiInfo],
    stats: &mut ProcessingStats,  // Assuming Stats is a struct; adjust based on actual type if it's a HashMap or other
) {
    for figi in warrant_securities.iter() {
        // Parse the name to extract underlying, issuer, and warrant_type
        // (Assuming a parse_warrant_name function exists; this is not changed)
        let (underlying, issuer, warrant_type) = parse_warrant_name(&figi.name);

        if underlying.is_empty() {
            continue;
        }

        // Outer map: key by underlying
        let underlying_map = warrants
            .entry(underlying.clone())
            .or_insert_with(HashMap::new);

        // Inner map: key by warrant_type
        let entry = underlying_map.entry(warrant_type.clone()).or_insert(WarrantInfo {
            underlying_company_name: underlying.clone(),
            issuer_company_name: issuer,
            warrant_type: warrant_type.clone(),
            warrants: HashMap::new(),
        });

        // Group by ISIN as before
        entry
            .warrants
            .entry(figi.isin.clone())
            .or_insert_with(Vec::new)
            .push(figi.clone());

        // Update stats (assuming stats has a 'warrants' field; adjust if needed)
        stats.warrants_added += 1;
    }
}

/// Process options into options HashMap
fn process_options(
    options: &mut HashMap<String, HashMap<String, OptionInfo>>,
    option_securities: &[FigiInfo],
    stats: &mut ProcessingStats,  // Assuming Stats is a struct; adjust based on actual type if it's a HashMap or other
) {
    for figi in option_securities.iter() {
        // Parse the name to extract underlying, issuer, and option_type
        // (Assuming a parse_option_name function exists; this is not changed)
        let (underlying, issuer, option_type) = parse_option_name(&figi.name);

        if underlying.is_empty() {
            continue;
        }

        // Outer map: key by underlying
        let underlying_map = options
            .entry(underlying.clone())
            .or_insert_with(HashMap::new);

        // Inner map: key by option_type
        let entry = underlying_map.entry(option_type.clone()).or_insert(OptionInfo {
            underlying_company_name: underlying.clone(),
            issuer_company_name: issuer,
            option_type: option_type.clone(),
            options: HashMap::new(),
        });

        // Group by ISIN as before
        entry
            .options
            .entry(figi.isin.clone())
            .or_insert_with(Vec::new)
            .push(figi.clone());

        // Update stats (assuming stats has an 'options' field; adjust if needed)
        stats.options_added += 1;
    }
}

/// Groups FigiInfo list by ISIN
fn group_by_isin(figi_infos: &[FigiInfo]) -> HashMap<String, Vec<FigiInfo>> {
    let mut grouped: HashMap<String, Vec<FigiInfo>> = HashMap::new();
    
    for figi_info in figi_infos {
        grouped.entry(figi_info.isin.clone())
            .or_insert_with(Vec::new)
            .push(figi_info.clone());
    }
    
    // Sort each group by FIGI for consistency
    for figis in grouped.values_mut() {
        figis.sort_by(|a, b| a.figi.cmp(&b.figi));
    }
    
    grouped
}

/// Merges two FigiInfo lists, deduplicating by FIGI
fn merge_figi_list(existing: &[FigiInfo], new_figis: &[FigiInfo]) -> Vec<FigiInfo> {
    let mut merged = existing.to_vec();
    let existing_figis: HashSet<String> = existing.iter()
        .map(|f| f.figi.clone())
        .collect();
    
    for new_figi in new_figis {
        if !existing_figis.contains(&new_figi.figi) {
            merged.push(new_figi.clone());
        }
    }
    
    // Sort by FIGI for consistency
    merged.sort_by(|a, b| a.figi.cmp(&b.figi));
    
    merged
}

/// Parse warrant name to extract underlying company, issuer, and warrant type
/// 
/// Examples:
/// - "VONTOBE-PW26 LEONARDO SPA" -> ("LEONARDO SPA", Some("VONTOBEL"), "put")
/// - "BAYER H-CW25 L'OREAL" -> ("L'OREAL", Some("BAYER H"), "call")
/// - "APPLE INC WARRANT" -> ("APPLE INC", None, "unknown")
fn parse_warrant_name(name: &str) -> (String, Option<String>, String) {
    let name_upper = name.to_uppercase();
    
    // Try to detect warrant type from code (PW=put, CW=call)
    let warrant_type = if name_upper.contains("-PW") || name_upper.contains(" PW") {
        "put".to_string()
    } else if name_upper.contains("-CW") || name_upper.contains(" CW") {
        "call".to_string()
    } else {
        "unknown".to_string()
    };
    
    // Try to split by warrant code pattern (e.g., "-PW26", "-CW25")
    if let Some(pos) = name.find("-PW") {
        let before = name[..pos].trim();
        let after_idx = name[pos..].find(' ').map(|i| pos + i + 1).unwrap_or(name.len());
        let after = if after_idx < name.len() {
            name[after_idx..].trim()
        } else {
            ""
        };
        
        return (
            after.to_string(),
            if !before.is_empty() { Some(before.to_string()) } else { None },
            warrant_type,
        );
    }
    
    if let Some(pos) = name.find("-CW") {
        let before = name[..pos].trim();
        let after_idx = name[pos..].find(' ').map(|i| pos + i + 1).unwrap_or(name.len());
        let after = if after_idx < name.len() {
            name[after_idx..].trim()
        } else {
            ""
        };
        
        return (
            after.to_string(),
            if !before.is_empty() { Some(before.to_string()) } else { None },
            warrant_type,
        );
    }
    
    // Fallback: return entire name as underlying
    (name.to_string(), None, warrant_type)
}

/// Parse option name to extract underlying company, issuer, and option type
/// 
/// Examples:
/// - "December 25 Calls on ALPHA GA" -> ("ALPHA GA", None, "call")
/// - "January 26 Puts on TESLA INC" -> ("TESLA INC", None, "put")
fn parse_option_name(name: &str) -> (String, Option<String>, String) {
    let name_upper = name.to_uppercase();
    
    // Detect option type
    let option_type = if name_upper.contains("CALL") {
        "call".to_string()
    } else if name_upper.contains("PUT") {
        "put".to_string()
    } else {
        "unknown".to_string()
    };
    
    // Try to extract underlying after "on"
    if let Some(pos) = name_upper.find(" ON ") {
        let underlying = name[pos + 4..].trim().to_string();
        return (underlying, None, option_type);
    }
    
    // Fallback: return entire name
    (name.to_string(), None, option_type)
}

/// Generic function to load from cache
async fn load_from_cache<T>(path: &str) -> anyhow::Result<Option<T>>
where
    T: serde::de::DeserializeOwned,
{
    let cache_file = Path::new(path);
    
    if !cache_file.exists() {
        return Ok(None);
    }
    
    let content = tokio_fs::read_to_string(cache_file).await
        .context(format!("Failed to read {}", path))?;
    
    let data: T = serde_json::from_str(&content)
        .context(format!("Failed to parse {}", path))?;
    
    Ok(Some(data))
}

/// Generic function to save to cache
async fn save_to_cache<T>(path: &str, data: &T) -> anyhow::Result<()>
where
    T: serde::Serialize,
{
    let cache_path = Path::new(path);
    let cache_dir = cache_path.parent().context("Invalid cache path")?;
    
    tokio_fs::create_dir_all(cache_dir).await
        .context(format!("Failed to create directory for {}", path))?;
    
    let json_str = serde_json::to_string_pretty(data)
        .context("Failed to serialize data")?;
    
    tokio_fs::write(cache_path, json_str).await
        .context(format!("Failed to write {}", path))?;
    
    println!("  ✓ Saved to {}", path);
    
    Ok(())
}

/// Loads all OpenFIGI mapping value lists (marketSecDes, micCode, securityType).
///
/// This function fetches the available values for each mapping parameter from the OpenFIGI API
/// and caches them as JSON files in `data/openfigi/`. If the files already exist and are recent
/// (less than 30 days old), they are reused instead of re-fetching.
///
/// # Returns
/// Ok(()) on success.
///
/// # Errors
/// Returns an error if API requests fail, JSON parsing fails, or file I/O fails.
pub async fn load_figi_type_lists() -> anyhow::Result<()> {
    println!("Loading OpenFIGI mapping value lists...");
    
    let client = OpenFigiClient::new().await?;
    
    // Create cache directory
    let dir = DataPaths::new(".")?;
    let cache_dir = dir.cache_openfigi_dir();
    tokio_fs::create_dir_all(cache_dir).await
        .context("Failed to create data/openfigi directory")?;
    
    // Fetch each type list
    get_figi_market_sec_des(&client, cache_dir).await?;
    get_figi_mic_code(&client, cache_dir).await?;
    get_figi_security_type(&client, cache_dir).await?;
    
    println!("OpenFIGI mapping value lists loaded successfully");
    Ok(())
}

/// Fetches and caches the list of valid marketSecDes values.
///
/// # Arguments
/// * `client` - The OpenFIGI client instance.
/// * `cache_dir` - Directory to save the cached JSON file.
///
/// # Returns
/// Ok(()) on success.
///
/// # Errors
/// Returns an error if the API request fails or file I/O fails.
async fn get_figi_market_sec_des(client: &OpenFigiClient, cache_dir: &Path) -> anyhow::Result<()> {
    let cache_file = cache_dir.join("marketSecDes.json");
    
    // Check if cache exists and is recent (< 30 days old)
    if should_use_cache(&cache_file).await? {
        println!("  Using cached marketSecDes values");
        return Ok(());
    }
    
    println!("  Fetching marketSecDes values from OpenFIGI API...");
    
    let resp = client.client
        .get("https://api.openfigi.com/v3/mapping/values/marketSecDes")
        .send()
        .await
        .context("Failed to fetch marketSecDes values")?;
    
    handle_rate_limit(&resp).await?;
    
    let values: Value = resp.json().await
        .context("Failed to parse marketSecDes response")?;
    
    // Save to cache
    let json_str = serde_json::to_string_pretty(&values)?;
    tokio_fs::write(&cache_file, json_str).await
        .context("Failed to write marketSecDes cache")?;
    
    println!("  ✓ Cached marketSecDes values");
    
    // Respect rate limits
    sleep(Duration::from_millis(if client.has_key { 240 } else { 2400 })).await;
    
    Ok(())
}

/// Fetches and caches the list of valid micCode values.
///
/// # Arguments
/// * `client` - The OpenFIGI client instance.
/// * `cache_dir` - Directory to save the cached JSON file.
///
/// # Returns
/// Ok(()) on success.
///
/// # Errors
/// Returns an error if the API request fails or file I/O fails.
async fn get_figi_mic_code(client: &OpenFigiClient, cache_dir: &Path) -> anyhow::Result<()> {
    let cache_file = cache_dir.join("micCode.json");
    
    if should_use_cache(&cache_file).await? {
        println!("  Using cached micCode values");
        return Ok(());
    }
    
    println!("  Fetching micCode values from OpenFIGI API...");
    
    let resp = client.client
        .get("https://api.openfigi.com/v3/mapping/values/micCode")
        .send()
        .await
        .context("Failed to fetch micCode values")?;
    
    handle_rate_limit(&resp).await?;
    
    let values: Value = resp.json().await
        .context("Failed to parse micCode response")?;
    
    let json_str = serde_json::to_string_pretty(&values)?;
    tokio_fs::write(&cache_file, json_str).await
        .context("Failed to write micCode cache")?;
    
    println!("  ✓ Cached micCode values");
    
    sleep(Duration::from_millis(if client.has_key { 240 } else { 2400 })).await;
    
    Ok(())
}

/// Fetches and caches the list of valid securityType values.
///
/// # Arguments
/// * `client` - The OpenFIGI client instance.
/// * `cache_dir` - Directory to save the cached JSON file.
///
/// # Returns
/// Ok(()) on success.
///
/// # Errors
/// Returns an error if the API request fails or file I/O fails.
async fn get_figi_security_type(client: &OpenFigiClient, cache_dir: &Path) -> anyhow::Result<()> {
    let cache_file = cache_dir.join("securityType.json");
    
    if should_use_cache(&cache_file).await? {
        println!("  Using cached securityType values");
        return Ok(());
    }
    
    println!("  Fetching securityType values from OpenFIGI API...");
    
    let resp = client.client
        .get("https://api.openfigi.com/v3/mapping/values/securityType")
        .send()
        .await
        .context("Failed to fetch securityType values")?;
    
    handle_rate_limit(&resp).await?;
    
    let values: Value = resp.json().await
        .context("Failed to parse securityType response")?;
    
    let json_str = serde_json::to_string_pretty(&values)?;
    tokio_fs::write(&cache_file, json_str).await
        .context("Failed to write securityType cache")?;
    
    println!("  ✓ Cached securityType values");
    
    sleep(Duration::from_millis(if client.has_key { 240 } else { 2400 })).await;
    
    Ok(())
}

/// Checks if a cache file exists and is less than 30 days old.
///
/// # Arguments
/// * `path` - Path to the cache file.
///
/// # Returns
/// True if the cache should be used, false if it needs refreshing.
async fn should_use_cache(path: &Path) -> anyhow::Result<bool> {
    if !path.exists() {
        return Ok(false);
    }
    
    let metadata = tokio_fs::metadata(path).await?;
    let modified = metadata.modified()?;
    let age = modified.elapsed().unwrap_or(std::time::Duration::from_secs(u64::MAX));
    
    // Cache is valid for 30 days
    Ok(age < std::time::Duration::from_secs(30 * 24 * 60 * 60))
}

/// Handles rate limit responses from the OpenFIGI API.
///
/// If a 429 status is received, this function sleeps for the duration specified
/// in the `ratelimit-reset` header (or 10 seconds by default).
///
/// # Arguments
/// * `resp` - The HTTP response to check.
///
/// # Returns
/// Ok(()) if no rate limit, or after waiting for the reset period.
///
/// # Errors
/// Returns an error if the response status indicates a non-rate-limit error.
async fn handle_rate_limit(resp: &reqwest::Response) -> anyhow::Result<()> {
    let status = resp.status();
    
    if status == 429 {
        let headers = resp.headers();
        let reset_sec = headers
            .get("ratelimit-reset")
            .and_then(|v| v.to_str().ok())
            .and_then(|s| s.parse::<u64>().ok())
            .unwrap_or(10);
        
        println!("  Rate limited—waiting {}s", reset_sec);
        sleep(std::time::Duration::from_secs(reset_sec.max(10))).await;
        
        return Err(anyhow!("Rate limited, please retry"));
    } else if status.is_client_error() || status.is_server_error() {
        return Err(anyhow!("OpenFIGI API error: {}", status));
    }
    
    Ok(())
}