// src/corporate/openfigi.rs
use super::{types::*};
use reqwest::Client as HttpClient;
use reqwest::header::{HeaderMap, HeaderValue};
use serde_json::{json, Value};
use std::collections::{HashMap, HashSet};
use std::fs::{File, OpenOptions};
use std::io::{BufRead, BufReader, Write};
use std::path::Path;
use std::time::Instant;
use tokio::time::{sleep, Duration};
use tokio::fs as tokio_fs;
use anyhow::{Context, anyhow};

#[derive(Clone)]
pub struct OpenFigiClient {
    client: HttpClient,
    has_key: bool,
}

impl OpenFigiClient {
    /// Creates a new OpenFIGI client, optionally with an API key.
    ///
    /// Loads the API key from the `OPENFIGI_API_KEY` environment variable if present.
    ///
    /// # Errors
    ///
    /// Returns an error if the HTTP client cannot be built or if the API key header is invalid.
    pub fn new() -> anyhow::Result<Self> {
        let api_key = dotenvy::var("OPENFIGI_API_KEY").ok();
        let has_key = api_key.is_some();

        let mut builder = HttpClient::builder()
            .user_agent("Mozilla/5.0 (compatible; OpenFIGI-Rust/1.0)")
            .timeout(Duration::from_secs(30));

        if let Some(key) = &api_key {
            let mut headers = HeaderMap::new();
            headers.insert("X-OPENFIGI-APIKEY", HeaderValue::from_str(key)?);
            builder = builder.default_headers(headers);
        }

        let client = builder.build().context("Failed to build HTTP client")?;

        println!(
            "OpenFIGI client initialized: {}",
            if has_key { "with API key" } else { "no key (limited mode)" }
        );

        Ok(Self { client, has_key })
    }

    /// Maps a batch of ISINs to FigiInfo structs, filtering for equities only.
    ///
    /// Batches requests according to rate limits (100 jobs/req with key, 5 without).
    /// Optimizes inter-request delays to approach the rate limit without exceeding it:
    /// - With key: ~240ms sleep per request (to sustain ~4 req/sec or 250 req/min).
    /// - Without key: 2.4s sleep (to sustain 25 req/min).
    /// Handles 429 rate limits with header-based backoff.
    /// Collects detailed FigiInfo from responses, using `exchCode` as proxy for `mic_code`.
    ///
    /// # Arguments
    ///
    /// * `isins` - Slice of ISIN strings to map (deduplicated internally if needed).
    ///
    /// # Returns
    ///
    /// A vector of `FigiInfo` structs for equity instruments.
    ///
    /// # Errors
    ///
    /// Returns an error on HTTP failures, JSON parsing issues, invalid API keys,
    /// or repeated rate limit violations after backoff.
    ///
    /// # Examples
    ///
    /// ```no_run
    /// # use anyhow::Result;
    /// # async fn example(client: &OpenFigiClient) -> Result<()> {
    /// let isins = vec!["US0378331005".to_string(), "US5949181045".to_string()];
    /// let figis = client.map_isins_to_figi_infos(&isins).await?;
    /// # Ok(())
    /// # }
    /// ```
    pub async fn map_isins_to_figi_infos(&self, isins: &[String]) -> anyhow::Result<Vec<FigiInfo>> {
        if isins.is_empty() {
            return Ok(vec![]);
        }

        let mut all_figi_infos = Vec::new();
        let chunk_size = if self.has_key { 100 } else { 5 };
        let inter_sleep = if self.has_key {
            Duration::from_millis(240) // ~4.16 req/sec (250/min)
        } else {
            Duration::from_millis(2400) // 25/min
        };

        let start_time = Instant::now();
        let mut req_count = 0;

        for chunk in isins.chunks(chunk_size) {
            let jobs: Vec<Value> = chunk.iter()
                .map(|isin| json!({
                    "idType": "ID_ISIN",
                    "idValue": isin,
                    "marketSecDes": "Equity",
                }))
                .collect();

            let resp = self.client
                .post("https://api.openfigi.com/v3/mapping")
                .header("Content-Type", "application/json")
                .json(&jobs)
                .send()
                .await
                .context("Failed to send mapping request")?;

            let status = resp.status();
            let headers = resp.headers().clone();
            let body = resp.text().await.context("Failed to read response body")?;

            if status.is_client_error() || status.is_server_error() {
                if status == 429 {
                    let reset_sec = headers
                        .get("ratelimit-reset")
                        .and_then(|v| v.to_str().ok())
                        .and_then(|s| s.parse::<u64>().ok())
                        .unwrap_or(10);
                    println!("Rate limited—backing off {}s", reset_sec);
                    sleep(Duration::from_secs(reset_sec.max(10))).await;
                    continue; // Retry the same chunk
                } else if status == 401 {
                    return Err(anyhow!("Invalid OpenFIGI API key: {}", body));
                } else if status == 413 {
                    return Err(anyhow!("Payload too large—reduce chunk size: {}", body));
                }
                return Err(anyhow!("OpenFIGI error {}: {}", status, body));
            }

            let results: Vec<Value> = serde_json::from_str(&body)
                .context("Failed to parse response JSON")?;

            for (isin, result) in chunk.iter().zip(results) {
                if let Some(data) = result["data"].as_array() {
                    for item in data {
                        let sec_type = item["securityType"].as_str().unwrap_or("");
                        let market_sec = item["marketSector"].as_str().unwrap_or("");
                        if market_sec == "Equity" &&
                           (sec_type.contains("Stock") || sec_type.contains("Share") || sec_type.contains("Equity") ||
                            sec_type.contains("Common") || sec_type.contains("Preferred") || sec_type == "ADR" || sec_type == "GDR") {
                            let figi = match item["figi"].as_str() {
                                Some(f) => f.to_string(),
                                None => continue,
                            };

                            let figi_info = FigiInfo {
                                isin: isin.clone(),
                                figi,
                                name: item["name"].as_str().unwrap_or("").to_string(),
                                ticker: item["ticker"].as_str().unwrap_or("").to_string(),
                                exch_code: item["micCode"].as_str().unwrap_or("").to_string(),
                                compositeFIGI: item["compositeFIGI"].as_str().unwrap_or("").to_string(),
                                securityType: sec_type.to_string(),
                                marketSector: market_sec.to_string(),
                                shareClassFIGI: item["shareClassFIGI"].as_str().unwrap_or("").to_string(),
                                securityType2: item["securityType2"].as_str().unwrap_or("").to_string(),
                                securityDescription: item["securityDescription"].as_str().unwrap_or("").to_string(),
                            };

                            all_figi_infos.push(figi_info);
                        }
                    }
                }
            }

            req_count += 1;
            if req_count % 25 == 0 {
                // Optional: Enforce 6-sec window for bursts
                let elapsed = start_time.elapsed();
                if self.has_key {
                    if elapsed < Duration::from_secs(6) {
                        sleep(Duration::from_secs(6) - elapsed).await;
                    }
                } else {
                    if elapsed < Duration::from_secs(6) {
                        sleep(Duration::from_secs(60) - elapsed).await;
                    }
                }
            } else {
                sleep(inter_sleep).await;
            }
        }

        Ok(all_figi_infos)
    }
}

/// Builds a LEI-to-FigiInfo map from the LEI-ISIN mapping, filtering for equities via OpenFIGI.
///
/// Attempts to load existing entries from "data/corporate/by_lei/lei_to_figi.jsonl" (JSON Lines format,
/// one LEI entry per line: {"lei": "ABC", "figis": [FigiInfo...]}). For any missing LEIs (compared to
/// `lei_to_isins`), fetches their FigiInfos and appends to the .jsonl file incrementally.
///
/// This design allows resumption after interruptions: on restart, already processed LEIs are skipped,
/// and only remaining ones are fetched. Processes LEIs in sorted order for deterministic behavior.
///
/// If no API key is present, skips building new entries and returns the loaded map (possibly partial).
///
/// # Arguments
///
/// * `lei_to_isins` - HashMap of LEI to Vec<ISIN> (used for fetching missing entries).
///
/// # Returns
///
/// The complete (or partial if interrupted) HashMap<LEI, Vec<FigiInfo>>.
///
/// # Errors
///
/// Returns an error if file I/O fails, JSON serialization/deserialization fails,
/// or if OpenFIGI queries fail during fetching.
pub async fn build_lei_to_figi_infos(lei_to_isins: &HashMap<String, Vec<String>>) -> anyhow::Result<HashMap<String, Vec<FigiInfo>>> {
    let data_dir = Path::new("data/corporate/by_lei");
    tokio_fs::create_dir_all(data_dir).await.context("Failed to create data directory")?;

    let path = data_dir.join("lei_to_figi.jsonl");
    let mut lei_to_figis: HashMap<String, Vec<FigiInfo>> = load_lei_to_figi_jsonl(&path)?;

    let client = OpenFigiClient::new()?;
    if !client.has_key {
        println!("No API key—using partial LEI→FIGI map with {} entries", lei_to_figis.len());
        return Ok(lei_to_figis);
    }

    // Sort LEIs for deterministic processing order
    let mut leis: Vec<_> = lei_to_isins.keys().cloned().collect();
    leis.sort();

    let mut processed = lei_to_figis.len();
    let total = leis.len();

    for lei in leis {
        if lei_to_figis.contains_key(&lei) {
            continue; // Skip already processed
        }

        let isins = match lei_to_isins.get(&lei) {
            Some(i) => i,
            None => continue,
        };

        let unique_isins: Vec<_> = isins.iter().cloned().collect::<HashSet<_>>().into_iter().collect();
        let equity_figi_infos = client.map_isins_to_figi_infos(&unique_isins).await?;

        let mut figis = equity_figi_infos;
        if !figis.is_empty() {
            figis.sort_by_key(|f| f.figi.clone());
            figis.dedup_by_key(|f| f.figi.clone());
        }

        // Append to .jsonl incrementally
        append_lei_to_figi_jsonl(&path, &lei, &figis).context("Failed to append to JSONL")?;

        // Insert into in-memory map
        lei_to_figis.insert(lei.clone(), figis);

        processed += 1;
        if processed % 100 == 0 {
            println!("Processed {}/{} LEIs → {} total equity FIGIs", processed, total, lei_to_figis.values().map(|v| v.len()).sum::<usize>());
        }

        tokio::time::sleep(Duration::from_millis(100)).await;
    }

    println!("Completed LEI→FIGI map: {} mappings (equity-only)", lei_to_figis.len());
    Ok(lei_to_figis)
}

/// Loads LEI-to-FigiInfo map from a JSON Lines file.
///
/// Each line is expected to be a JSON object: {"lei": "ABC", "figis": [FigiInfo...]}
///
/// # Arguments
///
/// * `path` - Path to the .jsonl file.
///
/// # Returns
///
/// The loaded HashMap<LEI, Vec<FigiInfo>>.
///
/// # Errors
///
/// Returns an error if the file cannot be opened or if any line fails to parse as JSON.
fn load_lei_to_figi_jsonl(path: &Path) -> anyhow::Result<HashMap<String, Vec<FigiInfo>>> {
    let mut map = HashMap::new();

    if !path.exists() {
        return Ok(map);
    }

    let file = File::open(path).context("Failed to open JSONL file for reading")?;
    let reader = BufReader::new(file);

    for (line_num, line) in reader.lines().enumerate() {
        let line = line.context(format!("Failed to read line {}", line_num + 1))?;
        if line.trim().is_empty() {
            continue;
        }

        let entry: Value = serde_json::from_str(&line).context(format!("Failed to parse JSON on line {}", line_num + 1))?;
        let lei = entry["lei"].as_str().context("Missing 'lei' field")?.to_string();
        let figis: Vec<FigiInfo> = serde_json::from_value(entry["figis"].clone()).context("Invalid 'figis' field")?;

        map.insert(lei, figis);
    }

    println!("Loaded LEI→FIGI map with {} entries from {}", map.len(), path.display());
    Ok(map)
}

/// Appends a single LEI entry to the JSON Lines file.
///
/// # Arguments
///
/// * `path` - Path to the .jsonl file.
/// * `lei` - The LEI key.
/// * `figis` - The Vec<FigiInfo> for this LEI.
///
/// # Errors
///
/// Returns an error if the file cannot be opened for append or if serialization fails.
fn append_lei_to_figi_jsonl(path: &Path, lei: &str, figis: &[FigiInfo]) -> anyhow::Result<()> {
    let mut file = OpenOptions::new()
        .create(true)
        .append(true)
        .open(path)
        .context("Failed to open JSONL file for append")?;

    let entry = json!({
        "lei": lei,
        "figis": figis,
    });

    let line = serde_json::to_string(&entry).context("Failed to serialize entry")? + "\n";
    file.write_all(line.as_bytes()).context("Failed to write to JSONL file")?;

    Ok(())
}

/// Loads or builds HashMaps for companies, warrants, and options.
///
/// This function:
/// 1. Attempts to load existing data from cache
/// 2. Processes new FIGI data and classifies by securityType:
///    - "Common Stock" → companies HashMap (grouped by ISIN)
///    - "Equity WRT" → warrants HashMap (parsed from name)
///    - "Equity Option" → options HashMap (parsed from name)
/// 3. Updates/extends existing entries
/// 4. Saves results to separate JSON files
///
/// # Arguments
/// * `figi_to_lei` - HashMap mapping LEI to Vec<FigiInfo>.
///
/// # Returns
/// A tuple of (companies, warrants, options) HashMaps.
///
/// # Errors
/// Returns an error if file I/O fails or JSON serialization fails.
pub async fn load_or_build_all_securities(
    figi_to_lei: &HashMap<String, Vec<FigiInfo>>
) -> anyhow::Result<(
    HashMap<String, CompanyInfo>,
    HashMap<String, HashMap<String, WarrantInfo>>,
    HashMap<String, HashMap<String, OptionInfo>>
)> {
    // Load existing data
    let mut companies = load_from_cache("data/corporate/by_name/common_stocks.json").await?
        .unwrap_or_else(HashMap::new);
    let mut warrants = load_from_cache("data/corporate/by_name/warrants.json").await?
        .unwrap_or_else(HashMap::new);
    let mut options = load_from_cache("data/corporate/by_name/options.json").await?
        .unwrap_or_else(HashMap::new);
    
    println!("Loaded existing data:");
    println!("  - Companies: {}", companies.len());
    println!("  - Warrants: {}", warrants.len());
    println!("  - Options: {}", options.len());
    
    let mut stats = ProcessingStats::new(companies.len(), warrants.len(), options.len());
    
    println!("Processing {} LEI entries from FIGI data...", figi_to_lei.len());
    
    for (_lei, figi_infos) in figi_to_lei.iter() {
        if figi_infos.is_empty() {
            continue;
        }
        
        // Group FigiInfos by security type
        let mut common_stocks = Vec::new();
        let mut warrant_securities = Vec::new();
        let mut option_securities = Vec::new();
        
        for figi_info in figi_infos {
            match figi_info.securityType.as_str() {
                "Common Stock" => common_stocks.push(figi_info.clone()),
                "Equity WRT" => warrant_securities.push(figi_info.clone()),
                "Equity Option" => option_securities.push(figi_info.clone()),
                _ => {} // Ignore other types
            }
        }
        
        // Process common stocks -> companies
        if !common_stocks.is_empty() {
            process_common_stocks(&mut companies, &common_stocks, &mut stats);
        }
        
        // Process warrants
        if !warrant_securities.is_empty() {
            process_warrants(&mut warrants, &warrant_securities, &mut stats);
        }
        
        // Process options
        if !option_securities.is_empty() {
            process_options(&mut options, &option_securities, &mut stats);
        }
    }
    
    stats.print_summary(companies.len(), warrants.len(), options.len());
    
    // Save all three HashMaps
    save_to_cache("data/corporate/by_name/common_stocks.json", &companies).await?;
    save_to_cache("data/corporate/by_name/warrants.json", &warrants).await?;
    save_to_cache("data/corporate/by_name/options.json", &options).await?;
    
    Ok((companies, warrants, options))
}

/// Statistics tracker for processing
#[derive(Debug)]
struct ProcessingStats {
    initial_companies: usize,
    initial_warrants: usize,
    initial_options: usize,
    companies_added: usize,
    companies_updated: usize,
    warrants_added: usize,
    warrants_updated: usize,
    options_added: usize,
    options_updated: usize,
}

impl ProcessingStats {
    fn new(companies: usize, warrants: usize, options: usize) -> Self {
        Self {
            initial_companies: companies,
            initial_warrants: warrants,
            initial_options: options,
            companies_added: 0,
            companies_updated: 0,
            warrants_added: 0,
            warrants_updated: 0,
            options_added: 0,
            options_updated: 0,
        }
    }
    
    fn print_summary(&self, final_companies: usize, final_warrants: usize, final_options: usize) {
        println!("\n=== Processing Statistics ===");
        println!("Companies:");
        println!("  - Initial: {}", self.initial_companies);
        println!("  - Added: {}", self.companies_added);
        println!("  - Updated: {}", self.companies_updated);
        println!("  - Total: {}", final_companies);
        println!("Warrants:");
        println!("  - Initial: {}", self.initial_warrants);
        println!("  - Added: {}", self.warrants_added);
        println!("  - Updated: {}", self.warrants_updated);
        println!("  - Total: {}", final_warrants);
        println!("Options:");
        println!("  - Initial: {}", self.initial_options);
        println!("  - Added: {}", self.options_added);
        println!("  - Updated: {}", self.options_updated);
        println!("  - Total: {}", final_options);
    }
}

/// Process common stocks into companies HashMap
fn process_common_stocks(
    companies: &mut HashMap<String, CompanyInfo>,
    figi_infos: &[FigiInfo],
    stats: &mut ProcessingStats,
) {
    let name = figi_infos[0].name.clone();
    if name.is_empty() {
        return;
    }
    
    // Group by ISIN
    let grouped_by_isin = group_by_isin(figi_infos);
    
    if let Some(existing) = companies.get_mut(&name) {
        // Update existing company
        let mut updated = false;
        for (isin, new_figis) in grouped_by_isin {
            if let Some(existing_figis) = existing.securities.get_mut(&isin) {
                let merged = merge_figi_list(existing_figis, &new_figis);
                if merged.len() > existing_figis.len() {
                    *existing_figis = merged;
                    updated = true;
                }
            } else {
                existing.securities.insert(isin.clone(), new_figis);
                updated = true;
            }
        }
        
        // Update primary ISIN if needed
        if existing.primary_isin.is_empty() || !existing.securities.contains_key(&existing.primary_isin) {
            if let Some(first_isin) = existing.securities.keys().next() {
                existing.primary_isin = first_isin.clone();
            }
        }
        
        if updated {
            stats.companies_updated += 1;
        }
    } else {
        // Add new company
        let primary_isin = grouped_by_isin.keys().next().cloned().unwrap_or_default();
        
        companies.insert(name.clone(), CompanyInfo {
            name,
            primary_isin,
            securities: grouped_by_isin,
        });
        
        stats.companies_added += 1;
    }
}

/// Process warrants into warrants HashMap
fn process_warrants(
    warrants: &mut HashMap<String, HashMap<String, WarrantInfo>>,
    warrant_securities: &[FigiInfo],
    stats: &mut ProcessingStats,  // Assuming Stats is a struct; adjust based on actual type if it's a HashMap or other
) {
    for figi in warrant_securities.iter() {
        // Parse the name to extract underlying, issuer, and warrant_type
        // (Assuming a parse_warrant_name function exists; this is not changed)
        let (underlying, issuer, warrant_type) = parse_warrant_name(&figi.name);

        if underlying.is_empty() {
            continue;
        }

        // Outer map: key by underlying
        let underlying_map = warrants
            .entry(underlying.clone())
            .or_insert_with(HashMap::new);

        // Inner map: key by warrant_type
        let entry = underlying_map.entry(warrant_type.clone()).or_insert(WarrantInfo {
            underlying_company_name: underlying.clone(),
            issuer_company_name: issuer,
            warrant_type: warrant_type.clone(),
            warrants: HashMap::new(),
        });

        // Group by ISIN as before
        entry
            .warrants
            .entry(figi.isin.clone())
            .or_insert_with(Vec::new)
            .push(figi.clone());

        // Update stats (assuming stats has a 'warrants' field; adjust if needed)
        stats.warrants_added += 1;
    }
}

/// Process options into options HashMap
fn process_options(
    options: &mut HashMap<String, HashMap<String, OptionInfo>>,
    option_securities: &[FigiInfo],
    stats: &mut ProcessingStats,  // Assuming Stats is a struct; adjust based on actual type if it's a HashMap or other
) {
    for figi in option_securities.iter() {
        // Parse the name to extract underlying, issuer, and option_type
        // (Assuming a parse_option_name function exists; this is not changed)
        let (underlying, issuer, option_type) = parse_option_name(&figi.name);

        if underlying.is_empty() {
            continue;
        }

        // Outer map: key by underlying
        let underlying_map = options
            .entry(underlying.clone())
            .or_insert_with(HashMap::new);

        // Inner map: key by option_type
        let entry = underlying_map.entry(option_type.clone()).or_insert(OptionInfo {
            underlying_company_name: underlying.clone(),
            issuer_company_name: issuer,
            option_type: option_type.clone(),
            options: HashMap::new(),
        });

        // Group by ISIN as before
        entry
            .options
            .entry(figi.isin.clone())
            .or_insert_with(Vec::new)
            .push(figi.clone());

        // Update stats (assuming stats has an 'options' field; adjust if needed)
        stats.options_added += 1;
    }
}

/// Groups FigiInfo list by ISIN
fn group_by_isin(figi_infos: &[FigiInfo]) -> HashMap<String, Vec<FigiInfo>> {
    let mut grouped: HashMap<String, Vec<FigiInfo>> = HashMap::new();
    
    for figi_info in figi_infos {
        grouped.entry(figi_info.isin.clone())
            .or_insert_with(Vec::new)
            .push(figi_info.clone());
    }
    
    // Sort each group by FIGI for consistency
    for figis in grouped.values_mut() {
        figis.sort_by(|a, b| a.figi.cmp(&b.figi));
    }
    
    grouped
}

/// Merges two FigiInfo lists, deduplicating by FIGI
fn merge_figi_list(existing: &[FigiInfo], new_figis: &[FigiInfo]) -> Vec<FigiInfo> {
    let mut merged = existing.to_vec();
    let existing_figis: HashSet<String> = existing.iter()
        .map(|f| f.figi.clone())
        .collect();
    
    for new_figi in new_figis {
        if !existing_figis.contains(&new_figi.figi) {
            merged.push(new_figi.clone());
        }
    }
    
    // Sort by FIGI for consistency
    merged.sort_by(|a, b| a.figi.cmp(&b.figi));
    
    merged
}

/// Parse warrant name to extract underlying company, issuer, and warrant type
/// 
/// Examples:
/// - "VONTOBE-PW26 LEONARDO SPA" -> ("LEONARDO SPA", Some("VONTOBEL"), "put")
/// - "BAYER H-CW25 L'OREAL" -> ("L'OREAL", Some("BAYER H"), "call")
/// - "APPLE INC WARRANT" -> ("APPLE INC", None, "unknown")
fn parse_warrant_name(name: &str) -> (String, Option<String>, String) {
    let name_upper = name.to_uppercase();
    
    // Try to detect warrant type from code (PW=put, CW=call)
    let warrant_type = if name_upper.contains("-PW") || name_upper.contains(" PW") {
        "put".to_string()
    } else if name_upper.contains("-CW") || name_upper.contains(" CW") {
        "call".to_string()
    } else {
        "unknown".to_string()
    };
    
    // Try to split by warrant code pattern (e.g., "-PW26", "-CW25")
    if let Some(pos) = name.find("-PW") {
        let before = name[..pos].trim();
        let after_idx = name[pos..].find(' ').map(|i| pos + i + 1).unwrap_or(name.len());
        let after = if after_idx < name.len() {
            name[after_idx..].trim()
        } else {
            ""
        };
        
        return (
            after.to_string(),
            if !before.is_empty() { Some(before.to_string()) } else { None },
            warrant_type,
        );
    }
    
    if let Some(pos) = name.find("-CW") {
        let before = name[..pos].trim();
        let after_idx = name[pos..].find(' ').map(|i| pos + i + 1).unwrap_or(name.len());
        let after = if after_idx < name.len() {
            name[after_idx..].trim()
        } else {
            ""
        };
        
        return (
            after.to_string(),
            if !before.is_empty() { Some(before.to_string()) } else { None },
            warrant_type,
        );
    }
    
    // Fallback: return entire name as underlying
    (name.to_string(), None, warrant_type)
}

/// Parse option name to extract underlying company, issuer, and option type
/// 
/// Examples:
/// - "December 25 Calls on ALPHA GA" -> ("ALPHA GA", None, "call")
/// - "January 26 Puts on TESLA INC" -> ("TESLA INC", None, "put")
fn parse_option_name(name: &str) -> (String, Option<String>, String) {
    let name_upper = name.to_uppercase();
    
    // Detect option type
    let option_type = if name_upper.contains("CALL") {
        "call".to_string()
    } else if name_upper.contains("PUT") {
        "put".to_string()
    } else {
        "unknown".to_string()
    };
    
    // Try to extract underlying after "on"
    if let Some(pos) = name_upper.find(" ON ") {
        let underlying = name[pos + 4..].trim().to_string();
        return (underlying, None, option_type);
    }
    
    // Fallback: return entire name
    (name.to_string(), None, option_type)
}

/// Generic function to load from cache
async fn load_from_cache<T>(path: &str) -> anyhow::Result<Option<T>>
where
    T: serde::de::DeserializeOwned,
{
    let cache_file = Path::new(path);
    
    if !cache_file.exists() {
        return Ok(None);
    }
    
    let content = tokio_fs::read_to_string(cache_file).await
        .context(format!("Failed to read {}", path))?;
    
    let data: T = serde_json::from_str(&content)
        .context(format!("Failed to parse {}", path))?;
    
    Ok(Some(data))
}

/// Generic function to save to cache
async fn save_to_cache<T>(path: &str, data: &T) -> anyhow::Result<()>
where
    T: serde::Serialize,
{
    let cache_path = Path::new(path);
    let cache_dir = cache_path.parent().context("Invalid cache path")?;
    
    tokio_fs::create_dir_all(cache_dir).await
        .context(format!("Failed to create directory for {}", path))?;
    
    let json_str = serde_json::to_string_pretty(data)
        .context("Failed to serialize data")?;
    
    tokio_fs::write(cache_path, json_str).await
        .context(format!("Failed to write {}", path))?;
    
    println!("  ✓ Saved to {}", path);
    
    Ok(())
}

/// Loads all OpenFIGI mapping value lists (marketSecDes, micCode, securityType).
///
/// This function fetches the available values for each mapping parameter from the OpenFIGI API
/// and caches them as JSON files in `data/openfigi/`. If the files already exist and are recent
/// (less than 30 days old), they are reused instead of re-fetching.
///
/// # Returns
/// Ok(()) on success.
///
/// # Errors
/// Returns an error if API requests fail, JSON parsing fails, or file I/O fails.
pub async fn load_figi_type_lists() -> anyhow::Result<()> {
    println!("Loading OpenFIGI mapping value lists...");
    
    let client = OpenFigiClient::new()?;
    
    // Create cache directory
    let cache_dir = Path::new("data/openfigi");
    tokio_fs::create_dir_all(cache_dir).await
        .context("Failed to create data/openfigi directory")?;
    
    // Fetch each type list
    get_figi_market_sec_des(&client, cache_dir).await?;
    get_figi_mic_code(&client, cache_dir).await?;
    get_figi_security_type(&client, cache_dir).await?;
    
    println!("OpenFIGI mapping value lists loaded successfully");
    Ok(())
}

/// Fetches and caches the list of valid marketSecDes values.
///
/// # Arguments
/// * `client` - The OpenFIGI client instance.
/// * `cache_dir` - Directory to save the cached JSON file.
///
/// # Returns
/// Ok(()) on success.
///
/// # Errors
/// Returns an error if the API request fails or file I/O fails.
async fn get_figi_market_sec_des(client: &OpenFigiClient, cache_dir: &Path) -> anyhow::Result<()> {
    let cache_file = cache_dir.join("marketSecDes.json");
    
    // Check if cache exists and is recent (< 30 days old)
    if should_use_cache(&cache_file).await? {
        println!("  Using cached marketSecDes values");
        return Ok(());
    }
    
    println!("  Fetching marketSecDes values from OpenFIGI API...");
    
    let resp = client.client
        .get("https://api.openfigi.com/v3/mapping/values/marketSecDes")
        .send()
        .await
        .context("Failed to fetch marketSecDes values")?;
    
    handle_rate_limit(&resp).await?;
    
    let values: Value = resp.json().await
        .context("Failed to parse marketSecDes response")?;
    
    // Save to cache
    let json_str = serde_json::to_string_pretty(&values)?;
    tokio_fs::write(&cache_file, json_str).await
        .context("Failed to write marketSecDes cache")?;
    
    println!("  ✓ Cached marketSecDes values");
    
    // Respect rate limits
    sleep(Duration::from_millis(if client.has_key { 240 } else { 2400 })).await;
    
    Ok(())
}

/// Fetches and caches the list of valid micCode values.
///
/// # Arguments
/// * `client` - The OpenFIGI client instance.
/// * `cache_dir` - Directory to save the cached JSON file.
///
/// # Returns
/// Ok(()) on success.
///
/// # Errors
/// Returns an error if the API request fails or file I/O fails.
async fn get_figi_mic_code(client: &OpenFigiClient, cache_dir: &Path) -> anyhow::Result<()> {
    let cache_file = cache_dir.join("micCode.json");
    
    if should_use_cache(&cache_file).await? {
        println!("  Using cached micCode values");
        return Ok(());
    }
    
    println!("  Fetching micCode values from OpenFIGI API...");
    
    let resp = client.client
        .get("https://api.openfigi.com/v3/mapping/values/micCode")
        .send()
        .await
        .context("Failed to fetch micCode values")?;
    
    handle_rate_limit(&resp).await?;
    
    let values: Value = resp.json().await
        .context("Failed to parse micCode response")?;
    
    let json_str = serde_json::to_string_pretty(&values)?;
    tokio_fs::write(&cache_file, json_str).await
        .context("Failed to write micCode cache")?;
    
    println!("  ✓ Cached micCode values");
    
    sleep(Duration::from_millis(if client.has_key { 240 } else { 2400 })).await;
    
    Ok(())
}

/// Fetches and caches the list of valid securityType values.
///
/// # Arguments
/// * `client` - The OpenFIGI client instance.
/// * `cache_dir` - Directory to save the cached JSON file.
///
/// # Returns
/// Ok(()) on success.
///
/// # Errors
/// Returns an error if the API request fails or file I/O fails.
async fn get_figi_security_type(client: &OpenFigiClient, cache_dir: &Path) -> anyhow::Result<()> {
    let cache_file = cache_dir.join("securityType.json");
    
    if should_use_cache(&cache_file).await? {
        println!("  Using cached securityType values");
        return Ok(());
    }
    
    println!("  Fetching securityType values from OpenFIGI API...");
    
    let resp = client.client
        .get("https://api.openfigi.com/v3/mapping/values/securityType")
        .send()
        .await
        .context("Failed to fetch securityType values")?;
    
    handle_rate_limit(&resp).await?;
    
    let values: Value = resp.json().await
        .context("Failed to parse securityType response")?;
    
    let json_str = serde_json::to_string_pretty(&values)?;
    tokio_fs::write(&cache_file, json_str).await
        .context("Failed to write securityType cache")?;
    
    println!("  ✓ Cached securityType values");
    
    sleep(Duration::from_millis(if client.has_key { 240 } else { 2400 })).await;
    
    Ok(())
}

/// Checks if a cache file exists and is less than 30 days old.
///
/// # Arguments
/// * `path` - Path to the cache file.
///
/// # Returns
/// True if the cache should be used, false if it needs refreshing.
async fn should_use_cache(path: &Path) -> anyhow::Result<bool> {
    if !path.exists() {
        return Ok(false);
    }
    
    let metadata = tokio_fs::metadata(path).await?;
    let modified = metadata.modified()?;
    let age = modified.elapsed().unwrap_or(std::time::Duration::from_secs(u64::MAX));
    
    // Cache is valid for 30 days
    Ok(age < std::time::Duration::from_secs(30 * 24 * 60 * 60))
}

/// Handles rate limit responses from the OpenFIGI API.
///
/// If a 429 status is received, this function sleeps for the duration specified
/// in the `ratelimit-reset` header (or 10 seconds by default).
///
/// # Arguments
/// * `resp` - The HTTP response to check.
///
/// # Returns
/// Ok(()) if no rate limit, or after waiting for the reset period.
///
/// # Errors
/// Returns an error if the response status indicates a non-rate-limit error.
async fn handle_rate_limit(resp: &reqwest::Response) -> anyhow::Result<()> {
    let status = resp.status();
    
    if status == 429 {
        let headers = resp.headers();
        let reset_sec = headers
            .get("ratelimit-reset")
            .and_then(|v| v.to_str().ok())
            .and_then(|s| s.parse::<u64>().ok())
            .unwrap_or(10);
        
        println!("  Rate limited—waiting {}s", reset_sec);
        sleep(std::time::Duration::from_secs(reset_sec.max(10))).await;
        
        return Err(anyhow!("Rate limited, please retry"));
    } else if status.is_client_error() || status.is_server_error() {
        return Err(anyhow!("OpenFIGI API error: {}", status));
    }
    
    Ok(())
}