diff --git a/src/corporate/openfigi.rs b/src/corporate/openfigi.rs index 3be1927..1d6f047 100644 --- a/src/corporate/openfigi.rs +++ b/src/corporate/openfigi.rs @@ -1067,21 +1067,49 @@ async fn setup_sector_directories( /// /// # Errors /// Returns an error if API requests fail, JSON parsing fails, or file I/O fails. -pub async fn load_figi_type_lists() -> anyhow::Result<()> { +pub async fn load_figi_type_lists(paths: &DataPaths) -> anyhow::Result<()> { logger::log_info("Loading OpenFIGI mapping value lists...").await; + + let state_path = paths.cache_dir().join("state.jsonl"); + let cache_openfigi_dir = paths.cache_openfigi_dir(); + tokio_fs::create_dir_all(cache_openfigi_dir).await + .context("Failed to create data/openfigi directory")?; + + /*if state_path.exists() { + let state_content = tokio::fs::read_to_string(&state_path).await?; + + for line in state_content.lines() { + if line.trim().is_empty() { + continue; + } + + if let Ok(state) = serde_json::from_str::(line) { + if state.get("yahoo_companies_cleansed_no_data").and_then(|v| v.as_bool()).unwrap_or(false) { + logger::log_info(" Yahoo companies cleansing already completed, reading existing file...").await; + + if output_path.exists() { + let output_content = tokio::fs::read_to_string(&output_path).await?; + let count = output_content.lines() + .filter(|line| !line.trim().is_empty()) + .count(); + + logger::log_info(&format!(" ✓ Found {} companies in companies_yahoo.jsonl", count)).await; + return Ok(count); + } else { + logger::log_warn(" State indicates completion but companies_yahoo.jsonl not found, re-running...").await; + break; + } + } + } + } + }*/ let client = OpenFigiClient::new().await?; - // Create cache directory - let dir = DataPaths::new(".")?; - let cache_dir = dir.cache_openfigi_dir(); - tokio_fs::create_dir_all(cache_dir).await - .context("Failed to create data/openfigi directory")?; - // Fetch each type list - get_figi_market_sec_des(&client, cache_dir).await?; - get_figi_mic_code(&client, cache_dir).await?; - get_figi_security_type(&client, cache_dir).await?; + get_figi_market_sec_des(&client, cache_openfigi_dir).await?; + get_figi_mic_code(&client, cache_openfigi_dir).await?; + get_figi_security_type(&client, cache_openfigi_dir).await?; logger::log_info("OpenFIGI mapping value lists loaded successfully").await; diff --git a/src/corporate/update.rs b/src/corporate/update.rs index 722dc42..e4f7b51 100644 --- a/src/corporate/update.rs +++ b/src/corporate/update.rs @@ -44,7 +44,7 @@ pub async fn run_full_update( check_shutdown!(shutdown_flag); logger::log_info("Step 2: Loading OpenFIGI metadata...").await; - load_figi_type_lists().await.ok(); + load_figi_type_lists(&paths).await.ok(); logger::log_info(" ✓ OpenFIGI metadata loaded").await; check_shutdown!(shutdown_flag);