diff --git a/src/corporate/openfigi.rs b/src/corporate/openfigi.rs index b1ed1e4..29d93d3 100644 --- a/src/corporate/openfigi.rs +++ b/src/corporate/openfigi.rs @@ -1,6 +1,6 @@ // src/corporate/openfigi.rs use super::{types::*}; -use reqwest::{Client as HttpClient, StatusCode}; +use reqwest::Client as HttpClient; use reqwest::header::{HeaderMap, HeaderValue}; use serde_json::{json, Value}; use std::collections::{HashMap, HashSet}; @@ -46,96 +46,74 @@ impl OpenFigiClient { let mut all_figis = Vec::new(); let chunk_size = if self.has_key { 100 } else { 5 }; - for (chunk_idx, chunk) in isins.chunks(chunk_size).enumerate() { - let mut retries = 0; - let mut success = false; + for chunk in isins.chunks(chunk_size) { + let jobs: Vec = chunk.iter() + .map(|isin| json!({ + "idType": "ID_ISIN", + "idValue": isin, + "marketSecDes": "Equity", // Pre-filter to equities + })) + .collect(); - while retries < 3 && !success { - let jobs: Vec = chunk.iter() - .map(|isin| json!({ - "idType": "ID_ISIN", - "idValue": isin, - "marketSecDes": "Equity", - })) - .collect(); + let resp = self.client + .post("https://api.openfigi.com/v3/mapping") + .header("Content-Type", "application/json") + .json(&jobs) + .send() + .await?; - let resp = self.client - .post("https://api.openfigi.com/v3/mapping") - .header("Content-Type", "application/json") - .json(&jobs) - .send() - .await?; + let status = resp.status(); + let headers = resp.headers().clone(); + let body = resp.text().await.unwrap_or_default(); - let status = resp.status(); - println!(" → OpenFIGI batch {}/{}: status {}", chunk_idx + 1, isins.len() / chunk_size + 1, status); + if status.is_client_error() || status.is_server_error() { + if status == 401 { + return Err(anyhow::anyhow!("Invalid OpenFIGI API key: {}", body)); + } else if status == 413 { + return Err(anyhow::anyhow!("Payload too large—reduce chunk size: {}", body)); + } else if status == 429 { + let reset = headers + .get("ratelimit-reset") + .and_then(|v| v.to_str().ok()) + .unwrap_or("10") + .parse::() + .unwrap_or(10); - match status { - StatusCode::OK => { - let results: Vec = resp.json().await?; - let mut chunk_figis = Vec::new(); - for (job, result) in chunk.iter().zip(results) { - if let Some(data) = result["data"].as_array() { - for item in data { - let sec_type = item["securityType"].as_str().unwrap_or(""); - let market_sec = item["marketSector"].as_str().unwrap_or(""); - if market_sec == "Equity" && - (sec_type.contains("Stock") || sec_type.contains("Share") || sec_type.contains("Equity") || - sec_type.contains("Common") || sec_type.contains("Preferred") || sec_type == "ADR" || sec_type == "GDR") { - if let Some(figi) = item["figi"].as_str() { - chunk_figis.push(figi.to_string()); - } - } - } - } else { - println!(" → Warning: No 'data' in response for ISIN {}", job); + println!("Rate limited—backing off {}s", reset); + sleep(Duration::from_secs(reset.max(10))).await; + continue; + } + + return Err(anyhow::anyhow!("OpenFIGI error {}: {}", status, body)); + } + + // JSON aus dem *Body-String* parsen + let results: Vec = serde_json::from_str(&body)?; + for (job, result) in chunk.iter().zip(results) { + if let Some(data) = result["data"].as_array() { + for item in data { + let sec_type = item["securityType"].as_str().unwrap_or(""); + let market_sec = item["marketSector"].as_str().unwrap_or(""); + if market_sec == "Equity" && + (sec_type.contains("Stock") || sec_type.contains("Share") || sec_type.contains("Equity") || + sec_type.contains("Common") || sec_type.contains("Preferred") || sec_type == "ADR" || sec_type == "GDR") { + if let Some(figi) = item["figi"].as_str() { + all_figis.push(figi.to_string()); } } - all_figis.extend(chunk_figis); - success = true; - } - StatusCode::TOO_MANY_REQUESTS => { // 429 - if let Some(reset_header) = resp.headers().get("ratelimit-reset") { - if let Ok(reset_secs) = reset_header.to_str().unwrap_or("10").parse::() { - println!(" → Rate limited (429) — backing off {}s", reset_secs); - sleep(Duration::from_secs(reset_secs.max(10))).await; - } - } else { - sleep(Duration::from_secs(30)).await; // Default backoff - } - retries += 1; - } - StatusCode::UNAUTHORIZED => { // 401 - return Err(anyhow::anyhow!("Invalid OpenFIGI API key — check .env")); - } - StatusCode::PAYLOAD_TOO_LARGE => { // 413 - println!(" → Payload too large (413) — reducing chunk size for next try"); - // Reduce chunk_size dynamically (stub: retry with half size) - sleep(Duration::from_secs(5)).await; - retries += 1; - } - _ if status.is_server_error() => { // 5xx - println!(" → Server error {} — retrying in {}s", status, 3u64.pow(retries as u32)); - sleep(Duration::from_secs(3u64.pow(retries as u32))).await; - retries += 1; - } - _ => { // 4xx client errors (not retryable) - let text = resp.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!("OpenFIGI client error {}: {}", status, text)); } } } - if !success { - println!(" → Failed chunk {} after 3 retries — skipping {} ISINs", chunk_idx + 1, chunk.len()); - // Don't crash — continue with partial results + // Rate limit respect: 6s between requests with key + if self.has_key { + sleep(Duration::from_secs(6)).await; + } else { + sleep(Duration::from_millis(500)).await; // Slower without key } - - // Inter-batch delay (respect limits) - sleep(if self.has_key { Duration::from_secs(3) } else { Duration::from_millis(1000) }).await; // Safer: 20s/min effective } - all_figis.dedup(); - println!(" → Mapped {} unique equity FIGIs from {} ISINs", all_figis.len(), isins.len()); + all_figis.dedup(); // Unique FIGIs per LEI Ok(all_figis) } }