donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

adding openfigi as identifier for company data

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2025-11-25 22:18:57 +01:00
parent eeae94e041
commit 9db4320c40
1 changed files with 56 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
src/corporate/openfigi.rs
View File

@@ -1,6 +1,6 @@
// src/corporate/openfigi.rs
use super::{types::*};
use reqwest::{Client as HttpClient, StatusCode};
use reqwest::Client as HttpClient;
use reqwest::header::{HeaderMap, HeaderValue};
use serde_json::{json, Value};
use std::collections::{HashMap, HashSet};
@@ -46,96 +46,74 @@ impl OpenFigiClient {
let mut all_figis = Vec::new();
let chunk_size = if self.has_key { 100 } else { 5 };
for (chunk_idx, chunk) in isins.chunks(chunk_size).enumerate() {
let mut retries = 0;
let mut success = false;
for chunk in isins.chunks(chunk_size) {
let jobs: Vec<Value> = chunk.iter()
.map(|isin| json!({
"idType": "ID_ISIN",
"idValue": isin,
"marketSecDes": "Equity", // Pre-filter to equities
}))
.collect();
while retries < 3 && !success {
let jobs: Vec<Value> = chunk.iter()
.map(|isin| json!({
"idType": "ID_ISIN",
"idValue": isin,
"marketSecDes": "Equity",
}))
.collect();
let resp = self.client
.post("https://api.openfigi.com/v3/mapping")
.header("Content-Type", "application/json")
.json(&jobs)
.send()
.await?;
let resp = self.client
.post("https://api.openfigi.com/v3/mapping")
.header("Content-Type", "application/json")
.json(&jobs)
.send()
.await?;
let status = resp.status();
let headers = resp.headers().clone();
let body = resp.text().await.unwrap_or_default();
let status = resp.status();
println!(" → OpenFIGI batch {}/{}: status {}", chunk_idx + 1, isins.len() / chunk_size + 1, status);
if status.is_client_error() || status.is_server_error() {
if status == 401 {
return Err(anyhow::anyhow!("Invalid OpenFIGI API key: {}", body));
} else if status == 413 {
return Err(anyhow::anyhow!("Payload too large—reduce chunk size: {}", body));
} else if status == 429 {
let reset = headers
.get("ratelimit-reset")
.and_then(|v| v.to_str().ok())
.unwrap_or("10")
.parse::<u64>()
.unwrap_or(10);
match status {
StatusCode::OK => {
let results: Vec<Value> = resp.json().await?;
let mut chunk_figis = Vec::new();
for (job, result) in chunk.iter().zip(results) {
if let Some(data) = result["data"].as_array() {
for item in data {
let sec_type = item["securityType"].as_str().unwrap_or("");
let market_sec = item["marketSector"].as_str().unwrap_or("");
if market_sec == "Equity" &&
(sec_type.contains("Stock") || sec_type.contains("Share") || sec_type.contains("Equity") ||
sec_type.contains("Common") || sec_type.contains("Preferred") || sec_type == "ADR" || sec_type == "GDR") {
if let Some(figi) = item["figi"].as_str() {
chunk_figis.push(figi.to_string());
}
}
}
} else {
println!(" → Warning: No 'data' in response for ISIN {}", job);
println!("Rate limited—backing off {}s", reset);
sleep(Duration::from_secs(reset.max(10))).await;
continue;
}
return Err(anyhow::anyhow!("OpenFIGI error {}: {}", status, body));
}
// JSON aus dem *Body-String* parsen
let results: Vec<Value> = serde_json::from_str(&body)?;
for (job, result) in chunk.iter().zip(results) {
if let Some(data) = result["data"].as_array() {
for item in data {
let sec_type = item["securityType"].as_str().unwrap_or("");
let market_sec = item["marketSector"].as_str().unwrap_or("");
if market_sec == "Equity" &&
(sec_type.contains("Stock") || sec_type.contains("Share") || sec_type.contains("Equity") ||
sec_type.contains("Common") || sec_type.contains("Preferred") || sec_type == "ADR" || sec_type == "GDR") {
if let Some(figi) = item["figi"].as_str() {
all_figis.push(figi.to_string());
}
}
all_figis.extend(chunk_figis);
success = true;
}
StatusCode::TOO_MANY_REQUESTS => { // 429
if let Some(reset_header) = resp.headers().get("ratelimit-reset") {
if let Ok(reset_secs) = reset_header.to_str().unwrap_or("10").parse::<u64>() {
println!(" → Rate limited (429) — backing off {}s", reset_secs);
sleep(Duration::from_secs(reset_secs.max(10))).await;
}
} else {
sleep(Duration::from_secs(30)).await; // Default backoff
}
retries += 1;
}
StatusCode::UNAUTHORIZED => { // 401
return Err(anyhow::anyhow!("Invalid OpenFIGI API key — check .env"));
}
StatusCode::PAYLOAD_TOO_LARGE => { // 413
println!(" → Payload too large (413) — reducing chunk size for next try");
// Reduce chunk_size dynamically (stub: retry with half size)
sleep(Duration::from_secs(5)).await;
retries += 1;
}
_ if status.is_server_error() => { // 5xx
println!(" → Server error {} — retrying in {}s", status, 3u64.pow(retries as u32));
sleep(Duration::from_secs(3u64.pow(retries as u32))).await;
retries += 1;
}
_ => { // 4xx client errors (not retryable)
let text = resp.text().await.unwrap_or_default();
return Err(anyhow::anyhow!("OpenFIGI client error {}: {}", status, text));
}
}
}
if !success {
println!(" → Failed chunk {} after 3 retries — skipping {} ISINs", chunk_idx + 1, chunk.len());
// Don't crash — continue with partial results
// Rate limit respect: 6s between requests with key
if self.has_key {
sleep(Duration::from_secs(6)).await;
} else {
sleep(Duration::from_millis(500)).await; // Slower without key
}
// Inter-batch delay (respect limits)
sleep(if self.has_key { Duration::from_secs(3) } else { Duration::from_millis(1000) }).await; // Safer: 20s/min effective
}
all_figis.dedup();
println!(" → Mapped {} unique equity FIGIs from {} ISINs", all_figis.len(), isins.len());
all_figis.dedup(); // Unique FIGIs per LEI
Ok(all_figis)
}
}