working api calls
This commit is contained in:
@@ -3,184 +3,232 @@ use super::{scraper::*, storage::*, helpers::*, types::*, aggregation::*, openfi
|
||||
use crate::config::Config;
|
||||
|
||||
use chrono::Local;
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
pub async fn run_full_update(client: &fantoccini::Client, config: &Config) -> anyhow::Result<()> {
|
||||
println!("Starting LEI-based corporate update");
|
||||
/// Hauptfunktion: Vollständiger Update-Durchlauf für alle Unternehmen (LEI-basiert)
|
||||
///
|
||||
/// Diese Funktion koordiniert den gesamten Update-Prozess:
|
||||
/// - Lädt GLEIF-Mappings
|
||||
/// - Baut FIGI-LEI-Map
|
||||
/// - Lädt bestehende Events
|
||||
/// - Verarbeitet jede Company: Ergänzt ISINs (abgeleitet aus FIGI), entdeckt Exchanges via FIGI,
|
||||
/// holt Prices & Earnings, aggregiert Daten
|
||||
/// - Speichert optimierte Events
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `config` - Konfiguration mit Startdaten etc.
|
||||
///
|
||||
/// # Returns
|
||||
/// `Ok(())` bei Erfolg, sonst `anyhow::Error` mit Kontext.
|
||||
///
|
||||
/// # Errors
|
||||
/// - Mapping-Laden fehlschlägt (Warning, fährt mit leer fort)
|
||||
/// - Company-Laden/Bauen fehlschlägt
|
||||
/// - Directory Creation oder Speichern fehlschlägt
|
||||
/// - Discovery/Fetch/Aggregation pro Company fehlschlägt (fortgesetzt bei Fehlern, mit Log)
|
||||
pub async fn run_full_update(config: &Config) -> anyhow::Result<()> {
|
||||
println!("=== Starting LEI-based corporate full update ===");
|
||||
|
||||
// 1. Download fresh GLEIF ISIN↔LEI mapping on every run
|
||||
// 1. Frisches GLEIF ISIN ↔ LEI Mapping laden (jeder Lauf neu)
|
||||
let lei_to_isins: HashMap<String, Vec<String>> = match load_isin_lei_csv().await {
|
||||
Ok(map) => map,
|
||||
Err(e) => {
|
||||
println!("Warning: Failed to load ISIN↔LEI mapping: {}", e);
|
||||
eprintln!("Warning: Could not load GLEIF ISIN↔LEI mapping: {}", e);
|
||||
HashMap::new()
|
||||
}
|
||||
};
|
||||
|
||||
let figi_to_lei: HashMap<String, String> = match build_figi_to_lei_map(&lei_to_isins).await {
|
||||
// 2. FIGI → LEI Map (optional, nur mit API-Key sinnvoll)
|
||||
let figi_to_lei= match build_lei_to_figi_infos(&lei_to_isins).await {
|
||||
Ok(map) => map,
|
||||
Err(e) => {
|
||||
println!("Warning: Failed to build FIGI→LEI map: {}", e);
|
||||
eprintln!("Warning: Could not build FIGI→LEI map: {}", e);
|
||||
HashMap::new()
|
||||
}
|
||||
};
|
||||
|
||||
let today = chrono::Local::now().format("%Y-%m-%d").to_string();
|
||||
let mut existing_events = load_existing_events().await?;
|
||||
|
||||
let mut companies: Vec<CompanyMetadata> = match load_or_build_companies_figi(&lei_to_isins, &figi_to_lei).await {
|
||||
Ok(comps) => comps,
|
||||
// 3. Bestehende Earnings-Events laden (für Change-Detection)
|
||||
let today = Local::now().format("%Y-%m-%d").to_string();
|
||||
let mut existing_events = match load_existing_events().await {
|
||||
Ok(events) => events,
|
||||
Err(e) => {
|
||||
println!("Error loading/building company metadata: {}", e);
|
||||
return Err(e);
|
||||
eprintln!("Warning: Could not load existing events: {}", e);
|
||||
HashMap::new()
|
||||
}
|
||||
}; // Vec<CompanyMetadata> with lei, isins, tickers
|
||||
};
|
||||
|
||||
for mut company in companies {
|
||||
println!("\nProcessing company: {} (LEI: {})", company.name, company.lei);
|
||||
// 4. Unternehmen laden / neu aufbauen (LEI + FIGI-Infos)
|
||||
let mut companies: Vec<CompanyMetadata> = load_or_build_companies_lei(&lei_to_isins).await?;
|
||||
|
||||
// === Enrich with ALL ISINs known to GLEIF (includes ADRs, GDRs, etc.) ===
|
||||
if let Some(all_isins) = lei_to_isins.get(&company.lei) {
|
||||
let mut seen = company.isins.iter().cloned().collect::<std::collections::HashSet<_>>();
|
||||
for isin in all_isins {
|
||||
if !seen.contains(isin) {
|
||||
company.isins.push(isin.clone());
|
||||
seen.insert(isin.clone());
|
||||
// 4.1 LEIs anreichern (falls missing, über bekannte ISINs aus FIGI suchen)
|
||||
//enrich_companies_with_leis(&mut companies, &lei_to_isins).await?;
|
||||
|
||||
// 5. Haupt-Loop: Jedes Unternehmen verarbeiten
|
||||
for company in companies.iter_mut() {
|
||||
let lei = &company.lei;
|
||||
let figi_infos = company.figi.as_ref().map_or(&[][..], |v| &v[..]);
|
||||
let name = figi_infos.first().map(|f| f.name.as_str()).unwrap_or("Unknown");
|
||||
println!("\nProcessing company: {} (LEI: {})", name, lei);
|
||||
|
||||
// --- 5.1 Alle bekannten ISINs aus GLEIF ergänzen ---
|
||||
let mut all_isins = lei_to_isins.get(lei).cloned().unwrap_or_default();
|
||||
let figi_isins: Vec<String> = figi_infos.iter().map(|f| f.isin.clone()).collect::<HashSet<_>>().into_iter().collect();
|
||||
all_isins.extend(figi_isins);
|
||||
all_isins.sort();
|
||||
all_isins.dedup(); // Unique ISINs
|
||||
|
||||
// --- 5.2 Verzeichnisstruktur anlegen & Metadaten speichern ---
|
||||
ensure_company_dirs(lei).await?;
|
||||
save_company_metadata(company).await?;
|
||||
|
||||
// --- 5.3 FIGI-Infos ermitteln (falls noch nicht vorhanden) ---
|
||||
let figi_infos = company.figi.get_or_insert_with(Vec::new);
|
||||
if figi_infos.is_empty() {
|
||||
println!(" No FIGI data yet → discovering exchanges via first known ISIN");
|
||||
let first_isin = all_isins.first().cloned().unwrap_or_default();
|
||||
if !first_isin.is_empty() {
|
||||
match discover_available_exchanges(&first_isin, "").await {
|
||||
Ok(discovered) => {
|
||||
figi_infos.extend(discovered);
|
||||
println!(" Discovered {} exchange(s) for first ISIN", figi_infos.len());
|
||||
}
|
||||
Err(e) => eprintln!(" Discovery failed for first ISIN: {}", e),
|
||||
}
|
||||
}
|
||||
} else {
|
||||
println!(" {} exchange(s) already known", figi_infos.len());
|
||||
}
|
||||
|
||||
// Ensure company directory exists (now uses LEI)
|
||||
//let figi_dir = format!("data/companies_by_figi/{}/", company.primary_figi);
|
||||
ensure_company_dirs(&company.lei).await?;
|
||||
save_company_metadata(&company).await?;
|
||||
|
||||
// === STEP 1: Discover additional exchanges using each known ISIN ===
|
||||
let mut all_tickers = company.tickers.clone();
|
||||
|
||||
if let Some(primary_ticker) = company.tickers.iter().find(|t| t.primary) {
|
||||
println!(" Discovering additional exchanges across {} ISIN(s)...", company.isins.len());
|
||||
|
||||
for isin in &company.isins {
|
||||
println!(" → Checking ISIN: {}", isin);
|
||||
match discover_available_exchanges(isin, &primary_ticker.ticker).await {
|
||||
Ok(discovered) => {
|
||||
if discovered.is_empty() {
|
||||
println!(" – No new exchanges found for {}", isin);
|
||||
} else {
|
||||
for disc in discovered {
|
||||
if !all_tickers.iter().any(|t| t.ticker == disc.ticker && t.exchange_mic == disc.exchange_mic) {
|
||||
println!(" New equity listing → {} ({}) via ISIN {}",
|
||||
disc.ticker, disc.exchange_mic, isin);
|
||||
all_tickers.push(disc);
|
||||
}
|
||||
}
|
||||
// --- 5.4 Weitere Exchanges über alle ISINs suchen ---
|
||||
let mut new_discovered = 0;
|
||||
for isin in &all_isins {
|
||||
if figi_infos.iter().any(|f| f.isin == *isin) {
|
||||
continue; // Schon bekannt
|
||||
}
|
||||
println!(" Discovering additional exchanges for ISIN {}", isin);
|
||||
match discover_available_exchanges(isin, "").await {
|
||||
Ok(mut found) => {
|
||||
for info in found.drain(..) {
|
||||
if !figi_infos.iter().any(|f| f.ticker == info.ticker && f.mic_code == info.mic_code) {
|
||||
figi_infos.push(info);
|
||||
new_discovered += 1;
|
||||
}
|
||||
}
|
||||
Err(e) => println!(" Discovery failed for {}: {}", isin, e),
|
||||
}
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(300)).await;
|
||||
Err(e) => eprintln!(" Discovery failed for {}: {}", isin, e),
|
||||
}
|
||||
}
|
||||
if new_discovered > 0 {
|
||||
println!(" +{} new exchange(s) discovered and added", new_discovered);
|
||||
}
|
||||
|
||||
// --- 5.5 AvailableExchange-Einträge anlegen (für Preis-Downloads) ---
|
||||
for figi in figi_infos.iter() {
|
||||
if let Err(e) = add_discovered_exchange(&figi.isin, figi).await {
|
||||
eprintln!(" Failed to record exchange {}: {}", figi.ticker, e);
|
||||
}
|
||||
}
|
||||
|
||||
// Save updated metadata if we found new listings
|
||||
if all_tickers.len() > company.tickers.len() {
|
||||
company.tickers = all_tickers.clone();
|
||||
save_company_metadata(&company).await?;
|
||||
println!(" Updated metadata: {} total tickers", all_tickers.len());
|
||||
}
|
||||
|
||||
// === STEP 2: Fetch data from ALL available tickers ===
|
||||
for ticker_info in &all_tickers {
|
||||
let ticker = &ticker_info.ticker;
|
||||
println!(" → Fetching: {} ({})", ticker, ticker_info.exchange_mic);
|
||||
|
||||
// --- 5.6 Preisdaten von allen Exchanges holen ---
|
||||
println!(" Fetching price data from {} exchange(s)...", figi_infos.len());
|
||||
let primary_isin = figi_infos.first().map(|f| f.isin.clone()).unwrap_or_default();
|
||||
for figi in figi_infos.iter() {
|
||||
let ticker = &figi.ticker;
|
||||
let mic = &figi.mic_code;
|
||||
let is_primary = figi.isin == primary_isin;
|
||||
let mut daily_success = false;
|
||||
let mut intraday_success = false;
|
||||
|
||||
// Earnings: only fetch from primary ticker to avoid duplicates
|
||||
if ticker_info.primary {
|
||||
if let Ok(new_events) = fetch_earnings_history(client, ticker).await {
|
||||
let result = process_batch(&new_events, &mut existing_events, &today);
|
||||
save_changes(&result.changes).await?;
|
||||
println!(" Earnings events: {}", new_events.len());
|
||||
if is_primary {
|
||||
match fetch_earnings_history(client, ticker).await {
|
||||
Ok(new_events) => {
|
||||
let result = process_batch(&new_events, &mut existing_events, &today);
|
||||
save_changes(&result.changes).await?;
|
||||
println!(" Earnings events: {}", new_events.len());
|
||||
}
|
||||
Err(e) => eprintln!(" Failed to fetch earnings for {}: {}", ticker, e),
|
||||
}
|
||||
}
|
||||
|
||||
// Daily prices
|
||||
if let Ok(prices) = fetch_daily_price_history(ticker, &config.corporate_start_date, &today).await {
|
||||
if !prices.is_empty() {
|
||||
save_prices_by_source(&company.lei, ticker, "daily", prices).await?;
|
||||
daily_success = true;
|
||||
match fetch_daily_price_history(ticker, &config.corporate_start_date, &today).await {
|
||||
Ok(prices) => {
|
||||
if !prices.is_empty() {
|
||||
save_prices_by_source(lei, ticker, "daily", prices).await?;
|
||||
daily_success = true;
|
||||
}
|
||||
}
|
||||
Err(e) => eprintln!(" Failed to fetch daily prices for {}: {}", ticker, e),
|
||||
}
|
||||
|
||||
// 5-minute intraday (last 60 days)
|
||||
let sixty_days_ago = (chrono::Local::now() - chrono::Duration::days(60))
|
||||
let sixty_days_ago = (Local::now() - chrono::Duration::days(60))
|
||||
.format("%Y-%m-%d")
|
||||
.to_string();
|
||||
|
||||
if let Ok(prices) = fetch_price_history_5min(ticker, &sixty_days_ago, &today).await {
|
||||
if !prices.is_empty() {
|
||||
save_prices_by_source(&company.lei, ticker, "5min", prices).await?;
|
||||
intraday_success = true;
|
||||
match fetch_price_history_5min(ticker, &sixty_days_ago, &today).await {
|
||||
Ok(prices) => {
|
||||
if !prices.is_empty() {
|
||||
save_prices_by_source(lei, ticker, "5min", prices).await?;
|
||||
intraday_success = true;
|
||||
}
|
||||
}
|
||||
Err(e) => eprintln!(" Failed to fetch 5min prices for {}: {}", ticker, e),
|
||||
}
|
||||
|
||||
// Update available_exchanges.json (now under LEI folder)
|
||||
update_available_exchange(
|
||||
&company.lei,
|
||||
ticker,
|
||||
&ticker_info.exchange_mic,
|
||||
daily_success,
|
||||
intraday_success,
|
||||
).await?;
|
||||
update_available_exchange(&figi.isin, ticker, mic, daily_success, intraday_success).await?;
|
||||
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(800)).await;
|
||||
}
|
||||
|
||||
// === STEP 3: Aggregate all sources into unified USD prices ===
|
||||
println!(" Aggregating multi-source price data (FX-adjusted)...");
|
||||
if let Err(e) = aggregate_best_price_data(&company.lei).await {
|
||||
println!(" Aggregation failed: {}", e);
|
||||
// --- 5.7 Aggregation aller Quellen → einheitliche USD-Preise ---
|
||||
println!(" Aggregating price data across all sources (FX-adjusted to USD)");
|
||||
if let Err(e) = aggregate_best_price_data(lei).await {
|
||||
eprintln!(" Aggregation failed: {}", e);
|
||||
} else {
|
||||
println!(" Aggregation complete");
|
||||
println!(" Aggregation completed successfully");
|
||||
}
|
||||
|
||||
// Metadaten erneut speichern (falls FIGIs hinzugefügt wurden)
|
||||
save_company_metadata(company).await?;
|
||||
}
|
||||
|
||||
// Final save of optimized earnings events
|
||||
// 6. Optimierte Earnings-Events final speichern
|
||||
save_optimized_events(existing_events).await?;
|
||||
println!("\nCorporate update complete (LEI-based)");
|
||||
println!("\n=== Corporate full update completed successfully ===");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn enrich_companies_with_leis(
|
||||
/// Companies mit LEIs anreichern
|
||||
async fn _enrich_companies_with_leis(
|
||||
companies: &mut Vec<CompanyMetadata>,
|
||||
lei_to_isins: &HashMap<String, Vec<String>>,
|
||||
) {
|
||||
) -> anyhow::Result<()> {
|
||||
for company in companies.iter_mut() {
|
||||
if company.lei.is_empty() {
|
||||
// Try to find LEI by any known ISIN
|
||||
for isin in &company.isins {
|
||||
for (lei, isins) in lei_to_isins {
|
||||
if isins.contains(isin) {
|
||||
company.lei = lei.clone();
|
||||
println!("Found real LEI {} for {}", lei, company.name);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !company.lei.is_empty() { break; }
|
||||
}
|
||||
if !company.lei.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Fallback: generate fake LEI if still missing
|
||||
if company.lei.is_empty() {
|
||||
company.lei = format!("FAKE{:019}", rand::random::<u64>());
|
||||
println!("No real LEI found → using fake for {}", company.name);
|
||||
|
||||
let figi_infos = company.figi.as_ref().map_or(&[][..], |v| &v[..]);
|
||||
let isins: Vec<String> = figi_infos.iter().map(|f| f.isin.clone()).collect::<HashSet<_>>().into_iter().collect();
|
||||
|
||||
// Try to find LEI by any known ISIN
|
||||
for isin in &isins {
|
||||
for (lei, isins) in lei_to_isins.iter() {
|
||||
if isins.contains(isin) {
|
||||
company.lei = lei.clone();
|
||||
let name = figi_infos.first().map(|f| f.name.as_str()).unwrap_or("Unknown");
|
||||
println!("Found real LEI {} for {}", lei, name);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !company.lei.is_empty() { break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct ProcessResult {
|
||||
pub changes: Vec<CompanyEventChange>,
|
||||
|
||||
Reference in New Issue
Block a user