donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

added logging

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2025-12-05 21:20:12 +01:00
parent f7083bf9f0
commit 58a498e694
17 changed files with 526 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
src/corporate/update.rs
View File

@@ -1,7 +1,9 @@
// src/corporate/update.rs
use super::{scraper::*, storage::*, helpers::*, types::*, aggregation::*, openfigi::*};
use crate::config::Config;
use crate::scraper::webdriver::ChromeDriverPool;
use crate::util::directories::DataPaths;
use crate::util::logger;
use crate::webdriver::webdriver::ChromeDriverPool;
use chrono::Local;
use std::collections::{HashMap};
@@ -24,50 +26,103 @@ use std::sync::Arc;
/// # Errors
/// Returns an error if any step in the update process fails.
pub async fn run_full_update(config: &Config, pool: &Arc<ChromeDriverPool>) -> anyhow::Result<()> {
println!("=== Starting LEI-based corporate full update ===");
let msg = "=== Starting LEI-based corporate full update ===";
println!("{}", msg);
logger::log_info(msg).await;
// Initialize paths
let paths = DataPaths::new(".")?;
// 1. Load fresh GLEIF ISIN ↔ LEI mapping
logger::log_info("Corporate Update: Loading GLEIF ISIN ↔ LEI mapping...").await;
let lei_to_isins: HashMap<String, Vec<String>> = match load_isin_lei_csv().await {
Ok(map) => map,
Ok(map) => {
let msg = format!("Corporate Update: Loaded GLEIF mapping with {} LEI entries", map.len());
println!("{}", msg);
logger::log_info(&msg).await;
map
}
Err(e) => {
eprintln!("Warning: Could not load GLEIF ISIN↔LEI mapping: {}", e);
let msg = format!("Corporate Update: Warning - Could not load GLEIF ISIN↔LEI mapping: {}", e);
eprintln!("{}", msg);
logger::log_warn(&msg).await;
HashMap::new()
}
};
// 2. Load OpenFIGI mapping value lists (cached)
logger::log_info("Corporate Update: Loading OpenFIGI type lists...").await;
if let Err(e) = load_figi_type_lists().await {
eprintln!("Warning: Could not load OpenFIGI type lists: {}", e);
let msg = format!("Corporate Update: Warning - Could not load OpenFIGI type lists: {}", e);
eprintln!("{}", msg);
logger::log_warn(&msg).await;
}
logger::log_info("Corporate Update: OpenFIGI type lists loaded").await;
// 3. Build FIGI → LEI map
// # Attributes
// * lei: Structuring the companies by legal dependencies [LEI -> Vec<ISIN>]
// * figi: metadata with ISIN as key
logger::log_info("Corporate Update: Building FIGI → LEI map...").await;
let figi_to_lei:HashMap<String, Vec<FigiInfo>> = match build_lei_to_figi_infos(&lei_to_isins).await {
Ok(map) => map,
Ok(map) => {
let msg = format!("Corporate Update: Built FIGI map with {} entries", map.len());
println!("{}", msg);
logger::log_info(&msg).await;
map
}
Err(e) => {
eprintln!("Warning: Could not build FIGI→LEI map: {}", e);
let msg = format!("Corporate Update: Warning - Could not build FIGI→LEI map: {}", e);
eprintln!("{}", msg);
logger::log_warn(&msg).await;
HashMap::new()
}
};
// 4. Load or build companies
let mut companies = load_or_build_all_securities(&figi_to_lei).await?;
println!("Processing {} companies", companies.0.len());
logger::log_info("Corporate Update: Loading/building company securities...").await;
let securities = load_or_build_all_securities(&figi_to_lei).await?;
let msg = format!("Corporate Update: Processing {} companies", securities.0.len());
println!("{}", msg);
logger::log_info(&msg).await;
// HashMap<Name, Vec<(ISIN, Ticker)>>
let companies: HashMap<String, Vec<(String, String)>> = securities.0
.iter()
.fold(HashMap::new(), |mut acc, security| {
let isin: Vec<String> = security.1.securities.values()
.flat_map(|figi_info| figi_info.iter().map(|x| x.isin.clone()))
.collect();
let ticker: Vec<String> = security.1.securities.values()
.flat_map(|figi_info| figi_info.iter().map(|x| x.ticker.clone()))
.collect();
acc.entry(security.1.name.clone())
.or_insert_with(Vec::new)
.push((isin.join(", "), ticker.join(", ")));
acc
});
logger::log_info(&format!("Corporate Update: Saving {} companies to JSONL", companies.len())).await;
save_companies_to_jsonl(&paths, &companies).await.expect("Failed to save companies List.");
logger::log_info("Corporate Update: Companies saved successfully").await;
// 5. Load existing earnings events (for change detection)
let today = Local::now().format("%Y-%m-%d").to_string();
let mut existing_events = match load_existing_events().await {
Ok(events) => events,
logger::log_info("Corporate Update: Loading existing events...").await;
let existing_events = match load_existing_events(&paths).await {
Ok(events) => {
let msg = format!("Corporate Update: Loaded {} existing events", events.len());
println!("{}", msg);
logger::log_info(&msg).await;
events
}
Err(e) => {
eprintln!("Warning: Could not load existing events: {}", e);
let msg = format!("Corporate Update: Warning - Could not load existing events: {}", e);
eprintln!("{}", msg);
logger::log_warn(&msg).await;
HashMap::new()
}
};
// 5. Use the provided pool (no need to create a new one)
let pool_size = pool.get_number_of_instances(); // Use the size from the shared pool
logger::log_info(&format!("Corporate Update: Using pool size: {}", pool_size)).await;
// Process companies in parallel using the shared pool
/*let results: Vec<_> = stream::iter(companies.into_iter())
@@ -88,10 +143,14 @@ pub async fn run_full_update(config: &Config, pool: &Arc<ChromeDriverPool>) -> a
}
}*/
save_optimized_events(existing_events).await?;
logger::log_info(&format!("Corporate Update: Saving {} events to optimized storage", existing_events.len())).await;
save_optimized_events(&paths, existing_events).await?;
logger::log_info("Corporate Update: Events saved successfully").await;
//save_changes(&all_changes).await?;
//println!("Corporate update complete — {} changes detected", all_changes.len());
let msg = "✓ Corporate update complete";
println!("{}", msg);
logger::log_info(msg).await;
Ok(())
}