donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

cleaned up update.rs eco and corp

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2026-01-09 19:52:26 +01:00
parent 8dd75f7bdf
commit ba841248f0
11 changed files with 55 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/corporate/update.rs
View File

@@ -1,5 +1,5 @@
// src/corporate/update.rs
use super::{scraper::*, storage::*, openfigi::*};
use super::{scraper::*, openfigi::*};
use crate::config::Config;
use crate::corporate::update_companies::build_companies_jsonl_streaming_parallel;
use crate::corporate::update_companies_cleanse::{companies_yahoo_cleansed_low_profile, companies_yahoo_cleansed_no_data};
@@ -22,7 +22,7 @@ pub async fn run_full_update(
pool: &Arc<ChromeDriverPool>,
shutdown_flag: &Arc<AtomicBool>,
) -> anyhow::Result<()> {
logger::log_info("=== Corporate Update (STREAMING MODE WITH DATA INTEGRITY) ===").await;
logger::log_info("=== Corporate Update ===").await;
let paths = DataPaths::new(".")?;
@@ -42,7 +42,7 @@ pub async fn run_full_update(
logger::log_warn("Shutdown detected after GLEIF download").await;
return Ok(());
}
if !shutdown_flag.load(Ordering::SeqCst) {
logger::log_info("Step 2: Loading OpenFIGI metadata...").await;
load_figi_type_lists().await.ok();
@@ -144,15 +144,7 @@ pub async fn run_full_update(
}
if !shutdown_flag.load(Ordering::SeqCst) {
logger::log_info("Step 11: Processing events (using index)...").await;
let _event_index = build_event_index(&paths).await?;
logger::log_info(" ✓ Event index built").await;
} else {
logger::log_warn("Shutdown detected, skipping event index build").await;
}
if !shutdown_flag.load(Ordering::SeqCst) {
logger::log_info("Step 12: Collecting FX rates...").await;
logger::log_info("Step 11: Collecting FX rates...").await;
let proxy_pool = pool.get_proxy_pool()
.ok_or_else(|| anyhow::anyhow!("ChromeDriverPool must have proxy rotation"))?;
@@ -166,14 +158,14 @@ pub async fn run_full_update(
}
if !shutdown_flag.load(Ordering::SeqCst) {
logger::log_info("Step 13: Collecting exchange information...").await;
logger::log_info("Step 12: Collecting exchange information...").await;
let exchange_count = collect_and_save_exchanges(&paths).await?;
logger::log_info(&format!(" ✓ Collected {} exchanges", exchange_count)).await;
} else {
logger::log_warn("Shutdown detected, skipping exchange collection").await;
}
logger::log_info("✅ Corporate update complete").await;
logger::log_info("=== Corporate update complete === ").await;
Ok(())
}