added helper functions to reduce bloat
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
// src/corporate/update.rs
|
||||
use super::{scraper::*, openfigi::*};
|
||||
use crate::config::Config;
|
||||
use crate::check_shutdown;
|
||||
use crate::corporate::update_companies::build_companies_jsonl_streaming_parallel;
|
||||
use crate::corporate::update_companies_cleanse::{companies_yahoo_cleansed_low_profile, companies_yahoo_cleansed_no_data};
|
||||
use crate::corporate::update_companies_enrich::enrich_companies_with_events;
|
||||
@@ -26,6 +27,8 @@ pub async fn run_full_update(
|
||||
|
||||
let paths = DataPaths::new(".")?;
|
||||
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
logger::log_info("Step 1: Downloading GLEIF CSV...").await;
|
||||
let gleif_csv_path = match download_isin_lei_csv().await? {
|
||||
Some(p) => {
|
||||
@@ -38,70 +41,49 @@ pub async fn run_full_update(
|
||||
}
|
||||
};
|
||||
|
||||
if shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_warn("Shutdown detected after GLEIF download").await;
|
||||
return Ok(());
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
logger::log_info("Step 2: Loading OpenFIGI metadata...").await;
|
||||
load_figi_type_lists().await.ok();
|
||||
logger::log_info(" ✓ OpenFIGI metadata loaded").await;
|
||||
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
logger::log_info("Step 3: Checking LEI-FIGI mapping status...").await;
|
||||
let all_mapped = ensure_all_leis_mapped(&gleif_csv_path, None).await?;
|
||||
|
||||
if !all_mapped {
|
||||
logger::log_warn(" ⚠ Some LEIs failed to map - continuing with partial data").await;
|
||||
} else {
|
||||
logger::log_info(" ✓ All LEIs successfully mapped").await;
|
||||
}
|
||||
|
||||
if !shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_info("Step 2: Loading OpenFIGI metadata...").await;
|
||||
load_figi_type_lists().await.ok();
|
||||
logger::log_info(" ✓ OpenFIGI metadata loaded").await;
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
logger::log_info("Step 4: Building securities map (streaming)...").await;
|
||||
let date_dir = find_most_recent_figi_date_dir(&paths).await?;
|
||||
|
||||
if let Some(date_dir) = date_dir {
|
||||
logger::log_info(&format!(" Using FIGI data from: {:?}", date_dir)).await;
|
||||
load_or_build_all_securities(&date_dir).await?;
|
||||
logger::log_info(" ✓ Securities map updated").await;
|
||||
} else {
|
||||
logger::log_warn("Shutdown detected, skipping event index build").await;
|
||||
logger::log_warn(" ✗ No FIGI data directory found").await;
|
||||
}
|
||||
|
||||
if !shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_info("Step 2: Loading OpenFIGI metadata...").await;
|
||||
load_figi_type_lists().await.ok();
|
||||
logger::log_info(" ✓ OpenFIGI metadata loaded").await;
|
||||
} else {
|
||||
logger::log_warn("Shutdown detected, skipping event index build").await;
|
||||
}
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
if !shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_info("Step 3: Checking LEI-FIGI mapping status...").await;
|
||||
let all_mapped = ensure_all_leis_mapped(&gleif_csv_path, None).await?;
|
||||
|
||||
if !all_mapped {
|
||||
logger::log_warn(" ⚠ Some LEIs failed to map - continuing with partial data").await;
|
||||
} else {
|
||||
logger::log_info(" ✓ All LEIs successfully mapped").await;
|
||||
}
|
||||
} else {
|
||||
logger::log_warn("Shutdown detected, skipping event index build").await;
|
||||
}
|
||||
logger::log_info("Step 5: Building companies.jsonl with parallel processing and validation...").await;
|
||||
let count = build_companies_jsonl_streaming_parallel(&paths, pool, shutdown_flag, config, &None).await?;
|
||||
logger::log_info(&format!(" ✓ Saved {} companies", count)).await;
|
||||
|
||||
if !shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_info("Step 4: Building securities map (streaming)...").await;
|
||||
let date_dir = find_most_recent_figi_date_dir(&paths).await?;
|
||||
|
||||
if let Some(date_dir) = date_dir {
|
||||
logger::log_info(&format!(" Using FIGI data from: {:?}", date_dir)).await;
|
||||
load_or_build_all_securities(&date_dir).await?;
|
||||
logger::log_info(" ✓ Securities map updated").await;
|
||||
} else {
|
||||
logger::log_warn(" ✗ No FIGI data directory found").await;
|
||||
}
|
||||
} else {
|
||||
logger::log_warn("Shutdown detected, skipping event index build").await;
|
||||
}
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
if !shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_info("Step 5: Building companies.jsonl with parallel processing and validation...").await;
|
||||
let count = build_companies_jsonl_streaming_parallel(&paths, pool, shutdown_flag, config, &None).await?;
|
||||
logger::log_info(&format!(" ✓ Saved {} companies", count)).await;
|
||||
} else {
|
||||
logger::log_warn("Shutdown detected, skipping event index build").await;
|
||||
}
|
||||
logger::log_info("Step 6: Cleansing companies with missing essential data...").await;
|
||||
let cleansed_count = companies_yahoo_cleansed_no_data(&paths).await?;
|
||||
logger::log_info(&format!(" ✓ {} companies found on Yahoo ready for further use in companies_yahoo.jsonl", cleansed_count)).await;
|
||||
|
||||
if !shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_info("Step 6: Cleansing companies with missing essential data...").await;
|
||||
let cleansed_count = companies_yahoo_cleansed_no_data(&paths).await?;
|
||||
logger::log_info(&format!(" ✓ {} companies found on Yahoo ready for further use in companies_yahoo.jsonl", cleansed_count)).await;
|
||||
} else {
|
||||
logger::log_warn("Shutdown detected, skipping event index build").await;
|
||||
}
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
let proxy_pool = pool.get_proxy_pool()
|
||||
.ok_or_else(|| anyhow::anyhow!("ChromeDriverPool must be created with VPN proxy rotation enabled"))?;
|
||||
@@ -110,60 +92,41 @@ pub async fn run_full_update(
|
||||
let yahoo_pool = Arc::new(YahooClientPool::new(proxy_pool, config, None).await?);
|
||||
logger::log_info(&format!("✓ YahooClientPool ready with {} clients", yahoo_pool.num_clients().await)).await;
|
||||
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
if !shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_info("Step 7: Cleansing companies with too low profile (with abort-safe persistence)...").await;
|
||||
let cleansed_count = companies_yahoo_cleansed_low_profile(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
|
||||
logger::log_info(&format!(" ✓ {} companies with sufficient profile ready for analytics", cleansed_count)).await;
|
||||
} else {
|
||||
logger::log_warn("Shutdown detected, skipping event index build").await;
|
||||
}
|
||||
logger::log_info("Step 7: Cleansing companies with too low profile (with abort-safe persistence)...").await;
|
||||
let cleansed_count = companies_yahoo_cleansed_low_profile(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
|
||||
logger::log_info(&format!(" ✓ {} companies with sufficient profile ready for analytics", cleansed_count)).await;
|
||||
|
||||
if !shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_info("Step 8: Enriching companies with Yahoo Events (with abort-safe persistence)...").await;
|
||||
let enriched_count = enrich_companies_with_events(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
|
||||
logger::log_info(&format!(" ✓ {} companies enriched with event data", enriched_count)).await;
|
||||
} else {
|
||||
logger::log_warn("Shutdown detected, skipping event index build").await;
|
||||
}
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
logger::log_info("Step 8: Enriching companies with Yahoo Events (with abort-safe persistence)...").await;
|
||||
let enriched_count = enrich_companies_with_events(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
|
||||
logger::log_info(&format!(" ✓ {} companies enriched with event data", enriched_count)).await;
|
||||
|
||||
if !shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_info("Step 9: Enriching companies with Yahoo Options (with abort-safe persistence)...").await;
|
||||
let options_count = enrich_companies_with_options(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
|
||||
logger::log_info(&format!(" ✓ {} companies enriched with options data", options_count)).await;
|
||||
} else {
|
||||
logger::log_warn("Shutdown detected, skipping event index build").await;
|
||||
}
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
if !shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_info("Step 10: Enriching companies with Yahoo Chart (with abort-safe persistence)...").await;
|
||||
let chart_count = enrich_companies_with_chart(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
|
||||
logger::log_info(&format!(" ✓ {} companies enriched with chart data", chart_count)).await;
|
||||
} else {
|
||||
logger::log_warn("Shutdown detected, skipping event index build").await;
|
||||
}
|
||||
logger::log_info("Step 9: Enriching companies with Yahoo Options (with abort-safe persistence)...").await;
|
||||
let options_count = enrich_companies_with_options(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
|
||||
logger::log_info(&format!(" ✓ {} companies enriched with options data", options_count)).await;
|
||||
|
||||
if !shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_info("Step 11: Collecting FX rates...").await;
|
||||
|
||||
let proxy_pool = pool.get_proxy_pool()
|
||||
.ok_or_else(|| anyhow::anyhow!("ChromeDriverPool must have proxy rotation"))?;
|
||||
|
||||
let yahoo_pool = Arc::new(YahooClientPool::new(proxy_pool, config, None).await?);
|
||||
|
||||
let fx_count = collect_fx_rates(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
|
||||
logger::log_info(&format!(" ✓ Collected {} FX rates", fx_count)).await;
|
||||
} else {
|
||||
logger::log_warn("Shutdown detected, skipping FX rates collection").await;
|
||||
}
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
if !shutdown_flag.load(Ordering::SeqCst) {
|
||||
logger::log_info("Step 12: Collecting exchange information...").await;
|
||||
let exchange_count = collect_and_save_exchanges(&paths).await?;
|
||||
logger::log_info(&format!(" ✓ Collected {} exchanges", exchange_count)).await;
|
||||
} else {
|
||||
logger::log_warn("Shutdown detected, skipping exchange collection").await;
|
||||
}
|
||||
logger::log_info("Step 10: Enriching companies with Yahoo Chart (with abort-safe persistence)...").await;
|
||||
let chart_count = enrich_companies_with_chart(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
|
||||
logger::log_info(&format!(" ✓ {} companies enriched with chart data", chart_count)).await;
|
||||
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
logger::log_info("Step 11: Collecting FX rates...").await;
|
||||
let fx_count = collect_fx_rates(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
|
||||
logger::log_info(&format!(" ✓ Collected {} FX rates", fx_count)).await;
|
||||
|
||||
check_shutdown!(shutdown_flag);
|
||||
|
||||
logger::log_info("Step 12: Collecting exchange information...").await;
|
||||
let exchange_count = collect_and_save_exchanges(&paths).await?;
|
||||
logger::log_info(&format!(" ✓ Collected {} exchanges", exchange_count)).await;
|
||||
|
||||
logger::log_info("=== Corporate update complete === ").await;
|
||||
Ok(())
|
||||
|
||||
Reference in New Issue
Block a user