donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

added event enrichment

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2026-01-08 00:35:10 +01:00
parent f9ce5bad99
commit 1720716144
6 changed files with 751 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/corporate/update.rs
View File

@@ -3,6 +3,7 @@ use super::{scraper::*, storage::*, openfigi::*};
use crate::config::Config;
use crate::corporate::update_companies::build_companies_jsonl_streaming_parallel;
use crate::corporate::update_companies_cleanse::{companies_yahoo_cleansed_low_profile, companies_yahoo_cleansed_no_data};
use crate::corporate::update_companies_enrich::enrich_companies_with_events;
use crate::util::directories::DataPaths;
use crate::util::logger;
use crate::scraper::webdriver::ChromeDriverPool;
@@ -87,16 +88,16 @@ pub async fn run_full_update(
return Ok(());
}
logger::log_info("Step 6: Cleansing up companies with missing essential data...").await;
logger::log_info("Step 6: Cleansing companies with missing essential data...").await;
let cleansed_count = companies_yahoo_cleansed_no_data(&paths).await?;
logger::log_info(&format!(" ✓ {} companies found on Yahoo ready for further use in companies_yahoo.jsonl", cleansed_count)).await;
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after companies.jsonl build").await;
logger::log_warn("Shutdown detected after no-data cleansing").await;
return Ok(());
}
logger::log_info("Step 7: Cleansing up companies with too low profile (with abort-safe persistence)...").await;
logger::log_info("Step 7: Cleansing companies with too low profile (with abort-safe persistence)...").await;
let proxy_pool = pool.get_proxy_pool()
.ok_or_else(|| anyhow::anyhow!("ChromeDriverPool must be created with VPN proxy rotation enabled"))?;
@@ -104,11 +105,25 @@ pub async fn run_full_update(
let yahoo_pool = Arc::new(YahooClientPool::new(proxy_pool, config, None).await?);
logger::log_info(&format!("✓ YahooClientPool ready with {} clients", yahoo_pool.num_clients().await)).await;
let cleansed_count = companies_yahoo_cleansed_low_profile(&paths, config, yahoo_pool, shutdown_flag).await?;
let cleansed_count = companies_yahoo_cleansed_low_profile(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
logger::log_info(&format!(" ✓ {} companies with sufficient profile ready for analytics", cleansed_count)).await;
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after low-profile cleansing").await;
return Ok(());
}
logger::log_info("Step 8: Enriching companies with Yahoo Events (with abort-safe persistence)...").await;
let enriched_count = enrich_companies_with_events(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
logger::log_info(&format!(" ✓ {} companies enriched with event data", enriched_count)).await;
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after event enrichment").await;
return Ok(());
}
if !shutdown_flag.load(Ordering::SeqCst) {
logger::log_info("Step 8: Processing events (using index)...").await;
logger::log_info("Step 9: Processing events (using index)...").await;
let _event_index = build_event_index(&paths).await?;
logger::log_info(" ✓ Event index built").await;
} else {