donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

added options chart enrichment

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2026-01-08 11:35:25 +01:00
parent 1720716144
commit ea128f6187
4 changed files with 820 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/corporate/update.rs
View File

@@ -4,6 +4,7 @@ use crate::config::Config;
use crate::corporate::update_companies::build_companies_jsonl_streaming_parallel;
use crate::corporate::update_companies_cleanse::{companies_yahoo_cleansed_low_profile, companies_yahoo_cleansed_no_data};
use crate::corporate::update_companies_enrich::enrich_companies_with_events;
use crate::corporate::update_companies_enrich_options_chart::{enrich_companies_with_options, enrich_companies_with_chart};
use crate::util::directories::DataPaths;
use crate::util::logger;
use crate::scraper::webdriver::ChromeDriverPool;
@@ -122,8 +123,26 @@ pub async fn run_full_update(
return Ok(());
}
logger::log_info("Step 9: Enriching companies with Yahoo Options (with abort-safe persistence)...").await;
let options_count = enrich_companies_with_options(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
logger::log_info(&format!(" ✓ {} companies enriched with options data", options_count)).await;
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after options enrichment").await;
return Ok(());
}
logger::log_info("Step 10: Enriching companies with Yahoo Chart (with abort-safe persistence)...").await;
let chart_count = enrich_companies_with_chart(&paths, config, yahoo_pool.clone(), shutdown_flag).await?;
logger::log_info(&format!(" ✓ {} companies enriched with chart data", chart_count)).await;
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after chart enrichment").await;
return Ok(());
}
if !shutdown_flag.load(Ordering::SeqCst) {
logger::log_info("Step 9: Processing events (using index)...").await;
logger::log_info("Step 11: Processing events (using index)...").await;
let _event_index = build_event_index(&paths).await?;
logger::log_info(" ✓ Event index built").await;
} else {