donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

added helper functions to reduce bloat

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2026-01-09 21:24:18 +01:00
parent ba841248f0
commit c6d301d434
14 changed files with 410 additions and 832 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/economic/update.rs
View File

@@ -1,8 +1,9 @@
// src/economic/update.rs
use super::{scraper::*, storage::*, helpers::*, types::*};
use crate::check_shutdown;
use crate::{config::Config, scraper::webdriver::{ScrapeTask, ChromeDriverPool}, util::directories::DataPaths, util::logger};
use chrono::{Local};
use std::sync::{Arc, atomic::{AtomicBool, Ordering}};
use std::sync::{Arc, atomic::{AtomicBool}};
use std::collections::HashMap;
/// Runs the full update for economic data using streaming to minimize memory usage
@@ -16,6 +17,8 @@ pub async fn run_full_update(config: &Config, pool: &Arc<ChromeDriverPool>, shut
logger::log_info("=== Economic Update ===").await;
check_shutdown!(shutdown_flag);
// Step 1: Build lightweight index instead of loading all events
logger::log_info("Step 1: Building event index...").await;
let chunks = scan_existing_chunks(&paths).await?;
@@ -23,10 +26,7 @@ pub async fn run_full_update(config: &Config, pool: &Arc<ChromeDriverPool>, shut
logger::log_info(&format!(" Economic Update: Indexed {} events from {} chunks",
event_index.len(), chunks.len())).await;
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after GLEIF download").await;
return Ok(());
}
check_shutdown!(shutdown_flag);
// Step 2: Determine start date
let start_date = if event_index.is_empty() {
@@ -54,20 +54,14 @@ pub async fn run_full_update(config: &Config, pool: &Arc<ChromeDriverPool>, shut
}
};
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after GLEIF download").await;
return Ok(());
}
check_shutdown!(shutdown_flag);
// Step 3: Scrape new events in batches
logger::log_info(&format!("Step 3: Scraping events from {} → {}", start_date, end_date)).await;
let new_events = scrape_all_economic_events(&start_date, &end_date, pool).await?;
logger::log_info(&format!(" Scraped {} new events", new_events.len())).await;
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after GLEIF download").await;
return Ok(());
}
check_shutdown!(shutdown_flag);
// Step 4: Process events in streaming fashion
logger::log_info(&format!("Step 4: Detecting changes")).await;
@@ -79,10 +73,7 @@ pub async fn run_full_update(config: &Config, pool: &Arc<ChromeDriverPool>, shut
logger::log_info(" Changes saved successfully").await;
}
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after GLEIF download").await;
return Ok(());
}
check_shutdown!(shutdown_flag);
// Step 5: Save consolidated events
logger::log_info(&format!("Step 5: Saving {} total events to chunks", updated_events.len())).await;