donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

added logging

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2025-12-05 21:20:12 +01:00
parent f7083bf9f0
commit 58a498e694
17 changed files with 526 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/economic/storage.rs
View File

@@ -1,12 +1,14 @@
// src/economic/storage.rs
use super::types::*;
use super::helpers::*;
use crate::util::directories::DataPaths;
use crate::util::logger;
use tokio::fs;
use chrono::{NaiveDate, Datelike};
use std::collections::HashMap;
pub async fn scan_existing_chunks() -> anyhow::Result<Vec<ChunkInfo>> {
let dir = std::path::Path::new("data/economic/events");
pub async fn scan_existing_chunks(paths: &DataPaths) -> anyhow::Result<Vec<ChunkInfo>> {
let dir = paths.economic_events_dir();
let mut chunks = Vec::new();
if dir.exists() {
@@ -29,6 +31,7 @@ pub async fn scan_existing_chunks() -> anyhow::Result<Vec<ChunkInfo>> {
}
}
chunks.sort_by_key(|c| c.start_date.clone());
logger::log_info(&format!("Economic Storage: Scanned {} event chunks", chunks.len())).await;
Ok(chunks)
}
@@ -41,25 +44,28 @@ pub async fn load_existing_events(chunks: &[ChunkInfo]) -> anyhow::Result<HashMa
map.insert(event_key(&e), e);
}
}
logger::log_info(&format!("Economic Storage: Loaded {} events from {} chunks", map.len(), chunks.len())).await;
Ok(map)
}
pub async fn save_optimized_chunks(events: HashMap<String, EconomicEvent>) -> anyhow::Result<()> {
let dir = std::path::Path::new("data/economic/events");
pub async fn save_optimized_chunks(paths: &DataPaths, events: HashMap<String, EconomicEvent>) -> anyhow::Result<()> {
let dir = paths.economic_events_dir();
fs::create_dir_all(dir).await?;
// Delete all old chunk files to prevent duplicates and overlaps
println!("Removing old chunks...");
logger::log_info("Economic Storage: Removing old chunk files...").await;
let mut entries = fs::read_dir(dir).await?;
let mut removed_count = 0;
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
if let Some(name) = path.file_name().and_then(|n| n.to_str()) {
if name.starts_with("chunk_") && path.extension().map(|e| e == "json").unwrap_or(false) {
fs::remove_file(&path).await?;
removed_count += 1;
}
}
}
logger::log_info(&format!("Economic Storage: Removed {} old chunk files", removed_count)).await;
let mut sorted: Vec<_> = events.into_values().collect();
sorted.sort_by_key(|e| e.date.clone());
@@ -77,6 +83,7 @@ pub async fn save_optimized_chunks(events: HashMap<String, EconomicEvent>) -> an
if !chunk.is_empty() {
save_chunk(&chunk, dir).await?;
}
logger::log_info(&format!("Economic Storage: Saved all event chunks to {:?}", dir)).await;
Ok(())
}
@@ -85,14 +92,20 @@ async fn save_chunk(events: &[EconomicEvent], dir: &std::path::Path) -> anyhow::
let end = events.iter().map(|e| &e.date).max().unwrap().clone();
let path = dir.join(format!("chunk_{}_{}.json", start, end));
fs::write(&path, serde_json::to_string_pretty(events)?).await?;
logger::log_info(&format!("Economic Storage: Saved chunk {} - {} ({} events)", start, end, events.len())).await;
Ok(())
}
pub async fn save_changes(changes: &[EventChange]) -> anyhow::Result<()> {
if changes.is_empty() { return Ok(()); }
let dir = std::path::Path::new("data/economic/events/changes");
pub async fn save_changes(paths: &DataPaths, changes: &[EventChange]) -> anyhow::Result<()> {
if changes.is_empty() {
logger::log_info("Economic Storage: No changes to save").await;
return Ok(());
}
let dir = paths.economic_changes_dir();
fs::create_dir_all(dir).await?;
logger::log_info(&format!("Economic Storage: Saving {} changes to {:?}", changes.len(), dir)).await;
let mut by_month: HashMap<String, Vec<EventChange>> = HashMap::new();
for c in changes {
if let Ok(d) = NaiveDate::parse_from_str(&c.date, "%Y-%m-%d") {
@@ -107,8 +120,10 @@ pub async fn save_changes(changes: &[EventChange]) -> anyhow::Result<()> {
let s = fs::read_to_string(&path).await?;
serde_json::from_str(&s).unwrap_or_default()
} else { vec![] };
all.extend(list);
all.extend(list.clone());
fs::write(&path, serde_json::to_string_pretty(&all)?).await?;
logger::log_info(&format!("Economic Storage: Saved {} changes for month {}", list.len(), month)).await;
}
logger::log_info("Economic Storage: All changes saved successfully").await;
Ok(())
}

50
src/economic/update.rs
View File

@@ -1,7 +1,6 @@
// src/economic/update.rs
use super::{scraper::*, storage::*, helpers::*, types::*};
use crate::{config::Config, scraper::webdriver::ScrapeTask};
use crate::scraper::webdriver::ChromeDriverPool;
use crate::{config::Config, webdriver::webdriver::{ScrapeTask, ChromeDriverPool}, util::directories::DataPaths, util::logger};
use chrono::{Local};
use std::sync::Arc;
@@ -14,38 +13,69 @@ use std::sync::Arc;
/// # Errors
/// Returns an error if scraping, loading, or saving fails.
pub async fn run_full_update(config: &Config, pool: &Arc<ChromeDriverPool>) -> anyhow::Result<()> {
let paths = DataPaths::new(".")?;
logger::log_info("Economic Update: Initializing...").await;
let today_str = chrono::Local::now().date_naive().format("%Y-%m-%d").to_string();
let end_date = config.target_end_date();
let chunks = scan_existing_chunks().await?;
logger::log_info(&format!("Economic Update: Scanning existing chunks from {:?}", paths.economic_events_dir())).await;
let chunks = scan_existing_chunks(&paths).await?;
let mut events = load_existing_events(&chunks).await?;
println!("Loaded {} events from {} chunks", events.len(), chunks.len());
let msg = format!("Economic Update: Loaded {} events from {} chunks", events.len(), chunks.len());
println!("{}", msg);
logger::log_info(&msg).await;
let start_date = if events.is_empty() {
logger::log_warn("Economic Update: No existing events found, starting from config date").await;
config.economic_start_date.clone()
} else if events.values().any(|e| e.date >= today_str) {
logger::log_info("Economic Update: Events exist for today, starting from today").await;
today_str.clone()
} else {
events.values()
let next = events.values()
.filter_map(|e| chrono::NaiveDate::parse_from_str(&e.date, "%Y-%m-%d").ok())
.max()
.and_then(|d| d.succ_opt())
.map(|d| d.format("%Y-%m-%d").to_string())
.unwrap_or(today_str.clone())
.unwrap_or(today_str.clone());
logger::log_info(&format!("Economic Update: Resuming from: {}", next)).await;
next
};
println!("Scraping economic events: {} → {}", start_date, end_date);
let msg = format!("Economic Update: Scraping events from {} → {}", start_date, end_date);
println!("{}", msg);
logger::log_info(&msg).await;
// Pass the pool to the scraping function
let new_events_all = scrape_all_economic_events(&start_date, &end_date, pool).await?;
let msg = format!("Economic Update: Scraped {} new events", new_events_all.len());
println!("{}", msg);
logger::log_info(&msg).await;
// Process all at once or in batches
let result = process_batch(&new_events_all, &mut events, &today_str);
let total_changes = result.changes.len();
save_changes(&result.changes).await?;
let msg = format!("Economic Update: Detected {} changes", total_changes);
println!("{}", msg);
logger::log_info(&msg).await;
if total_changes > 0 {
logger::log_info(&format!("Economic Update: Saving {} changes to log", total_changes)).await;
save_changes(&paths, &result.changes).await?;
logger::log_info("Economic Update: Changes saved successfully").await;
}
save_optimized_chunks(events).await?;
println!("Economic update complete — {} changes detected", total_changes);
logger::log_info(&format!("Economic Update: Saving {} total events to chunks", events.len())).await;
save_optimized_chunks(&paths, events).await?;
let msg = format!("✓ Economic update complete — {} changes detected", total_changes);
println!("{}", msg);
logger::log_info(&msg).await;
Ok(())
}