donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Compare commits

base: donpat1to:d26e833d934d131c4a02e5db02bb47785a0e5b94
Branches Tags
donpat1to:main donpat1to:feature/browser-vpn
..
compare: donpat1to:5e81959322a672875cf1ba7fb1e99d2f44d0cb7e
Branches Tags
donpat1to:main donpat1to:feature/browser-vpn

5 Commits
d26e833d93 ... 5e81959322

Author SHA1 Message Date
donpat1to
5e81959322 updated yahoo company extraction js to get the most data rich row 2025-12-19 14:43:36 +01:00
donpat1to
b366f366e6 added atomic writer action for ctr c abort 2025-12-19 14:12:56 +01:00
donpat1to
cd91de253b added pool rotation to chromedriver pool 2025-12-18 15:59:56 +01:00
donpat1to
c51b36c125 added session detection with requests per task 2025-12-18 14:01:51 +01:00
donpat1to
9c66f0d361 added parallelized scraping instances for company yahoo ticker seeding 2025-12-18 13:05:23 +01:00
28 changed files with 4284 additions and 541 deletions
Expand all files Collapse all files

38
.env.example
View File

@@ -3,46 +3,38 @@
# This file configures the behavior of the WebScraper application
# Copy to .env and adjust values as needed
# ===== ECONOMIC DATA =====
# Start date for economic event scraping
OPENFIGI_API_KEY=
# Economic calendar start (usually the earliest available on finanzen.net)
ECONOMIC_START_DATE=2007-02-13
# How far into the future to look ahead for economic events (in months)
ECONOMIC_LOOKAHEAD_MONTHS=3
# ===== CORPORATE DATA =====
# Start date for corporate earnings/data scraping
# Corporate earnings & price history start
CORPORATE_START_DATE=2010-01-01
# ===== PERFORMANCE & CONCURRENCY =====
# Maximum number of parallel ChromeDriver instances
# Higher = more concurrent tasks, but higher resource usage
MAX_PARALLEL_INSTANCES=3
# How far into the future we scrape economic events (in months)
ECONOMIC_LOOKAHEAD_MONTHS=3
# Maximum tasks per ChromeDriver instance before recycling
# 0 = unlimited (instance lives for entire application runtime)
MAX_TASKS_PER_INSTANCE=0
# Maximum number of parallel scraping tasks (default: 10)
MAX_PARALLEL_TASKS=10
# ===== VPN ROTATION (ProtonVPN Integration) =====
# Enable automatic VPN rotation between sessions?
# If false, all traffic goes through system without VPN tunneling
ENABLE_VPN_ROTATION=false
# Comma-separated list of ProtonVPN servers to rotate through
# Examples:
# "US-Free#1,US-Free#2,UK-Free#1"
# "US,UK,JP,DE,NL"
# NOTE: Must have ENABLE_VPN_ROTATION=true for this to take effect
VPN_SERVERS=
ENABLE_VPN_ROTATION=true
# Number of tasks per VPN session before rotating to new server/IP
# 0 = rotate between economic and corporate phases (one phase = one IP)
# 5 = rotate every 5 tasks
# NOTE: Must have ENABLE_VPN_ROTATION=true for this to take effect
TASKS_PER_VPN_SESSION=0
TASKS_PER_VPN_SESSION=50
# ===== LOGGING =====
# Set via RUST_LOG environment variable:
# RUST_LOG=info cargo run
# RUST_LOG=debug cargo run
# Leave empty or unset for default logging level
MAX_REQUESTS_PER_SESSION=25
MIN_REQUEST_INTERVAL_MS=300
MAX_RETRY_ATTEMPTS=3

212
Cargo.lock generated
View File

@@ -110,6 +110,17 @@ dependencies = [
"tokio",
]
[[package]]
name = "async-trait"
version = "0.1.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.110",
]
[[package]]
name = "atomic-waker"
version = "1.1.2"
@@ -122,6 +133,64 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "axum"
version = "0.7.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
dependencies = [
"async-trait",
"axum-core",
"base64 0.22.1",
"bytes",
"futures-util",
"http 1.3.1",
"http-body 1.0.1",
"http-body-util",
"hyper 1.8.1",
"hyper-util",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"rustversion",
"serde",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sha1",
"sync_wrapper",
"tokio",
"tokio-tungstenite 0.24.0",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "axum-core"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http 1.3.1",
"http-body 1.0.1",
"http-body-util",
"mime",
"pin-project-lite",
"rustversion",
"sync_wrapper",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "base64"
version = "0.21.7"
@@ -660,34 +729,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "event_backtest_engine"
version = "0.1.0"
dependencies = [
"anyhow",
"chrono",
"csv",
"dotenvy",
"fantoccini",
"flate2",
"futures",
"once_cell",
"rand 0.9.2",
"rayon",
"regex",
"reqwest",
"scraper",
"serde",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber",
"url",
"walkdir",
"yfinance-rs",
"zip",
]
[[package]]
name = "fantoccini"
version = "0.20.0"
@@ -1099,6 +1140,7 @@ dependencies = [
"http 1.3.1",
"http-body 1.0.1",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"pin-utils",
@@ -1522,6 +1564,12 @@ dependencies = [
"regex-automata",
]
[[package]]
name = "matchit"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
[[package]]
name = "memchr"
version = "2.7.6"
@@ -2684,6 +2732,17 @@ dependencies = [
"serde_core",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
dependencies = [
"itoa",
"serde",
"serde_core",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@@ -3100,6 +3159,30 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-tungstenite"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38"
dependencies = [
"futures-util",
"log",
"tokio",
"tungstenite 0.21.0",
]
[[package]]
name = "tokio-tungstenite"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9"
dependencies = [
"futures-util",
"log",
"tokio",
"tungstenite 0.24.0",
]
[[package]]
name = "tokio-tungstenite"
version = "0.28.0"
@@ -3113,7 +3196,7 @@ dependencies = [
"rustls-pki-types",
"tokio",
"tokio-rustls 0.26.4",
"tungstenite",
"tungstenite 0.28.0",
]
[[package]]
@@ -3172,6 +3255,7 @@ dependencies = [
"tokio",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
@@ -3210,6 +3294,7 @@ version = "0.1.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
dependencies = [
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
@@ -3271,6 +3356,43 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "tungstenite"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1"
dependencies = [
"byteorder",
"bytes",
"data-encoding",
"http 1.3.1",
"httparse",
"log",
"rand 0.8.5",
"sha1",
"thiserror 1.0.69",
"url",
"utf-8",
]
[[package]]
name = "tungstenite"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a"
dependencies = [
"byteorder",
"bytes",
"data-encoding",
"http 1.3.1",
"httparse",
"log",
"rand 0.8.5",
"sha1",
"thiserror 1.0.69",
"utf-8",
]
[[package]]
name = "tungstenite"
version = "0.28.0"
@@ -3484,6 +3606,36 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "web_scraper"
version = "0.1.0"
dependencies = [
"anyhow",
"axum",
"chrono",
"csv",
"dotenvy",
"fantoccini",
"flate2",
"futures",
"once_cell",
"rand 0.9.2",
"rayon",
"regex",
"reqwest",
"scraper",
"serde",
"serde_json",
"tokio",
"tokio-tungstenite 0.21.0",
"tracing",
"tracing-subscriber",
"url",
"walkdir",
"yfinance-rs",
"zip",
]
[[package]]
name = "webdriver"
version = "0.50.0"
@@ -3798,7 +3950,7 @@ dependencies = [
"serde_json",
"thiserror 2.0.17",
"tokio",
"tokio-tungstenite",
"tokio-tungstenite 0.28.0",
"url",
]

6
Cargo.toml
View File

@@ -1,5 +1,5 @@
[package]
name = "event_backtest_engine"
name = "web_scraper"
version = "0.1.0"
edition = "2024"
authors = ["Your Name <you@example.com>"]
@@ -54,3 +54,7 @@ once_cell = "1.21.3"
# Parallel processing (for batch tickers)
futures = "0.3"
rayon = "1.10" # optional: for parallel price downloads
# Web server for dashboard
axum = { version = "0.7", features = ["ws"] }
tokio-tungstenite = "0.21" # For WebSocket support

84
src/config.rs
View File

@@ -1,32 +1,52 @@
// src/config.rs - FIXED VERSION
use anyhow::{Context, Result};
use chrono::{self};
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Config {
// Economic calendar start (usually the earliest available on finanzen.net)
pub economic_start_date: String, // e.g. "2007-02-13"
// Corporate earnings & price history start
pub corporate_start_date: String, // e.g. "2000-01-01" or "2010-01-01"
// How far into the future we scrape economic events
pub economic_lookahead_months: u32, // default: 3
/// Maximum number of parallel scraping tasks (default: 10).
/// This limits concurrency to protect system load and prevent website spamming.
pub economic_start_date: String,
pub corporate_start_date: String,
pub economic_lookahead_months: u32,
#[serde(default = "default_max_parallel_instances")]
pub max_parallel_instances: usize,
pub max_tasks_per_instance: usize,
/// VPN rotation configuration
/// If set to "true", enables automatic VPN rotation between sessions
#[serde(default)]
#[serde(default = "default_enable_vpn_rotation")]
pub enable_vpn_rotation: bool,
// IMPROVEMENT: Reduzierte Defaults für weniger aggressive Scraping
#[serde(default = "default_max_requests_per_session")]
pub max_requests_per_session: usize,
#[serde(default = "default_min_request_interval_ms")]
pub min_request_interval_ms: u64,
#[serde(default = "default_max_retry_attempts")]
pub max_retry_attempts: u32,
}
fn default_enable_vpn_rotation() -> bool {
false
}
fn default_max_parallel_instances() -> usize {
4
}
fn default_max_requests_per_session() -> usize {
10
}
fn default_min_request_interval_ms() -> u64 {
1200
}
fn default_max_retry_attempts() -> u32 { 3 }
impl Default for Config {
fn default() -> Self {
Self {
@@ -35,26 +55,19 @@ impl Default for Config {
economic_lookahead_months: 3,
max_parallel_instances: default_max_parallel_instances(),
max_tasks_per_instance: 0,
max_requests_per_session: default_max_requests_per_session(),
min_request_interval_ms: default_min_request_interval_ms(),
max_retry_attempts: default_max_retry_attempts(),
enable_vpn_rotation: false,
}
}
}
impl Config {
/// Loads the configuration from environment variables using dotenvy.
///
/// This function loads a `.env` file if present (via `dotenvy::dotenv()`),
/// then retrieves each configuration value from environment variables.
/// If a variable is missing, it falls back to the default value.
/// Variable names are uppercase with underscores (e.g., ECONOMIC_START_DATE).
///
/// # Returns
/// The loaded Config on success.
///
/// # Errors
/// Returns an error if parsing fails (e.g., invalid integer for lookahead months).
/// Loads configuration from environment variables using dotenvy.
pub fn load() -> Result<Self> {
// Load .env file if it exists; ignore if not found (dotenvy::dotenv returns Ok if no file)
let _ = dotenvy::dotenv().context("Failed to load .env file (optional)")?;
let economic_start_date = dotenvy::var("ECONOMIC_START_DATE")
@@ -68,13 +81,14 @@ impl Config {
.parse()
.context("Failed to parse ECONOMIC_LOOKAHEAD_MONTHS as u32")?;
// IMPROVEMENT: Reduzierte Defaults
let max_parallel_instances: usize = dotenvy::var("MAX_PARALLEL_INSTANCES")
.unwrap_or_else(|_| "10".to_string())
.unwrap_or_else(|_| "4".to_string()) // Geändert von 10
.parse()
.context("Failed to parse MAX_PARALLEL_INSTANCES as usize")?;
let max_tasks_per_instance: usize = dotenvy::var("MAX_TASKS_PER_INSTANCE")
.unwrap_or_else(|_| "0".to_string())
.unwrap_or_else(|_| "5".to_string()) // Geändert von 0
.parse()
.context("Failed to parse MAX_TASKS_PER_INSTANCE as usize")?;
@@ -83,6 +97,21 @@ impl Config {
.parse::<bool>()
.context("Failed to parse ENABLE_VPN_ROTATION as bool")?;
let max_requests_per_session: usize = dotenvy::var("MAX_REQUESTS_PER_SESSION")
.unwrap_or_else(|_| "10".to_string()) // Geändert von 25
.parse()
.context("Failed to parse MAX_REQUESTS_PER_SESSION as usize")?;
let min_request_interval_ms: u64 = dotenvy::var("MIN_REQUEST_INTERVAL_MS")
.unwrap_or_else(|_| "1200".to_string()) // Geändert von 300
.parse()
.context("Failed to parse MIN_REQUEST_INTERVAL_MS as u64")?;
let max_retry_attempts: u32 = dotenvy::var("MAX_RETRY_ATTEMPTS")
.unwrap_or_else(|_| "3".to_string())
.parse()
.context("Failed to parse MAX_RETRY_ATTEMPTS as u32")?;
Ok(Self {
economic_start_date,
corporate_start_date,
@@ -90,6 +119,9 @@ impl Config {
max_parallel_instances,
max_tasks_per_instance,
enable_vpn_rotation,
max_requests_per_session,
min_request_interval_ms,
max_retry_attempts,
})
}

346
src/corporate/atomic_writer.rs Normal file
View File

@@ -0,0 +1,346 @@
// src/corporate/atomic_writer.rs
//
// Atomic JSONL writer that prevents partial/corrupted results from being written
use anyhow::Result;
use serde::Serialize;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use tokio::fs::{File, OpenOptions};
use tokio::io::AsyncWriteExt;
use tokio::sync::mpsc;
/// Command to write or validate data
#[derive(Debug)]
pub enum WriteCommand<T> {
/// Stage a result for writing (held in memory until committed)
Stage { id: String, data: T },
/// Commit staged result to disk (atomic write)
Commit { id: String },
/// Rollback staged result (discard without writing)
Rollback { id: String },
/// Commit all pending staged results and flush
CommitAll,
/// Shutdown writer gracefully (only commits valid staged results)
Shutdown,
}
/// Result of a write operation
#[derive(Debug)]
pub struct WriteResult {
pub id: String,
pub success: bool,
pub error: Option<String>,
}
/// Atomic writer that prevents partial results from being written
pub struct AtomicJsonlWriter<T> {
file: File,
staged: HashMap<String, T>,
committed_count: usize,
rollback_count: usize,
}
impl<T: Serialize + Clone> AtomicJsonlWriter<T> {
pub async fn new(path: PathBuf) -> Result<Self> {
// Ensure parent directory exists
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent).await?;
}
let file = OpenOptions::new()
.create(true)
.append(true)
.open(&path)
.await?;
crate::util::logger::log_info(&format!(
"Atomic writer initialized: {:?}",
path
)).await;
Ok(Self {
file,
staged: HashMap::new(),
committed_count: 0,
rollback_count: 0,
})
}
/// Stage data for writing (held in memory, not yet written)
pub async fn stage(&mut self, id: String, data: T) {
crate::util::logger::log_info(&format!(
"Staging result for: {} (total staged: {})",
id,
self.staged.len() + 1
)).await;
self.staged.insert(id, data);
}
/// Commit a staged result to disk (atomic write)
pub async fn commit(&mut self, id: &str) -> Result<()> {
if let Some(data) = self.staged.remove(id) {
// Serialize to JSON
let json_line = serde_json::to_string(&data)?;
// Write atomically (single syscall)
self.file.write_all(json_line.as_bytes()).await?;
self.file.write_all(b"\n").await?;
self.file.flush().await?;
self.committed_count += 1;
crate::util::logger::log_info(&format!(
"✓ Committed result for: {} (total committed: {})",
id, self.committed_count
)).await;
Ok(())
} else {
Err(anyhow::anyhow!("No staged result found for id: {}", id))
}
}
/// Rollback a staged result (discard without writing)
pub async fn rollback(&mut self, id: &str) {
if self.staged.remove(id).is_some() {
self.rollback_count += 1;
crate::util::logger::log_warn(&format!(
"⚠ Rolled back result for: {} (total rollbacks: {})",
id, self.rollback_count
)).await;
}
}
/// Commit all staged results
pub async fn commit_all(&mut self) -> Result<usize> {
let ids: Vec<String> = self.staged.keys().cloned().collect();
let mut committed = 0;
for id in ids {
if let Ok(()) = self.commit(&id).await {
committed += 1;
}
}
Ok(committed)
}
/// Rollback all staged results (discard everything)
pub async fn rollback_all(&mut self) -> usize {
let count = self.staged.len();
self.staged.clear();
self.rollback_count += count;
crate::util::logger::log_warn(&format!(
"⚠ Rolled back all {} staged results",
count
)).await;
count
}
/// Get statistics
pub fn stats(&self) -> WriterStats {
WriterStats {
staged_count: self.staged.len(),
committed_count: self.committed_count,
rollback_count: self.rollback_count,
}
}
}
#[derive(Debug, Clone)]
pub struct WriterStats {
pub staged_count: usize,
pub committed_count: usize,
pub rollback_count: usize,
}
/// Managed writer service that runs in its own task
pub struct AtomicWriterService<T> {
rx: mpsc::UnboundedReceiver<WriteCommand<T>>,
writer: AtomicJsonlWriter<T>,
shutdown_flag: Arc<AtomicBool>,
}
impl<T: Serialize + Clone> AtomicWriterService<T> {
pub async fn new(
path: PathBuf,
rx: mpsc::UnboundedReceiver<WriteCommand<T>>,
shutdown_flag: Arc<AtomicBool>,
) -> Result<Self> {
let writer = AtomicJsonlWriter::new(path).await?;
Ok(Self {
rx,
writer,
shutdown_flag,
})
}
/// Main service loop
pub async fn run(mut self) {
crate::util::logger::log_info("Atomic writer service started").await;
while let Some(cmd) = self.rx.recv().await {
// Check for shutdown flag
if self.shutdown_flag.load(Ordering::SeqCst) {
crate::util::logger::log_warn(
"Shutdown detected - processing only Commit/Rollback commands"
).await;
// Only process commit/rollback commands during shutdown
match cmd {
WriteCommand::Commit { id } => {
if let Err(e) = self.writer.commit(&id).await {
crate::util::logger::log_error(&format!(
"Failed to commit {}: {}",
id, e
)).await;
}
}
WriteCommand::Rollback { id } => {
self.writer.rollback(&id).await;
}
WriteCommand::CommitAll => {
match self.writer.commit_all().await {
Ok(count) => {
crate::util::logger::log_info(&format!(
"Committed {} results during shutdown",
count
)).await;
}
Err(e) => {
crate::util::logger::log_error(&format!(
"Failed to commit all: {}",
e
)).await;
}
}
}
WriteCommand::Shutdown => break,
_ => {
// Ignore Stage commands during shutdown
crate::util::logger::log_warn(
"Ignoring new Stage command during shutdown"
).await;
}
}
continue;
}
// Normal operation
match cmd {
WriteCommand::Stage { id, data } => {
self.writer.stage(id, data).await;
}
WriteCommand::Commit { id } => {
if let Err(e) = self.writer.commit(&id).await {
crate::util::logger::log_error(&format!(
"Failed to commit {}: {}",
id, e
)).await;
}
}
WriteCommand::Rollback { id } => {
self.writer.rollback(&id).await;
}
WriteCommand::CommitAll => {
match self.writer.commit_all().await {
Ok(count) => {
crate::util::logger::log_info(&format!(
"Committed all {} staged results",
count
)).await;
}
Err(e) => {
crate::util::logger::log_error(&format!(
"Failed to commit all: {}",
e
)).await;
}
}
}
WriteCommand::Shutdown => break,
}
}
// Final shutdown - rollback any remaining staged items
let stats = self.writer.stats();
if stats.staged_count > 0 {
crate::util::logger::log_warn(&format!(
"⚠ Shutdown with {} uncommitted results - rolling back",
stats.staged_count
)).await;
self.writer.rollback_all().await;
}
crate::util::logger::log_info(&format!(
"Atomic writer service stopped. Final stats: {} committed, {} rolled back",
stats.committed_count,
stats.rollback_count
)).await;
}
}
/// Handle for sending write commands
#[derive(Clone)]
pub struct AtomicWriterHandle<T> {
tx: mpsc::UnboundedSender<WriteCommand<T>>,
}
impl<T> AtomicWriterHandle<T> {
pub fn new(tx: mpsc::UnboundedSender<WriteCommand<T>>) -> Self {
Self { tx }
}
/// Stage data for writing (does not write immediately)
pub fn stage(&self, id: String, data: T) {
let _ = self.tx.send(WriteCommand::Stage { id, data });
}
/// Commit staged data to disk
pub fn commit(&self, id: String) {
let _ = self.tx.send(WriteCommand::Commit { id });
}
/// Rollback staged data (discard)
pub fn rollback(&self, id: String) {
let _ = self.tx.send(WriteCommand::Rollback { id });
}
/// Commit all staged data
pub fn commit_all(&self) {
let _ = self.tx.send(WriteCommand::CommitAll);
}
/// Shutdown writer gracefully
pub fn shutdown(&self) {
let _ = self.tx.send(WriteCommand::Shutdown);
}
}
/// Create atomic writer service
pub async fn create_atomic_writer<T: Serialize + Clone + Send + 'static>(
path: PathBuf,
shutdown_flag: Arc<AtomicBool>,
) -> Result<(AtomicWriterHandle<T>, tokio::task::JoinHandle<()>)> {
let (tx, rx) = mpsc::unbounded_channel();
let service = AtomicWriterService::new(path, rx, shutdown_flag).await?;
let handle = tokio::spawn(async move {
service.run().await;
});
Ok((AtomicWriterHandle::new(tx), handle))
}

14
src/corporate/helpers.rs
View File

@@ -2,6 +2,8 @@
use super::types::*;
use chrono::{Local, NaiveDate};
use std::collections::{HashMap, HashSet};
use rand::rngs::StdRng;
use rand::prelude::{Rng, SeedableRng, IndexedRandom};
pub fn event_key(e: &CompanyEvent) -> String {
format!("{}|{}|{}", e.ticker, e.date, e.time)
@@ -68,3 +70,15 @@ pub fn parse_yahoo_date(s: &str) -> anyhow::Result<NaiveDate> {
.or_else(|_| NaiveDate::parse_from_str(s, "%b %d, %Y"))
.map_err(|_| anyhow::anyhow!("Bad date: {s}"))
}
/// Send-safe random range
pub fn random_range(min: u64, max: u64) -> u64 {
let mut rng = StdRng::from_rng(&mut rand::rng());
rng.gen_range(min..max)
}
/// Send-safe random choice
pub fn choose_random<T: Clone>(items: &[T]) -> T {
let mut rng = StdRng::from_rng(&mut rand::rng());
items.choose(&mut rng).unwrap().clone()
}

3
src/corporate/mod.rs
View File

@@ -8,5 +8,8 @@ pub mod aggregation;
pub mod fx;
pub mod openfigi;
pub mod yahoo;
pub mod update_parallel;
pub mod page_validation;
pub mod atomic_writer;
pub use update::run_full_update;

180
src/corporate/page_validation.rs Normal file
View File

@@ -0,0 +1,180 @@
// src/corporate/page_validation.rs
//
// Utilities to ensure page state is correct before extraction
use anyhow::{anyhow, Result};
use fantoccini::Client;
use tokio::time::{sleep, Duration};
/// Validates that the browser navigated to the expected URL
///
/// This prevents extracting data from a stale page when navigation fails silently
pub async fn verify_navigation(
client: &Client,
expected_url_fragment: &str,
max_attempts: u32,
) -> Result<()> {
for attempt in 1..=max_attempts {
let current_url = client.current_url().await?;
let current = current_url.as_str();
if current.contains(expected_url_fragment) {
crate::util::logger::log_info(&format!(
"✓ Navigation verified: {} (attempt {})",
current, attempt
)).await;
return Ok(());
}
if attempt < max_attempts {
crate::util::logger::log_warn(&format!(
"Navigation mismatch (attempt {}): expected '{}', got '{}'. Retrying...",
attempt, expected_url_fragment, current
)).await;
sleep(Duration::from_millis(500)).await;
}
}
let current_url = client.current_url().await?;
Err(anyhow!(
"Navigation verification failed: expected URL containing '{}', but got '{}'",
expected_url_fragment,
current_url.as_str()
))
}
/// Clears browser state by navigating to a blank page
///
/// Use this when a navigation fails or times out to ensure clean slate
pub async fn clear_browser_state(client: &Client) -> Result<()> {
crate::util::logger::log_info("Clearing browser state with about:blank").await;
// Navigate to blank page to clear any stale content
client.goto("about:blank").await?;
// Brief wait to ensure page clears
sleep(Duration::from_millis(200)).await;
Ok(())
}
/// Validates that expected content exists on the page before extraction
///
/// This adds an extra safety check that the page actually loaded
pub async fn verify_page_content(
client: &Client,
content_checks: Vec<ContentCheck>,
) -> Result<()> {
for check in content_checks {
match check {
ContentCheck::ElementExists(selector) => {
let exists: bool = client
.execute(
&format!(
"return !!document.querySelector('{}');",
selector.replace("'", "\\'")
),
vec![],
)
.await?
.as_bool()
.unwrap_or(false);
if !exists {
return Err(anyhow!(
"Expected element '{}' not found on page",
selector
));
}
}
ContentCheck::TextContains(text) => {
let page_text: String = client
.execute("return document.body.innerText;", vec![])
.await?
.as_str()
.unwrap_or("")
.to_string();
if !page_text.contains(&text) {
return Err(anyhow!(
"Expected text '{}' not found on page",
text
));
}
}
}
}
Ok(())
}
#[derive(Debug, Clone)]
pub enum ContentCheck {
/// Verify that a CSS selector exists
ElementExists(String),
/// Verify that page body contains text
TextContains(String),
}
/// Safe navigation wrapper that validates and clears state on failure
pub async fn navigate_with_validation(
client: &Client,
url: &str,
expected_url_fragment: &str,
timeout_secs: u64,
) -> Result<()> {
use tokio::time::timeout;
// Attempt navigation with timeout
let nav_result = timeout(
Duration::from_secs(timeout_secs),
client.goto(url)
).await;
match nav_result {
Ok(Ok(_)) => {
// Navigation succeeded, verify we're on correct page
verify_navigation(client, expected_url_fragment, 3).await?;
Ok(())
}
Ok(Err(e)) => {
// Navigation failed - clear state before returning error
crate::util::logger::log_error(&format!(
"Navigation failed: {}. Clearing browser state...",
e
)).await;
clear_browser_state(client).await.ok(); // Best effort
Err(anyhow!("Navigation failed: {}", e))
}
Err(_) => {
// Navigation timed out - clear state before returning error
crate::util::logger::log_error(&format!(
"Navigation timeout after {}s. Clearing browser state...",
timeout_secs
)).await;
clear_browser_state(client).await.ok(); // Best effort
Err(anyhow!("Navigation timeout"))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_content_check_variants() {
let check1 = ContentCheck::ElementExists("table".to_string());
let check2 = ContentCheck::TextContains("Yahoo Finance".to_string());
match check1 {
ContentCheck::ElementExists(sel) => assert_eq!(sel, "table"),
_ => panic!("Wrong variant"),
}
match check2 {
ContentCheck::TextContains(text) => assert_eq!(text, "Yahoo Finance"),
_ => panic!("Wrong variant"),
}
}
}

485
src/corporate/update.rs
View File

@@ -1,7 +1,7 @@
// src/corporate/update.rs - ABORT-SAFE VERSION WITH JSONL LOG
// src/corporate/update.rs - UPDATED WITH DATA INTEGRITY FIXES
use super::{scraper::*, storage::*, helpers::*, types::*, openfigi::*, yahoo::*};
use crate::config::Config;
use crate::corporate::update_parallel::build_companies_jsonl_streaming_parallel;
use crate::util::directories::DataPaths;
use crate::util::logger;
use crate::scraper::webdriver::ChromeDriverPool;
@@ -11,12 +11,13 @@ use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
/// UPDATED: Main corporate update entry point with shutdown awareness
pub async fn run_full_update(
_config: &Config,
pool: &Arc<ChromeDriverPool>,
shutdown_flag: &Arc<AtomicBool>,
) -> anyhow::Result<()> {
logger::log_info("=== Corporate Update (STREAMING MODE) ===").await;
logger::log_info("=== Corporate Update (STREAMING MODE WITH DATA INTEGRITY) ===").await;
let paths = DataPaths::new(".")?;
@@ -33,6 +34,7 @@ pub async fn run_full_update(
};
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after GLEIF download").await;
return Ok(());
}
@@ -41,6 +43,7 @@ pub async fn run_full_update(
logger::log_info(" ✓ OpenFIGI metadata loaded").await;
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after OpenFIGI load").await;
return Ok(());
}
@@ -54,6 +57,7 @@ pub async fn run_full_update(
}
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after LEI-FIGI mapping").await;
return Ok(());
}
@@ -69,36 +73,45 @@ pub async fn run_full_update(
}
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected after securities map build").await;
return Ok(());
}
logger::log_info("Step 5: Building companies.jsonl (streaming with abort-safe persistence)...").await;
let count = build_companies_jsonl_streaming(&paths, pool, shutdown_flag).await?;
logger::log_info("Step 5: Building companies.jsonl with parallel processing and validation...").await;
let count = build_companies_jsonl_streaming_parallel(&paths, pool, shutdown_flag).await?;
logger::log_info(&format!(" ✓ Saved {} companies", count)).await;
if !shutdown_flag.load(Ordering::SeqCst) {
logger::log_info("Step 6: Processing events (using index)...").await;
let _event_index = build_event_index(&paths).await?;
logger::log_info(" ✓ Event index built").await;
} else {
logger::log_warn("Shutdown detected, skipping event index build").await;
}
logger::log_info("✓ Corporate update complete").await;
Ok(())
}
/// Abort-safe incremental JSONL persistence with atomic checkpoints
/// UPDATED: Serial version with validation (kept for compatibility/debugging)
///
/// Implements the data_updating_rule.md specification:
/// - Append-only JSONL log for all updates
/// - fsync after each write batch
/// - Atomic checkpoints via temp file + rename
/// - Crash recovery by loading checkpoint + replaying log
/// - Partial lines ignored during recovery
async fn build_companies_jsonl_streaming(
/// This is the non-parallel version that processes companies sequentially.
/// Updated with same validation and shutdown checks as parallel version.
///
/// Use this for:
/// - Debugging issues with specific companies
/// - Environments where parallel processing isn't desired
/// - Testing validation logic without concurrency complexity
async fn build_companies_jsonl_streaming_serial(
paths: &DataPaths,
pool: &Arc<ChromeDriverPool>,
shutdown_flag: &Arc<AtomicBool>,
) -> anyhow::Result<usize> {
// Configuration constants
const CHECKPOINT_INTERVAL: usize = 50;
const FSYNC_BATCH_SIZE: usize = 10;
const FSYNC_INTERVAL_SECS: u64 = 10;
let path = DataPaths::new(".")?;
let corporate_path = path.data_dir().join("corporate").join("by_name");
let securities_path = corporate_path.join("common_stocks.json");
@@ -118,51 +131,42 @@ async fn build_companies_jsonl_streaming(
tokio::fs::create_dir_all(parent).await?;
}
// === RECOVERY PHASE 1: Load last checkpoint ===
// === RECOVERY PHASE: Load checkpoint + replay log ===
let mut existing_companies: HashMap<String, CompanyCrossPlatformInfo> = HashMap::new();
let mut processed_names: std::collections::HashSet<String> = std::collections::HashSet::new();
if companies_path.exists() {
logger::log_info("Loading checkpoint from companies.jsonl...").await;
let existing_content = tokio::fs::read_to_string(&companies_path).await?;
for line in existing_content.lines() {
if line.trim().is_empty() {
continue;
}
// Only process complete lines (ending with proper JSON closing brace)
// This ensures we don't process partial writes from crashed processes
if !line.ends_with('}') {
logger::log_warn(&format!("Skipping incomplete checkpoint line: {}", &line[..line.len().min(50)])).await;
continue;
}
match serde_json::from_str::<CompanyCrossPlatformInfo>(line) {
Ok(company) => {
processed_names.insert(company.name.clone());
existing_companies.insert(company.name.clone(), company);
}
Err(e) => {
logger::log_warn(&format!("Failed to parse checkpoint line: {}", e)).await;
logger::log_warn(&format!("Skipping invalid checkpoint line: {}", e)).await;
}
}
}
logger::log_info(&format!("Loaded checkpoint with {} companies", existing_companies.len())).await;
}
// === RECOVERY PHASE 2: Replay log after checkpoint ===
if log_path.exists() {
logger::log_info("Replaying update log...").await;
let log_content = tokio::fs::read_to_string(&log_path).await?;
let mut replayed = 0;
for line in log_content.lines() {
if line.trim().is_empty() {
continue;
}
// Only replay complete lines (crash-safe: incomplete lines are ignored)
// A line is considered complete only if it ends with '\n' and valid JSON
if !line.ends_with('}') {
logger::log_warn(&format!("Skipping incomplete log line: {}", &line[..line.len().min(50)])).await;
continue;
}
match serde_json::from_str::<CompanyCrossPlatformInfo>(line) {
Ok(company) => {
processed_names.insert(company.name.clone());
@@ -170,7 +174,7 @@ async fn build_companies_jsonl_streaming(
replayed += 1;
}
Err(e) => {
logger::log_warn(&format!("Failed to parse log line: {}", e)).await;
logger::log_warn(&format!("Skipping invalid log line: {}", e)).await;
}
}
}
@@ -179,193 +183,151 @@ async fn build_companies_jsonl_streaming(
}
}
// === APPEND-ONLY LOG: Open in append mode with O_APPEND semantics ===
// === OPEN LOG FILE ===
use tokio::fs::OpenOptions;
use tokio::io::AsyncWriteExt;
let mut log_file = OpenOptions::new()
.create(true)
.append(true) // O_APPEND - atomic append operations
.append(true)
.open(&log_path)
.await?;
let mut count = existing_companies.len();
let mut updated_count = 0;
let mut new_count = 0;
let checkpoint_interval = 50; // Create atomic checkpoint every 50 updates
let mut writes_since_fsync = 0;
let mut last_fsync = std::time::Instant::now();
let mut updates_since_checkpoint = 0;
let mut count = 0;
let mut new_count = 0;
let mut updated_count = 0;
use tokio::io::AsyncWriteExt;
logger::log_info(&format!("Processing {} companies sequentially...", securities.len())).await;
for (name, company_info) in securities.iter() {
// === PROCESS COMPANIES SEQUENTIALLY ===
for (name, company_info) in securities.clone() {
// Check shutdown before each company
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_info("Shutdown requested - stopping company processing").await;
logger::log_warn(&format!(
"Shutdown detected at company: {} (progress: {}/{})",
name, count, count + securities.len()
)).await;
break;
}
// Skip if already processed (from checkpoint or log replay)
if processed_names.contains(name) {
continue;
}
let existing_entry = existing_companies.get(name).cloned();
let existing_entry = existing_companies.get(&name).cloned();
let is_update = existing_entry.is_some();
let mut isin_tickers_map: HashMap<String, Vec<String>> =
existing_entry
.as_ref()
.map(|e| e.isin_tickers_map.clone())
.unwrap_or_default();
// Process company with validation
match process_single_company_serial(
name.clone(),
company_info,
existing_entry,
pool,
shutdown_flag,
).await {
Ok(Some(company_entry)) => {
// Write to log
let line = serde_json::to_string(&company_entry)?;
log_file.write_all(line.as_bytes()).await?;
log_file.write_all(b"\n").await?;
let mut sector = existing_entry.as_ref().and_then(|e| e.sector.clone());
let mut exchange = existing_entry.as_ref().and_then(|e| e.exchange.clone());
writes_since_fsync += 1;
let mut unique_isin_ticker_pairs: HashMap<String, Vec<String>> = HashMap::new();
// Batched + time-based fsync
let should_fsync = writes_since_fsync >= FSYNC_BATCH_SIZE
|| last_fsync.elapsed().as_secs() >= FSYNC_INTERVAL_SECS;
for figi_infos in company_info.securities.values() {
for figi_info in figi_infos {
if !figi_info.isin.is_empty() {
let tickers = unique_isin_ticker_pairs
.entry(figi_info.isin.clone())
.or_insert_with(Vec::new);
if should_fsync {
log_file.flush().await?;
log_file.sync_data().await?;
writes_since_fsync = 0;
last_fsync = std::time::Instant::now();
}
if !figi_info.ticker.is_empty() && !tickers.contains(&figi_info.ticker) {
tickers.push(figi_info.ticker.clone());
// Update in-memory state
processed_names.insert(name.clone());
existing_companies.insert(name.clone(), company_entry);
count += 1;
updates_since_checkpoint += 1;
if is_update {
updated_count += 1;
} else {
new_count += 1;
}
// Periodic checkpoint
if updates_since_checkpoint >= CHECKPOINT_INTERVAL {
if writes_since_fsync > 0 {
log_file.flush().await?;
log_file.sync_data().await?;
writes_since_fsync = 0;
last_fsync = std::time::Instant::now();
}
}
}
}
for (isin, figi_tickers) in unique_isin_ticker_pairs {
if shutdown_flag.load(Ordering::SeqCst) {
break;
}
logger::log_info(&format!("Creating checkpoint at {} companies...", count)).await;
let tickers = isin_tickers_map
.entry(isin.clone())
.or_insert_with(Vec::new);
let checkpoint_tmp = companies_path.with_extension("jsonl.tmp");
let mut checkpoint_file = tokio::fs::File::create(&checkpoint_tmp).await?;
for figi_ticker in figi_tickers {
if !tickers.contains(&figi_ticker) {
tickers.push(figi_ticker);
}
}
let has_yahoo_ticker = tickers.iter().any(|t| t.starts_with("YAHOO:"));
if !has_yahoo_ticker && !shutdown_flag.load(Ordering::SeqCst) {
logger::log_info(&format!("Fetching Yahoo details for {} (ISIN: {})", name, isin)).await;
match scrape_company_details_by_isin(pool, &isin).await {
Ok(Some(details)) => {
logger::log_info(&format!("✓ Found Yahoo ticker {} for ISIN {}", details.ticker, isin)).await;
tickers.push(format!("YAHOO:{}", details.ticker));
if sector.is_none() && details.sector.is_some() {
sector = details.sector.clone();
logger::log_info(&format!(" Sector: {}", details.sector.as_ref().unwrap())).await;
}
if exchange.is_none() && details.exchange.is_some() {
exchange = details.exchange.clone();
logger::log_info(&format!(" Exchange: {}", details.exchange.as_ref().unwrap())).await;
}
},
Ok(None) => {
logger::log_warn(&format!("◯ No search results for ISIN {}", isin)).await;
tickers.push("YAHOO:NO_RESULTS".to_string());
},
Err(e) => {
if shutdown_flag.load(Ordering::SeqCst) {
break;
}
logger::log_warn(&format!("✗ Yahoo lookup error for ISIN {}: {}", isin, e)).await;
for company in existing_companies.values() {
let line = serde_json::to_string(company)?;
checkpoint_file.write_all(line.as_bytes()).await?;
checkpoint_file.write_all(b"\n").await?;
}
checkpoint_file.flush().await?;
checkpoint_file.sync_all().await?;
drop(checkpoint_file);
tokio::fs::rename(&checkpoint_tmp, &companies_path).await?;
drop(log_file);
tokio::fs::remove_file(&log_path).await.ok();
log_file = OpenOptions::new()
.create(true)
.append(true)
.open(&log_path)
.await?;
updates_since_checkpoint = 0;
logger::log_info("✓ Checkpoint created and log cleared").await;
}
if count % 10 == 0 {
logger::log_info(&format!(
"Progress: {} companies ({} new, {} updated)",
count, new_count, updated_count
)).await;
}
}
Ok(None) => {
// Company had no ISINs or was skipped
logger::log_info(&format!("Skipped company: {} (no ISINs)", name)).await;
}
Err(e) => {
logger::log_warn(&format!("Error processing company {}: {}", name, e)).await;
}
}
if shutdown_flag.load(Ordering::SeqCst) {
break;
}
if !isin_tickers_map.is_empty() {
let company_entry = CompanyCrossPlatformInfo {
name: name.clone(),
isin_tickers_map,
sector,
exchange,
};
// === APPEND-ONLY: Write single-line JSON with fsync ===
// This guarantees the line is either fully written or not at all
let line = serde_json::to_string(&company_entry)?;
log_file.write_all(line.as_bytes()).await?;
log_file.write_all(b"\n").await?;
// Time-based fsync
if writes_since_fsync > 0 && last_fsync.elapsed().as_secs() >= FSYNC_INTERVAL_SECS {
log_file.flush().await?;
// Critical: fsync to ensure durability before considering write successful
// This prevents data loss on power failure or kernel panic
log_file.sync_data().await?;
// Update in-memory state ONLY after successful fsync
processed_names.insert(name.clone());
existing_companies.insert(name.clone(), company_entry);
count += 1;
updates_since_checkpoint += 1;
if is_update {
updated_count += 1;
} else {
new_count += 1;
}
// === ATOMIC CHECKPOINT: Periodically create checkpoint ===
// This reduces recovery time by snapshotting current state
if updates_since_checkpoint >= checkpoint_interval {
logger::log_info(&format!("Creating checkpoint at {} companies...", count)).await;
let checkpoint_tmp = companies_path.with_extension("jsonl.tmp");
let mut checkpoint_file = tokio::fs::File::create(&checkpoint_tmp).await?;
// Write all current state to temporary checkpoint file
for company in existing_companies.values() {
let line = serde_json::to_string(company)?;
checkpoint_file.write_all(line.as_bytes()).await?;
checkpoint_file.write_all(b"\n").await?;
}
checkpoint_file.flush().await?;
checkpoint_file.sync_all().await?;
drop(checkpoint_file);
// Atomic rename - this is the commit point
// After this succeeds, the checkpoint is visible
tokio::fs::rename(&checkpoint_tmp, &companies_path).await?;
// Clear log after successful checkpoint
// Any entries before this point are now captured in the checkpoint
drop(log_file);
tokio::fs::remove_file(&log_path).await.ok();
log_file = OpenOptions::new()
.create(true)
.append(true)
.open(&log_path)
.await?;
updates_since_checkpoint = 0;
logger::log_info("✓ Checkpoint created and log cleared").await;
}
if count % 10 == 0 {
logger::log_info(&format!("Progress: {} companies ({} new, {} updated)", count, new_count, updated_count)).await;
tokio::task::yield_now().await;
}
writes_since_fsync = 0;
last_fsync = std::time::Instant::now();
}
}
// === FINAL CHECKPOINT: Write complete final state ===
// This ensures we don't need to replay the log on next startup
// === FSYNC PENDING WRITES ===
if writes_since_fsync > 0 {
logger::log_info(&format!("Fsyncing {} pending writes...", writes_since_fsync)).await;
log_file.flush().await?;
log_file.sync_data().await?;
logger::log_info("✓ Pending writes saved").await;
}
// === FINAL CHECKPOINT ===
if !shutdown_flag.load(Ordering::SeqCst) && updates_since_checkpoint > 0 {
logger::log_info("Creating final checkpoint...").await;
@@ -382,21 +344,172 @@ async fn build_companies_jsonl_streaming(
checkpoint_file.sync_all().await?;
drop(checkpoint_file);
// Atomic rename makes final checkpoint visible
tokio::fs::rename(&checkpoint_tmp, &companies_path).await?;
// Clean up log
drop(log_file);
tokio::fs::remove_file(&log_path).await.ok();
logger::log_info("✓ Final checkpoint created").await;
}
logger::log_info(&format!("Completed: {} total companies ({} new, {} updated)", count, new_count, updated_count)).await;
logger::log_info(&format!(
"Completed: {} total companies ({} new, {} updated)",
count, new_count, updated_count
)).await;
Ok(count)
}
/// UPDATED: Process single company serially with validation
async fn process_single_company_serial(
name: String,
company_info: CompanyInfo,
existing_entry: Option<CompanyCrossPlatformInfo>,
pool: &Arc<ChromeDriverPool>,
shutdown_flag: &Arc<AtomicBool>,
) -> anyhow::Result<Option<CompanyCrossPlatformInfo>> {
// Check shutdown at start
if shutdown_flag.load(Ordering::SeqCst) {
return Ok(None);
}
let mut isin_tickers_map: HashMap<String, Vec<String>> =
existing_entry
.as_ref()
.map(|e| e.isin_tickers_map.clone())
.unwrap_or_default();
let mut sector = existing_entry.as_ref().and_then(|e| e.sector.clone());
let mut exchange = existing_entry.as_ref().and_then(|e| e.exchange.clone());
// Collect unique ISIN-ticker pairs
let mut unique_isin_ticker_pairs: HashMap<String, Vec<String>> = HashMap::new();
for figi_infos in company_info.securities.values() {
for figi_info in figi_infos {
if !figi_info.isin.is_empty() {
let tickers = unique_isin_ticker_pairs
.entry(figi_info.isin.clone())
.or_insert_with(Vec::new);
if !figi_info.ticker.is_empty() && !tickers.contains(&figi_info.ticker) {
tickers.push(figi_info.ticker.clone());
}
}
}
}
// Process each ISIN with validation
for (isin, figi_tickers) in unique_isin_ticker_pairs {
// Check shutdown before each ISIN
if shutdown_flag.load(Ordering::SeqCst) {
return Ok(None);
}
let tickers = isin_tickers_map
.entry(isin.clone())
.or_insert_with(Vec::new);
for figi_ticker in figi_tickers {
if !tickers.contains(&figi_ticker) {
tickers.push(figi_ticker);
}
}
let has_yahoo_ticker = tickers.iter().any(|t| t.starts_with("YAHOO:"));
if !has_yahoo_ticker {
logger::log_info(&format!("Fetching Yahoo details for {} (ISIN: {})", name, isin)).await;
// Use validated scraping with retry
match scrape_with_retry_serial(pool, &isin, 3, shutdown_flag).await {
Ok(Some(details)) => {
logger::log_info(&format!(
"✓ Found Yahoo ticker {} for ISIN {} (company: {})",
details.ticker, isin, name
)).await;
tickers.push(format!("YAHOO:{}", details.ticker));
if sector.is_none() && details.sector.is_some() {
sector = details.sector.clone();
}
if exchange.is_none() && details.exchange.is_some() {
exchange = details.exchange.clone();
}
},
Ok(None) => {
logger::log_warn(&format!("◯ No search results for ISIN {} (company: {})", isin, name)).await;
tickers.push("YAHOO:NO_RESULTS".to_string());
},
Err(e) => {
if shutdown_flag.load(Ordering::SeqCst) {
return Ok(None);
}
logger::log_warn(&format!(
"✗ Yahoo lookup error for ISIN {} (company: {}): {}",
isin, name, e
)).await;
}
}
}
}
// Final shutdown check
if shutdown_flag.load(Ordering::SeqCst) {
return Ok(None);
}
if !isin_tickers_map.is_empty() {
Ok(Some(CompanyCrossPlatformInfo {
name,
isin_tickers_map,
sector,
exchange,
}))
} else {
Ok(None)
}
}
/// UPDATED: Scrape with retry for serial processing
async fn scrape_with_retry_serial(
pool: &Arc<ChromeDriverPool>,
isin: &str,
max_retries: u32,
shutdown_flag: &Arc<AtomicBool>,
) -> anyhow::Result<Option<YahooCompanyDetails>> {
let mut retries = 0;
loop {
if shutdown_flag.load(Ordering::SeqCst) {
return Err(anyhow::anyhow!("Aborted due to shutdown"));
}
match scrape_company_details_by_isin(pool, isin, shutdown_flag).await {
Ok(result) => return Ok(result),
Err(e) => {
if retries >= max_retries {
return Err(e);
}
let backoff_ms = 1000 * 2u64.pow(retries);
let jitter_ms = random_range(0, 500);
let total_delay = backoff_ms + jitter_ms;
logger::log_warn(&format!(
"Retry {}/{} for ISIN {} after {}ms: {}",
retries + 1, max_retries, isin, total_delay, e
)).await;
tokio::time::sleep(tokio::time::Duration::from_millis(total_delay)).await;
retries += 1;
}
}
}
}
async fn find_most_recent_figi_date_dir(paths: &DataPaths) -> anyhow::Result<Option<std::path::PathBuf>> {
let map_cache_dir = paths.cache_gleif_openfigi_map_dir();

578
src/corporate/update_parallel.rs Normal file
View File

@@ -0,0 +1,578 @@
// src/corporate/update_parallel.rs - UPDATED WITH DATA INTEGRITY FIXES
// PARALLELIZED VERSION with atomic commits and validation
//
// Key improvements over original:
// - Page validation to prevent stale content extraction
// - Shutdown-aware task processing
// - Better error recovery with browser state cleanup
// - All original fsync and checkpoint logic preserved
use super::{types::*, yahoo::*, helpers::*};
use crate::util::directories::DataPaths;
use crate::util::logger;
use crate::scraper::webdriver::ChromeDriverPool;
use rand::Rng;
use tokio::sync::mpsc;
use tokio::io::AsyncWriteExt;
use tokio::fs::OpenOptions;
use tokio::time::sleep;
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use futures::stream::{FuturesUnordered, StreamExt};
use anyhow::{anyhow, Context, Result};
/// Represents a write command to be serialized through the log writer
enum LogCommand {
Write(CompanyCrossPlatformInfo),
Checkpoint,
Shutdown,
}
/// Result from processing a single company
struct CompanyProcessResult {
company: CompanyCrossPlatformInfo,
is_update: bool,
}
/// UPDATED: Abort-safe incremental JSONL persistence with validation
///
/// New safety features:
/// - Page validation before extraction
/// - Shutdown checks at all critical points
/// - Browser state cleanup on errors
/// - All writes still atomic with fsync
pub async fn build_companies_jsonl_streaming_parallel(
paths: &DataPaths,
pool: &Arc<ChromeDriverPool>,
shutdown_flag: &Arc<AtomicBool>,
) -> anyhow::Result<usize> {
// Configuration constants
const CHECKPOINT_INTERVAL: usize = 50;
const FSYNC_BATCH_SIZE: usize = 10;
const FSYNC_INTERVAL_SECS: u64 = 10;
const CONCURRENCY_LIMIT: usize = 100;
let path = DataPaths::new(".")?;
let corporate_path = path.data_dir().join("corporate").join("by_name");
let securities_path = corporate_path.join("common_stocks.json");
if !securities_path.exists() {
logger::log_warn("No common_stocks.json found").await;
return Ok(0);
}
let content = tokio::fs::read_to_string(securities_path).await?;
let securities: HashMap<String, CompanyInfo> = serde_json::from_str(&content)?;
let companies_path = paths.data_dir().join("companies.jsonl");
let log_path = paths.data_dir().join("companies_updates.log");
if let Some(parent) = companies_path.parent() {
tokio::fs::create_dir_all(parent).await?;
}
// === RECOVERY PHASE: Load checkpoint + replay log ===
let mut existing_companies: HashMap<String, CompanyCrossPlatformInfo> = HashMap::new();
let mut processed_names: std::collections::HashSet<String> = std::collections::HashSet::new();
if companies_path.exists() {
logger::log_info("Loading checkpoint from companies.jsonl...").await;
let existing_content = tokio::fs::read_to_string(&companies_path).await?;
for line in existing_content.lines() {
if line.trim().is_empty() {
continue;
}
match serde_json::from_str::<CompanyCrossPlatformInfo>(line) {
Ok(company) => {
processed_names.insert(company.name.clone());
existing_companies.insert(company.name.clone(), company);
}
Err(e) => {
logger::log_warn(&format!("Skipping invalid checkpoint line: {}", e)).await;
}
}
}
logger::log_info(&format!("Loaded checkpoint with {} companies", existing_companies.len())).await;
}
if log_path.exists() {
logger::log_info("Replaying update log...").await;
let log_content = tokio::fs::read_to_string(&log_path).await?;
let mut replayed = 0;
for line in log_content.lines() {
if line.trim().is_empty() {
continue;
}
match serde_json::from_str::<CompanyCrossPlatformInfo>(line) {
Ok(company) => {
processed_names.insert(company.name.clone());
existing_companies.insert(company.name.clone(), company);
replayed += 1;
}
Err(e) => {
logger::log_warn(&format!("Skipping invalid log line: {}", e)).await;
}
}
}
if replayed > 0 {
logger::log_info(&format!("Replayed {} updates from log", replayed)).await;
}
}
// === SETUP LOG WRITER TASK ===
let (write_tx, mut write_rx) = mpsc::channel::<LogCommand>(1000);
let log_file_init = OpenOptions::new()
.create(true)
.append(true)
.open(&log_path)
.await?;
let companies_path_clone = companies_path.clone();
let log_path_clone = log_path.clone();
let existing_companies_writer = Arc::new(tokio::sync::Mutex::new(existing_companies.clone()));
let write_tx_for_writer = write_tx.clone();
let writer_task = tokio::spawn(async move {
let mut log_file = log_file_init;
let mut writes_since_fsync = 0;
let mut last_fsync = std::time::Instant::now();
let mut updates_since_checkpoint = 0;
let mut count = 0;
let mut new_count = 0;
let mut updated_count = 0;
while let Some(cmd) = write_rx.recv().await {
match cmd {
LogCommand::Write(company) => {
// Write to log
let line = serde_json::to_string(&company).unwrap();
if let Err(e) = log_file.write_all(line.as_bytes()).await {
logger::log_error(&format!("Failed to write to log: {}", e)).await;
break;
}
if let Err(e) = log_file.write_all(b"\n").await {
logger::log_error(&format!("Failed to write newline: {}", e)).await;
break;
}
writes_since_fsync += 1;
updates_since_checkpoint += 1;
count += 1;
// Update in-memory state
let mut existing_companies = existing_companies_writer.lock().await;
let is_update = existing_companies.contains_key(&company.name);
existing_companies.insert(company.name.clone(), company);
drop(existing_companies);
if is_update {
updated_count += 1;
} else {
new_count += 1;
}
// Batched + time-based fsync
let should_fsync = writes_since_fsync >= FSYNC_BATCH_SIZE
|| last_fsync.elapsed().as_secs() >= FSYNC_INTERVAL_SECS;
if should_fsync {
if let Err(e) = log_file.flush().await {
logger::log_error(&format!("Failed to flush: {}", e)).await;
break;
}
if let Err(e) = log_file.sync_data().await {
logger::log_error(&format!("Failed to fsync: {}", e)).await;
break;
}
writes_since_fsync = 0;
last_fsync = std::time::Instant::now();
}
}
LogCommand::Checkpoint => {
if let Err(e) = log_file.flush().await {
logger::log_error(&format!("Failed to flush before checkpoint: {}", e)).await;
break;
}
if let Err(e) = log_file.sync_data().await {
logger::log_error(&format!("Failed to fsync before checkpoint: {}", e)).await;
break;
}
let existing_companies = existing_companies_writer.lock().await;
let companies_vec: Vec<_> = existing_companies.values().cloned().collect();
drop(existing_companies);
let temp_path = companies_path_clone.with_extension("tmp");
match tokio::fs::File::create(&temp_path).await {
Ok(mut temp_file) => {
let mut checkpoint_ok = true;
for company in &companies_vec {
if let Ok(line) = serde_json::to_string(company) {
if temp_file.write_all(line.as_bytes()).await.is_err() ||
temp_file.write_all(b"\n").await.is_err() {
checkpoint_ok = false;
break;
}
}
}
if checkpoint_ok {
if temp_file.flush().await.is_ok() &&
temp_file.sync_data().await.is_ok() {
drop(temp_file);
if tokio::fs::rename(&temp_path, &companies_path_clone).await.is_ok() {
if tokio::fs::remove_file(&log_path_clone).await.is_ok() {
logger::log_info(&format!(
"✓ Checkpoint created ({} companies), log cleared",
companies_vec.len()
)).await;
if let Ok(new_log) = OpenOptions::new()
.create(true)
.append(true)
.open(&log_path_clone)
.await {
log_file = new_log;
}
}
}
}
}
}
Err(e) => {
logger::log_error(&format!("Failed to create checkpoint temp file: {}", e)).await;
}
}
updates_since_checkpoint = 0;
}
LogCommand::Shutdown => {
logger::log_info("Writer shutting down...").await;
break;
}
}
// Periodic checkpoint trigger
if updates_since_checkpoint >= CHECKPOINT_INTERVAL {
let _ = write_tx.send(LogCommand::Checkpoint).await;
}
}
// Final fsync
let _ = log_file.flush().await;
let _ = log_file.sync_data().await;
logger::log_info(&format!(
"Writer finished: {} total ({} new, {} updated)",
count, new_count, updated_count
)).await;
(count, new_count, updated_count)
});
// === PARALLEL PROCESSING PHASE ===
logger::log_info(&format!(
"Starting parallel processing of {} companies (concurrency limit: {})",
securities.len(),
CONCURRENCY_LIMIT
)).await;
let mut processing_tasks = FuturesUnordered::new();
let mut processed = 0;
let total = securities.len();
for (name, company_info) in securities.into_iter() {
// Check shutdown before creating new tasks
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected, stopping task creation").await;
break;
}
// Wait if we hit concurrency limit
while processing_tasks.len() >= CONCURRENCY_LIMIT {
if let Some(result) = processing_tasks.next().await {
match result {
Ok(Ok(Some(company_result))) => {
let company_result: CompanyProcessResult = company_result;
let _ = write_tx_for_writer.send(LogCommand::Write(company_result.company)).await?;
processed += 1;
}
Ok(Ok(None)) => {
processed += 1;
}
Ok(Err(e)) => {
logger::log_warn(&format!("Company processing error: {}", e)).await;
processed += 1;
}
Err(e) => {
logger::log_error(&format!("Task panic: {}", e)).await;
processed += 1;
}
}
}
if shutdown_flag.load(Ordering::SeqCst) {
break;
}
}
if shutdown_flag.load(Ordering::SeqCst) {
break;
}
// Spawn new task
let pool = pool.clone();
let shutdown_flag = shutdown_flag.clone();
let existing_entry = existing_companies.get(&name).cloned();
let task = tokio::spawn(async move {
process_single_company_validated(
name,
company_info,
existing_entry,
&pool,
&shutdown_flag
).await
});
processing_tasks.push(task);
if processed % 10 == 0 && processed > 0 {
logger::log_info(&format!("Progress: {}/{} companies processed", processed, total)).await;
}
}
// Wait for remaining tasks
logger::log_info(&format!(
"Waiting for {} remaining tasks to complete...",
processing_tasks.len()
)).await;
while let Some(result) = processing_tasks.next().await {
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn("Shutdown detected during final task wait").await;
break;
}
match result {
Ok(Ok(Some(company_result))) => {
if write_tx_for_writer.send(LogCommand::Write(company_result.company)).await.is_err() {
logger::log_error("Writer task died").await;
break;
}
processed += 1;
}
Ok(Ok(None)) => {
processed += 1;
}
Ok(Err(e)) => {
logger::log_warn(&format!("Company processing error: {}", e)).await;
processed += 1;
}
Err(e) => {
logger::log_error(&format!("Task panic: {}", e)).await;
processed += 1;
}
}
}
// Signal writer to finish
let _ = write_tx_for_writer.send(LogCommand::Checkpoint).await;
let _ = write_tx_for_writer.send(LogCommand::Shutdown).await;
drop(write_tx_for_writer);
// Wait for writer to finish
let (final_count, final_new, final_updated) = writer_task.await
.unwrap_or((0, 0, 0));
logger::log_info(&format!(
"Completed: {} total companies ({} new, {} updated)",
final_count, final_new, final_updated
)).await;
Ok(final_count)
}
/// Scrape with retry, validation, and shutdown awareness
async fn scrape_with_retry(
pool: &Arc<ChromeDriverPool>,
isin: &str,
max_retries: u32,
shutdown_flag: &Arc<AtomicBool>,
) -> Result<Option<YahooCompanyDetails>> {
let mut retries = 0;
loop {
// Check shutdown before each attempt
if shutdown_flag.load(Ordering::SeqCst) {
return Err(anyhow!("Aborted due to shutdown"));
}
match scrape_company_details_by_isin(pool, isin, shutdown_flag).await {
Ok(result) => return Ok(result),
Err(e) => {
if retries >= max_retries {
logger::log_error(&format!(
"All {} retries exhausted for ISIN {}: {}",
max_retries, isin, e
)).await;
return Err(e);
}
let backoff_ms = 1000 * 2u64.pow(retries);
let jitter_ms = random_range(0, 500);
let total_delay = backoff_ms + jitter_ms;
logger::log_warn(&format!(
"Retry {}/{} for ISIN {} after {}ms: {}",
retries + 1, max_retries, isin, total_delay, e
)).await;
sleep(Duration::from_millis(total_delay)).await;
retries += 1;
}
}
}
}
/// UPDATED: Process single company with validation and shutdown checks
async fn process_single_company_validated(
name: String,
company_info: CompanyInfo,
existing_entry: Option<CompanyCrossPlatformInfo>,
pool: &Arc<ChromeDriverPool>,
shutdown_flag: &Arc<AtomicBool>,
) -> anyhow::Result<Option<CompanyProcessResult>> {
// Check shutdown at start
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn(&format!("Shutdown detected, skipping company: {}", name)).await;
return Ok(None);
}
let is_update = existing_entry.is_some();
let mut isin_tickers_map: HashMap<String, Vec<String>> =
existing_entry
.as_ref()
.map(|e| e.isin_tickers_map.clone())
.unwrap_or_default();
let mut sector = existing_entry.as_ref().and_then(|e| e.sector.clone());
let mut exchange = existing_entry.as_ref().and_then(|e| e.exchange.clone());
// Collect unique ISIN-ticker pairs
let mut unique_isin_ticker_pairs: HashMap<String, Vec<String>> = HashMap::new();
for figi_infos in company_info.securities.values() {
for figi_info in figi_infos {
if !figi_info.isin.is_empty() {
let tickers = unique_isin_ticker_pairs
.entry(figi_info.isin.clone())
.or_insert_with(Vec::new);
if !figi_info.ticker.is_empty() && !tickers.contains(&figi_info.ticker) {
tickers.push(figi_info.ticker.clone());
}
}
}
}
// Process each ISIN with validation
for (isin, figi_tickers) in unique_isin_ticker_pairs {
// Check shutdown before each ISIN
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn(&format!(
"Shutdown detected while processing company: {}",
name
)).await;
break;
}
let tickers = isin_tickers_map
.entry(isin.clone())
.or_insert_with(Vec::new);
for figi_ticker in figi_tickers {
if !tickers.contains(&figi_ticker) {
tickers.push(figi_ticker);
}
}
let has_yahoo_ticker = tickers.iter().any(|t| t.starts_with("YAHOO:"));
if !has_yahoo_ticker {
logger::log_info(&format!("Fetching Yahoo details for {} (ISIN: {})", name, isin)).await;
match scrape_with_retry(pool, &isin, 3, shutdown_flag).await {
Ok(Some(details)) => {
logger::log_info(&format!(
"✓ Found Yahoo ticker {} for ISIN {} (company: {})",
details.ticker, isin, name
)).await;
tickers.push(format!("YAHOO:{}", details.ticker));
if sector.is_none() && details.sector.is_some() {
sector = details.sector.clone();
logger::log_info(&format!(" Sector: {}", details.sector.as_ref().unwrap())).await;
}
if exchange.is_none() && details.exchange.is_some() {
exchange = details.exchange.clone();
logger::log_info(&format!(" Exchange: {}", details.exchange.as_ref().unwrap())).await;
}
},
Ok(None) => {
logger::log_warn(&format!("◯ No search results for ISIN {} (company: {})", isin, name)).await;
tickers.push("YAHOO:NO_RESULTS".to_string());
},
Err(e) => {
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn(&format!("Shutdown during scrape for ISIN {}", isin)).await;
break;
}
logger::log_warn(&format!(
"✗ Yahoo lookup error for ISIN {} (company: {}): {}",
isin, name, e
)).await;
// Continue with next ISIN
}
}
}
}
// Final shutdown check before returning result
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn(&format!(
"Shutdown detected, discarding incomplete result for: {}",
name
)).await;
return Ok(None);
}
if !isin_tickers_map.is_empty() {
let company_entry = CompanyCrossPlatformInfo {
name: name.clone(),
isin_tickers_map,
sector,
exchange,
};
Ok(Some(CompanyProcessResult {
company: company_entry,
is_update,
}))
} else {
logger::log_warn(&format!("No ISINs found for company: {}", name)).await;
Ok(None)
}
}

273
src/corporate/yahoo.rs
View File

@@ -1,17 +1,15 @@
// src/corporate/yahoo.rs
use super::{types::*, helpers::*};
// src/corporate/yahoo.rs - UPDATED WITH DATA INTEGRITY FIXES
use super::{types::*, helpers::*, page_validation::*};
use crate::{scraper::webdriver::*, util::{directories::DataPaths}};
use event_backtest_engine::logger;
use crate::logger;
use fantoccini::{Client, Locator};
use rand::Rng;
use serde::{Deserialize, Serialize};
use tokio::{time::{Duration as TokioDuration, sleep}};
use std::{sync::Arc};
use tokio::time::{Duration as TokioDuration, sleep, timeout};
use std::{sync::Arc, sync::atomic::{AtomicBool, Ordering}};
use anyhow::{anyhow, Result};
const YAHOO_COMPANY_EXTRACTION_JS: &str = include_str!("yahoo_company_extraction.js");
/// Mapping existing
/// getting historical stock price data daily (xxxx - 2025) and hourly (last 30 days)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum YahooTickerResult {
@@ -21,6 +19,16 @@ pub enum YahooTickerResult {
AmbiguousResults,
}
#[derive(Debug, Deserialize)]
pub struct ExtractionMetadata {
#[serde(rename = "selectedRowIndex")]
pub selected_row_index: usize,
#[serde(rename = "validFieldCount")]
pub valid_field_count: usize,
#[serde(rename = "totalRows")]
pub total_rows: usize,
}
#[derive(Debug, Deserialize)]
pub struct ExtractionResult {
status: String,
@@ -29,6 +37,8 @@ pub struct ExtractionResult {
exchange: Option<String>,
#[serde(default)]
error_message: Option<String>,
#[serde(default)]
metadata: Option<ExtractionMetadata>,
}
impl YahooTickerResult {
@@ -53,47 +63,227 @@ impl YahooTickerResult {
}
}
/// UPDATED: Scrape company details with full validation and shutdown support
pub async fn scrape_company_details_by_isin(
pool: &Arc<ChromeDriverPool>,
isin: &str,
shutdown_flag: &Arc<AtomicBool>,
) -> anyhow::Result<Option<YahooCompanyDetails>> {
let isin = isin.to_string();
pool.execute(format!("https://finance.yahoo.com/lookup/?s={}", isin), move |client| {
let isin = isin.clone();
// Check shutdown before starting
if shutdown_flag.load(Ordering::SeqCst) {
logger::log_warn(&format!("Shutdown detected, skipping ISIN: {}", isin)).await;
return Ok(None);
}
let isin_owned = isin.to_string();
let shutdown_clone = Arc::clone(shutdown_flag);
let url = format!("https://finance.yahoo.com/lookup/?s={}", isin);
pool.execute(url.clone(), move |client| {
let isin = isin_owned.clone();
let shutdown = shutdown_clone.clone();
Box::pin(async move {
sleep(TokioDuration::from_millis(1000)).await;
// Check shutdown during task execution
if shutdown.load(Ordering::SeqCst) {
return Err(anyhow!("Task aborted due to shutdown"));
}
// Random delay
let delay = rand::rng().random_range(800..1500);
sleep(TokioDuration::from_millis(delay)).await;
// Reject cookies
reject_yahoo_cookies(&client).await?;
sleep(TokioDuration::from_millis(1000)).await;
extract_company_details(&client, &isin).await
// Check shutdown again
if shutdown.load(Ordering::SeqCst) {
return Err(anyhow!("Task aborted due to shutdown"));
}
// CRITICAL: Validate navigation succeeded
let expected_fragment = format!("lookup/?s={}", isin);
match verify_navigation(&client, &expected_fragment, 5).await {
Ok(_) => {
logger::log_info(&format!("✓ Navigation validated for ISIN: {}", isin)).await;
}
Err(e) => {
logger::log_error(&format!(
"Navigation verification failed for ISIN {}: {}",
isin, e
)).await;
// Clear browser state before returning error
clear_browser_state(&client).await.ok();
return Err(e);
}
}
// Additional content validation
let page_ready: bool = client
.execute(
r#"
const table = document.querySelector('#main-content-wrapper > section > section.container.yf-1omxedn > div.tableContainer.yf-1omxedn > div > table');
const noData = document.querySelector('#main-content-wrapper > section > div.noData.yf-1omxedn');
return !!(table || noData);
"#,
vec![],
)
.await?
.as_bool()
.unwrap_or(false);
if !page_ready {
logger::log_error(&format!(
"Page content not ready for ISIN {} - neither table nor no-data element found",
isin
)).await;
clear_browser_state(&client).await.ok();
return Err(anyhow!("Page content not ready"));
}
logger::log_info(&format!("✓ Page content validated for ISIN: {}", isin)).await;
// Check shutdown before extraction
if shutdown.load(Ordering::SeqCst) {
return Err(anyhow!("Task aborted due to shutdown"));
}
// Random delay before extraction
let delay = rand::rng().random_range(800..1500);
sleep(TokioDuration::from_millis(delay)).await;
// Now safe to extract
extract_company_details_validated(&client, &isin).await
})
}).await
}
/// UPDATED: Extract with additional URL validation
async fn extract_company_details_validated(
client: &Client,
isin: &str,
) -> Result<Option<YahooCompanyDetails>> {
// Double-check URL is still correct before extraction
let current_url = client.current_url().await?;
if !current_url.as_str().contains(isin) {
logger::log_error(&format!(
"URL mismatch before extraction: expected ISIN '{}' in URL, got '{}'",
isin,
current_url.as_str()
)).await;
clear_browser_state(client).await.ok();
return Err(anyhow!("URL mismatch - possible stale page"));
}
// Run extraction
let result = extract_company_details(client, isin).await?;
// Validate extraction result
if let Some(ref details) = result {
logger::log_info(&format!(
"✓ Extracted ticker '{}' for ISIN {} (sector: {:?}, exchange: {:?})",
details.ticker, isin, details.sector, details.exchange
)).await;
} else {
logger::log_info(&format!(
"No ticker found for ISIN {} (legitimately not found)",
isin
)).await;
}
Ok(result)
}
pub async fn extract_company_details(
client: &Client,
_isin: &str,
) -> Result<Option<YahooCompanyDetails>> {
// Wait for page to load - look for either the table or the no-data element
let wait_result: Result<Result<bool, anyhow::Error>> = timeout(
TokioDuration::from_secs(30),
async {
for _ in 0..60 {
let has_content: bool = client
.execute(
r#"
const table = document.querySelector('#main-content-wrapper > section > section.container.yf-1omxedn > div.tableContainer.yf-1omxedn > div > table');
const noData = document.querySelector('#main-content-wrapper > section > div.noData.yf-1omxedn');
return !!(table || noData);
"#,
vec![],
)
.await
.map_err(|e| anyhow!("Execute error: {}", e))?
.as_bool()
.unwrap_or(false);
if has_content {
return Ok(true);
}
sleep(TokioDuration::from_millis(500)).await;
}
Ok(false)
},
)
.await
.map_err(|_| anyhow!("Timeout waiting for Yahoo Finance page to load"));
match wait_result {
Err(_) => {
return Err(anyhow!("Timeout waiting for Yahoo Finance page to load"));
},
Ok(Err(e)) => {
return Err(anyhow!("Error checking page content: {}", e));
},
Ok(Ok(false)) => {
logger::log_warn("Page content not found after waiting, attempting extraction anyway").await;
},
Ok(Ok(true)) => {
logger::log_info("Page content detected, proceeding with extraction").await;
}
}
// Execute the JavaScript extraction script
let result = client.execute(YAHOO_COMPANY_EXTRACTION_JS, vec![]).await?;
// Log the raw result for debugging
logger::log_info(&format!("JavaScript extraction raw result: {:?}", result)).await;
// Check if result is null
if result.is_null() {
return Err(anyhow!("JavaScript returned null - page may not be fully loaded or script failed"));
}
// Parse the JSON result
let extraction: ExtractionResult = serde_json::from_value(result)
.map_err(|e| anyhow!("Failed to parse extraction result: {}", e))?;
let extraction: ExtractionResult = serde_json::from_value(result.clone())
.map_err(|e| {
let result_str = serde_json::to_string_pretty(&result).unwrap_or_else(|_| format!("{:?}", result));
anyhow!("Failed to parse extraction result: {}. Raw result: {}", e, result_str)
})?;
match extraction.status.as_str() {
"found" => {
if let Some(ticker) = extraction.ticker {
if let Some(ref metadata) = extraction.metadata {
logger::log_info(&format!(
"Selected row {} with {} valid fields out of {} total rows",
metadata.selected_row_index,
metadata.valid_field_count,
metadata.total_rows
)).await;
}
Ok(Some(YahooCompanyDetails {
ticker,
sector: extraction.sector,
exchange: extraction.exchange,
}))
} else {
Ok(None)
Err(anyhow!("Status 'found' but no ticker present"))
}
},
"no_results" => Ok(None),
"not_found" => Ok(None),
"error" => {
let error_msg = extraction.error_message.unwrap_or_else(|| "Unknown error".to_string());
Err(anyhow!("JavaScript extraction error: {}", error_msg))
@@ -116,19 +306,6 @@ pub async fn get_all_tickers_from_companies_jsonl(paths: &DataPaths) -> anyhow::
Ok(tickers)
}
/// Fetches earnings events for a ticker using a dedicated ScrapeTask.
///
/// This function creates and executes a ScrapeTask to navigate to the Yahoo Finance earnings calendar,
/// reject cookies, and extract the events.
///
/// # Arguments
/// * `ticker` - The stock ticker symbol.
///
/// # Returns
/// A vector of CompanyEvent structs on success.
///
/// # Errors
/// Returns an error if the task execution fails, e.g., chromedriver spawn or navigation issues.
pub async fn fetch_earnings_with_pool(
pool: &Arc<ChromeDriverPool>,
ticker: &str,
@@ -147,40 +324,6 @@ pub async fn fetch_earnings_with_pool(
}).await
}
/// Extracts earnings events from the currently loaded Yahoo Finance earnings calendar page.
///
/// This function assumes the client is already navigated to the correct URL (e.g.,
/// https://finance.yahoo.com/calendar/earnings?symbol={ticker}) and cookies are handled.
///
/// It waits for the earnings table, extracts rows, parses cells into CompanyEvent structs,
/// and handles date parsing, float parsing, and optional fields.
///
/// # Arguments
/// * `client` - The fantoccini Client with the page loaded.
/// * `ticker` - The stock ticker symbol for the events.
///
/// # Returns
/// A vector of CompanyEvent on success.
///
/// # Errors
/// Returns an error if:
/// - Table or elements not found.
/// - Date or float parsing fails.
/// - WebDriver operations fail.
///
/// # Examples
///
/// ```no_run
/// use fantoccini::Client;
/// use crate::corporate::scraper::extract_earnings;
///
/// #[tokio::main]
/// async fn main() -> Result<()> {
/// // Assume client is set up and navigated
/// let events = extract_earnings(&client, "AAPL").await?;
/// Ok(())
/// }
/// ```
pub async fn extract_earnings_events(client: &Client, ticker: &str) -> Result<Vec<CompanyEvent>> {
// Wait for the table to load
let table = client

214
src/corporate/yahoo_company_extraction.js
View File

@@ -1,61 +1,214 @@
// yahoo_company_extraction.js
// JavaScript extraction script for Yahoo Finance company details
// Used to extract ticker, sector, and exchange from Yahoo Finance search results
// Only ticker is mandatory - sector and exchange are optional fields
(function() {
// Example selectors:
// with results:
// document.querySelector("#main-content-wrapper > section > section.container.yf-1omxedn > div.tableContainer.yf-1omxedn > div > table")
// document.querySelector("#\\30 > td:nth-child(1) > span > div > a")
// document.querySelector("#\\30 > td:nth-child(2) > span > div")
// document.querySelector("#\\30 > td:nth-child(3) > span > div")
// document.querySelector("#\\30 > td:nth-child(4) > span > div > a")
// document.querySelector("#\\30 > td:nth-child(5) > span > div")
// document.querySelector("#\\30 > td:nth-child(6) > span > div")
// row with no result:
// document.querySelector("#\\32 > td:nth-child(4) > span > p")
// no results:
// document.querySelector("#main-content-wrapper > section > div.noData.yf-1omxedn")
// Using a wrapper to ensure the result is properly captured
var extractionResult = (function() {
try {
// Check for "No results found" message
const noDataElement = document.querySelector('.noData');
// Check for "No results found" message using exact selector
const noDataElement = document.querySelector('#main-content-wrapper > section > div.noData.yf-1omxedn');
if (noDataElement) {
return { status: 'no_results', ticker: null, sector: null, exchange: null };
}
// Find the results table
const table = document.querySelector('table.markets-table');
// Find the results table using exact selector
const table = document.querySelector('#main-content-wrapper > section > section.container.yf-1omxedn > div.tableContainer.yf-1omxedn > div > table');
if (!table) {
return { status: 'no_results', ticker: null, sector: null, exchange: null };
}
// Find the first row in tbody
const firstRow = table.querySelector('tbody tr');
if (!firstRow) {
// Find all rows in tbody
const allRows = table.querySelectorAll('tbody tr');
if (!allRows || allRows.length === 0) {
return { status: 'no_results', ticker: null, sector: null, exchange: null };
}
// Extract ticker from first column (td:nth-child(1))
const tickerCell = firstRow.querySelector('td:nth-child(1)');
const ticker = tickerCell ? tickerCell.textContent.trim() : '';
if (!ticker) {
return { status: 'not_found', ticker: null, sector: null, exchange: null };
// Helper function to safely extract text content
function extractText(element) {
if (!element) return '';
const text = element.textContent.trim();
return text;
}
// Extract sector from column 4 (td:nth-child(4) > span > div > a)
const sectorCell = firstRow.querySelector('td:nth-child(4) span div a');
let sector = sectorCell ? sectorCell.textContent.trim() : '';
// Helper function to check if a cell actually contains data
// Multiple indicators are used to determine if data is present
function hasValidData(cellElement) {
if (!cellElement) return false;
// Normalize empty/invalid values to null
if (!sector || sector === '-' || sector === 'N/A') {
sector = null;
// Indicator 1: Check if the cell contains a <p> tag (Yahoo uses this for "no data")
const pTag = cellElement.querySelector('p');
if (pTag) return false;
// Indicator 2: Check the direct child structure
// Valid data cells have: td > span > div or td > span > div > a
// Invalid data cells have: td > span > p
const span = cellElement.querySelector('span');
if (span) {
const directChildren = Array.from(span.children);
// If the only or first child is a <p>, it's likely "no data"
if (directChildren.length > 0 && directChildren[0].tagName === 'P') {
return false;
}
}
// Indicator 3: Check text content
const text = extractText(cellElement);
if (!text) return false;
const normalized = text.toLowerCase().trim();
// Common "no data" indicators
const noDataIndicators = [
'-',
'n/a',
'na',
'none',
'not available',
'no data',
'--',
'—', // em dash
'–', // en dash
];
if (noDataIndicators.includes(normalized)) {
return false;
}
// Indicator 4: Check for common CSS classes that indicate empty state
const classIndicators = ['empty', 'no-data', 'na', 'null', 'undefined'];
const classList = cellElement.className || '';
for (const indicator of classIndicators) {
if (classList.includes(indicator)) {
return false;
}
}
// Indicator 5: Check if cell has an anchor tag (usually indicates real data)
const hasLink = cellElement.querySelector('a') !== null;
// Indicator 6: Check if there's actual substantial content
// If text is very short (1-2 chars) and not alphanumeric, it's likely not real data
if (text.length <= 2 && !/[a-zA-Z0-9]/.test(text)) {
return false;
}
// If we passed all checks, consider it valid data
return true;
}
// Extract exchange from column 6 (td:nth-child(6) > span)
const exchangeCell = firstRow.querySelector('td:nth-child(6) span');
let exchange = exchangeCell ? exchangeCell.textContent.trim() : '';
// Helper function to extract and normalize data from a cell
function extractCellData(cellElement) {
if (!cellElement) return null;
if (!hasValidData(cellElement)) return null;
// Normalize empty/invalid values to null
if (!exchange || exchange === '-' || exchange === 'N/A') {
exchange = null;
const text = extractText(cellElement);
return text || null;
}
// Helper function to extract and normalize data from a row
function extractRowData(row) {
// Extract ticker from column 1 (td:nth-child(1))
const tickerCell = row.querySelector('td:nth-child(1)');
const ticker = extractCellData(tickerCell);
// Extract sector from column 4 (td:nth-child(4))
const sectorCell = row.querySelector('td:nth-child(4)');
const sector = extractCellData(sectorCell);
// Extract exchange from column 6 (td:nth-child(6))
const exchangeCell = row.querySelector('td:nth-child(6)');
const exchange = extractCellData(exchangeCell);
return { ticker, sector, exchange };
}
// Helper function to count non-null fields (data completeness counter)
function countValidFields(data) {
let count = 0;
if (data.ticker) count++;
if (data.sector) count++;
if (data.exchange) count++;
return count;
}
// Helper function to score a row (prioritize rows with more complete data)
function scoreRow(data) {
let score = 0;
// Ticker is mandatory and gets highest weight
if (data.ticker) score += 100;
// Sector and exchange are nice-to-have
if (data.sector) score += 10;
if (data.exchange) score += 10;
return score;
}
// Extract data from all rows and find the one with most complete data
let bestRow = null;
let maxScore = -1;
let rowIndex = 0;
for (const row of allRows) {
const data = extractRowData(row);
const score = scoreRow(data);
// Select row with highest score (most complete data)
// If tied, first row wins
if (score > maxScore) {
bestRow = data;
maxScore = score;
bestRow.rowIndex = rowIndex;
bestRow.validFieldCount = countValidFields(data);
bestRow.score = score;
}
rowIndex++;
}
// Ticker is mandatory - return error status if not found
if (!bestRow || !bestRow.ticker) {
return {
status: 'error',
error_message: 'No ticker found in any row',
ticker: null,
sector: null,
exchange: null
};
}
// Return success with ticker (mandatory) and optional sector/exchange
// Include metadata about which row was selected and how many valid fields it had
return {
status: 'found',
ticker: ticker,
sector: sector,
exchange: exchange
ticker: bestRow.ticker,
sector: bestRow.sector,
exchange: bestRow.exchange,
metadata: {
selectedRowIndex: bestRow.rowIndex,
validFieldCount: bestRow.validFieldCount,
score: bestRow.score,
totalRows: allRows.length
}
};
} catch (error) {
// Only catch unexpected errors during extraction
return {
status: 'error',
error_message: error.toString(),
@@ -65,3 +218,6 @@
};
}
})();
// Return the result explicitly
return extractionResult;

2
src/economic/scraper.rs
View File

@@ -1,6 +1,6 @@
// src/economic/scraper.rs
use super::types::{EconomicEvent};
use event_backtest_engine::logger;
use crate::logger;
use fantoccini::Client;
use tokio::time::{sleep, Duration};

2
src/economic/storage.rs
View File

@@ -84,7 +84,7 @@ pub async fn load_events_in_batches(
Ok(all_events.into_iter())
}
/// NEW: Build a lightweight index instead of loading all events
/// Build a lightweight index instead of loading all events
#[derive(Debug, Clone)]
pub struct EventIndex {
pub key: String,

6
src/lib.rs
View File

@@ -6,10 +6,12 @@
pub mod config;
pub mod scraper;
pub mod util;
pub mod monitoring;
pub mod economic;
pub mod corporate;
// Re-export commonly used types for convenience
pub use monitoring::{init_monitoring, ConfigSnapshot, MonitoringEvent};
pub use config::Config;
pub use scraper::webdriver::{ChromeDriverPool, ChromeInstance, ScrapeTask};
pub use util::directories::DataPaths;
pub use util::logger;
pub use util::opnv;

77
src/main.rs
View File

@@ -1,30 +1,70 @@
// src/main.rs
mod config;
mod corporate;
mod economic;
mod util;
mod scraper;
use web_scraper::{*, scraper, economic, corporate};
use anyhow::Result;
use config::Config;
use web_scraper::config::Config;
use scraper::docker_vpn_proxy::{DockerVpnProxyPool, cleanup_all_proxy_containers};
use scraper::webdriver::ChromeDriverPool;
use util::directories::DataPaths;
use util::{logger, opnv};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::process::Command;
#[tokio::main]
async fn main() -> Result<()> {
let output = if cfg!(target_os = "windows") {
Command::new("cmd")
.args(["/C", "docker desktop start"])
.output()
.expect("failed to execute process")
} else {
Command::new("sh")
.arg("-c")
.arg("echo hello")
.output()
.expect("failed to execute process")
};
let _start_docker_desktop = output.stdout;
cleanup_all_proxy_containers().await.ok();
let config = Config::load().map_err(|err| {
eprintln!("Failed to load config: {}", err);
err
})?;
let config = match Config::load() {
Ok(cfg) => cfg,
Err(_) => {
eprintln!("Using default configuration");
Config::default()
}
};
let paths = DataPaths::new(".")?;
// Initialize monitoring system
let config_snapshot = ConfigSnapshot {
max_parallel_instances: config.max_parallel_instances,
max_tasks_per_instance: config.max_tasks_per_instance,
enable_vpn_rotation: config.enable_vpn_rotation,
max_requests_per_session: config.max_requests_per_session,
min_request_interval_ms: config.min_request_interval_ms,
max_retry_attempts: config.max_retry_attempts,
};
let (monitoring_handle, _monitoring_task) = init_monitoring(
config_snapshot,
paths.logs_dir().to_path_buf(),
3030, // Dashboard port
).await?;
// Emit pool initialization event
monitoring_handle.emit(monitoring::MonitoringEvent::PoolInitialized {
pool_size: config.max_parallel_instances,
with_proxy: config.enable_vpn_rotation,
with_rotation: config.max_tasks_per_instance > 0,
});
logger::log_info("Monitoring dashboard available at http://localhost:3030").await;
logger::init_debug_logger(paths.logs_dir()).await.ok();
logger::log_info("=== Event Backtest Engine Started ===").await;
logger::log_info(&format!(
@@ -40,7 +80,8 @@ async fn main() -> Result<()> {
// === Step 1: Fetch VPNBook configs ===
let proxy_pool: Option<Arc<DockerVpnProxyPool>> = if config.enable_vpn_rotation {
logger::log_info("VPN Rotation Enabled – Fetching latest VPNBook configs").await;
let temp_pool = Arc::new(ChromeDriverPool::new_with_proxy_and_task_limit(config.max_parallel_instances, None, config.max_tasks_per_instance).await?);
let temp_pool = Arc::new(ChromeDriverPool::new_with_proxy_and_task_limit(None, &config, Some(monitoring_handle.clone())).await?);
let (username, password, _files) = opnv::fetch_vpnbook_configs(&temp_pool, paths.cache_dir()).await?;
logger::log_info(&format!("VPNBook credentials → User: {}", username)).await;
@@ -56,6 +97,16 @@ async fn main() -> Result<()> {
let pp = Arc::new(DockerVpnProxyPool::new(paths.cache_openvpn_dir(), username, password).await?);
logger::log_info(&format!("All {} Docker proxy containers started and ready", pp.num_proxies())).await;
for i in 0..pp.num_proxies() {
if let Some(proxy_info) = pp.get_proxy_info(i) {
monitoring_handle.emit(monitoring::MonitoringEvent::ProxyConnected {
container_name: proxy_info.container_name.clone(),
ip_address: proxy_info.ip_address.clone(),
port: proxy_info.port,
});
}
}
Some(pp)
}
} else {
@@ -71,9 +122,9 @@ async fn main() -> Result<()> {
let pool = Arc::new(
if task_limit > 0 {
ChromeDriverPool::new_with_proxy_and_task_limit(pool_size, proxy_pool.clone(), task_limit).await?
ChromeDriverPool::new_with_proxy_and_task_limit(proxy_pool.clone(), &config, Some(monitoring_handle.clone())).await?
} else {
ChromeDriverPool::new_with_proxy(pool_size, proxy_pool.clone()).await?
ChromeDriverPool::new_with_proxy_and_task_limit(proxy_pool.clone(), &config, Some(monitoring_handle.clone())).await?
}
);

644
src/monitoring/dashboard.html Normal file
View File

@@ -0,0 +1,644 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Scraper Monitoring Dashboard</title>
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: 'Courier New', monospace;
background: #1a1a1a;
color: #f0f0f0;
padding: 20px;
font-size: 13px;
}
.header {
text-align: center;
padding: 20px;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
border-radius: 8px;
margin-bottom: 20px;
}
.header h1 {
font-size: 28px;
margin-bottom: 5px;
}
.header .uptime {
font-size: 14px;
opacity: 0.9;
}
.section {
background: #2a2a2a;
border: 2px solid #444;
padding: 15px;
margin-bottom: 20px;
border-radius: 5px;
}
.section-title {
font-size: 16px;
font-weight: bold;
margin-bottom: 12px;
padding-bottom: 8px;
border-bottom: 2px solid #667eea;
display: flex;
align-items: center;
gap: 8px;
}
/* Config Section */
.config-grid {
display: grid;
grid-template-columns: repeat(3, 1fr);
gap: 15px;
}
.config-item {
background: #333;
padding: 12px;
border-radius: 4px;
border-left: 3px solid #667eea;
}
.config-label {
color: #888;
font-size: 11px;
text-transform: uppercase;
margin-bottom: 5px;
}
.config-value {
color: #4CAF50;
font-size: 18px;
font-weight: bold;
}
/* Instance Grid */
.instance-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(450px, 1fr));
gap: 15px;
margin-top: 10px;
}
.instance-box {
background: #333;
border: 2px solid #555;
border-radius: 5px;
padding: 0;
display: flex;
gap: 0;
overflow: hidden;
transition: border-color 0.3s;
}
.instance-box.status-idle {
border-color: #666;
}
.instance-box.status-active {
border-color: #4CAF50;
box-shadow: 0 0 10px rgba(76, 175, 80, 0.3);
}
.instance-box.status-renewing {
border-color: #FF9800;
box-shadow: 0 0 10px rgba(255, 152, 0, 0.3);
}
.instance-box.status-error {
border-color: #f44336;
box-shadow: 0 0 10px rgba(244, 67, 54, 0.3);
}
.instance-side,
.proxy-side {
flex: 1;
padding: 12px;
}
.instance-side {
background: #3a3a3a;
border-right: 1px solid #555;
}
.proxy-side {
background: #2a3a4a;
}
.side-header {
font-weight: bold;
font-size: 14px;
margin-bottom: 10px;
padding-bottom: 5px;
border-bottom: 1px solid #555;
display: flex;
align-items: center;
gap: 5px;
}
.status-badge {
display: inline-block;
padding: 2px 8px;
border-radius: 3px;
font-size: 11px;
font-weight: bold;
text-transform: uppercase;
}
.status-badge.idle {
background: #666;
color: #fff;
}
.status-badge.active {
background: #4CAF50;
color: #fff;
}
.status-badge.renewing {
background: #FF9800;
color: #fff;
}
.status-badge.error {
background: #f44336;
color: #fff;
}
.metric-row {
display: flex;
justify-content: space-between;
padding: 4px 0;
font-size: 12px;
border-bottom: 1px solid #444;
}
.metric-row:last-child {
border-bottom: none;
}
.metric-label {
color: #888;
}
.metric-value {
color: #4CAF50;
font-weight: bold;
}
.metric-value.warning {
color: #FF9800;
}
.metric-value.danger {
color: #f44336;
}
.current-url {
margin-top: 8px;
padding-top: 8px;
border-top: 1px solid #555;
font-size: 11px;
color: #aaa;
word-wrap: break-word;
}
.no-proxy {
text-align: center;
color: #666;
padding: 30px 10px;
font-style: italic;
}
/* Global Stats */
.stats-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
gap: 12px;
margin-top: 10px;
}
.stat-box {
background: #333;
padding: 15px;
border-radius: 5px;
text-align: center;
border-left: 4px solid #667eea;
}
.stat-value {
font-size: 28px;
font-weight: bold;
color: #4CAF50;
margin-bottom: 5px;
}
.stat-label {
font-size: 11px;
color: #888;
text-transform: uppercase;
}
/* Logs */
.log-container {
max-height: 300px;
overflow-y: auto;
background: #1a1a1a;
padding: 10px;
border-radius: 4px;
font-size: 12px;
}
.log-container::-webkit-scrollbar {
width: 8px;
}
.log-container::-webkit-scrollbar-track {
background: #2a2a2a;
}
.log-container::-webkit-scrollbar-thumb {
background: #667eea;
border-radius: 4px;
}
.log-entry {
padding: 4px 0;
border-bottom: 1px solid #333;
display: flex;
gap: 10px;
}
.log-entry:last-child {
border-bottom: none;
}
.log-time {
color: #666;
font-weight: bold;
min-width: 70px;
}
.log-message {
flex: 1;
}
.log-message.info {
color: #4CAF50;
}
.log-message.warn {
color: #FF9800;
}
.log-message.error {
color: #f44336;
}
.connection-status {
position: fixed;
top: 20px;
right: 20px;
padding: 8px 15px;
border-radius: 20px;
font-size: 12px;
font-weight: bold;
z-index: 1000;
}
.connection-status.connected {
background: #4CAF50;
color: white;
}
.connection-status.disconnected {
background: #f44336;
color: white;
}
@keyframes pulse {
0%, 100% {
opacity: 1;
}
50% {
opacity: 0.5;
}
}
.pulse {
animation: pulse 2s infinite;
}
</style>
</head>
<body>
<div class="connection-status" id="connection-status">
Connecting...
</div>
<div class="header">
<h1>🚀 Scraper Monitoring Dashboard</h1>
<div class="uptime" id="uptime">Uptime: Loading...</div>
</div>
<!-- Config Section -->
<div class="section">
<div class="section-title">⚙️ CONFIGURATION</div>
<div class="config-grid" id="config"></div>
</div>
<!-- Pool Status Section -->
<div class="section">
<div class="section-title">🔧 POOL STATUS</div>
<div class="instance-grid" id="instances"></div>
</div>
<!-- Global Metrics Section -->
<div class="section">
<div class="section-title">📊 GLOBAL METRICS</div>
<div class="stats-grid" id="global-stats"></div>
</div>
<!-- Logs Section -->
<div class="section">
<div class="section-title">📝 RECENT LOGS</div>
<div class="log-container" id="logs"></div>
</div>
<script>
let ws = null;
let reconnectInterval = null;
function connect() {
ws = new WebSocket('ws://' + window.location.host + '/ws');
ws.onopen = () => {
console.log('WebSocket connected');
updateConnectionStatus(true);
if (reconnectInterval) {
clearInterval(reconnectInterval);
reconnectInterval = null;
}
};
ws.onmessage = (event) => {
try {
const state = JSON.parse(event.data);
updateDashboard(state);
} catch (error) {
console.error('Failed to parse message:', error);
}
};
ws.onclose = () => {
console.log('WebSocket disconnected');
updateConnectionStatus(false);
// Attempt to reconnect every 3 seconds
if (!reconnectInterval) {
reconnectInterval = setInterval(() => {
console.log('Attempting to reconnect...');
connect();
}, 3000);
}
};
ws.onerror = (error) => {
console.error('WebSocket error:', error);
};
}
function updateConnectionStatus(connected) {
const status = document.getElementById('connection-status');
if (connected) {
status.textContent = '● Connected';
status.className = 'connection-status connected';
} else {
status.textContent = '● Disconnected';
status.className = 'connection-status disconnected pulse';
}
}
function updateDashboard(state) {
updateConfig(state.config);
updateInstances(state.instances);
updateGlobalStats(state.global);
updateLogs(state.logs);
}
function updateConfig(config) {
const container = document.getElementById('config');
container.innerHTML = `
<div class="config-item">
<div class="config-label">Parallel Instances</div>
<div class="config-value">${config.max_parallel_instances}</div>
</div>
<div class="config-item">
<div class="config-label">Tasks per Instance</div>
<div class="config-value">${config.max_tasks_per_instance || 'Unlimited'}</div>
</div>
<div class="config-item">
<div class="config-label">VPN Rotation</div>
<div class="config-value">${config.enable_vpn_rotation ? '✓ Enabled' : '✗ Disabled'}</div>
</div>
<div class="config-item">
<div class="config-label">Requests per Session</div>
<div class="config-value">${config.max_requests_per_session}</div>
</div>
<div class="config-item">
<div class="config-label">Min Request Interval</div>
<div class="config-value">${config.min_request_interval_ms}ms</div>
</div>
<div class="config-item">
<div class="config-label">Max Retry Attempts</div>
<div class="config-value">${config.max_retry_attempts}</div>
</div>
`;
}
function updateInstances(instances) {
const container = document.getElementById('instances');
if (!instances || instances.length === 0) {
container.innerHTML = '<div style="text-align: center; padding: 40px; color: #666;">No instances available</div>';
return;
}
container.innerHTML = instances.map(inst => {
const statusClass = `status-${inst.status}`;
const proxy = inst.connected_proxy;
const successRate = inst.total_requests > 0
? ((inst.success_count / inst.total_requests) * 100).toFixed(1)
: '0.0';
return `
<div class="instance-box ${statusClass}">
<div class="instance-side">
<div class="side-header">
🖥️ Instance #${inst.id}
<span class="status-badge ${inst.status}">${inst.status}</span>
</div>
<div class="metric-row">
<span class="metric-label">Current Tasks</span>
<span class="metric-value ${inst.tasks_current_session >= inst.tasks_max ? 'warning' : ''}">
${inst.tasks_current_session}/${inst.tasks_max}
</span>
</div>
<div class="metric-row">
<span class="metric-label">Session Requests</span>
<span class="metric-value">${inst.session_requests}</span>
</div>
<div class="metric-row">
<span class="metric-label">Total Requests</span>
<span class="metric-value">${inst.total_requests}</span>
</div>
<div class="metric-row">
<span class="metric-label">Success / Fail</span>
<span class="metric-value">${inst.success_count} / ${inst.failure_count}</span>
</div>
<div class="metric-row">
<span class="metric-label">Success Rate</span>
<span class="metric-value ${successRate < 50 ? 'danger' : successRate < 80 ? 'warning' : ''}">
${successRate}%
</span>
</div>
<div class="metric-row">
<span class="metric-label">Last Activity</span>
<span class="metric-value">${inst.last_activity}</span>
</div>
${inst.current_task ? `
<div class="current-url">
<strong>Current URL:</strong><br>
${escapeHtml(inst.current_task)}
</div>
` : ''}
</div>
${proxy ? `
<div class="proxy-side">
<div class="side-header">
📡 ${proxy.container_name}
<span class="status-badge ${proxy.status}">${proxy.status}</span>
</div>
<div class="metric-row">
<span class="metric-label">IP Address</span>
<span class="metric-value">${proxy.ip_address}</span>
</div>
<div class="metric-row">
<span class="metric-label">Port</span>
<span class="metric-value">${proxy.port}</span>
</div>
<div class="metric-row">
<span class="metric-label">Status</span>
<span class="metric-value">${proxy.status}</span>
</div>
</div>
` : `
<div class="proxy-side">
<div class="no-proxy">
🌐<br>
Direct Connection<br>
(No Proxy)
</div>
</div>
`}
</div>
`;
}).join('');
}
function updateGlobalStats(global) {
const container = document.getElementById('global-stats');
const uptime = document.getElementById('uptime');
uptime.textContent = `Uptime: ${formatUptime(global.uptime_seconds)}`;
container.innerHTML = `
<div class="stat-box">
<div class="stat-value">${global.total_requests}</div>
<div class="stat-label">Total Requests</div>
</div>
<div class="stat-box">
<div class="stat-value">${global.success_rate.toFixed(1)}%</div>
<div class="stat-label">Success Rate</div>
</div>
<div class="stat-box">
<div class="stat-value">${global.successful_requests}</div>
<div class="stat-label">Successful</div>
</div>
<div class="stat-box">
<div class="stat-value">${global.failed_requests}</div>
<div class="stat-label">Failed</div>
</div>
<div class="stat-box">
<div class="stat-value">${global.session_renewals}</div>
<div class="stat-label">Session Renewals</div>
</div>
<div class="stat-box">
<div class="stat-value">${global.rotation_events}</div>
<div class="stat-label">Rotation Events</div>
</div>
<div class="stat-box">
<div class="stat-value">${global.navigation_timeouts}</div>
<div class="stat-label">Timeouts</div>
</div>
<div class="stat-box">
<div class="stat-value">${global.bot_detection_hits}</div>
<div class="stat-label">Bot Detection</div>
</div>
<div class="stat-box">
<div class="stat-value">${global.proxy_failures}</div>
<div class="stat-label">Proxy Failures</div>
</div>
`;
}
function updateLogs(logs) {
const container = document.getElementById('logs');
const wasScrolledToBottom = container.scrollHeight - container.scrollTop === container.clientHeight;
container.innerHTML = logs.map(log => `
<div class="log-entry">
<span class="log-time">${log.timestamp}</span>
<span class="log-message ${log.level}">${escapeHtml(log.message)}</span>
</div>
`).join('');
// Auto-scroll to bottom if user was already at bottom
if (wasScrolledToBottom) {
container.scrollTop = container.scrollHeight;
}
}
function formatUptime(seconds) {
const hours = Math.floor(seconds / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
const secs = seconds % 60;
return `${hours}h ${minutes}m ${secs}s`;
}
function escapeHtml(text) {
const map = {
'&': '&amp;',
'<': '&lt;',
'>': '&gt;',
'"': '&quot;',
"'": '&#039;'
};
return text.replace(/[&<>"']/g, m => map[m]);
}
// Initialize connection
connect();
</script>
</body>
</html>

129
src/monitoring/events.rs Normal file
View File

@@ -0,0 +1,129 @@
// src/monitoring/events.rs
use super::metrics::ProxyInfo;
/// Events emitted by the scraper system
#[derive(Debug, Clone)]
pub enum MonitoringEvent {
// Pool initialization
PoolInitialized {
pool_size: usize,
with_proxy: bool,
with_rotation: bool,
},
// Instance lifecycle
InstanceCreated {
instance_id: usize,
max_tasks: usize,
proxy: Option<ProxyInfo>,
},
InstanceStatusChanged {
instance_id: usize,
status: InstanceStatusChange,
},
// Task execution
TaskStarted {
instance_id: usize,
url: String,
},
TaskCompleted {
instance_id: usize,
success: bool,
duration_ms: u64,
error: Option<String>,
},
NavigationTimeout {
instance_id: usize,
url: String,
},
BotDetectionTriggered {
instance_id: usize,
url: String,
},
// Session management
SessionStarted {
instance_id: usize,
proxy: Option<ProxyInfo>,
},
SessionRenewed {
instance_id: usize,
old_request_count: usize,
reason: RenewalReason,
new_proxy: Option<ProxyInfo>,
},
SessionRequestIncremented {
instance_id: usize,
new_count: usize,
},
// Proxy events
ProxyConnected {
container_name: String,
ip_address: String,
port: u16,
},
ProxyFailed {
container_name: String,
error: String,
},
ProxyRotated {
instance_id: usize,
old_proxy: Option<String>,
new_proxy: String,
},
// Pool rotation events
RotationTriggered {
reason: String,
},
// Logging
LogMessage {
level: LogLevel,
message: String,
},
}
#[derive(Debug, Clone)]
pub enum InstanceStatusChange {
Idle,
Active,
Renewing,
Error(String),
}
#[derive(Debug, Clone)]
pub enum RenewalReason {
TaskLimit,
RequestLimit,
Error,
Manual,
}
#[derive(Debug, Clone)]
pub enum LogLevel {
Info,
Warn,
Error,
}
impl std::fmt::Display for RenewalReason {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
RenewalReason::TaskLimit => write!(f, "task_limit"),
RenewalReason::RequestLimit => write!(f, "request_limit"),
RenewalReason::Error => write!(f, "error"),
RenewalReason::Manual => write!(f, "manual"),
}
}
}

103
src/monitoring/logger.rs Normal file
View File

@@ -0,0 +1,103 @@
// src/monitoring/logger.rs
use super::metrics::SessionSummary;
use chrono::Local;
use std::path::PathBuf;
use tokio::fs::OpenOptions;
use tokio::io::AsyncWriteExt;
use tokio::sync::Mutex;
/// Logs session summaries to JSONL files
pub struct SessionLogger {
log_dir: PathBuf,
file: Mutex<Option<tokio::fs::File>>,
}
impl SessionLogger {
pub fn new(log_dir: PathBuf) -> Self {
Self {
log_dir,
file: Mutex::new(None),
}
}
/// Log a completed session summary
pub async fn log_session(&self, summary: &SessionSummary) {
if let Err(e) = self.write_session(summary).await {
eprintln!("Failed to log session: {}", e);
}
}
async fn write_session(&self, summary: &SessionSummary) -> anyhow::Result<()> {
let mut file_guard = self.file.lock().await;
// Open file if not already open
if file_guard.is_none() {
let filename = format!(
"sessions_{}.jsonl",
Local::now().format("%Y%m%d")
);
let filepath = self.log_dir.join(filename);
tokio::fs::create_dir_all(&self.log_dir).await?;
let file = OpenOptions::new()
.create(true)
.append(true)
.open(&filepath)
.await?;
*file_guard = Some(file);
}
if let Some(file) = file_guard.as_mut() {
let json_line = serde_json::to_string(summary)?;
file.write_all(json_line.as_bytes()).await?;
file.write_all(b"\n").await?;
file.flush().await?;
}
Ok(())
}
}
/// Logs metrics snapshots periodically
pub struct MetricsLogger {
log_dir: PathBuf,
}
impl MetricsLogger {
pub fn new(log_dir: PathBuf) -> Self {
Self { log_dir }
}
/// Log a metrics snapshot
pub async fn log_metrics(&self, state: &super::metrics::DashboardState) -> anyhow::Result<()> {
let filename = format!(
"metrics_{}.jsonl",
Local::now().format("%Y%m%d")
);
let filepath = self.log_dir.join(filename);
tokio::fs::create_dir_all(&self.log_dir).await?;
let mut file = OpenOptions::new()
.create(true)
.append(true)
.open(&filepath)
.await?;
let snapshot = serde_json::json!({
"timestamp": Local::now().format("%Y-%m-%d %H:%M:%S").to_string(),
"global": state.global,
"instance_count": state.instances.len(),
"proxy_count": state.proxies.len(),
});
let json_line = serde_json::to_string(&snapshot)?;
file.write_all(json_line.as_bytes()).await?;
file.write_all(b"\n").await?;
file.flush().await?;
Ok(())
}
}

252
src/monitoring/metrics.rs Normal file
View File

@@ -0,0 +1,252 @@
// src/monitoring/metrics.rs
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Instant;
/// Complete dashboard state sent to web clients
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DashboardState {
pub config: ConfigSnapshot,
pub instances: Vec<InstanceMetrics>,
pub proxies: Vec<ProxyMetrics>,
pub global: GlobalMetrics,
pub logs: Vec<LogEntry>,
}
/// Snapshot of configuration settings
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ConfigSnapshot {
pub max_parallel_instances: usize,
pub max_tasks_per_instance: usize,
pub enable_vpn_rotation: bool,
pub max_requests_per_session: usize,
pub min_request_interval_ms: u64,
pub max_retry_attempts: u32,
}
/// Metrics for a single ChromeDriver instance
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct InstanceMetrics {
pub id: usize,
pub status: InstanceStatus,
pub current_task: Option<String>,
pub tasks_current_session: usize,
pub tasks_max: usize,
pub session_requests: usize,
pub total_requests: usize,
pub success_count: usize,
pub failure_count: usize,
pub connected_proxy: Option<ProxyInfo>,
pub last_activity: String, // Timestamp
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum InstanceStatus {
Idle,
Active,
Renewing,
Error,
}
/// Information about a proxy connection
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProxyInfo {
pub container_name: String,
pub ip_address: String,
pub port: u16,
pub status: ProxyStatus,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum ProxyStatus {
Connected,
Disconnected,
}
/// Metrics for a proxy
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProxyMetrics {
pub container_name: String,
pub ip_address: String,
pub port: u16,
pub status: ProxyStatus,
pub instances_using: Vec<usize>,
}
/// Global pool metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GlobalMetrics {
pub total_requests: usize,
pub successful_requests: usize,
pub failed_requests: usize,
pub success_rate: f64,
pub session_renewals: usize,
pub rotation_events: usize,
pub navigation_timeouts: usize,
pub bot_detection_hits: usize,
pub proxy_failures: usize,
pub uptime_seconds: u64,
}
/// Log entry for display in dashboard
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LogEntry {
pub timestamp: String,
pub level: LogLevel,
pub message: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum LogLevel {
Info,
Warn,
Error,
}
/// Internal state tracked by monitoring service
#[derive(Debug, Clone)]
pub struct MonitoringState {
pub instances: HashMap<usize, InstanceState>,
pub proxies: HashMap<String, ProxyState>,
pub global: GlobalState,
pub start_time: Instant,
}
#[derive(Debug, Clone)]
pub struct InstanceState {
pub id: usize,
pub status: InstanceStatus,
pub current_task: Option<String>,
pub tasks_current_session: usize,
pub tasks_max: usize,
pub session_requests: usize,
pub total_requests: usize,
pub success_count: usize,
pub failure_count: usize,
pub connected_proxy: Option<ProxyInfo>,
pub last_activity: Instant,
}
#[derive(Debug, Clone)]
pub struct ProxyState {
pub container_name: String,
pub ip_address: String,
pub port: u16,
pub status: ProxyStatus,
pub instances_using: Vec<usize>,
}
#[derive(Debug, Clone)]
pub struct GlobalState {
pub total_requests: usize,
pub successful_requests: usize,
pub failed_requests: usize,
pub session_renewals: usize,
pub rotation_events: usize,
pub navigation_timeouts: usize,
pub bot_detection_hits: usize,
pub proxy_failures: usize,
}
impl MonitoringState {
pub fn new() -> Self {
Self {
instances: HashMap::new(),
proxies: HashMap::new(),
global: GlobalState {
total_requests: 0,
successful_requests: 0,
failed_requests: 0,
session_renewals: 0,
rotation_events: 0,
navigation_timeouts: 0,
bot_detection_hits: 0,
proxy_failures: 0,
},
start_time: Instant::now(),
}
}
/// Convert internal state to dashboard state for web clients
pub fn to_dashboard_state(&self, config: ConfigSnapshot, logs: Vec<LogEntry>) -> DashboardState {
let instances: Vec<InstanceMetrics> = self
.instances
.values()
.map(|inst| InstanceMetrics {
id: inst.id,
status: inst.status.clone(),
current_task: inst.current_task.clone(),
tasks_current_session: inst.tasks_current_session,
tasks_max: inst.tasks_max,
session_requests: inst.session_requests,
total_requests: inst.total_requests,
success_count: inst.success_count,
failure_count: inst.failure_count,
connected_proxy: inst.connected_proxy.clone(),
last_activity: format_timestamp(inst.last_activity),
})
.collect();
let proxies: Vec<ProxyMetrics> = self
.proxies
.values()
.map(|proxy| ProxyMetrics {
container_name: proxy.container_name.clone(),
ip_address: proxy.ip_address.clone(),
port: proxy.port,
status: proxy.status.clone(),
instances_using: proxy.instances_using.clone(),
})
.collect();
let success_rate = if self.global.total_requests > 0 {
(self.global.successful_requests as f64 / self.global.total_requests as f64) * 100.0
} else {
0.0
};
let global = GlobalMetrics {
total_requests: self.global.total_requests,
successful_requests: self.global.successful_requests,
failed_requests: self.global.failed_requests,
success_rate,
session_renewals: self.global.session_renewals,
rotation_events: self.global.rotation_events,
navigation_timeouts: self.global.navigation_timeouts,
bot_detection_hits: self.global.bot_detection_hits,
proxy_failures: self.global.proxy_failures,
uptime_seconds: self.start_time.elapsed().as_secs(),
};
DashboardState {
config,
instances,
proxies,
global,
logs,
}
}
}
fn format_timestamp(instant: Instant) -> String {
use chrono::Local;
// This is a placeholder - in real impl we'd track actual wall-clock time
Local::now().format("%H:%M:%S").to_string()
}
/// Session completion summary for logging
#[derive(Debug, Clone, Serialize)]
pub struct SessionSummary {
pub instance_id: usize,
pub session_start: String,
pub session_end: String,
pub duration_seconds: u64,
pub total_requests: usize,
pub successful_requests: usize,
pub failed_requests: usize,
pub proxy_info: Option<ProxyInfo>,
pub renewal_reason: String, // "task_limit", "request_limit", "error"
}

78
src/monitoring/mod.rs Normal file
View File

@@ -0,0 +1,78 @@
// src/monitoring/mod.rs
//! Monitoring system for tracking scraper performance and health
//!
//! This module provides:
//! - Real-time metrics collection
//! - Web-based dashboard
//! - Session logging
//! - Minimal performance overhead
pub mod metrics;
pub mod events;
pub mod service;
pub mod webserver;
pub mod logger;
pub use events::{MonitoringEvent,RenewalReason, InstanceStatusChange};
pub use metrics::{ConfigSnapshot, ProxyInfo, ProxyStatus};
pub use service::{MonitoringService, MonitoringHandle};
pub use webserver::WebServer;
use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::{mpsc, RwLock};
/// Initialize the complete monitoring system
pub async fn init_monitoring(
config_snapshot: ConfigSnapshot,
log_dir: PathBuf,
dashboard_port: u16,
) -> anyhow::Result<(MonitoringHandle, tokio::task::JoinHandle<()>)> {
// Create channel for events
let (tx, rx) = mpsc::unbounded_channel();
// Create monitoring service
let service = MonitoringService::new(config_snapshot, rx, log_dir);
let service_arc = Arc::new(RwLock::new(service));
// Start monitoring service task
let service_clone = Arc::clone(&service_arc);
let monitoring_task = tokio::spawn(async move {
println!("🚀 MONITORING TASK STARTED!");
// Take ownership of the service
let mut service = {
let mut guard = service_clone.write().await;
std::mem::replace(
&mut *guard,
MonitoringService::new(
ConfigSnapshot {
max_parallel_instances: 0,
max_tasks_per_instance: 0,
enable_vpn_rotation: false,
max_requests_per_session: 0,
min_request_interval_ms: 0,
max_retry_attempts: 0,
},
mpsc::unbounded_channel().1,
PathBuf::new(),
),
)
};
println!("✅ ABOUT TO RUN SERVICE!");
service.run().await;
});
// Start web server
let webserver = WebServer::new(Arc::clone(&service_arc), dashboard_port);
tokio::spawn(async move {
if let Err(e) = webserver.run().await {
eprintln!("Web server error: {}", e);
}
});
// Create handle for emitting events
let handle = MonitoringHandle::new(tx);
Ok((handle, monitoring_task))
}

341
src/monitoring/service.rs Normal file
View File

@@ -0,0 +1,341 @@
// src/monitoring/service.rs
use super::events::*;
use super::metrics::*;
use super::logger::SessionLogger;
use std::collections::VecDeque;
use std::sync::Arc;
use std::time::Instant;
use tokio::sync::{mpsc, RwLock};
use chrono::Local;
const MAX_LOGS: usize = 100;
/// Monitoring service that collects events and maintains state
pub struct MonitoringService {
state: Arc<RwLock<MonitoringState>>,
config: ConfigSnapshot,
logs: Arc<RwLock<VecDeque<LogEntry>>>,
session_logger: Arc<SessionLogger>,
event_rx: mpsc::UnboundedReceiver<MonitoringEvent>,
}
impl MonitoringService {
pub fn new(
config: ConfigSnapshot,
event_rx: mpsc::UnboundedReceiver<MonitoringEvent>,
log_dir: std::path::PathBuf,
) -> Self {
Self {
state: Arc::new(RwLock::new(MonitoringState::new())),
config,
logs: Arc::new(RwLock::new(VecDeque::with_capacity(MAX_LOGS))),
session_logger: Arc::new(SessionLogger::new(log_dir)),
event_rx,
}
}
/// Get current dashboard state for web clients
pub async fn get_dashboard_state(&self) -> DashboardState {
let state = self.state.read().await;
let logs = self.logs.read().await;
state.to_dashboard_state(
self.config.clone(),
logs.iter().cloned().collect(),
)
}
/// Main event processing loop
pub async fn run(mut self) {
while let Some(event) = self.event_rx.recv().await {
self.process_event(event).await;
}
}
async fn process_event(&self, event: MonitoringEvent) {
match event {
MonitoringEvent::PoolInitialized { pool_size, with_proxy, with_rotation } => {
self.log_info(format!(
"Pool initialized: {} instances, proxy={}, rotation={}",
pool_size, with_proxy, with_rotation
)).await;
}
MonitoringEvent::InstanceCreated { instance_id, max_tasks, proxy } => {
let mut state = self.state.write().await;
state.instances.insert(
instance_id,
InstanceState {
id: instance_id,
status: InstanceStatus::Idle,
current_task: None,
tasks_current_session: 0,
tasks_max: max_tasks,
session_requests: 0,
total_requests: 0,
success_count: 0,
failure_count: 0,
connected_proxy: proxy.clone(),
last_activity: Instant::now(),
},
);
if let Some(proxy_info) = proxy {
state.proxies.entry(proxy_info.container_name.clone()).or_insert_with(|| {
ProxyState {
container_name: proxy_info.container_name.clone(),
ip_address: proxy_info.ip_address.clone(),
port: proxy_info.port,
status: ProxyStatus::Connected,
instances_using: vec![instance_id],
}
}).instances_using.push(instance_id);
}
self.log_info(format!("Instance #{} created", instance_id)).await;
}
MonitoringEvent::InstanceStatusChanged { instance_id, status } => {
let mut state = self.state.write().await;
if let Some(inst) = state.instances.get_mut(&instance_id) {
inst.status = match status {
InstanceStatusChange::Idle => InstanceStatus::Idle,
InstanceStatusChange::Active => InstanceStatus::Active,
InstanceStatusChange::Renewing => InstanceStatus::Renewing,
InstanceStatusChange::Error(_) => InstanceStatus::Error,
};
inst.last_activity = Instant::now();
}
}
MonitoringEvent::TaskStarted { instance_id, url } => {
let mut state = self.state.write().await;
if let Some(inst) = state.instances.get_mut(&instance_id) {
inst.status = InstanceStatus::Active;
inst.current_task = Some(url.clone());
inst.last_activity = Instant::now();
}
state.global.total_requests += 1;
self.log_info(format!("Instance #{} started task: {}", instance_id, url)).await;
}
MonitoringEvent::TaskCompleted { instance_id, success, duration_ms, error } => {
let mut state = self.state.write().await;
if let Some(inst) = state.instances.get_mut(&instance_id) {
inst.current_task = None;
inst.status = InstanceStatus::Idle;
inst.total_requests += 1;
inst.last_activity = Instant::now();
if success {
inst.success_count += 1;
state.global.successful_requests += 1;
} else {
inst.failure_count += 1;
state.global.failed_requests += 1;
}
}
if success {
self.log_info(format!(
"Instance #{} completed task in {}ms",
instance_id, duration_ms
)).await;
} else {
self.log_error(format!(
"Instance #{} failed task: {}",
instance_id,
error.unwrap_or_else(|| "unknown error".to_string())
)).await;
}
}
MonitoringEvent::NavigationTimeout { instance_id, url } => {
let mut state = self.state.write().await;
state.global.navigation_timeouts += 1;
self.log_warn(format!(
"Instance #{} navigation timeout: {}",
instance_id, url
)).await;
}
MonitoringEvent::BotDetectionTriggered { instance_id, url } => {
let mut state = self.state.write().await;
state.global.bot_detection_hits += 1;
self.log_warn(format!(
"Instance #{} bot detection triggered: {}",
instance_id, url
)).await;
}
MonitoringEvent::SessionStarted { instance_id, proxy } => {
let mut state = self.state.write().await;
if let Some(inst) = state.instances.get_mut(&instance_id) {
inst.session_requests = 0;
inst.tasks_current_session = 0;
inst.connected_proxy = proxy;
inst.last_activity = Instant::now();
}
self.log_info(format!("Instance #{} started new session", instance_id)).await;
}
MonitoringEvent::SessionRenewed { instance_id, old_request_count, reason, new_proxy } => {
// Log the completed session
let session_summary = {
let state = self.state.read().await;
if let Some(inst) = state.instances.get(&instance_id) {
Some(SessionSummary {
instance_id,
session_start: "N/A".to_string(), // We'd need to track this
session_end: Local::now().format("%Y-%m-%d %H:%M:%S").to_string(),
duration_seconds: 0, // We'd need to track session start time
total_requests: old_request_count,
successful_requests: inst.success_count,
failed_requests: inst.failure_count,
proxy_info: inst.connected_proxy.clone(),
renewal_reason: reason.to_string(),
})
} else {
None
}
};
if let Some(summary) = session_summary {
self.session_logger.log_session(&summary).await;
}
// Update state for new session
let mut state = self.state.write().await;
if let Some(inst) = state.instances.get_mut(&instance_id) {
inst.session_requests = 0;
inst.tasks_current_session = 0;
inst.connected_proxy = new_proxy;
inst.last_activity = Instant::now();
}
state.global.session_renewals += 1;
self.log_info(format!(
"Instance #{} renewed session (reason: {}, {} requests)",
instance_id, reason, old_request_count
)).await;
}
MonitoringEvent::SessionRequestIncremented { instance_id, new_count } => {
let mut state = self.state.write().await;
if let Some(inst) = state.instances.get_mut(&instance_id) {
inst.session_requests = new_count;
inst.last_activity = Instant::now();
}
}
MonitoringEvent::ProxyConnected { container_name, ip_address, port } => {
let mut state = self.state.write().await;
state.proxies.insert(
container_name.clone(),
ProxyState {
container_name: container_name.clone(),
ip_address: ip_address.clone(),
port,
status: ProxyStatus::Connected,
instances_using: vec![],
},
);
self.log_info(format!(
"Proxy {} connected: {}:{}",
container_name, ip_address, port
)).await;
}
MonitoringEvent::ProxyFailed { container_name, error } => {
let mut state = self.state.write().await;
if let Some(proxy) = state.proxies.get_mut(&container_name) {
proxy.status = ProxyStatus::Disconnected;
}
state.global.proxy_failures += 1;
self.log_error(format!(
"Proxy {} failed: {}",
container_name, error
)).await;
}
MonitoringEvent::ProxyRotated { instance_id, old_proxy, new_proxy } => {
self.log_info(format!(
"Instance #{} rotated proxy: {} -> {}",
instance_id,
old_proxy.unwrap_or_else(|| "none".to_string()),
new_proxy
)).await;
}
MonitoringEvent::RotationTriggered { reason } => {
let mut state = self.state.write().await;
state.global.rotation_events += 1;
self.log_info(format!("Pool rotation triggered: {}", reason)).await;
}
MonitoringEvent::LogMessage { level, message } => {
match level {
crate::monitoring::events::LogLevel::Info => self.log_info(message).await,
crate::monitoring::events::LogLevel::Warn => self.log_warn(message).await,
crate::monitoring::events::LogLevel::Error => self.log_error(message).await,
}
}
}
}
async fn log_info(&self, message: String) {
self.add_log(LogEntry {
timestamp: Local::now().format("%H:%M:%S").to_string(),
level: super::metrics::LogLevel::Info,
message,
}).await;
}
async fn log_warn(&self, message: String) {
self.add_log(LogEntry {
timestamp: Local::now().format("%H:%M:%S").to_string(),
level: super::metrics::LogLevel::Warn,
message,
}).await;
}
async fn log_error(&self, message: String) {
self.add_log(LogEntry {
timestamp: Local::now().format("%H:%M:%S").to_string(),
level: super::metrics::LogLevel::Error,
message,
}).await;
}
async fn add_log(&self, entry: LogEntry) {
let mut logs = self.logs.write().await;
if logs.len() >= MAX_LOGS {
logs.pop_front();
}
logs.push_back(entry);
}
}
/// Handle for emitting monitoring events
#[derive(Clone)]
pub struct MonitoringHandle {
tx: mpsc::UnboundedSender<MonitoringEvent>,
}
impl MonitoringHandle {
pub fn new(tx: mpsc::UnboundedSender<MonitoringEvent>) -> Self {
Self { tx }
}
/// Emit a monitoring event (non-blocking)
pub fn emit(&self, event: MonitoringEvent) {
// Ignore send errors (monitoring should never block application)
let _ = self.tx.send(event);
}
}

77
src/monitoring/webserver.rs Normal file
View File

@@ -0,0 +1,77 @@
// src/monitoring/webserver.rs
use super::service::MonitoringService;
use axum::{
extract::{
ws::{Message, WebSocket, WebSocketUpgrade},
State,
},
response::{Html, IntoResponse, Response},
routing::get,
Router,
};
use std::sync::Arc;
use tokio::sync::RwLock;
use tokio::time::{interval, Duration};
const UPDATE_INTERVAL_MS: u64 = 1000; // 1 second updates
pub struct WebServer {
service: Arc<RwLock<MonitoringService>>,
port: u16,
}
impl WebServer {
pub fn new(service: Arc<RwLock<MonitoringService>>, port: u16) -> Self {
Self { service, port }
}
pub async fn run(self) -> anyhow::Result<()> {
let app = Router::new()
.route("/", get(dashboard_handler))
.route("/ws", get(websocket_handler))
.with_state(self.service);
let addr = format!("0.0.0.0:{}", self.port);
println!("📊 Dashboard available at: http://localhost:{}", self.port);
let listener = tokio::net::TcpListener::bind(&addr).await?;
axum::serve(listener, app).await?;
Ok(())
}
}
async fn dashboard_handler() -> impl IntoResponse {
Html(include_str!("dashboard.html"))
}
async fn websocket_handler(
ws: WebSocketUpgrade,
State(service): State<Arc<RwLock<MonitoringService>>>,
) -> Response {
ws.on_upgrade(|socket| handle_socket(socket, service))
}
async fn handle_socket(mut socket: WebSocket, service: Arc<RwLock<MonitoringService>>) {
let mut ticker = interval(Duration::from_millis(UPDATE_INTERVAL_MS));
loop {
ticker.tick().await;
let service_guard = service.read().await;
let state = service_guard.get_dashboard_state().await;
drop(service_guard);
match serde_json::to_string(&state) {
Ok(json) => {
if socket.send(Message::Text(json)).await.is_err() {
break; // Client disconnected
}
}
Err(e) => {
eprintln!("Failed to serialize dashboard state: {}", e);
break;
}
}
}
}

19
src/scraper/docker_vpn_proxy.rs
View File

@@ -342,6 +342,25 @@ impl DockerVpnProxyPool {
}
Ok(())
}
/// Get ProxyInfo for monitoring dashboard
pub fn get_proxy_info(&self, index: usize) -> Option<crate::monitoring::ProxyInfo> {
if index >= self.container_names.len() {
return None;
}
Some(crate::monitoring::ProxyInfo {
container_name: self.container_names[index].clone(),
ip_address: "127.0.0.1".to_string(), // SOCKS5 proxy on localhost
port: self.proxy_ports[index],
status: crate::monitoring::ProxyStatus::Connected,
})
}
/// Get container name by index
pub fn get_container_name(&self, index: usize) -> Option<String> {
self.container_names.get(index).cloned()
}
}
pub async fn cleanup_all_proxy_containers() -> Result<()> {

14
src/scraper/helpers.rs Normal file
View File

@@ -0,0 +1,14 @@
use rand::rngs::StdRng;
use rand::prelude::{Rng, SeedableRng, IndexedRandom};
/// Send-safe random range
pub fn random_range(min: u64, max: u64) -> u64 {
let mut rng = StdRng::from_rng(&mut rand::rng());
rng.random_range(min..max)
}
/// Send-safe random choice
pub fn choose_random<T: Clone>(items: &[T]) -> T {
let mut rng = StdRng::from_rng(&mut rand::rng());
items.choose(&mut rng).unwrap().clone()
}

1
src/scraper/mod.rs
View File

@@ -1,2 +1,3 @@
pub mod webdriver;
pub mod docker_vpn_proxy;
pub mod helpers;

539
src/scraper/webdriver.rs
View File

@@ -1,4 +1,5 @@
// src/scraper/webdriver.rs
use super::helpers::*;
use anyhow::{anyhow, Context, Result};
use fantoccini::{Client, ClientBuilder};
@@ -6,12 +7,14 @@ use serde_json::{Map, Value};
use std::pin::Pin;
use std::process::Stdio;
use std::sync::Arc;
use std::time::Instant;
use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::process::{Child, Command};
use tokio::task::JoinHandle;
use tokio::sync::{Mutex, Semaphore};
use tokio::time::{sleep, timeout, Duration};
use crate::scraper::docker_vpn_proxy::{DockerVpnProxyPool};
use crate::Config;
/// Manages a pool of ChromeDriver instances for parallel scraping with optional VPN binding.
pub struct ChromeDriverPool {
@@ -23,25 +26,31 @@ pub struct ChromeDriverPool {
rotation_enabled: bool,
/// Index for round-robin instance selection (when rotation is enabled)
next_instance: Arc<Mutex<usize>>,
last_request_time: Arc<Mutex<Instant>>,
min_request_interval_ms: u64,
monitoring: Option<crate::monitoring::MonitoringHandle>,
}
impl ChromeDriverPool {
/// Creates a new pool without any proxy (direct connection).
pub async fn _new(pool_size: usize) -> Result<Self> {
Self::new_with_proxy_and_task_limit(pool_size, None, 0).await
pub async fn _new(config: &Config, monitoring: Option<crate::monitoring::MonitoringHandle>,) -> Result<Self> {
Self::new_with_proxy_and_task_limit(None, config, monitoring).await
}
/// Creates a new pool with task-per-instance limit but no proxy.
pub async fn _new_with_task_limit(pool_size: usize, max_tasks_per_instance: usize) -> Result<Self> {
Self::new_with_proxy_and_task_limit(pool_size, None, max_tasks_per_instance).await
pub async fn _new_with_task_limit(config: &Config, monitoring: Option<crate::monitoring::MonitoringHandle>,) -> Result<Self> {
Self::new_with_proxy_and_task_limit(None, config, monitoring).await
}
/// Creates a new pool where each Chrome instance uses a different SOCKS5 proxy from the Docker pool.
pub async fn new_with_proxy(
pool_size: usize,
proxy_pool: Option<Arc<DockerVpnProxyPool>>,
config: &Config,
monitoring: Option<crate::monitoring::MonitoringHandle>,
) -> Result<Self> {
Self::new_with_proxy_and_task_limit(pool_size, proxy_pool, 0).await
Self::new_with_proxy_and_task_limit(proxy_pool, config, monitoring).await
}
/// Full constructor: supports proxy + task limiting + rotation.
@@ -55,10 +64,13 @@ impl ChromeDriverPool {
///
/// Uses the minimum of these constraints to determine actual pool size.
pub async fn new_with_proxy_and_task_limit(
pool_size_limit: usize,
proxy_pool: Option<Arc<DockerVpnProxyPool>>,
max_tasks_per_instance: usize,
config: &Config,
monitoring: Option<crate::monitoring::MonitoringHandle>,
) -> Result<Self> {
let pool_size_limit = config.max_parallel_instances;
let task_per_instance_limit = config.max_tasks_per_instance;
// Determine actual pool size based on available resources
let actual_pool_size = if let Some(ref pp) = proxy_pool {
let available_proxies = pp.num_proxies();
@@ -72,7 +84,7 @@ impl ChromeDriverPool {
}
// Rotation is enabled when task limiting is active
let rotation_enabled = max_tasks_per_instance > 0;
let rotation_enabled = task_per_instance_limit > 0;
let mut instances = Vec::with_capacity(actual_pool_size);
@@ -91,34 +103,77 @@ impl ChromeDriverPool {
}
for i in 0..actual_pool_size {
let proxy_url = proxy_pool
.as_ref()
.map(|pp| pp.get_proxy_url(i));
let instance = ChromeInstance::new(proxy_url, max_tasks_per_instance).await?;
// Pass the entire proxy_pool and the index
let instance = ChromeInstance::new(
proxy_pool.clone(), // Clone the Arc
i, // This instance's proxy index
config,
monitoring.clone(),
).await?;
crate::util::logger::log_info(&format!(" Instance {} ready", i + 1)).await;
instances.push(Arc::new(Mutex::new(instance)));
}
// Emit instance created events
for (i, instance) in instances.iter().enumerate() {
if let Some(ref mon) = monitoring {
let guard = instance.lock().await;
// Extract proxy info if available
let proxy_info = if let Some(ref pp) = proxy_pool {
pp.get_proxy_info(i % pp.num_proxies())
} else {
guard.proxy_url.as_ref().and_then(|url| {
// Parse proxy URL manually if no pool
// Format: socks5://localhost:10801
if let Some(port_str) = url.split(':').last() {
if let Ok(port) = port_str.parse::<u16>() {
return Some(crate::monitoring::ProxyInfo {
container_name: format!("proxy-{}", i),
ip_address: "127.0.0.1".to_string(),
port,
status: crate::monitoring::ProxyStatus::Connected,
});
}
}
None
})
};
mon.emit(crate::monitoring::MonitoringEvent::InstanceCreated {
instance_id: i,
max_tasks: guard.max_tasks_per_instance,
proxy: proxy_info.clone(), // ✅ Now includes actual proxy info
});
// Also emit ProxyConnected event if proxy exists
if let Some(ref proxy) = proxy_info {
mon.emit(crate::monitoring::MonitoringEvent::ProxyConnected {
container_name: proxy.container_name.clone(),
ip_address: proxy.ip_address.clone(),
port: proxy.port,
});
}
drop(guard);
}
}
let min_request_interval_ms = config.min_request_interval_ms;
Ok(Self {
instances,
semaphore: Arc::new(Semaphore::new(actual_pool_size)),
proxy_pool,
rotation_enabled,
next_instance: Arc::new(Mutex::new(0)),
last_request_time: Arc::new(Mutex::new(Instant::now())),
min_request_interval_ms,
monitoring,
})
}
/// Execute a scraping task using an available instance from the pool.
///
/// When rotation is enabled:
/// - Uses only half of the instances at a time
/// - Rotates to the other half when an instance reaches its task limit
/// - Cycles through instances in round-robin fashion within the active half
///
/// When rotation is disabled:
/// - Uses all instances with random selection
pub async fn execute<T, F, Fut>(&self, url: String, parse: F) -> Result<T>
where
T: Send + 'static,
@@ -127,108 +182,146 @@ impl ChromeDriverPool {
{
let _permit = self.semaphore.acquire().await.map_err(|_| anyhow!("Pool closed"))?;
let index = if self.rotation_enabled {
// Rotation mode: use only half of instances at a time
let total_instances = self.instances.len();
let half_size = (total_instances + 1) / 2; // Round up for odd numbers
{
let mut last_time = self.last_request_time.lock().await;
let elapsed = last_time.elapsed().as_millis() as u64;
let mut next_idx = self.next_instance.lock().await;
let base_idx = *next_idx;
let mut selected_idx = base_idx;
let mut found_in_current_half = false;
if elapsed < self.min_request_interval_ms {
let wait_ms = self.min_request_interval_ms - elapsed;
drop(last_time); // Lock vor Sleep freigeben!
// Try to find an available instance in the current half
for offset in 0..half_size {
let candidate_idx = (base_idx + offset) % half_size;
sleep(Duration::from_millis(wait_ms)).await;
// Check if this instance has reached its task limit
let instance = &self.instances[candidate_idx];
let guard = instance.lock().await;
if guard.max_tasks_per_instance == 0 ||
guard.task_count < guard.max_tasks_per_instance {
// This instance is available
*next_idx = (candidate_idx + 1) % half_size;
selected_idx = candidate_idx;
found_in_current_half = true;
drop(guard);
break;
} else {
drop(guard);
}
let mut last_time = self.last_request_time.lock().await;
*last_time = Instant::now();
} else {
*last_time = Instant::now();
}
if !found_in_current_half {
// All instances in current half are at limit, switch to other half
crate::util::logger::log_info(
"Current half saturated, rotating to other half of instances"
).await;
let other_half_start = half_size;
let other_half_size = total_instances - half_size;
// Find available instance in other half
let mut found_in_other_half = false;
for offset in 0..other_half_size {
let candidate_idx = other_half_start + offset;
let instance = &self.instances[candidate_idx];
let guard = instance.lock().await;
if guard.max_tasks_per_instance == 0 ||
guard.task_count < guard.max_tasks_per_instance {
// Switch to this half for future requests
*next_idx = offset;
selected_idx = candidate_idx;
found_in_other_half = true;
drop(guard);
break;
} else {
drop(guard);
}
}
if !found_in_other_half {
// All instances saturated - use round-robin anyway
selected_idx = *next_idx % total_instances;
*next_idx = (*next_idx + 1) % total_instances;
}
}
drop(next_idx);
selected_idx
} else {
// Non-rotation mode: random selection as before
rand::random_range(..self.instances.len())
};
let instance = self.instances[index].clone();
let mut guard = instance.lock().await;
guard.increment_task_count();
if guard.max_tasks_per_instance > 0 {
crate::util::logger::log_info(&format!(
"Instance {} task count: {}/{}",
index,
guard.get_task_count(),
guard.max_tasks_per_instance
))
.await;
}
let client = guard.new_session().await?;
let random_index = random_range(0, self.instances.len() as u64) as usize;
// Index-Auswahl (vereinfacht, siehe unten für vollständige Rotation)
let index = if self.rotation_enabled {
self.get_rotated_index().await?
} else {
random_index
};
drop(guard); // release lock early
if let Some(ref mon) = self.monitoring {
mon.emit(crate::monitoring::MonitoringEvent::TaskStarted {
instance_id: index,
url: url.clone(),
});
mon.emit(crate::monitoring::MonitoringEvent::InstanceStatusChanged {
instance_id: index,
status: crate::monitoring::InstanceStatusChange::Active,
});
}
crate::util::logger::log_info(&format!("Scraping {} ...", url)).await;
client.goto(&url).await.context("Navigation failed")?;
let instance = &self.instances[index];
let mut guard = instance.lock().await;
let result = timeout(Duration::from_secs(90), parse(client))
.await
.context("Parse timeout")??;
// NEU: Session mit automatischer Erneuerung holen!
let client = guard.get_or_renew_session().await?;
Ok(result)
guard.increment_task_count();
let (task_count, session_requests) = guard.get_session_stats().await;
crate::util::logger::log_info(&format!(
"Instance {} executing task (tasks: {}/{}, session requests: {})",
index, task_count, guard.max_tasks_per_instance, session_requests
)).await;
drop(guard); // Lock freigeben vor Navigation
let start_time = Instant::now();
// Navigation mit Timeout
let navigation_result = timeout(
Duration::from_secs(60),
client.goto(&url)
).await;
match navigation_result {
Ok(Ok(_)) => {
if let Some(ref mon) = self.monitoring {
mon.emit(crate::monitoring::MonitoringEvent::TaskCompleted {
instance_id: index,
success: navigation_result.is_ok(),
duration_ms: start_time.elapsed().as_millis() as u64,
error: navigation_result.as_ref().err().map(|e| e.to_string()),
});
mon.emit(crate::monitoring::MonitoringEvent::InstanceStatusChanged {
instance_id: index,
status: crate::monitoring::InstanceStatusChange::Idle,
});
}
crate::util::logger::log_info(&format!("✓ Navigated to {}", url)).await;
// Parse-Funktion ausführen
parse(client).await
}
Ok(Err(e)) => {
crate::util::logger::log_error(&format!("Navigation failed: {}", e)).await;
Err(anyhow!("Navigation failed: {}", e))
}
Err(_) => {
if let Some(ref mon) = self.monitoring {
mon.emit(crate::monitoring::MonitoringEvent::NavigationTimeout {
instance_id: index,
url: url.clone(),
});
}
crate::util::logger::log_error("Navigation timeout (60s)").await;
Err(anyhow!("Navigation timeout"))
}
}
}
async fn get_rotated_index(&self) -> Result<usize> {
let total = self.instances.len();
let half_size = total / 2;
if half_size == 0 {
return Ok(0); // Pool zu klein für Rotation
}
let mut next_idx = self.next_instance.lock().await;
let current_half_start = if *next_idx < half_size { 0 } else { half_size };
let current_half_end = if *next_idx < half_size { half_size } else { total };
// Suche verfügbare Instanz in aktueller Hälfte
for offset in 0..(current_half_end - current_half_start) {
let candidate_idx = current_half_start + ((*next_idx + offset) % half_size);
let instance = &self.instances[candidate_idx];
let guard = instance.lock().await;
if guard.max_tasks_per_instance == 0 ||
guard.task_count < guard.max_tasks_per_instance {
*next_idx = (candidate_idx + 1) % total;
drop(guard);
return Ok(candidate_idx);
}
}
// Aktuelle Hälfte voll → Zur anderen wechseln
crate::util::logger::log_info("Current half saturated, rotating to other half").await;
let new_half_start = if current_half_start == 0 { half_size } else { 0 };
let new_half_end = if current_half_start == 0 { total } else { half_size };
// Alte Hälfte zurücksetzen (für nächste Rotation)
for i in current_half_start..current_half_end {
let mut instance = self.instances[i].lock().await;
instance.reset_task_count();
}
*next_idx = new_half_start;
drop(next_idx);
Ok(new_half_start)
}
/// Gracefully shut down all ChromeDriver processes and Docker proxy containers.
@@ -273,12 +366,28 @@ pub struct ChromeInstance {
task_count: usize,
max_tasks_per_instance: usize,
proxy_url: Option<String>,
current_session: Arc<Mutex<Option<Client>>>, // Current active session
session_request_count: Arc<Mutex<usize>>,
max_requests_per_session: usize, // z.B. 25
proxy_pool: Option<Arc<DockerVpnProxyPool>>, // Referernce to the proxy pool
current_proxy_index: Arc<Mutex<usize>>, // Current proxy index in use
instance_id: usize,
monitoring: Option<crate::monitoring::MonitoringHandle>,
}
impl ChromeInstance {
pub async fn new(proxy_url: Option<String>, max_tasks_per_instance: usize) -> Result<Self> {
pub async fn new(proxy_pool: Option<Arc<DockerVpnProxyPool>>, instance_id: usize, config: &Config, monitoring: Option<crate::monitoring::MonitoringHandle>) -> Result<Self> {
let (base_url, process, stderr_handle) = Self::spawn_chromedriver().await?;
// Get proxy URL if proxy pool is provided
let proxy_url = proxy_pool.as_ref().map(|pp| pp.get_proxy_url(instance_id));
let max_tasks_per_instance = config.max_tasks_per_instance;
let max_requests_per_session = config.max_requests_per_session;
Ok(Self {
base_url,
process,
@@ -286,17 +395,170 @@ impl ChromeInstance {
task_count: 0,
max_tasks_per_instance,
proxy_url,
current_session: Arc::new(Mutex::new(None)),
session_request_count: Arc::new(Mutex::new(0)),
max_requests_per_session,
proxy_pool,
current_proxy_index: Arc::new(Mutex::new(instance_id)),
instance_id,
monitoring,
})
}
pub async fn new_session(&self) -> Result<Client> {
pub async fn get_or_renew_session(&self) -> Result<Client> {
let mut session_opt = self.current_session.lock().await;
let mut request_count = self.session_request_count.lock().await;
let old_request_count = *request_count;
// Session erneuern wenn:
// 1. Keine Session vorhanden
// 2. Request-Limit erreicht
let needs_renewal = session_opt.is_none() || *request_count >= self.max_requests_per_session;
if needs_renewal {
if let Some(ref mon) = self.monitoring {
mon.emit(crate::monitoring::MonitoringEvent::InstanceStatusChanged {
instance_id: self.instance_id,
status: crate::monitoring::InstanceStatusChange::Renewing,
});
}
// Alte Session schließen
if let Some(old_session) = session_opt.take() {
crate::util::logger::log_info("Closing old session").await;
let _ = old_session.close().await;
// Kurze Pause zwischen Sessions
let random_delay = random_range(500, 1000);
sleep(Duration::from_millis(random_delay)).await;
}
// Neue Session mit frischem User-Agent erstellen
crate::util::logger::log_info(&format!(
"Creating new session (requests in last session: {})",
*request_count
)).await;
let new_session = self.create_fresh_session().await?;
*session_opt = Some(new_session.clone());
*request_count = 0;
if let Some(ref mon) = self.monitoring {
let reason = if *request_count >= self.max_requests_per_session {
crate::monitoring::RenewalReason::RequestLimit
} else {
crate::monitoring::RenewalReason::TaskLimit
};
// Get updated proxy info
let new_proxy_info = if let Some(ref pp) = self.proxy_pool {
let proxy_idx = *self.current_proxy_index.lock().await;
pp.get_proxy_info(proxy_idx)
} else {
self.proxy_url.as_ref().and_then(|url| {
if let Some(port_str) = url.split(':').last() {
if let Ok(port) = port_str.parse::<u16>() {
return Some(crate::monitoring::ProxyInfo {
container_name: format!("proxy-{}", self.instance_id),
ip_address: "127.0.0.1".to_string(),
port,
status: crate::monitoring::ProxyStatus::Connected,
});
}
}
None
})
};
mon.emit(crate::monitoring::MonitoringEvent::SessionRenewed {
instance_id: self.instance_id,
old_request_count: *request_count,
reason: crate::monitoring::RenewalReason::RequestLimit,
new_proxy: new_proxy_info,
});
}
Ok(new_session)
} else {
// Existierende Session verwenden
*request_count += 1;
Ok(session_opt.as_ref().unwrap().clone())
}
}
async fn create_fresh_session(&self) -> Result<Client> {
// Hole aktuellen Proxy-URL ohne self zu mutieren
let proxy_url = if let Some(ref pool) = self.proxy_pool {
let mut proxy_idx = self.current_proxy_index.lock().await;
*proxy_idx = (*proxy_idx + 1) % pool.num_proxies();
let url = pool.get_proxy_url(*proxy_idx);
crate::util::logger::log_info(&format!(
"Using proxy {} for new session",
*proxy_idx
)).await;
Some(url)
} else {
self.proxy_url.clone()
};
let user_agent = Self::chrome_user_agent();
let capabilities = self.chrome_args_with_ua(user_agent, &proxy_url);
ClientBuilder::native()
.capabilities(self.chrome_args())
.capabilities(capabilities)
.connect(&self.base_url)
.await
.context("Failed to connect to ChromeDriver")
}
fn chrome_args_with_ua(&self, user_agent: &str, proxy_url: &Option<String>) -> Map<String, Value> {
let mut args = vec![
"--headless=new".to_string(),
"--disable-gpu".to_string(),
"--no-sandbox".to_string(),
"--disable-dev-shm-usage".to_string(),
"--disable-infobars".to_string(),
"--disable-extensions".to_string(),
"--disable-popup-blocking".to_string(),
"--disable-notifications".to_string(),
"--disable-autofill".to_string(),
"--disable-sync".to_string(),
"--disable-default-apps".to_string(),
"--disable-translate".to_string(),
"--disable-blink-features=AutomationControlled".to_string(),
format!("--user-agent={}", user_agent),
];
if let Some(proxy) = proxy_url {
args.push(format!("--proxy-server={}", proxy));
}
let caps = serde_json::json!({
"goog:chromeOptions": {
"args": args,
"excludeSwitches": ["enable-logging", "enable-automation"],
"prefs": {
"profile.default_content_setting_values.notifications": 2
}
}
});
caps.as_object().cloned().unwrap()
}
pub fn reset_task_count(&mut self) {
self.task_count = 0;
}
pub async fn get_session_stats(&self) -> (usize, usize) {
let request_count = *self.session_request_count.lock().await;
(self.task_count, request_count)
}
pub fn increment_task_count(&mut self) {
self.task_count += 1;
}
@@ -362,40 +624,15 @@ impl ChromeInstance {
Err(anyhow!("ChromeDriver failed to start within 30s"))
}
fn chrome_args(&self) -> Map<String, Value> {
let mut args = vec![
"--headless=new".to_string(),
"--disable-gpu".to_string(),
"--no-sandbox".to_string(),
"--disable-dev-shm-usage".to_string(),
"--disable-infobars".to_string(),
"--disable-extensions".to_string(),
"--disable-popup-blocking".to_string(),
"--disable-notifications".to_string(),
"--disable-logging".to_string(),
"--disable-autofill".to_string(),
"--disable-sync".to_string(),
"--disable-default-apps".to_string(),
"--disable-translate".to_string(),
"--window-size=1920,1080".to_string(),
"--disable-blink-features=AutomationControlled".to_string(),
"--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36".to_string()
pub fn chrome_user_agent() -> &'static str {
static UAS: &[&str] = &[
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.91 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.122 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.129 Safari/537.36",
"Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.6367.91 Safari/537.36",
];
if let Some(ref proxy) = self.proxy_url {
let proxy = proxy.clone();
let proxy_formatted = format!("--proxy-server={}", proxy);
args.push(proxy_formatted);
}
let caps = serde_json::json!({
"goog:chromeOptions": {
"args": args,
"excludeSwitches": ["enable-logging", "enable-automation"],
"prefs": {
"profile.default_content_setting_values.notifications": 2
}
}
});
caps.as_object().cloned().unwrap()
let random_user_agent = choose_random(UAS);
random_user_agent
}
}

82
src/util/logger.rs
View File

@@ -5,6 +5,8 @@ use tokio::sync::Mutex;
use std::fs::{self, OpenOptions};
use std::io::Write;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
static LOGGER: Lazy<Mutex<Option<DebugLogger>>> = Lazy::new(|| Mutex::new(None));
@@ -76,3 +78,83 @@ pub async fn log_warn(msg: &str) {
pub async fn log_error(msg: &str) {
log_detailed("ERROR", msg).await;
}
struct PoolLogger {
file: std::fs::File,
log_path: PathBuf,
}
impl PoolLogger {
fn new(log_dir: &std::path::Path) -> std::io::Result<Self> {
fs::create_dir_all(log_dir)?;
let filename = format!("webdriver_{}.log", Local::now().format("%Y%m%d_%H%M%S"));
let log_path = log_dir.join(&filename);
let file = OpenOptions::new()
.create(true)
.append(true)
.open(&log_path)?;
Ok(Self { file, log_path })
}
async fn log(&mut self, msg: &str) {
let line = format!("[{}] {}\n", Local::now().format("%H:%M:%S"), msg);
let _ = self.file.write_all(line.as_bytes());
let _ = self.file.flush();
println!("{}", line.trim_end());
}
}
pub struct PoolMetrics {
pub total_requests: Arc<AtomicUsize>,
pub successful_requests: Arc<AtomicUsize>,
pub failed_requests: Arc<AtomicUsize>,
pub session_renewals: Arc<AtomicUsize>,
pub rotation_events: Arc<AtomicUsize>,
pub retries: Arc<AtomicUsize>,
// IMPROVEMENT: Neue Metriken für besseres Monitoring
pub navigation_timeouts: Arc<AtomicUsize>,
pub bot_detection_hits: Arc<AtomicUsize>,
pub proxy_failures: Arc<AtomicUsize>,
}
impl PoolMetrics {
pub fn new() -> Self {
Self {
total_requests: Arc::new(AtomicUsize::new(0)),
successful_requests: Arc::new(AtomicUsize::new(0)),
failed_requests: Arc::new(AtomicUsize::new(0)),
session_renewals: Arc::new(AtomicUsize::new(0)),
rotation_events: Arc::new(AtomicUsize::new(0)),
retries: Arc::new(AtomicUsize::new(0)),
navigation_timeouts: Arc::new(AtomicUsize::new(0)),
bot_detection_hits: Arc::new(AtomicUsize::new(0)),
proxy_failures: Arc::new(AtomicUsize::new(0)),
}
}
pub async fn log_stats(&self) {
let total = self.total_requests.load(Ordering::Relaxed);
let success = self.successful_requests.load(Ordering::Relaxed);
// FIX: Prefix unused variable with underscore
let _failed = self.failed_requests.load(Ordering::Relaxed);
let renewals = self.session_renewals.load(Ordering::Relaxed);
let rotations = self.rotation_events.load(Ordering::Relaxed);
let retries = self.retries.load(Ordering::Relaxed);
let timeouts = self.navigation_timeouts.load(Ordering::Relaxed);
let bot_hits = self.bot_detection_hits.load(Ordering::Relaxed);
let proxy_fails = self.proxy_failures.load(Ordering::Relaxed);
let success_rate = if total > 0 {
(success as f64 / total as f64) * 100.0
} else {
0.0
};
crate::util::logger::log_info(&format!(
"Pool Metrics: {} total requests, {:.1}% success rate, {} renewals, {} rotations, {} retries, {} timeouts, {} bot detections, {} proxy failures",
total, success_rate, renewals, rotations, retries, timeouts, bot_hits, proxy_fails
)).await;
}
}