donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

added update_rule for incremental change

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2025-12-15 23:47:28 +01:00
parent d744769138
commit d26e833d93
10 changed files with 566 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
src/main.rs
View File

@@ -13,18 +13,17 @@ use scraper::webdriver::ChromeDriverPool;
use util::directories::DataPaths;
use util::{logger, opnv};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
#[tokio::main]
async fn main() -> Result<()> {
cleanup_all_proxy_containers().await.ok();
// Load configuration from .env
let config = Config::load().map_err(|err| {
eprintln!("Failed to load config: {}", err);
err
})?;
// Initialize paths and logger
let paths = DataPaths::new(".")?;
logger::init_debug_logger(paths.logs_dir()).await.ok();
logger::log_info("=== Event Backtest Engine Started ===").await;
@@ -35,61 +34,36 @@ async fn main() -> Result<()> {
config.enable_vpn_rotation
)).await;
// === Step 1: Fetch fresh VPNBook credentials and .ovpn files (if rotation enabled) ===
// Simple shutdown flag
let shutdown_flag = Arc::new(AtomicBool::new(false));
// === Step 1: Fetch VPNBook configs ===
let proxy_pool: Option<Arc<DockerVpnProxyPool>> = if config.enable_vpn_rotation {
logger::log_info("VPN Rotation Enabled — Fetching latest VPNBook configs").await;
// We only need 1 Chrome instance to scrape vpnbook.com (no proxy yet)
let temp_pool = Arc::new(ChromeDriverPool::new(1).await?);
logger::log_info("VPN Rotation Enabled – Fetching latest VPNBook configs").await;
let temp_pool = Arc::new(ChromeDriverPool::new_with_proxy_and_task_limit(config.max_parallel_instances, None, config.max_tasks_per_instance).await?);
let (username, password, _files) = opnv::fetch_vpnbook_configs(&temp_pool, paths.cache_dir()).await?;
logger::log_info(&format!("VPNBook credentials → User: {}", username)).await;
// Count how many distinct servers (subfolders) we have in cache/openvpn/
let server_count = std::fs::read_dir(paths.cache_openvpn_dir())?
.filter(|e| e.as_ref().unwrap().path().is_dir())
.count();
if server_count == 0 {
logger::log_warn("No VPN servers found — continuing without VPN").await;
logger::log_warn("No VPN servers found – continuing without VPN").await;
None
} else {
logger::log_info(&format!("Found {} VPN servers — starting Docker proxy containers", server_count)).await;
let pp = Arc::new(
DockerVpnProxyPool::new(paths.cache_openvpn_dir(), username, password).await?
);
// Verify all proxies are working before proceeding
logger::log_info("Verifying all proxy connections...").await;
let mut all_working = true;
for i in 0..pp.num_proxies() {
match pp.test_proxy_connection(i).await {
Ok(ip) => {
logger::log_info(&format!(" Proxy {}: working with IP: {}", i + 1, ip)).await;
}
Err(e) => {
logger::log_error(&format!(" Proxy {}: FAILED - {}", i + 1, e)).await;
all_working = false;
}
}
}
if !all_working {
logger::log_warn("Some proxies failed, but continuing with working ones...").await;
} else {
logger::log_info("All proxies verified and ready!").await;
}
logger::log_info(&format!("Found {} VPN servers – starting Docker proxy containers", server_count)).await;
let pp = Arc::new(DockerVpnProxyPool::new(paths.cache_openvpn_dir(), username, password).await?);
logger::log_info(&format!("All {} Docker proxy containers started and ready", pp.num_proxies())).await;
Some(pp)
}
} else {
logger::log_info("VPN rotation disabled — using direct connection").await;
logger::log_info("VPN rotation disabled – using direct connection").await;
None
};
// === Step 2: Initialize the main ChromeDriver pool (with proxy if enabled) ===
// === Step 2: Initialize ChromeDriver pool ===
let pool_size = config.max_parallel_instances;
let task_limit = config.max_tasks_per_instance;
@@ -105,17 +79,23 @@ async fn main() -> Result<()> {
logger::log_info(&format!("ChromeDriver pool ready with {} instances", pool_size)).await;
// === Step 3: Graceful Ctrl+C handler ===
// === Step 3: Ctrl+C handler ===
{
let shutdown_flag_clone = Arc::clone(&shutdown_flag);
let pool_clone = Arc::clone(&pool);
let proxy_clone = proxy_pool.clone();
tokio::spawn(async move {
tokio::signal::ctrl_c().await.ok();
logger::log_info("Ctrl+C received – shutting down gracefully...").await;
// Set flag first
shutdown_flag_clone.store(true, Ordering::SeqCst);
// Wait a bit for tasks to notice
tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
logger::log_info("Ctrl+C received — shutting down gracefully...").await;
// Now works: &*pool_clone derefs Arc → &ChromeDriverPool
// Cleanup
if let Err(e) = (&*pool_clone).shutdown().await {
logger::log_error(&format!("Error during pool shutdown: {}", e)).await;
}
@@ -129,31 +109,34 @@ async fn main() -> Result<()> {
}
let _ = cleanup_all_proxy_containers().await;
std::process::exit(0);
});
}
// === Step 4: Run the actual scraping jobs ===
// === Step 4: Run scraping jobs ===
logger::log_info("--- Starting ECONOMIC data update ---").await;
economic::run_full_update(&config, &pool).await?;
logger::log_info("Economic update completed").await;
logger::log_info("--- Starting CORPORATE data update ---").await;
corporate::run_full_update(&config, &pool).await?;
logger::log_info("Corporate update completed").await;
// === Step 5: Final cleanup ===
logger::log_info("Shutting down ChromeDriver pool...").await;
pool.shutdown().await?;
if let Some(pp) = proxy_pool {
logger::log_info("Stopping Docker VPN proxy containers...").await;
pp.shutdown().await?;
// CLEANUP ANY LEFTOVER CONTAINERS FROM PREVIOUS RUNS
cleanup_all_proxy_containers().await.ok();
if !shutdown_flag.load(Ordering::SeqCst) {
logger::log_info("--- Starting CORPORATE data update ---").await;
corporate::run_full_update(&config, &pool, &shutdown_flag).await?;
logger::log_info("Corporate update completed").await;
}
logger::log_info("=== Application finished successfully ===").await;
// === Step 5: Final cleanup ===
if !shutdown_flag.load(Ordering::SeqCst) {
logger::log_info("Shutting down ChromeDriver pool...").await;
pool.shutdown().await?;
if let Some(pp) = proxy_pool {
logger::log_info("Stopping Docker VPN proxy containers...").await;
pp.shutdown().await?;
cleanup_all_proxy_containers().await.ok();
}
logger::log_info("=== Application finished successfully ===").await;
}
Ok(())
}