donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

capabable spawning multiple openvpn instances

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2025-12-11 00:36:46 +01:00
parent c9da56e8e9
commit 470f0922ed
17 changed files with 3000 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/config.rs
View File

@@ -22,12 +22,6 @@ pub struct Config {
#[serde(default)]
pub enable_vpn_rotation: bool,
/// Comma-separated list of VPN servers/country codes to rotate through.
/// Example: "US-Free#1,UK-Free#1,JP-Free#1" or "US,JP,DE"
/// If empty, VPN rotation is disabled.
#[serde(default)]
pub vpn_servers: String,
/// Number of tasks per session before rotating VPN
/// If set to 0, rotates VPN between economic and corporate phases
#[serde(default = "default_tasks_per_session")]
@@ -51,7 +45,6 @@ impl Default for Config {
max_parallel_instances: default_max_parallel_instances(),
max_tasks_per_instance: 0,
enable_vpn_rotation: false,
vpn_servers: String::new(),
tasks_per_vpn_session: default_tasks_per_session(),
}
}
@@ -100,9 +93,6 @@ impl Config {
.parse::<bool>()
.context("Failed to parse ENABLE_VPN_ROTATION as bool")?;
let vpn_servers = dotenvy::var("VPN_SERVERS")
.unwrap_or_else(|_| String::new());
let tasks_per_vpn_session: usize = dotenvy::var("TASKS_PER_VPN_SESSION")
.unwrap_or_else(|_| "0".to_string())
.parse()
@@ -115,7 +105,6 @@ impl Config {
max_parallel_instances,
max_tasks_per_instance,
enable_vpn_rotation,
vpn_servers,
tasks_per_vpn_session,
})
}

13
src/lib.rs
View File

@@ -5,4 +5,15 @@
pub mod config;
pub mod scraper;
pub mod util;
pub mod util;
// Re-export commonly used types for convenience
pub use config::Config;
pub use scraper::webdriver::{ChromeDriverPool, ChromeInstance, ScrapeTask};
pub use scraper::vpn_manager::{VpnInstance, VpnPool};
pub use util::directories::DataPaths;
pub use util::logger;
pub use util::opnv;
#[cfg(target_os = "windows")]
pub use scraper::forcebindip::ForceBindIpManager;

132
src/main.rs
View File

@@ -8,23 +8,28 @@ mod scraper;
use anyhow::Result;
use config::Config;
use scraper::webdriver::ChromeDriverPool;
use scraper::vpn_manager::VpnPool;
use util::directories::DataPaths;
use util::{logger, opnv};
use std::sync::Arc;
/// The entry point of the application.
///
/// This function loads the configuration, initializes a shared ChromeDriver pool,
/// fetches the latest VPNBook OpenVPN configurations if VPN rotation is enabled,
/// This function loads the configuration, optionally initializes a VPN pool,
/// initializes a shared ChromeDriver pool bound to the VPN pool (if enabled),
/// and sequentially runs the full updates for corporate and economic data.
/// Sequential execution helps prevent resource exhaustion from concurrent
/// chromedriver instances and avoids spamming the target websites with too many requests.
///
/// If VPN rotation is enabled:
/// 1. Fetches latest VPNBook OpenVPN configurations
/// 2. Creates a VPN pool and connects all VPN instances
/// 3. Binds each ChromeDriver instance to a different VPN for IP rotation
/// 4. Performs periodic health checks to reconnect unhealthy VPN instances
///
/// # Errors
///
/// Returns an error if configuration loading fails, pool initialization fails,
/// VPN fetching fails (if enabled), or if either update function encounters an issue
/// (e.g., network errors, scraping failures, or chromedriver spawn failures like "program not found").
/// (e.g., network errors, scraping failures, or chromedriver spawn failures).
#[tokio::main]
async fn main() -> Result<()> {
let config = Config::load().map_err(|err| {
@@ -41,27 +46,100 @@ async fn main() -> Result<()> {
})?;
logger::log_info("=== Application started ===").await;
logger::log_info(&format!("Config: economic_start_date={}, corporate_start_date={}, lookahead_months={}, max_parallel_instances={}, enable_vpn_rotation={}",
config.economic_start_date, config.corporate_start_date, config.economic_lookahead_months, config.max_parallel_instances, config.enable_vpn_rotation)).await;
logger::log_info(&format!("Config: economic_start_date={}, corporate_start_date={}, lookahead_months={}, max_parallel_instances={}, enable_vpn_rotation={}, max_tasks_per_instance={}",
config.economic_start_date, config.corporate_start_date, config.economic_lookahead_months, config.max_parallel_instances, config.enable_vpn_rotation, config.max_tasks_per_instance)).await;
// Initialize the shared ChromeDriver pool once
// Initialize VPN pool if enabled
let vpn_pool = if config.enable_vpn_rotation {
logger::log_info("=== VPN Rotation Enabled ===").await;
logger::log_info("--- Fetching latest VPNBook OpenVPN configurations ---").await;
let (username, password, _files) =
util::opnv::fetch_vpnbook_configs(&Arc::new(ChromeDriverPool::new(1).await?), paths.cache_dir()).await?;
let amount_of_openvpn_servers = _files.len();
logger::log_info(&format!("✓ Fetched VPN credentials - Username: {}", username)).await;
// Create VPN pool
let openvpn_dir = paths.cache_dir().join("openvpn");
logger::log_info("--- Initializing VPN Pool ---").await;
let vp = Arc::new(VpnPool::new(
&openvpn_dir,
username,
password,
true, // enable rotation
config.tasks_per_vpn_session,
amount_of_openvpn_servers,
).await?);
// Connect all VPN instances (gracefully handles failures)
logger::log_info("--- Connecting to VPN servers ---").await;
match vp.connect_all().await {
Ok(()) => {
logger::log_info("✓ VPN initialization complete").await;
Some(vp)
}
Err(e) => {
logger::log_warn(&format!(
"⚠ VPN initialization failed: {}. Continuing without VPN.",
e
)).await;
None
}
}
} else {
None
};
// Initialize the shared ChromeDriver pool with VPN pool
let pool_size = config.max_parallel_instances;
logger::log_info(&format!("Initializing ChromeDriver pool with size: {}", pool_size)).await;
let max_tasks_per_instance = config.max_tasks_per_instance;
logger::log_info(&format!(
"Initializing ChromeDriver pool with size: {}{}",
pool_size,
if max_tasks_per_instance > 0 { &format!(" (max {} tasks/instance)", max_tasks_per_instance) } else { "" }
)).await;
let pool = Arc::new(
if max_tasks_per_instance > 0 {
ChromeDriverPool::new_with_vpn_and_task_limit(pool_size, vpn_pool.clone(), max_tasks_per_instance).await?
} else if vpn_pool.is_some() {
ChromeDriverPool::new_with_vpn(pool_size, vpn_pool.clone()).await?
} else {
ChromeDriverPool::new(pool_size).await?
}
);
let pool = Arc::new(ChromeDriverPool::new(pool_size).await?);
logger::log_info("✓ ChromeDriver pool initialized successfully").await;
// Fetch VPNBook configs if VPN rotation is enabled
if config.enable_vpn_rotation {
logger::log_info("--- Fetching latest VPNBook OpenVPN configurations ---").await;
let (username, password, files) =
util::opnv::fetch_vpnbook_configs(&pool, paths.cache_dir()).await?;
logger::log_info(&format!("Fetched VPN username: {}, password: {}", username, password)).await;
for file in &files {
logger::log_info(&format!("Extracted OVPN: {:?}", file)).await;
}
// Optionally, store username/password for rotation use (e.g., in a file or global state)
// For now, just log them; extend as needed for rotation integration
// Spawn background Ctrl-C handler to gracefully shutdown pool and VPNs
{
let pool_for_signal = Arc::clone(&pool);
let vpn_for_signal = vpn_pool.clone();
tokio::spawn(async move {
if let Err(e) = tokio::signal::ctrl_c().await {
let _ = util::logger::log_error(&format!("Ctrl-C handler failed to install: {}", e)).await;
return;
}
let _ = util::logger::log_info("Ctrl-C received — initiating graceful shutdown").await;
if let Err(e) = pool_for_signal.shutdown().await {
let _ = util::logger::log_warn(&format!("Error shutting down ChromeDriver pool: {}", e)).await;
}
if let Some(vp) = vpn_for_signal {
if let Err(e) = vp.disconnect_all().await {
let _ = util::logger::log_warn(&format!("Error disconnecting VPNs: {}", e)).await;
}
}
let _ = util::logger::log_info("Graceful shutdown complete (from Ctrl-C)").await;
// Exit the process now that cleanup is done
std::process::exit(0);
});
}
// Run economic update first, passing the shared pool
@@ -74,6 +152,18 @@ async fn main() -> Result<()> {
corporate::run_full_update(&config, &pool).await?;
logger::log_info("✓ Corporate data update completed").await;
// Shutdown ChromeDriver pool before disconnecting VPNs so instances can
// cleanly terminate any network-bound processes.
logger::log_info("--- Shutting down ChromeDriver pool ---").await;
pool.shutdown().await?;
logger::log_info("✓ ChromeDriver pool shutdown complete").await;
// Disconnect all VPN instances if enabled
if let Some(vp) = vpn_pool {
logger::log_info("--- Disconnecting VPN instances ---").await;
vp.disconnect_all().await?;
}
logger::log_info("=== Application completed successfully ===").await;
Ok(())
}

7
src/scraper/create_tapctls.sh Normal file
View File

@@ -0,0 +1,7 @@
# Als Administrator ausführen
cd "C:\Program Files\OpenVPN\bin"
# 10 TAP-Adapter hinzufügen
for ($i=2; $i -le 10; $i++) {
.\tapctl.exe create --name "OpenVPN-TAP-$i"
}

163
src/scraper/forcebindip.rs Normal file
View File

@@ -0,0 +1,163 @@
// src/scraper/forcebindip.rs
use anyhow::{anyhow, Context, Result};
use std::path::{Path, PathBuf};
use std::process::Command;
/// Manages ForceBindIP integration for binding processes to specific IP addresses
pub struct ForceBindIpManager {
forcebindip_path: PathBuf,
}
impl ForceBindIpManager {
/// Creates a new ForceBindIP manager
///
/// On Windows, looks for ForceBindIP.exe in common locations or PATH
/// On other platforms, returns an error as ForceBindIP is Windows-only
pub fn new() -> Result<Self> {
#[cfg(target_os = "windows")]
{
let possible_paths = vec![
PathBuf::from("ForceBindIP.exe"),
PathBuf::from("tools/ForceBindIP.exe"),
PathBuf::from("C:/Program Files/ForceBindIP/ForceBindIP.exe"),
PathBuf::from("C:/Program Files (x86)/ForceBindIP/ForceBindIP.exe"),
];
for path in possible_paths {
if path.exists() {
return Ok(Self {
forcebindip_path: path,
});
}
}
// Try to find in PATH
if let Ok(output) = Command::new("where").arg("ForceBindIP.exe").output() {
if output.status.success() {
let path_str = String::from_utf8_lossy(&output.stdout);
let path = PathBuf::from(path_str.trim());
if path.exists() {
return Ok(Self {
forcebindip_path: path,
});
}
}
}
Err(anyhow!(
"ForceBindIP.exe not found. Please download from http://r1ch.net/projects/forcebindip \
and place it in the project directory or add to PATH"
))
}
#[cfg(not(target_os = "windows"))]
{
Err(anyhow!(
"ForceBindIP is only available on Windows. For Linux/macOS, consider using \
network namespaces or other routing mechanisms"
))
}
}
/// Creates a command that will run the given program bound to the specified IP
///
/// # Arguments
/// * `bind_ip` - The IP address to bind to
/// * `program` - Path to the program to execute
/// * `args` - Arguments to pass to the program
///
/// # Returns
/// A configured Command ready to be spawned
pub fn create_bound_command(
&self,
bind_ip: &str,
program: &Path,
args: &[&str],
) -> Command {
let mut cmd = Command::new(&self.forcebindip_path);
// ForceBindIP syntax: ForceBindIP.exe [IP] [program] [args...]
cmd.arg(bind_ip)
.arg(program);
for arg in args {
cmd.arg(arg);
}
cmd
}
/// Verifies that ForceBindIP is working by testing with a simple command
pub async fn verify_installation(&self) -> Result<()> {
#[cfg(target_os = "windows")]
{
// Test by running a simple command
let output = Command::new(&self.forcebindip_path)
.arg("0.0.0.0")
.arg("cmd.exe")
.arg("/c")
.arg("echo test")
.output()
.context("Failed to execute ForceBindIP verification")?;
if !output.status.success() {
return Err(anyhow!(
"ForceBindIP verification failed. stderr: {}",
String::from_utf8_lossy(&output.stderr)
));
}
Ok(())
}
#[cfg(not(target_os = "windows"))]
{
Err(anyhow!("ForceBindIP verification not available on non-Windows platforms"))
}
}
/// Returns the path to the ForceBindIP executable
pub fn path(&self) -> &Path {
&self.forcebindip_path
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
#[cfg(target_os = "windows")]
fn test_forcebindip_manager_creation() {
// This test will only pass if ForceBindIP is actually installed
// In CI/CD, you might want to skip this or mock it
match ForceBindIpManager::new() {
Ok(manager) => {
println!("ForceBindIP found at: {:?}", manager.path());
}
Err(e) => {
println!("ForceBindIP not found (expected in dev environments): {}", e);
}
}
}
#[test]
fn test_command_creation() {
#[cfg(target_os = "windows")]
{
if let Ok(manager) = ForceBindIpManager::new() {
let cmd = manager.create_bound_command(
"192.168.1.1",
Path::new("test.exe"),
&["--arg1", "--arg2"],
);
// Verify the command is constructed correctly
let cmd_str = format!("{:?}", cmd);
assert!(cmd_str.contains("192.168.1.1"));
assert!(cmd_str.contains("test.exe"));
}
}
}
}

135
src/scraper/install_tap_adapter.ps1 Normal file
View File

@@ -0,0 +1,135 @@
# install_tap_adapters.ps1
# Installs additional TAP-Windows adapters for parallel OpenVPN connections
# MUST BE RUN AS ADMINISTRATOR
$ErrorActionPreference = "Stop"
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "TAP Adapter Installation Script" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host ""
# Check if running as Administrator
$currentPrincipal = New-Object Security.Principal.WindowsPrincipal([Security.Principal.WindowsIdentity]::GetCurrent())
$isAdmin = $currentPrincipal.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)
if (-not $isAdmin) {
Write-Host "ERROR: This script must be run as Administrator!" -ForegroundColor Red
Write-Host ""
Write-Host "To run as Administrator:" -ForegroundColor Yellow
Write-Host " 1. Right-click PowerShell" -ForegroundColor Yellow
Write-Host " 2. Select 'Run as Administrator'" -ForegroundColor Yellow
Write-Host " 3. Run this script again" -ForegroundColor Yellow
Write-Host ""
Read-Host "Press Enter to exit"
exit 1
}
Write-Host "✓ Running with Administrator privileges" -ForegroundColor Green
Write-Host ""
# Check for OpenVPN installation
$tapctlPath = "C:\Program Files\OpenVPN\bin\tapctl.exe"
if (-not (Test-Path $tapctlPath)) {
Write-Host "ERROR: OpenVPN not found!" -ForegroundColor Red
Write-Host ""
Write-Host "Expected location: $tapctlPath" -ForegroundColor Yellow
Write-Host ""
Write-Host "Please install OpenVPN from:" -ForegroundColor Yellow
Write-Host "https://openvpn.net/community-downloads/" -ForegroundColor Cyan
Write-Host ""
Read-Host "Press Enter to exit"
exit 1
}
Write-Host "✓ OpenVPN found at: $tapctlPath" -ForegroundColor Green
Write-Host ""
# Count existing TAP adapters
Write-Host "Checking existing TAP adapters..." -ForegroundColor Cyan
$existingAdapters = Get-NetAdapter | Where-Object { $_.InterfaceDescription -like "*TAP*" }
$existingCount = $existingAdapters.Count
Write-Host " Found $existingCount existing TAP adapter(s)" -ForegroundColor Yellow
if ($existingCount -ge 10) {
Write-Host ""
Write-Host "✓ You already have $existingCount TAP adapters (sufficient)" -ForegroundColor Green
Write-Host ""
Read-Host "Press Enter to exit"
exit 0
}
Write-Host ""
Write-Host "Installing additional TAP adapters..." -ForegroundColor Cyan
Write-Host " Target: 10 total adapters" -ForegroundColor Yellow
Write-Host " To install: $(10 - $existingCount) adapters" -ForegroundColor Yellow
Write-Host ""
$targetCount = 10
$successCount = 0
$failCount = 0
for ($i = ($existingCount + 1); $i -le $targetCount; $i++) {
$adapterName = "OpenVPN-TAP-$i"
Write-Host "[$i/$targetCount] Creating $adapterName..." -ForegroundColor Cyan
try {
$output = & $tapctlPath create --name $adapterName 2>&1
if ($LASTEXITCODE -eq 0) {
Write-Host " ✓ Successfully created $adapterName" -ForegroundColor Green
$successCount++
} else {
Write-Host " ⚠ Failed to create $adapterName (exit code: $LASTEXITCODE)" -ForegroundColor Red
Write-Host " Output: $output" -ForegroundColor Gray
$failCount++
}
} catch {
Write-Host " ✗ Error creating $adapterName : $_" -ForegroundColor Red
$failCount++
}
# Small delay to prevent resource conflicts
Start-Sleep -Milliseconds 500
}
Write-Host ""
Write-Host "========================================" -ForegroundColor Cyan
Write-Host "Installation Summary" -ForegroundColor Cyan
Write-Host "========================================" -ForegroundColor Cyan
Write-Host " Successfully created: $successCount adapter(s)" -ForegroundColor Green
Write-Host " Failed: $failCount adapter(s)" -ForegroundColor $(if ($failCount -gt 0) { "Red" } else { "Gray" })
Write-Host ""
# Verify final count
Write-Host "Verifying installation..." -ForegroundColor Cyan
Start-Sleep -Seconds 2
$finalAdapters = Get-NetAdapter | Where-Object { $_.InterfaceDescription -like "*TAP*" }
$finalCount = $finalAdapters.Count
Write-Host ""
Write-Host "Total TAP adapters now: $finalCount" -ForegroundColor $(if ($finalCount -ge 10) { "Green" } else { "Yellow" })
Write-Host ""
if ($finalCount -ge 10) {
Write-Host "✓ Installation complete! You now have sufficient TAP adapters." -ForegroundColor Green
Write-Host " You can now run up to $(($finalCount * 3/4)) VPN connections in parallel." -ForegroundColor Cyan
} elseif ($finalCount -gt $existingCount) {
Write-Host "⚠ Partial success. Added $(($finalCount - $existingCount)) adapter(s)." -ForegroundColor Yellow
Write-Host " You can run up to $(($finalCount * 3/4)) VPN connections in parallel." -ForegroundColor Cyan
Write-Host " Consider running this script again if you need more." -ForegroundColor Yellow
} else {
Write-Host "✗ No adapters were added. Check error messages above." -ForegroundColor Red
}
Write-Host ""
Write-Host "Adapter List:" -ForegroundColor Cyan
$finalAdapters | ForEach-Object {
Write-Host " • $($_.Name) ($($_.InterfaceDescription))" -ForegroundColor Gray
}
Write-Host ""
Read-Host "Press Enter to exit"

4
src/scraper/mod.rs
View File

@@ -1 +1,5 @@
pub mod webdriver;
pub mod vpn_manager;
#[cfg(target_os = "windows")]
pub mod forcebindip;

1422
src/scraper/vpn_manager.rs Normal file
View File

File diff suppressed because it is too large Load Diff

397
src/scraper/vpn_rotation_system.md Normal file
View File

@@ -0,0 +1,397 @@
# VPN Rotation System - Setup Checklist
## 🚀 Quick Setup (5 Minutes)
Follow these steps to get your VPN rotation system up and running:
### ✅ Step 1: Install OpenVPN
**Windows:**
```powershell
# Download installer
# https://openvpn.net/community-downloads/
# Install to default location
# Add to PATH: C:\Program Files\OpenVPN\bin
# Verify installation
openvpn --version
```
**Linux (Ubuntu/Debian):**
```bash
sudo apt-get update
sudo apt-get install openvpn
openvpn --version
```
**macOS:**
```bash
brew install openvpn
openvpn --version
```
### ✅ Step 2: Install ForceBindIP (Windows Only)
```powershell
# Download from: http://r1ch.net/projects/forcebindip
# Extract ForceBindIP.exe and place in one of:
# Option 1: Project root
.\ForceBindIP.exe
# Option 2: Tools directory
.\tools\ForceBindIP.exe
# Option 3: Add to PATH
C:\Program Files\ForceBindIP\ForceBindIP.exe
# Verify installation
ForceBindIP.exe
```
**Linux/macOS Users:**
- ForceBindIP is Windows-only
- Use network namespaces (Linux) or alternative routing
- See documentation for workarounds
### ✅ Step 3: Update Cargo.toml
Add these dependencies if not already present:
```toml
[dependencies]
anyhow = "1.0"
tokio = { version = "1.0", features = ["full"] }
fantoccini = "0.19"
reqwest = { version = "0.11", features = ["blocking"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
chrono = "0.4"
once_cell = "1.19"
dotenvy = "0.15"
url = "2.5"
zip = "0.6"
```
### ✅ Step 4: Configure Environment
Create or update `.env` file in project root:
```bash
# Required: Date ranges
ECONOMIC_START_DATE=2007-02-13
CORPORATE_START_DATE=2010-01-01
ECONOMIC_LOOKAHEAD_MONTHS=3
# Required: Parallelism
MAX_PARALLEL_INSTANCES=5
MAX_TASKS_PER_INSTANCE=0
# VPN Configuration
ENABLE_VPN_ROTATION=true
TASKS_PER_VPN_SESSION=50
```
**Configuration Presets:**
**Conservative (Recommended for first run):**
```bash
MAX_PARALLEL_INSTANCES=3
TASKS_PER_VPN_SESSION=100
```
**Balanced:**
```bash
MAX_PARALLEL_INSTANCES=5
TASKS_PER_VPN_SESSION=50
```
**Aggressive (Use with caution):**
```bash
MAX_PARALLEL_INSTANCES=10
TASKS_PER_VPN_SESSION=25
```
### ✅ Step 5: Add VPN Module Files
Copy these files to your project:
```
src/
├── scraper/
│ ├── mod.rs (update with: pub mod vpn_manager; pub mod forcebindip;)
│ ├── vpn_manager.rs (new file - from artifact)
│ ├── forcebindip.rs (new file - from artifact)
│ └── webdriver.rs (replace with VPN-enabled version)
├── util/
│ ├── mod.rs (already includes opnv)
│ ├── opnv.rs (already present)
│ └── ...
├── main.rs (replace with VPN-enabled version)
└── lib.rs (update to expose VPN modules)
```
### ✅ Step 6: Verify Directory Structure
Ensure these directories exist (will be auto-created):
```
project/
├── cache/
│ ├── openvpn/ (VPN configs stored here)
│ └── temp_vpn_zips/ (temporary, auto-cleaned)
├── logs/ (application logs)
├── data/
│ ├── economic/
│ └── corporate/
└── chromedriver-win64/
└── chromedriver.exe
```
### ✅ Step 7: Test Installation
**Test 1: OpenVPN**
```bash
openvpn --version
# Should output version info
```
**Test 2: ForceBindIP (Windows)**
```powershell
ForceBindIP.exe 127.0.0.1 cmd.exe /c echo test
# Should output: test
```
**Test 3: Build Project**
```bash
cargo build --release
# Should compile without errors
```
**Test 4: Dry Run (No VPN)**
```bash
# Temporarily disable VPN
# Set in .env: ENABLE_VPN_ROTATION=false
cargo run --release
# Should initialize ChromeDriver pool and run
```
### ✅ Step 8: First VPN-Enabled Run
```bash
# Enable VPN in .env
ENABLE_VPN_ROTATION=true
TASKS_PER_VPN_SESSION=0 # Start with phase rotation only
# Run application
cargo run --release
# Watch logs
tail -f logs/backtest_*.log
```
**Expected Output:**
```
[HH:MM:SS] [INFO] === Application started ===
[HH:MM:SS] [INFO] === VPN Rotation Enabled ===
[HH:MM:SS] [INFO] --- Fetching latest VPNBook OpenVPN configurations ---
[HH:MM:SS] [INFO] ✓ Fetched VPN credentials - Username: vpnbook
[HH:MM:SS] [INFO] ✓ Downloaded 6 .ovpn configuration files
[HH:MM:SS] [INFO] --- Initializing VPN Pool ---
[HH:MM:SS] [INFO] Found 6 OpenVPN configurations
[HH:MM:SS] [INFO] --- Connecting to VPN servers ---
[HH:MM:SS] [INFO] Starting VPN connection for ca149.vpnbook.com
[HH:MM:SS] [INFO] ✓ VPN ca149.vpnbook.com connected with IP: 142.4.217.133
...
[HH:MM:SS] [INFO] ✓ ChromeDriver pool initialized successfully
```
## 🎯 Common Issues and Solutions
### Issue: "openvpn: command not found"
```bash
# Windows: Add to PATH
setx PATH "%PATH%;C:\Program Files\OpenVPN\bin"
# Linux: Install package
sudo apt-get install openvpn
# Verify
which openvpn
```
### Issue: "ForceBindIP.exe not found"
```powershell
# Place in project root
curl -o ForceBindIP.exe http://r1ch.net/projects/forcebindip/ForceBindIP.exe
# Or add to PATH
setx PATH "%PATH%;C:\path\to\ForceBindIP"
```
### Issue: VPN Connection Timeout
```bash
# Try different config file
# VPNBook offers multiple servers/protocols
# Look in cache/openvpn/ after first fetch
# Files named like:
# - vpnbook-ca149-tcp80.ovpn (TCP port 80 - most compatible)
# - vpnbook-ca149-tcp443.ovpn (TCP port 443 - works through most firewalls)
# - vpnbook-ca149-udp53.ovpn (UDP port 53 - faster but may be blocked)
# Check firewall settings
# - Allow OpenVPN.exe through Windows Firewall
# - Allow outbound connections on ports 80, 443, 53, 1194
```
### Issue: "Failed to spawn chromedriver"
```bash
# Verify chromedriver path
ls chromedriver-win64/chromedriver.exe
# Check Chrome/ChromeDriver version match
chromedriver.exe --version
# Chrome version should be compatible
# Update ChromeDriver if needed
# Download from: https://chromedriver.chromium.org/
```
### Issue: "Semaphore closed"
```bash
# Reduce parallelism in .env
MAX_PARALLEL_INSTANCES=3
# Or increase system resources
# Check Task Manager / Activity Monitor
```
## 📊 Performance Tuning
### Optimize for Speed
```bash
MAX_PARALLEL_INSTANCES=10
TASKS_PER_VPN_SESSION=100
# More instances, less frequent rotation
# Risk: More aggressive, may hit rate limits
```
### Optimize for Stealth
```bash
MAX_PARALLEL_INSTANCES=2
TASKS_PER_VPN_SESSION=10
# Fewer instances, frequent rotation
# Risk: Slower, but more IP diversity
```
### Optimize for Stability
```bash
MAX_PARALLEL_INSTANCES=5
TASKS_PER_VPN_SESSION=50
# Balanced approach (recommended)
```
## 🔍 Monitoring and Logs
### Key Log Files
```
logs/
└── backtest_YYYYMMDD_HHMMSS.log
```
### Important Log Patterns
**Successful VPN Connection:**
```
[INFO] ✓ VPN ca149.vpnbook.com connected with IP: 142.4.217.133
```
**VPN Rotation:**
```
[INFO] ✓ VPN ca149.vpnbook.com rotated: 142.4.217.133 -> 142.4.217.201
```
**Health Issues:**
```
[WARN] ⚠ Health check failed for VPN us1.vpnbook.com
[INFO] Attempting to reconnect unhealthy VPN: us1.vpnbook.com
```
**Binding ChromeDriver:**
```
[INFO] Binding ChromeDriver to VPN IP: 142.4.217.133
```
### Monitor Real-Time
```bash
# Linux/macOS
tail -f logs/backtest_*.log
# Windows PowerShell
Get-Content logs\backtest_*.log -Wait -Tail 50
```
### Search Logs
```bash
# Count successful connections
grep "connected with IP" logs/*.log | wc -l
# Find errors
grep ERROR logs/*.log
# Track rotations
grep "rotated:" logs/*.log
# Find failed tasks
grep "failed" logs/*.log
```
## 🚦 Next Steps
1. **✅ Complete Setup**: Verify all checkboxes above
2. **🧪 Test Run**: Run with `TASKS_PER_VPN_SESSION=0` first
3. **📊 Monitor**: Watch logs during first run
4. **⚙️ Tune**: Adjust configuration based on results
5. **🔄 Iterate**: Increase parallelism gradually
6. **📈 Scale**: Once stable, increase to production levels
## 📚 Additional Resources
- **VPNBook Website**: https://www.vpnbook.com/freevpn
- **OpenVPN Docs**: https://openvpn.net/community-resources/
- **ForceBindIP**: http://r1ch.net/projects/forcebindip
- **ChromeDriver**: https://chromedriver.chromium.org/
## 🆘 Getting Help
If you encounter issues:
1. **Check Prerequisites**: Verify all software is installed
2. **Review Logs**: Look in `logs/` directory
3. **Test Components**: Test OpenVPN and ForceBindIP independently
4. **Simplify**: Start with `ENABLE_VPN_ROTATION=false`
5. **Document Error**: Note exact error message and context
## 🎉 Success Criteria
You're ready to proceed when you see:
```
✓ OpenVPN installed and in PATH
✓ ForceBindIP.exe accessible (Windows)
✓ Project compiles successfully
✓ VPN configurations fetched
✓ All VPNs connected
✓ ChromeDriver pool initialized
✓ First scraping task completed
```
**Congratulations! Your VPN rotation system is now operational.**
---
*Last Updated: December 2024*
*Version: 1.0*

352
src/scraper/webdriver.rs
View File

@@ -8,52 +8,104 @@ use std::process::Stdio;
use std::sync::Arc;
use tokio::io::{AsyncBufReadExt, BufReader};
use tokio::process::{Child, Command};
use tokio::task::JoinHandle;
use tokio::sync::{Mutex, Semaphore};
use tokio::time::{sleep, timeout, Duration};
/// Manages a pool of ChromeDriver instances for parallel scraping.
///
/// This struct maintains multiple ChromeDriver processes and allows controlled
/// concurrent access via a semaphore. Instances are reused across tasks to avoid
/// the overhead of spawning new processes.
use super::vpn_manager::{VpnInstance, VpnPool};
#[cfg(target_os = "windows")]
use super::forcebindip::ForceBindIpManager;
/// Manages a pool of ChromeDriver instances for parallel scraping with optional VPN binding.
pub struct ChromeDriverPool {
instances: Vec<Arc<Mutex<ChromeInstance>>>,
semaphore: Arc<Semaphore>,
tasks_per_instance: usize,
vpn_pool: Option<Arc<VpnPool>>,
#[cfg(target_os = "windows")]
forcebindip: Option<Arc<ForceBindIpManager>>,
}
impl ChromeDriverPool {
/// Creates a new pool with the specified number of ChromeDriver instances.
///
/// # Arguments
/// * `pool_size` - Number of concurrent ChromeDriver instances to maintain
/// Creates a new pool with the specified number of ChromeDriver instances (no VPN).
pub async fn new(pool_size: usize) -> Result<Self> {
Self::new_with_vpn_and_task_limit(pool_size, None, 0).await
}
/// Creates a new ChromeDriver pool with task-per-instance tracking.
pub async fn new_with_task_limit(
pool_size: usize,
max_tasks_per_instance: usize,
) -> Result<Self> {
Self::new_with_vpn_and_task_limit(pool_size, None, max_tasks_per_instance).await
}
/// Creates a new pool with VPN support.
pub async fn new_with_vpn(
pool_size: usize,
vpn_pool: Option<Arc<VpnPool>>,
) -> Result<Self> {
Self::new_with_vpn_and_task_limit(pool_size, vpn_pool, 0).await
}
/// Creates a new pool with VPN support and task-per-instance limits.
pub async fn new_with_vpn_and_task_limit(
pool_size: usize,
vpn_pool: Option<Arc<VpnPool>>,
max_tasks_per_instance: usize,
) -> Result<Self> {
let mut instances = Vec::with_capacity(pool_size);
println!(
"Initializing ChromeDriver pool with {} instances...",
pool_size
);
for i in 0..pool_size {
match ChromeInstance::new().await {
Ok(instance) => {
println!(" ✓ Instance {} ready", i + 1);
instances.push(Arc::new(Mutex::new(instance)));
#[cfg(target_os = "windows")]
let forcebindip = if vpn_pool.is_some() {
match ForceBindIpManager::new() {
Ok(manager) => {
crate::util::logger::log_info("✓ ForceBindIP manager initialized").await;
Some(Arc::new(manager))
}
Err(e) => {
eprintln!(" ✗ Failed to create instance {}: {}", i + 1, e);
// Clean up already created instances
drop(instances);
return Err(e);
crate::util::logger::log_warn(&format!(
"⚠ ForceBindIP not available: {}. Proceeding without IP binding.",
e
)).await;
None
}
}
} else {
None
};
crate::util::logger::log_info(&format!(
"Initializing ChromeDriver pool with {} instances{}{}...",
pool_size,
if vpn_pool.is_some() { " (VPN-enabled)" } else { "" },
if max_tasks_per_instance > 0 { &format!(" (max {} tasks/instance)", max_tasks_per_instance) } else { "" }
)).await;
for i in 0..pool_size {
// If VPN pool exists, acquire a VPN instance for this ChromeDriver
let vpn_instance = if let Some(ref vp) = vpn_pool {
Some(vp.acquire().await?)
} else {
None
};
#[cfg(target_os = "windows")]
let instance = ChromeInstance::new_with_task_limit(vpn_instance, forcebindip.clone(), max_tasks_per_instance).await?;
#[cfg(not(target_os = "windows"))]
let instance = ChromeInstance::new_with_task_limit(vpn_instance, max_tasks_per_instance).await?;
crate::util::logger::log_info(&format!(" ✓ Instance {} ready", i + 1)).await;
instances.push(Arc::new(Mutex::new(instance)));
}
Ok(Self {
instances,
semaphore: Arc::new(Semaphore::new(pool_size)),
tasks_per_instance: 0,
vpn_pool,
#[cfg(target_os = "windows")]
forcebindip,
})
}
@@ -72,9 +124,31 @@ impl ChromeDriverPool {
.map_err(|_| anyhow!("Semaphore closed"))?;
// Find an available instance (round-robin or first available)
let instance = self.instances[0].clone(); // Simple: use first, could be round-robin
let instance = self.instances[0].clone();
let mut guard = instance.lock().await;
// Track task count
guard.increment_task_count();
// Get VPN info before creating session
let vpn_info = if let Some(ref vpn) = guard.vpn_instance {
let vpn_guard = vpn.lock().await;
Some(format!("{} ({})",
vpn_guard.hostname(),
vpn_guard.external_ip().unwrap_or("unknown")))
} else {
None
};
// Log task count if limit is set
if guard.max_tasks_per_instance > 0 {
crate::util::logger::log_info(&format!(
"Instance task count: {}/{}",
guard.get_task_count(),
guard.max_tasks_per_instance
)).await;
}
// Create a new session for this task
let client = guard.new_session().await?;
@@ -82,40 +156,137 @@ impl ChromeDriverPool {
drop(guard);
// Navigate and parse
if let Some(ref info) = vpn_info {
crate::util::logger::log_info(&format!("Scraping {} via VPN: {}", url, info)).await;
}
client.goto(&url).await.context("Failed to navigate")?;
let result = timeout(Duration::from_secs(60), parse(client))
.await
.context("Parse function timed out after 60s")??;
// Handle VPN rotation if needed
if let Some(ref vpn_pool) = self.vpn_pool {
let mut guard = instance.lock().await;
if let Some(ref vpn) = guard.vpn_instance {
vpn_pool.rotate_if_needed(vpn.clone()).await?;
guard.reset_task_count(); // Reset task count on VPN rotation
}
}
Ok(result)
}
pub fn get_number_of_instances(&self) -> usize {
self.instances.len()
}
/// Returns whether VPN is enabled for this pool
pub fn is_vpn_enabled(&self) -> bool {
self.vpn_pool.is_some()
}
/// Gracefully shutdown all ChromeDriver instances in the pool.
pub async fn shutdown(&self) -> Result<()> {
crate::util::logger::log_info("Shutting down ChromeDriverPool instances...").await;
for inst in &self.instances {
crate::util::logger::log_info("Shutting down a ChromeDriver instance...").await;
let mut guard = inst.lock().await;
if let Err(e) = guard.shutdown().await {
crate::util::logger::log_warn(&format!("Error shutting down instance: {}", e)).await;
}
}
crate::util::logger::log_info("All ChromeDriver instances shut down").await;
Ok(())
}
}
/// Represents a single instance of chromedriver process.
/// Represents a single instance of chromedriver process, optionally bound to a VPN.
pub struct ChromeInstance {
process: Child,
base_url: String,
vpn_instance: Option<Arc<Mutex<VpnInstance>>>,
task_count: usize,
max_tasks_per_instance: usize,
// Optional join handle for background stderr logging task
stderr_log: Option<JoinHandle<()>>,
}
impl ChromeInstance {
/// Creates a new ChromeInstance by spawning chromedriver with random port.
///
/// This spawns `chromedriver --port=0` to avoid port conflicts, reads stdout to extract
/// the listening address, and waits for the success message. If timeout occurs or
/// spawning fails, returns an error with context.
///
/// # Errors
///
/// Returns an error if chromedriver fails to spawn (e.g., not in PATH, version mismatch),
/// if the process exits early, or if the address/success message isn't found within 30s.
pub async fn new() -> Result<Self> {
let mut command = Command::new("chromedriver-win64/chromedriver.exe");
/// Creates a new ChromeInstance, optionally bound to a VPN IP.
#[cfg(target_os = "windows")]
pub async fn new(
vpn_instance: Option<Arc<Mutex<VpnInstance>>>,
forcebindip: Option<Arc<ForceBindIpManager>>,
) -> Result<Self> {
Self::new_with_task_limit(vpn_instance, forcebindip, 0).await
}
/// Creates a new ChromeInstance with task-per-instance limit, bound to a VPN IP if provided.
#[cfg(target_os = "windows")]
pub async fn new_with_task_limit(
vpn_instance: Option<Arc<Mutex<VpnInstance>>>,
forcebindip: Option<Arc<ForceBindIpManager>>,
max_tasks_per_instance: usize,
) -> Result<Self> {
let bind_ip = if let Some(ref vpn) = vpn_instance {
let vpn_guard = vpn.lock().await;
vpn_guard.external_ip().map(|s| s.to_string())
} else {
None
};
let mut command = if let (Some(ip), Some(fb)) = (&bind_ip, &forcebindip) {
// Use ForceBindIP to bind ChromeDriver to specific VPN IP
crate::util::logger::log_info(&format!("Binding ChromeDriver to VPN IP: {}", ip)).await;
let mut std_cmd = fb.create_bound_command(
ip,
std::path::Path::new("chromedriver-win64/chromedriver.exe"),
&["--port=0"],
);
Command::from(std_cmd)
} else {
let mut cmd = Command::new("chromedriver-win64/chromedriver.exe");
cmd.arg("--port=0");
cmd
};
command.stdout(Stdio::piped()).stderr(Stdio::piped());
let mut process = command
.spawn()
.context("Failed to spawn chromedriver. Ensure it's installed and in PATH.")?;
let (base_url, stderr_handle) = Self::wait_for_chromedriver_start(&mut process).await?;
Ok(Self {
process,
base_url,
vpn_instance,
task_count: 0,
max_tasks_per_instance,
stderr_log: stderr_handle,
})
}
/// Creates a new ChromeInstance on non-Windows platforms (no ForceBindIP support).
#[cfg(not(target_os = "windows"))]
pub async fn new(vpn_instance: Option<Arc<Mutex<VpnInstance>>>) -> Result<Self> {
Self::new_with_task_limit(vpn_instance, 0).await
}
/// Creates a new ChromeInstance on non-Windows platforms with task-per-instance limit.
#[cfg(not(target_os = "windows"))]
pub async fn new_with_task_limit(vpn_instance: Option<Arc<Mutex<VpnInstance>>>, max_tasks_per_instance: usize) -> Result<Self> {
if vpn_instance.is_some() {
crate::util::logger::log_warn(
"⚠ VPN binding requested but ForceBindIP is not available on this platform"
).await;
}
let mut command = Command::new("chromedriver");
command
.arg("--port=0") // Use random available port to support pooling
.arg("--port=0")
.stdout(Stdio::piped())
.stderr(Stdio::piped());
@@ -123,20 +294,38 @@ impl ChromeInstance {
.spawn()
.context("Failed to spawn chromedriver. Ensure it's installed and in PATH.")?;
let (base_url, stderr_handle) = Self::wait_for_chromedriver_start(&mut process).await?;
Ok(Self {
process,
base_url,
vpn_instance,
task_count: 0,
max_tasks_per_instance,
stderr_log: stderr_handle,
})
}
/// Waits for ChromeDriver to start and extracts the listening address.
async fn wait_for_chromedriver_start(process: &mut Child) -> Result<(String, Option<JoinHandle<()>>)> {
let mut stdout =
BufReader::new(process.stdout.take().context("Failed to capture stdout")?).lines();
let mut stderr =
BufReader::new(process.stderr.take().context("Failed to capture stderr")?).lines();
let stderr_reader = process.stderr.take().context("Failed to capture stderr")?;
let start_time = std::time::Instant::now();
let mut address: Option<String> = None;
let mut success = false;
// Log stderr in background for debugging
tokio::spawn(async move {
while let Ok(Some(line)) = stderr.next_line().await {
eprintln!("ChromeDriver stderr: {}", line);
// Log stderr in background for debugging and return the JoinHandle so we can
// abort/await it during shutdown.
let stderr_handle: JoinHandle<()> = tokio::spawn(async move {
let mut stderr_lines = BufReader::new(stderr_reader).lines();
while let Ok(Some(line)) = stderr_lines.next_line().await {
let trimmed = line.trim();
if !trimmed.is_empty() {
crate::util::logger::log_info(&format!("ChromeDriver stderr: {}", trimmed)).await;
}
}
});
@@ -152,10 +341,7 @@ impl ChromeInstance {
}
if let (Some(addr), true) = (&address, success) {
return Ok(Self {
process,
base_url: addr.clone(),
});
return Ok((addr.clone(), Some(stderr_handle)));
}
}
@@ -164,11 +350,13 @@ impl ChromeInstance {
// Cleanup on failure
let _ = process.kill().await;
Err(anyhow!("Timeout: ChromeDriver did not start within 30 seconds. Check version match with Chrome browser and system resources."))
// If we timed out, abort stderr logging task
stderr_handle.abort();
let _ = stderr_handle.await;
Err(anyhow!("Timeout: ChromeDriver did not start within 30 seconds"))
}
/// Creates a new browser session (client) from this ChromeDriver instance.
/// Each session is independent and can be closed without affecting the driver.
pub async fn new_session(&self) -> Result<Client> {
ClientBuilder::native()
.capabilities(Self::chrome_args())
@@ -177,11 +365,47 @@ impl ChromeInstance {
.context("Failed to create new session")
}
/// Increments task counter and returns whether limit has been reached
pub fn increment_task_count(&mut self) -> bool {
if self.max_tasks_per_instance > 0 {
self.task_count += 1;
self.task_count >= self.max_tasks_per_instance
} else {
false
}
}
/// Resets task counter (called when VPN is rotated)
pub fn reset_task_count(&mut self) {
self.task_count = 0;
}
/// Returns current task count for this instance
pub fn get_task_count(&self) -> usize {
self.task_count
}
/// Gracefully shutdown the chromedriver process and background log tasks.
pub async fn shutdown(&mut self) -> Result<()> {
// Abort and await stderr logging task if present
if let Some(handle) = self.stderr_log.take() {
handle.abort();
let _ = handle.await;
}
// Try to terminate the child process
let _ = self.process.start_kill();
// Await the process to ensure resources are released
let _ = self.process.wait().await;
Ok(())
}
fn chrome_args() -> Map<String, Value> {
let args = serde_json::json!({
"goog:chromeOptions": {
"args": [
"--headless=new",
"--headless",
"--disable-gpu",
"--no-sandbox",
"--disable-dev-shm-usage",
@@ -191,13 +415,14 @@ impl ChromeInstance {
"--disable-notifications",
"--disable-logging",
"--disable-autofill",
"--disable-features=TranslateUI,OptimizationGuideModelDownloading",
"--disable-sync",
"--disable-default-apps",
"--disable-translate",
"--window-size=1920,1080",
"--disable-blink-features=AutomationControlled",
"--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
"--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
],
"excludeSwitches": ["enable-logging", "enable-automation"],
"useAutomationExtension": false,
"prefs": {
"profile.default_content_setting_values.notifications": 2
}
@@ -209,10 +434,6 @@ impl ChromeInstance {
}
}
/// Parses the ChromeDriver address from a log line.
///
/// Looks for the "Starting ChromeDriver ... on port XXXX" line and extracts the port.
/// Returns `Some("http://localhost:XXXX")` if found, else `None`.
fn parse_chromedriver_address(line: &str) -> Option<String> {
if line.contains("Starting ChromeDriver") {
if let Some(port_str) = line.split("on port ").nth(1) {
@@ -223,7 +444,6 @@ fn parse_chromedriver_address(line: &str) -> Option<String> {
}
}
}
// Fallback for other formats (e.g., explicit port mentions)
for word in line.split_whitespace() {
if let Ok(port) = word.trim_matches(|c: char| !c.is_numeric()).parse::<u16>() {
if port > 1024 && port < 65535 && line.to_lowercase().contains("port") {
@@ -236,14 +456,13 @@ fn parse_chromedriver_address(line: &str) -> Option<String> {
impl Drop for ChromeInstance {
fn drop(&mut self) {
// Signal child to terminate. Do NOT block here; shutdown should be
// performed with the async `shutdown()` method when possible.
let _ = self.process.start_kill();
std::thread::sleep(std::time::Duration::from_millis(100));
}
}
/// Simplified task execution - now uses the pool pattern.
///
/// For backwards compatibility with existing code.
/// Simplified task execution - uses the pool pattern.
pub struct ScrapeTask<T> {
url: String,
parse: Box<
@@ -263,7 +482,6 @@ impl<T: Send + 'static> ScrapeTask<T> {
}
}
/// Executes using a provided pool (more efficient for multiple tasks).
pub async fn execute_with_pool(self, pool: &ChromeDriverPool) -> Result<T> {
let url = self.url;
let parse = self.parse;
@@ -271,4 +489,4 @@ impl<T: Send + 'static> ScrapeTask<T> {
pool.execute(url, move |client| async move { (parse)(client).await })
.await
}
}
}

2
src/util/directories.rs
View File

@@ -1,8 +1,6 @@
use std::path::{Path, PathBuf};
use std::fs;
use crate::util::opnv;
/// Central configuration for all data paths
pub struct DataPaths {
base_dir: PathBuf,