added pool rotation to chromedriver pool

This commit is contained in:
2025-12-18 15:59:56 +01:00
parent c51b36c125
commit cd91de253b
5 changed files with 314 additions and 170 deletions

View File

@@ -3,46 +3,38 @@
# This file configures the behavior of the WebScraper application
# Copy to .env and adjust values as needed
# ===== ECONOMIC DATA =====
# Start date for economic event scraping
OPENFIGI_API_KEY=
# Economic calendar start (usually the earliest available on finanzen.net)
ECONOMIC_START_DATE=2007-02-13
# How far into the future to look ahead for economic events (in months)
ECONOMIC_LOOKAHEAD_MONTHS=3
# ===== CORPORATE DATA =====
# Start date for corporate earnings/data scraping
# Corporate earnings & price history start
CORPORATE_START_DATE=2010-01-01
# ===== PERFORMANCE & CONCURRENCY =====
# Maximum number of parallel ChromeDriver instances
# Higher = more concurrent tasks, but higher resource usage
MAX_PARALLEL_INSTANCES=3
# How far into the future we scrape economic events (in months)
ECONOMIC_LOOKAHEAD_MONTHS=3
# Maximum tasks per ChromeDriver instance before recycling
# 0 = unlimited (instance lives for entire application runtime)
MAX_TASKS_PER_INSTANCE=0
# Maximum number of parallel scraping tasks (default: 10)
MAX_PARALLEL_TASKS=10
# ===== VPN ROTATION (ProtonVPN Integration) =====
# Enable automatic VPN rotation between sessions?
# If false, all traffic goes through system without VPN tunneling
ENABLE_VPN_ROTATION=false
# Comma-separated list of ProtonVPN servers to rotate through
# Examples:
# "US-Free#1,US-Free#2,UK-Free#1"
# "US,UK,JP,DE,NL"
# NOTE: Must have ENABLE_VPN_ROTATION=true for this to take effect
VPN_SERVERS=
ENABLE_VPN_ROTATION=true
# Number of tasks per VPN session before rotating to new server/IP
# 0 = rotate between economic and corporate phases (one phase = one IP)
# 5 = rotate every 5 tasks
# NOTE: Must have ENABLE_VPN_ROTATION=true for this to take effect
TASKS_PER_VPN_SESSION=0
TASKS_PER_VPN_SESSION=50
# ===== LOGGING =====
# Set via RUST_LOG environment variable:
# RUST_LOG=info cargo run
# RUST_LOG=debug cargo run
# Leave empty or unset for default logging level
MAX_REQUESTS_PER_SESSION=25
MIN_REQUEST_INTERVAL_MS=300
MAX_RETRY_ATTEMPTS=3