donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

added hard reset for navigation timeout after 3 hours

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2025-12-22 00:31:28 +01:00
parent c01b47000f
commit fb0876309f
12 changed files with 1036 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.env.example
View File

@@ -14,8 +14,8 @@ CORPORATE_START_DATE=2010-01-01
# How far into the future we scrape economic events (in months)
ECONOMIC_LOOKAHEAD_MONTHS=3
# Maximum number of parallel scraping tasks (default: 10)
MAX_PARALLEL_TASKS=10
# Maximum number of parallel scraping tasks (default: 4)
MAX_PARALLEL_INSTANCES=10
# ===== VPN ROTATION (ProtonVPN Integration) =====
# Enable automatic VPN rotation between sessions?
@@ -37,4 +37,6 @@ TASKS_PER_VPN_SESSION=50
MAX_REQUESTS_PER_SESSION=25
MIN_REQUEST_INTERVAL_MS=300
MAX_RETRY_ATTEMPTS=3
MAX_RETRY_ATTEMPTS=3
PROXY_INSTANCES_PER_CERTIFICATE=2