added pool rotation to chromedriver pool

This commit is contained in:
2025-12-18 15:59:56 +01:00
parent c51b36c125
commit cd91de253b
5 changed files with 314 additions and 170 deletions

View File

@@ -3,6 +3,7 @@ use super::{types::*, helpers::*};
use crate::{scraper::webdriver::*, util::{directories::DataPaths}};
use event_backtest_engine::logger;
use fantoccini::{Client, Locator};
use rand::Rng;
use serde::{Deserialize, Serialize};
use tokio::time::{Duration as TokioDuration, sleep, timeout};
use std::{sync::Arc};
@@ -73,9 +74,16 @@ pub async fn scrape_company_details_by_isin(
pool.execute(format!("https://finance.yahoo.com/lookup/?s={}", isin), move |client| {
let isin = isin.clone();
Box::pin(async move {
sleep(TokioDuration::from_millis(1000)).await;
// Random Delay between 800-1500ms
let delay = rand::rng().random_range(800..1500);
sleep(TokioDuration::from_millis(delay)).await;
reject_yahoo_cookies(&client).await?;
sleep(TokioDuration::from_millis(1000)).await;
// Random Delay
let delay = rand::rng().random_range(800..1500);
sleep(TokioDuration::from_millis(delay)).await;
extract_company_details(&client, &isin).await
})
}).await