added pool rotation to chromedriver pool
This commit is contained in:
@@ -3,6 +3,7 @@ use super::{types::*, helpers::*};
|
||||
use crate::{scraper::webdriver::*, util::{directories::DataPaths}};
|
||||
use event_backtest_engine::logger;
|
||||
use fantoccini::{Client, Locator};
|
||||
use rand::Rng;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::time::{Duration as TokioDuration, sleep, timeout};
|
||||
use std::{sync::Arc};
|
||||
@@ -73,9 +74,16 @@ pub async fn scrape_company_details_by_isin(
|
||||
pool.execute(format!("https://finance.yahoo.com/lookup/?s={}", isin), move |client| {
|
||||
let isin = isin.clone();
|
||||
Box::pin(async move {
|
||||
sleep(TokioDuration::from_millis(1000)).await;
|
||||
// Random Delay between 800-1500ms
|
||||
let delay = rand::rng().random_range(800..1500);
|
||||
sleep(TokioDuration::from_millis(delay)).await;
|
||||
|
||||
reject_yahoo_cookies(&client).await?;
|
||||
sleep(TokioDuration::from_millis(1000)).await;
|
||||
|
||||
// Random Delay
|
||||
let delay = rand::rng().random_range(800..1500);
|
||||
sleep(TokioDuration::from_millis(delay)).await;
|
||||
|
||||
extract_company_details(&client, &isin).await
|
||||
})
|
||||
}).await
|
||||
|
||||
Reference in New Issue
Block a user