donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

removed seperate description scraping

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2025-11-16 19:20:09 +01:00
parent 3df871f69f
commit 6302c8749a
1 changed files with 0 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/main.rs
View File

@@ -135,63 +135,6 @@ async fn extract_all_data_via_js(client: &fantoccini::Client) -> anyhow::Result<
Ok(vec![])
}
async fn extract_event_descriptions_via_js(client: &fantoccini::Client) -> anyhow::Result<HashMap<String, String>> {
println!("Extracting event descriptions via JavaScript (3 YELLOW stars only)...");
let description_script = r#"
const descriptions = {};
// Find all description rows (they have class starting with 'teletrader')
const descRows = document.querySelectorAll('tr td[class*="teletrader"]');
for (const descRow of descRows) {
// Get the description text from the <p> tag
const descPara = descRow.querySelector('p');
if (descPara) {
const description = descPara.textContent?.trim() || '';
// Find the corresponding event name by looking for the row above
let eventRow = descRow.parentElement.previousElementSibling;
if (eventRow) {
// Check if this is a 3 YELLOW star event
const importanceCell = eventRow.querySelector('td:nth-child(4)');
if (importanceCell) {
// Count ONLY YELLOW stars
const yellowStarCount = importanceCell.querySelectorAll('.icon--star.font-color-yellow').length;
// Only process events with 3 YELLOW stars
if (yellowStarCount === 3) {
const eventCell = eventRow.querySelector('td:nth-child(5)');
if (eventCell) {
const eventName = eventCell.textContent?.trim() || '';
if (eventName) {
descriptions[eventName] = description;
}
}
}
}
}
}
}
return descriptions;
"#;
let result = client.execute(description_script, vec![]).await?;
let mut event_type_map = HashMap::new();
if let Some(desc_obj) = result.as_object() {
for (key, value) in desc_obj {
if let Some(desc_text) = value.as_str() {
event_type_map.entry(key.clone()).or_insert(desc_text.to_string());
}
}
}
println!("Extracted {} event descriptions (3 YELLOW stars only)", event_type_map.len());
Ok(event_type_map)
}
async fn check_data_consistency(events: &[EconomicEvent]) {
println!("\n=== DATA CONSISTENCY CHECKS ===");