removed seperate description scraping
This commit is contained in:
57
src/main.rs
57
src/main.rs
@@ -135,63 +135,6 @@ async fn extract_all_data_via_js(client: &fantoccini::Client) -> anyhow::Result<
|
|||||||
Ok(vec![])
|
Ok(vec![])
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn extract_event_descriptions_via_js(client: &fantoccini::Client) -> anyhow::Result<HashMap<String, String>> {
|
|
||||||
println!("Extracting event descriptions via JavaScript (3 YELLOW stars only)...");
|
|
||||||
|
|
||||||
let description_script = r#"
|
|
||||||
const descriptions = {};
|
|
||||||
|
|
||||||
// Find all description rows (they have class starting with 'teletrader')
|
|
||||||
const descRows = document.querySelectorAll('tr td[class*="teletrader"]');
|
|
||||||
|
|
||||||
for (const descRow of descRows) {
|
|
||||||
// Get the description text from the <p> tag
|
|
||||||
const descPara = descRow.querySelector('p');
|
|
||||||
if (descPara) {
|
|
||||||
const description = descPara.textContent?.trim() || '';
|
|
||||||
|
|
||||||
// Find the corresponding event name by looking for the row above
|
|
||||||
let eventRow = descRow.parentElement.previousElementSibling;
|
|
||||||
if (eventRow) {
|
|
||||||
// Check if this is a 3 YELLOW star event
|
|
||||||
const importanceCell = eventRow.querySelector('td:nth-child(4)');
|
|
||||||
if (importanceCell) {
|
|
||||||
// Count ONLY YELLOW stars
|
|
||||||
const yellowStarCount = importanceCell.querySelectorAll('.icon--star.font-color-yellow').length;
|
|
||||||
|
|
||||||
// Only process events with 3 YELLOW stars
|
|
||||||
if (yellowStarCount === 3) {
|
|
||||||
const eventCell = eventRow.querySelector('td:nth-child(5)');
|
|
||||||
if (eventCell) {
|
|
||||||
const eventName = eventCell.textContent?.trim() || '';
|
|
||||||
if (eventName) {
|
|
||||||
descriptions[eventName] = description;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return descriptions;
|
|
||||||
"#;
|
|
||||||
|
|
||||||
let result = client.execute(description_script, vec![]).await?;
|
|
||||||
|
|
||||||
let mut event_type_map = HashMap::new();
|
|
||||||
if let Some(desc_obj) = result.as_object() {
|
|
||||||
for (key, value) in desc_obj {
|
|
||||||
if let Some(desc_text) = value.as_str() {
|
|
||||||
event_type_map.entry(key.clone()).or_insert(desc_text.to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
println!("Extracted {} event descriptions (3 YELLOW stars only)", event_type_map.len());
|
|
||||||
Ok(event_type_map)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn check_data_consistency(events: &[EconomicEvent]) {
|
async fn check_data_consistency(events: &[EconomicEvent]) {
|
||||||
println!("\n=== DATA CONSISTENCY CHECKS ===");
|
println!("\n=== DATA CONSISTENCY CHECKS ===");
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user