donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

added logging

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2025-12-05 21:20:12 +01:00
parent f7083bf9f0
commit 58a498e694
17 changed files with 526 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files

249
src/corporate/scraper.rs
View File

@@ -1,7 +1,7 @@
// src/corporate/scraper.rs
use super::{types::*, helpers::*, openfigi::*};
//use crate::corporate::openfigi::OpenFigiClient;
use crate::{scraper::webdriver::*};
use crate::{webdriver::webdriver::*};
use fantoccini::{Client, Locator};
use scraper::{Html, Selector};
use chrono::{DateTime, Duration, NaiveDate, Utc};
@@ -15,160 +15,6 @@ use anyhow::{anyhow, Result};
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";
/// Discover all exchanges where this ISIN trades by querying Yahoo Finance and enriching with OpenFIGI API calls.
///
/// # Arguments
/// * `isin` - The ISIN to search for.
/// * `known_ticker` - A known ticker symbol for fallback or initial check.
///
/// # Returns
/// A vector of FigiInfo structs containing enriched data from API calls.
///
/// # Errors
/// Returns an error if HTTP requests fail, JSON parsing fails, or OpenFIGI API responds with an error.
pub async fn discover_available_exchanges(isin: &str, known_ticker: &str) -> anyhow::Result<Vec<FigiInfo>> {
println!(" Discovering exchanges for ISIN {}", isin);
let mut potential: Vec<(String, PrimaryInfo)> = Vec::new();
// Try the primary ticker first
if let Ok(info) = check_ticker_exists(known_ticker).await {
potential.push((known_ticker.to_string(), info));
}
// Search for ISIN directly on Yahoo to find other listings
let search_url = format!(
"https://query2.finance.yahoo.com/v1/finance/search?q={}&quotesCount=20&newsCount=0",
isin
);
let resp = HttpClient::new()
.get(&search_url)
.header("User-Agent", USER_AGENT)
.send()
.await?;
let json = resp.json::<Value>().await?;
if let Some(quotes) = json["quotes"].as_array() {
for quote in quotes {
// First: filter by quoteType directly from search results (faster rejection)
let quote_type = quote["quoteType"].as_str().unwrap_or("");
if quote_type.to_uppercase() != "EQUITY" {
continue; // Skip bonds, ETFs, mutual funds, options, etc.
}
if let Some(symbol) = quote["symbol"].as_str() {
// Avoid duplicates
if potential.iter().any(|(s, _)| s == symbol) {
continue;
}
// Double-check with full quote data (some search results are misleading)
if let Ok(info) = check_ticker_exists(symbol).await {
potential.push((symbol.to_string(), info));
}
}
}
}
if potential.is_empty() {
return Ok(vec![]);
}
// Enrich with OpenFIGI API
let client = OpenFigiClient::new()?;
let mut discovered_figis = Vec::new();
if !client.has_key() {
// Fallback without API key - create FigiInfo with default/empty fields
for (symbol, info) in potential {
println!(" Found equity listing: {} on {} ({}) - no FIGI (fallback mode)", symbol, info.exchange_mic, info.currency);
let figi_info = FigiInfo {
isin: info.isin,
figi: String::new(),
name: info.name,
ticker: symbol,
mic_code: info.exchange_mic,
currency: info.currency,
compositeFIGI: String::new(),
securityType: String::new(),
marketSector: String::new(),
shareClassFIGI: String::new(),
securityType2: String::new(),
securityDescription: String::new(),
};
discovered_figis.push(figi_info);
}
return Ok(discovered_figis);
}
// With API key, batch the mapping requests
let chunk_size = 100;
for chunk in potential.chunks(chunk_size) {
let mut jobs = vec![];
for (symbol, info) in chunk {
jobs.push(json!({
"idType": "TICKER",
"idValue": symbol,
"micCode": info.exchange_mic,
"marketSecDes": "Equity",
}));
}
let resp = client.get_figi_client()
.post("https://api.openfigi.com/v3/mapping")
.header("Content-Type", "application/json")
.json(&jobs)
.send()
.await?;
if !resp.status().is_success() {
return Err(anyhow::anyhow!("OpenFIGI mapping failed with status: {}", resp.status()));
}
let parsed: Vec<Value> = resp.json().await?;
for (i, item) in parsed.iter().enumerate() {
let (symbol, info) = &chunk[i];
if let Some(data) = item["data"].as_array() {
if let Some(entry) = data.first() {
let market_sec = entry["marketSector"].as_str().unwrap_or("");
if market_sec != "Equity" {
continue;
}
println!(" Found equity listing: {} on {} ({}) - FIGI: {}", symbol, info.exchange_mic, info.currency, entry["figi"]);
let figi_info = FigiInfo {
isin: info.isin.clone(),
figi: entry["figi"].as_str().unwrap_or("").to_string(),
name: entry["name"].as_str().unwrap_or(&info.name).to_string(),
ticker: symbol.clone(),
mic_code: info.exchange_mic.clone(),
currency: info.currency.clone(),
compositeFIGI: entry["compositeFIGI"].as_str().unwrap_or("").to_string(),
securityType: entry["securityType"].as_str().unwrap_or("").to_string(),
marketSector: market_sec.to_string(),
shareClassFIGI: entry["shareClassFIGI"].as_str().unwrap_or("").to_string(),
securityType2: entry["securityType2"].as_str().unwrap_or("").to_string(),
securityDescription: entry["securityDescription"].as_str().unwrap_or("").to_string(),
};
discovered_figis.push(figi_info);
} else {
println!(" No data returned for ticker {} on MIC {}", symbol, info.exchange_mic);
}
} else if let Some(error) = item["error"].as_str() {
println!(" OpenFIGI error for ticker {}: {}", symbol, error);
}
}
// Respect rate limit (6 seconds between requests with key)
sleep(TokioDuration::from_secs(6)).await;
}
Ok(discovered_figis)
}
/// Check if a ticker exists on Yahoo Finance and return core metadata.
///
/// This function calls the public Yahoo Finance quoteSummary endpoint and extracts:
@@ -305,33 +151,6 @@ pub async fn check_ticker_exists(ticker: &str) -> anyhow::Result<PrimaryInfo> {
})
}
/// Convert Yahoo's exchange name to MIC code (best effort)
fn exchange_name_to_mic(name: &str) -> String {
match name {
"NMS" | "NasdaqGS" | "NASDAQ" => "XNAS",
"NYQ" | "NYSE" => "XNYS",
"LSE" | "London" => "XLON",
"FRA" | "Frankfurt" | "GER" | "XETRA" => "XFRA",
"PAR" | "Paris" => "XPAR",
"AMS" | "Amsterdam" => "XAMS",
"MIL" | "Milan" => "XMIL",
"JPX" | "Tokyo" => "XJPX",
"HKG" | "Hong Kong" => "XHKG",
"SHH" | "Shanghai" => "XSHG",
"SHZ" | "Shenzhen" => "XSHE",
"TOR" | "Toronto" => "XTSE",
"ASX" | "Australia" => "XASX",
"SAU" | "Saudi" => "XSAU",
"SWX" | "Switzerland" => "XSWX",
"BSE" | "Bombay" => "XBSE",
"NSE" | "NSI" => "XNSE",
"TAI" | "Taiwan" => "XTAI",
"SAO" | "Sao Paulo" => "BVMF",
"MCE" | "Madrid" => "XMAD",
_ => name, // Fallback to name itself
}.to_string()
}
/// Fetches earnings events for a ticker using a dedicated ScrapeTask.
///
/// This function creates and executes a ScrapeTask to navigate to the Yahoo Finance earnings calendar,
@@ -670,51 +489,63 @@ pub async fn _fetch_latest_gleif_isin_lei_mapping_url(client: &Client) -> anyhow
pub async fn download_isin_lei_csv() -> anyhow::Result<Option<String>> {
let url = "https://mapping.gleif.org/api/v2/isin-lei/9315e3e3-305a-4e71-b062-46714740fa8d/download";
let zip_path = "data/gleif/isin_lei.zip";
let csv_path = "data/gleif/isin_lei.csv";
if let Err(e) = std::fs::create_dir_all("data") {
if let Err(e) = std::fs::create_dir_all("data/gleif") {
println!("Failed to create data directory: {e}");
return Ok(None);
}
// Download ZIP
let bytes = match reqwest::Client::builder()
// Download ZIP and get the filename from Content-Disposition header
let client = match reqwest::Client::builder()
.user_agent(USER_AGENT)
.timeout(std::time::Duration::from_secs(30))
.build()
.and_then(|c| Ok(c))
{
Ok(client) => match client.get(url).send().await {
Ok(resp) if resp.status().is_success() => match resp.bytes().await {
Ok(b) => b,
Err(e) => {
println!("Failed to read ZIP bytes: {e}");
return Ok(None);
}
},
Ok(resp) => {
println!("Server returned HTTP {}", resp.status());
return Ok(None);
}
Err(e) => {
println!("Failed to download ISIN/LEI ZIP: {e}");
return Ok(None);
}
},
Ok(c) => c,
Err(e) => {
println!("Failed to create HTTP client: {e}");
return Ok(None);
}
};
if let Err(e) = tokio::fs::write(zip_path, &bytes).await {
let resp = match client.get(url).send().await {
Ok(r) if r.status().is_success() => r,
Ok(resp) => {
println!("Server returned HTTP {}", resp.status());
return Ok(None);
}
Err(e) => {
println!("Failed to download ISIN/LEI ZIP: {e}");
return Ok(None);
}
};
// Extract filename from Content-Disposition header or use default
let filename = resp
.headers()
.get("content-disposition")
.and_then(|h| h.to_str().ok())
.and_then(|s| s.split("filename=").nth(1).map(|f| f.trim_matches('"').to_string()))
.unwrap_or_else(|| "isin_lei.zip".to_string());
let bytes = match resp.bytes().await {
Ok(b) => b,
Err(e) => {
println!("Failed to read ZIP bytes: {e}");
return Ok(None);
}
};
let zip_path = format!("data/gleif/{}", filename);
let csv_path = format!("data/gleif/{}", filename.replace(".zip", ".csv"));
if let Err(e) = tokio::fs::write(&zip_path, &bytes).await {
println!("Failed to write ZIP file: {e}");
return Ok(None);
}
// Extract CSV
let archive = match std::fs::File::open(zip_path)
let archive = match std::fs::File::open(&zip_path)
.map(ZipArchive::new)
{
Ok(Ok(a)) => a,
@@ -756,12 +587,12 @@ pub async fn download_isin_lei_csv() -> anyhow::Result<Option<String>> {
return Ok(None);
}
if let Err(e) = tokio::fs::write(csv_path, &csv_bytes).await {
if let Err(e) = tokio::fs::write(&csv_path, &csv_bytes).await {
println!("Failed to save CSV file: {e}");
return Ok(None);
}
Ok(Some(csv_path.to_string()))
Ok(Some(csv_path))
}