donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

added corporate quarterly announcments for the last 4 years

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2025-11-23 16:02:23 +01:00
parent cd3f47d91f
commit 462f7ca672
12 changed files with 1457 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/config.rs
View File

@@ -15,7 +15,7 @@ impl Default for Config {
fn default() -> Self {
Self {
economic_start_date: "2007-02-13".to_string(),
corporate_start_date: "2010-01-01".to_string(),
corporate_start_date: "2007-01-01".to_string(),
economic_lookahead_months: 3,
}
}

9
src/continents.json
View File

@@ -1,9 +0,0 @@
[
"afrika",
"asien",
"europa",
"nordamerika",
"suedamerika",
"antarktis",
"ozeanien"
]

52
src/corporate/helpers.rs Normal file
View File

@@ -0,0 +1,52 @@
// src/corporate/helpers.rs
use super::types::*;
use chrono::{Local, NaiveDate};
use std::collections::{HashMap, HashSet};
pub fn event_key(e: &CompanyEvent) -> String {
format!("{}|{}|{}", e.ticker, e.date, e.time)
}
pub fn detect_changes(old: &CompanyEvent, new: &CompanyEvent, today: &str) -> Vec<CompanyEventChange> {
let mut changes = Vec::new();
let ts = Local::now().format("%Y-%m-%d %H:%M:%S").to_string();
if new.date.as_str() <= today { return changes; }
if old.time != new.time {
changes.push(CompanyEventChange {
ticker: new.ticker.clone(),
date: new.date.clone(),
field_changed: "time".to_string(),
old_value: old.time.clone(),
new_value: new.time.clone(),
detected_at: ts.clone(),
});
}
if old.eps_forecast != new.eps_forecast {
changes.push(CompanyEventChange {
ticker: new.ticker.clone(),
date: new.date.clone(),
field_changed: "eps_forecast".to_string(),
old_value: format!("{:?}", old.eps_forecast),
new_value: format!("{:?}", new.eps_forecast),
detected_at: ts.clone(),
});
}
if old.eps_actual != new.eps_actual {
changes.push(CompanyEventChange {
ticker: new.ticker.clone(),
date: new.date.clone(),
field_changed: "eps_actual".to_string(),
old_value: format!("{:?}", old.eps_actual),
new_value: format!("{:?}", new.eps_actual),
detected_at: ts.clone(),
});
}
// Add similar for revenue if applicable
changes
}

1
src/corporate/mod.rs
View File

@@ -3,6 +3,7 @@ pub mod types;
pub mod scraper;
pub mod storage;
pub mod update;
pub mod helpers;
pub use types::*;
pub use update::run_full_update;

70
src/corporate/scraper.rs
View File

@@ -2,8 +2,10 @@
use super::types::{CompanyEvent, CompanyPrice};
use fantoccini::{Client, Locator};
use scraper::{Html, Selector};
use chrono::{NaiveDate, Datelike};
use chrono::{NaiveDate};
use tokio::time::{sleep, Duration};
use yfinance_rs::{YfClient, Ticker, Range, Interval};
use yfinance_rs::core::conversions::money_to_f64;
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36";
@@ -54,9 +56,9 @@ pub async fn fetch_earnings_history(client: &Client, ticker: &str) -> anyhow::Re
let cols: Vec<String> = row.select(&Selector::parse("td").unwrap())
.map(|td| td.text().collect::<Vec<_>>().join(" ").trim().to_string())
.collect();
if cols.len() < 6 { continue; }
if cols.len() < 4 { continue; }
let full_date = &cols[2];
let full_date = &cols[0];
let parts: Vec<&str> = full_date.split(" at ").collect();
let raw_date = parts[0].trim();
let time_str = if parts.len() > 1 { parts[1].trim() } else { "" };
@@ -66,8 +68,8 @@ pub async fn fetch_earnings_history(client: &Client, ticker: &str) -> anyhow::Re
Err(_) => continue,
};
let eps_forecast = parse_float(&cols[3]);
let eps_actual = if cols[4] == "-" { None } else { parse_float(&cols[4]) };
let eps_forecast = parse_float(&cols[1]);
let eps_actual = if cols[2] == "-" { None } else { parse_float(&cols[2]) };
let surprise_pct = if let (Some(f), Some(a)) = (eps_forecast, eps_actual) {
if f.abs() > 0.001 { Some((a - f) / f.abs() * 100.0) } else { None }
@@ -85,7 +87,7 @@ pub async fn fetch_earnings_history(client: &Client, ticker: &str) -> anyhow::Re
ticker: ticker.to_string(),
date: date.format("%Y-%m-%d").to_string(),
time,
period: "".to_string(), // No period info available, set to empty
period: "".to_string(),
eps_forecast,
eps_actual,
revenue_forecast: None,
@@ -98,38 +100,46 @@ pub async fn fetch_earnings_history(client: &Client, ticker: &str) -> anyhow::Re
Ok(events)
}
pub async fn fetch_price_history(client: &Client, ticker: &str, start: &str, end: &str) -> anyhow::Result<Vec<CompanyPrice>> {
let start_ts = NaiveDate::parse_from_str(start, "%Y-%m-%d")?
.and_hms_opt(0, 0, 0).unwrap().and_utc()
.timestamp();
pub async fn fetch_price_history(
ticker: &str,
start: &str,
end: &str,
) -> anyhow::Result<Vec<CompanyPrice>> {
let client = YfClient::default();
let tk = Ticker::new(&client, ticker);
let end_ts = NaiveDate::parse_from_str(end, "%Y-%m-%d")?
.succ_opt().unwrap()
.and_hms_opt(0, 0, 0).unwrap().and_utc()
.timestamp();
// We request the maximum range – the library will automatically respect Yahoo's limits
let history = tk
.history(Some(Range::Max), Some(Interval::D1), true)
.await
.map_err(|e| anyhow::anyhow!("Yahoo Finance API error for {ticker}: {e:?}"))?;
let url = format!(
"https://query1.finance.yahoo.com/v7/finance/download/{ticker}?period1={start_ts}&period2={end_ts}&interval=1d&events=history&includeAdjustedClose=true"
);
let mut prices = Vec::with_capacity(history.len());
client.goto(&url).await?;
let csv = client.source().await?;
for candle in history {
let date_str = candle.ts.format("%Y-%m-%d").to_string();
// Filter by user-defined start / end
if date_str < (*start).to_string() || date_str > (*end).to_string() {
continue;
}
let mut prices = Vec::new();
for line in csv.lines().skip(1) {
let cols: Vec<&str> = line.split(',').collect();
if cols.len() < 7 { continue; }
prices.push(CompanyPrice {
ticker: ticker.to_string(),
date: cols[0].to_string(),
open: cols[1].parse()?,
high: cols[2].parse()?,
low: cols[3].parse()?,
close: cols[4].parse()?,
adj_close: cols[5].parse()?,
volume: cols[6].parse()?,
date: date_str,
open: money_to_f64(&candle.open),
high: money_to_f64(&candle.high),
low: money_to_f64(&candle.low),
// close_unadj is the raw (non-adjusted) close; close is the adjusted one
close: money_to_f64(&candle.close_unadj.unwrap_or(candle.close.clone())),
adj_close: money_to_f64(&candle.close),
volume: candle.volume.unwrap_or(0),
});
}
// Sort just in case (normally already sorted)
prices.sort_by_key(|p| p.date.clone());
Ok(prices)
}

91
src/corporate/storage.rs
View File

@@ -1,11 +1,11 @@
// src/corporate/storage.rs
use super::types::{CompanyEvent, CompanyPrice};
use std::collections::{HashMap, HashSet};
use super::types::{CompanyEvent, CompanyPrice, CompanyEventChange};
use super::helpers::*;
use tokio::fs;
use chrono::{Local, NaiveDate};
use chrono::{Local, NaiveDate, Datelike};
use std::collections::HashMap;
/// Load all events from disk into a HashMap<ticker|date, event>
async fn load_all_events_map() -> anyhow::Result<HashMap<String, CompanyEvent>> {
pub async fn load_existing_events() -> anyhow::Result<HashMap<String, CompanyEvent>> {
let mut map = HashMap::new();
let dir = std::path::Path::new("corporate_events");
if !dir.exists() {
@@ -16,11 +16,12 @@ async fn load_all_events_map() -> anyhow::Result<HashMap<String, CompanyEvent>>
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
if path.extension().and_then(|s| s.to_str()) == Some("json") {
let content = fs::read_to_string(&path).await?;
if let Ok(events) = serde_json::from_str::<Vec<CompanyEvent>>(&content) {
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
if name.starts_with("events_") && name.len() == 17 { // events_yyyy-mm.json
let content = fs::read_to_string(&path).await?;
let events: Vec<CompanyEvent> = serde_json::from_str(&content)?;
for event in events {
let key = format!("{}|{}", event.ticker, event.date);
map.insert(key, event);
map.insert(event_key(&event), event);
}
}
}
@@ -28,34 +29,68 @@ async fn load_all_events_map() -> anyhow::Result<HashMap<String, CompanyEvent>>
Ok(map)
}
/// Merge new events with existing ones and save back to disk
pub async fn merge_and_save_events(ticker: &str, new_events: Vec<CompanyEvent>) -> anyhow::Result<()> {
let mut existing = load_all_events_map().await?;
// Insert or update
for event in new_events {
let key = format!("{}|{}", event.ticker, event.date);
existing.insert(key, event);
}
// Convert back to Vec and save (simple single file for now)
let all_events: Vec<CompanyEvent> = existing.into_values().collect();
pub async fn save_optimized_events(events: HashMap<String, CompanyEvent>) -> anyhow::Result<()> {
let dir = std::path::Path::new("corporate_events");
fs::create_dir_all(dir).await?;
let path = dir.join("all_events.json");
let json = serde_json::to_string_pretty(&all_events)?;
fs::write(&path, json).await?;
// Delete old files
let mut entries = fs::read_dir(dir).await?;
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
if name.starts_with("events_") && path.extension().map(|e| e == "json").unwrap_or(false) {
fs::remove_file(&path).await?;
}
}
let mut sorted: Vec<_> = events.into_values().collect();
sorted.sort_by_key(|e| (e.ticker.clone(), e.date.clone()));
let mut by_month: HashMap<String, Vec<CompanyEvent>> = HashMap::new();
for e in sorted {
if let Ok(d) = NaiveDate::parse_from_str(&e.date, "%Y-%m-%d") {
let key = format!("{}-{:02}", d.year(), d.month());
by_month.entry(key).or_default().push(e);
}
}
for (month, list) in by_month {
let path = dir.join(format!("events_{}.json", month));
fs::write(&path, serde_json::to_string_pretty(&list)?).await?;
}
Ok(())
}
/// Save price history for a single ticker (overwrite old file)
pub async fn save_prices_for_ticker(ticker: &str, prices: Vec<CompanyPrice>) -> anyhow::Result<()> {
pub async fn save_changes(changes: &[CompanyEventChange]) -> anyhow::Result<()> {
if changes.is_empty() { return Ok(()); }
let dir = std::path::Path::new("corporate_event_changes");
fs::create_dir_all(dir).await?;
let mut by_month: HashMap<String, Vec<CompanyEventChange>> = HashMap::new();
for c in changes {
if let Ok(d) = NaiveDate::parse_from_str(&c.date, "%Y-%m-%d") {
let key = format!("{}-{:02}", d.year(), d.month());
by_month.entry(key).or_default().push(c.clone());
}
}
for (month, list) in by_month {
let path = dir.join(format!("changes_{}.json", month));
let mut all = if path.exists() {
let s = fs::read_to_string(&path).await?;
serde_json::from_str(&s).unwrap_or_default()
} else { vec![] };
all.extend(list);
fs::write(&path, serde_json::to_string_pretty(&all)?).await?;
}
Ok(())
}
pub async fn save_prices_for_ticker(ticker: &str, mut prices: Vec<CompanyPrice>) -> anyhow::Result<()> {
let dir = std::path::Path::new("corporate_prices");
fs::create_dir_all(dir).await?;
let path = dir.join(format!("{}.json", ticker));
// Optional: sort by date
let mut prices = prices;
prices.sort_by_key(|p| p.date.clone());
let json = serde_json::to_string_pretty(&prices)?;

2
src/corporate/types.rs
View File

@@ -31,7 +31,7 @@ pub struct CompanyPrice {
pub struct CompanyEventChange {
pub ticker: String,
pub date: String,
pub field: String, // "time", "eps_forecast", "eps_actual", "new_event"
pub field_changed: String, // "time", "eps_forecast", "eps_actual", "new_event"
pub old_value: String,
pub new_value: String,
pub detected_at: String,

74
src/corporate/update.rs
View File

@@ -1,31 +1,83 @@
// src/corporate/update.rs
use super::{scraper::*, storage::*, types::*};
use super::{scraper::*, storage::*, helpers::*, types::*};
use crate::config::Config;
use chrono::Local;
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
pub async fn run_full_update(client: &fantoccini::Client, tickers: Vec<String>, config: &Config) -> anyhow::Result<()> {
println!("Updating {} tickers (prices from {})", tickers.len(), config.corporate_start_date);
let today = chrono::Local::now().format("%Y-%m-%d").to_string();
for ticker in tickers {
let mut existing = load_existing_events().await?;
for ticker in &tickers {
print!(" → {:6} ", ticker);
// Earnings
if let Ok(events) = fetch_earnings_history(client, &ticker).await {
merge_and_save_events(&ticker, events.clone()).await?;
println!("{} earnings", events.len());
if let Ok(new_events) = fetch_earnings_history(client, ticker).await {
let result = process_batch(&new_events, &mut existing, &today);
save_changes(&result.changes).await?;
println!("{} earnings, {} changes", new_events.len(), result.changes.len());
}
// Prices – now using config.corporate_start_date
if let Ok(prices) = fetch_price_history(client, &ticker, &config.corporate_start_date, &today).await {
save_prices_for_ticker(&ticker, prices).await?;
if let Ok(prices) = fetch_price_history(ticker, &config.corporate_start_date, &today).await {
save_prices_for_ticker(ticker, prices).await?;
}
tokio::time::sleep(tokio::time::Duration::from_millis(250)).await;
}
save_optimized_events(existing).await?;
Ok(())
}
pub struct ProcessResult {
pub changes: Vec<CompanyEventChange>,
}
pub fn process_batch(
new_events: &[CompanyEvent],
existing: &mut HashMap<String, CompanyEvent>,
today: &str,
) -> ProcessResult {
let mut changes = Vec::new();
for new in new_events {
let key = event_key(new);
if let Some(old) = existing.get(&key) {
changes.extend(detect_changes(old, new, today));
existing.insert(key, new.clone());
continue;
}
// Check for time change on same date
let date_key = format!("{}|{}", new.ticker, new.date);
let mut found_old = None;
for (k, e) in existing.iter() {
if format!("{}|{}", e.ticker, e.date) == date_key && k != &key {
found_old = Some((k.clone(), e.clone()));
break;
}
}
if let Some((old_key, old_event)) = found_old {
if new.date.as_str() > today {
changes.push(CompanyEventChange {
ticker: new.ticker.clone(),
date: new.date.clone(),
field_changed: "time".to_string(),
old_value: old_event.time.clone(),
new_value: new.time.clone(),
detected_at: Local::now().format("%Y-%m-%d %H:%M:%S").to_string(),
});
}
existing.remove(&old_key);
}
existing.insert(key, new.clone());
}
ProcessResult { changes }
}

54
src/countries.json
View File

@@ -1,54 +0,0 @@
[
"aegypten",
"frankreich",
"litauen",
"schweiz",
"argentinien",
"griechenland",
"mexiko",
"singapur",
"australien",
"hongkong",
"neuseeland",
"slowakei",
"bahrain",
"indien",
"niederlande",
"spanien",
"belgien",
"indonesien",
"norwegen",
"suedafrika",
"brasilien",
"irland",
"oesterreich",
"suedkorea",
"chile",
"island",
"peru",
"taiwan",
"china",
"italien",
"philippinen",
"tschechien",
"daenemark",
"japan",
"polen",
"tuerkei",
"deutschland",
"kanada",
"portugal",
"ungarn",
"estland",
"katar",
"rumaenien",
"usa",
"eurozone",
"kolumbien",
"russland",
"vereinigte-arabische-emirate",
"finnland",
"lettland",
"schweden",
"vereinigtes-koenigreich"
]