donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

removed crossplatformcompany from types

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2026-01-14 14:49:00 +01:00
parent 93fbefc9d4
commit f4b20f824d
8 changed files with 83 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
src/corporate/update_companies_cleanse.rs
View File

@@ -20,15 +20,15 @@ use tokio::sync::mpsc;
/// Result of processing a single company
#[derive(Debug, Clone)]
pub enum CompanyProcessResult {
Valid(CompanyCrossPlatformData),
Valid(CompanyData),
FilteredLowCap { name: String, market_cap: f64 },
FilteredNoPrice { name: String },
Failed { company: CompanyCrossPlatformData, error: String, is_transient: bool },
Failed { company: CompanyData, error: String, is_transient: bool },
}
/// Represents a write command to be serialized through the log writer
enum LogCommand {
Write(CompanyCrossPlatformData),
Write(CompanyData),
Checkpoint,
Shutdown,
}
@@ -81,7 +81,7 @@ pub async fn companies_yahoo_cleansed_no_data(paths: &DataPaths) -> Result<usize
total_count += 1;
let company: CompanyCrossPlatformData = match serde_json::from_str(&line) {
let company: CompanyData = match serde_json::from_str(&line) {
Ok(c) => c,
Err(e) => {
logger::log_warn(&format!(" Failed to parse company on line {}: {}", total_count, e)).await;
@@ -90,13 +90,17 @@ pub async fn companies_yahoo_cleansed_no_data(paths: &DataPaths) -> Result<usize
};
let has_valid_yahoo = company.isin_tickers_map
.values()
.flatten()
.any(|ticker| {
ticker.starts_with("YAHOO:")
&& ticker != "YAHOO:NO_RESULTS"
&& ticker != "YAHOO:ERROR"
});
.as_ref()
.map(|map| {
map.values()
.flatten()
.any(|ticker| {
ticker.starts_with("YAHOO:")
&& ticker != "YAHOO:NO_RESULTS"
&& ticker != "YAHOO:ERROR"
})
})
.unwrap_or(false);
if has_valid_yahoo {
let json_line = serde_json::to_string(&company)?;
@@ -194,7 +198,7 @@ pub async fn companies_yahoo_cleansed_low_profile(
logger::log_info(" Cleansing companies with low Yahoo profile...").await;
// === RECOVERY PHASE: Load checkpoint + replay log ===
let mut existing_companies: HashMap<String, CompanyCrossPlatformData> = HashMap::new();
let mut existing_companies: HashMap<String, CompanyData> = HashMap::new();
let mut processed_names: std::collections::HashSet<String> = std::collections::HashSet::new();
if checkpoint_path.exists() {
@@ -206,7 +210,7 @@ pub async fn companies_yahoo_cleansed_low_profile(
continue; // Skip incomplete lines
}
match serde_json::from_str::<CompanyCrossPlatformData>(line) {
match serde_json::from_str::<CompanyData>(line) {
Ok(company) => {
processed_names.insert(company.name.clone());
existing_companies.insert(company.name.clone(), company);
@@ -229,7 +233,7 @@ pub async fn companies_yahoo_cleansed_low_profile(
continue; // Skip incomplete lines
}
match serde_json::from_str::<CompanyCrossPlatformData>(line) {
match serde_json::from_str::<CompanyData>(line) {
Ok(company) => {
processed_names.insert(company.name.clone());
existing_companies.insert(company.name.clone(), company);
@@ -251,7 +255,7 @@ pub async fn companies_yahoo_cleansed_low_profile(
logger::log_info(&format!("Loaded {} companies from input", input_companies.len())).await;
// === BUILD PENDING LIST (smart skip logic) ===
let mut pending: Vec<CompanyCrossPlatformData> = input_companies
let mut pending: Vec<CompanyData> = input_companies
.into_iter()
.filter(|company| company_needs_processing(company, &existing_companies))
.collect();
@@ -608,7 +612,7 @@ pub async fn companies_yahoo_cleansed_low_profile(
/// Helper function to spawn a validation task (reduces code duplication)
fn spawn_validation_task(
company: CompanyCrossPlatformData,
company: CompanyData,
yahoo_pool: &Arc<YahooClientPool>,
paths: &Arc<DataPaths>,
write_tx: &mpsc::Sender<LogCommand>,
@@ -688,7 +692,7 @@ fn spawn_validation_task(
/// Process a single company with full error categorization
async fn process_company_with_validation(
company: &CompanyCrossPlatformData,
company: &CompanyData,
yahoo_pool: &Arc<YahooClientPool>,
paths: &DataPaths,
) -> CompanyProcessResult {
@@ -897,8 +901,8 @@ async fn save_company_core_data(
/// Check if a company needs processing (validation check)
fn company_needs_processing(
company: &CompanyCrossPlatformData,
existing_companies: &HashMap<String, CompanyCrossPlatformData>,
company: &CompanyData,
existing_companies: &HashMap<String, CompanyData>,
) -> bool {
// If company exists in cleaned output, skip it
!existing_companies.contains_key(&company.name)