donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

moved structs to types.rs

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2026-01-12 18:50:44 +01:00
parent c0c9bc0ed9
commit 29d8f1d89e
15 changed files with 120 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/corporate/update_companies_cleanse.rs
View File

@@ -20,15 +20,15 @@ use tokio::sync::mpsc;
/// Result of processing a single company
#[derive(Debug, Clone)]
pub enum CompanyProcessResult {
Valid(CompanyCrossPlatformInfo),
Valid(CompanyCrossPlatformData),
FilteredLowCap { name: String, market_cap: f64 },
FilteredNoPrice { name: String },
Failed { company: CompanyCrossPlatformInfo, error: String, is_transient: bool },
Failed { company: CompanyCrossPlatformData, error: String, is_transient: bool },
}
/// Represents a write command to be serialized through the log writer
enum LogCommand {
Write(CompanyCrossPlatformInfo),
Write(CompanyCrossPlatformData),
Checkpoint,
Shutdown,
}
@@ -81,7 +81,7 @@ pub async fn companies_yahoo_cleansed_no_data(paths: &DataPaths) -> Result<usize
total_count += 1;
let company: CompanyCrossPlatformInfo = match serde_json::from_str(&line) {
let company: CompanyCrossPlatformData = match serde_json::from_str(&line) {
Ok(c) => c,
Err(e) => {
logger::log_warn(&format!(" Failed to parse company on line {}: {}", total_count, e)).await;
@@ -194,7 +194,7 @@ pub async fn companies_yahoo_cleansed_low_profile(
logger::log_info(" Cleansing companies with low Yahoo profile...").await;
// === RECOVERY PHASE: Load checkpoint + replay log ===
let mut existing_companies: HashMap<String, CompanyCrossPlatformInfo> = HashMap::new();
let mut existing_companies: HashMap<String, CompanyCrossPlatformData> = HashMap::new();
let mut processed_names: std::collections::HashSet<String> = std::collections::HashSet::new();
if checkpoint_path.exists() {
@@ -206,7 +206,7 @@ pub async fn companies_yahoo_cleansed_low_profile(
continue; // Skip incomplete lines
}
match serde_json::from_str::<CompanyCrossPlatformInfo>(line) {
match serde_json::from_str::<CompanyCrossPlatformData>(line) {
Ok(company) => {
processed_names.insert(company.name.clone());
existing_companies.insert(company.name.clone(), company);
@@ -229,7 +229,7 @@ pub async fn companies_yahoo_cleansed_low_profile(
continue; // Skip incomplete lines
}
match serde_json::from_str::<CompanyCrossPlatformInfo>(line) {
match serde_json::from_str::<CompanyCrossPlatformData>(line) {
Ok(company) => {
processed_names.insert(company.name.clone());
existing_companies.insert(company.name.clone(), company);
@@ -251,7 +251,7 @@ pub async fn companies_yahoo_cleansed_low_profile(
logger::log_info(&format!("Loaded {} companies from input", input_companies.len())).await;
// === BUILD PENDING LIST (smart skip logic) ===
let mut pending: Vec<CompanyCrossPlatformInfo> = input_companies
let mut pending: Vec<CompanyCrossPlatformData> = input_companies
.into_iter()
.filter(|company| company_needs_processing(company, &existing_companies))
.collect();
@@ -608,7 +608,7 @@ pub async fn companies_yahoo_cleansed_low_profile(
/// Helper function to spawn a validation task (reduces code duplication)
fn spawn_validation_task(
company: CompanyCrossPlatformInfo,
company: CompanyCrossPlatformData,
yahoo_pool: &Arc<YahooClientPool>,
paths: &Arc<DataPaths>,
write_tx: &mpsc::Sender<LogCommand>,
@@ -688,7 +688,7 @@ fn spawn_validation_task(
/// Process a single company with full error categorization
async fn process_company_with_validation(
company: &CompanyCrossPlatformInfo,
company: &CompanyCrossPlatformData,
yahoo_pool: &Arc<YahooClientPool>,
paths: &DataPaths,
) -> CompanyProcessResult {
@@ -897,8 +897,8 @@ async fn save_company_core_data(
/// Check if a company needs processing (validation check)
fn company_needs_processing(
company: &CompanyCrossPlatformInfo,
existing_companies: &HashMap<String, CompanyCrossPlatformInfo>,
company: &CompanyCrossPlatformData,
existing_companies: &HashMap<String, CompanyCrossPlatformData>,
) -> bool {
// If company exists in cleaned output, skip it
!existing_companies.contains_key(&company.name)