donpat1to/WebScraper
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

moved structs to types.rs

Browse Source
This commit is contained in:
Patrick Mahnke-Hartmann
2026-01-12 18:50:44 +01:00
parent c0c9bc0ed9
commit 29d8f1d89e
15 changed files with 120 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/corporate/update_companies_enrich.rs
View File

@@ -29,7 +29,7 @@ enum LogCommand {
/// Type alias for enrichment function
type EnrichmentFn = Arc<
dyn Fn(CompanyCrossPlatformInfo, Arc<YahooClientPool>, DataPaths)
dyn Fn(CompanyCrossPlatformData, Arc<YahooClientPool>, DataPaths)
-> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send>>
+ Send
+ Sync
@@ -104,7 +104,7 @@ pub async fn enrich_companies_with_events(
logger::log_info(&format!("Found {} companies to process", total_companies)).await;
// Filter companies that need enrichment
let pending_companies: Vec<CompanyCrossPlatformInfo> = companies
let pending_companies: Vec<CompanyCrossPlatformData> = companies
.into_iter()
.filter(|company| !enriched_companies.contains(&company.name))
.collect();
@@ -140,7 +140,7 @@ pub async fn enrich_companies_with_events(
let failed_count = Arc::new(AtomicUsize::new(0));
// Log writer channel with batching and fsync
let (log_tx, mut log_rx) = mpsc::channel::<LogCommand>(1000);
let (log_tx, log_rx) = mpsc::channel::<LogCommand>(1000);
// Spawn log writer task
let log_writer_handle = spawn_log_writer(
@@ -283,7 +283,7 @@ async fn track_events_completion(
/// Enrich a single company with event data
async fn enrich_company_with_events(
company: &CompanyCrossPlatformInfo,
company: &CompanyCrossPlatformData,
yahoo_pool: &Arc<YahooClientPool>,
paths: &DataPaths,
) -> anyhow::Result<()> {
@@ -438,7 +438,7 @@ pub async fn enrich_companies_with_option(
logger::log_info(&format!("Found {} companies to process", total_companies)).await;
// Filter companies that need enrichment
let pending_companies: Vec<CompanyCrossPlatformInfo> = companies
let pending_companies: Vec<CompanyCrossPlatformData> = companies
.into_iter()
.filter(|company| !enriched_companies.contains(&company.name))
.collect();
@@ -474,7 +474,7 @@ pub async fn enrich_companies_with_option(
let failed_count = Arc::new(AtomicUsize::new(0));
// Log writer channel with batching and fsync
let (log_tx, mut log_rx) = mpsc::channel::<LogCommand>(1000);
let (log_tx, log_rx) = mpsc::channel::<LogCommand>(1000);
// Spawn log writer task
let log_writer_handle = spawn_log_writer(
@@ -605,7 +605,7 @@ async fn track_option_completion(
/// Enrich a single company with option data
async fn enrich_company_with_option(
company: &CompanyCrossPlatformInfo,
company: &CompanyCrossPlatformData,
yahoo_pool: &Arc<YahooClientPool>,
paths: &DataPaths,
) -> anyhow::Result<()> {
@@ -697,7 +697,7 @@ pub async fn enrich_companies_with_chart(
logger::log_info(&format!("Found {} companies to process", total_companies)).await;
// Filter companies that need enrichment
let pending_companies: Vec<CompanyCrossPlatformInfo> = companies
let pending_companies: Vec<CompanyCrossPlatformData> = companies
.into_iter()
.filter(|company| !enriched_companies.contains(&company.name))
.collect();
@@ -733,7 +733,7 @@ pub async fn enrich_companies_with_chart(
let failed_count = Arc::new(AtomicUsize::new(0));
// Log writer channel with batching and fsync
let (log_tx, mut log_rx) = mpsc::channel::<LogCommand>(1000);
let (log_tx, log_rx) = mpsc::channel::<LogCommand>(1000);
// Spawn log writer task
let log_writer_handle = spawn_log_writer(
@@ -864,7 +864,7 @@ async fn track_chart_completion(
/// Enrich a single company with chart data
async fn enrich_company_with_chart(
company: &CompanyCrossPlatformInfo,
company: &CompanyCrossPlatformData,
yahoo_pool: &Arc<YahooClientPool>,
paths: &DataPaths,
) -> anyhow::Result<()> {
@@ -1005,7 +1005,7 @@ fn spawn_log_writer(
/// - `shutdown_flag`: Flag to signal shutdown
/// - `enrichment_fn`: The specific enrichment function to call (events, option, chart, etc.)
fn spawn_enrichment_task(
company: CompanyCrossPlatformInfo,
company: CompanyCrossPlatformData,
yahoo_pool: Arc<YahooClientPool>,
paths: DataPaths,
processed_count: Arc<AtomicUsize>,