fixed yahoo api calls for cleansing low profile data
This commit is contained in:
@@ -0,0 +1,34 @@
|
|||||||
|
// src/corporate/update_companies_enrich.rs
|
||||||
|
use super::{helpers::*, types::*};
|
||||||
|
use crate::config::Config;
|
||||||
|
use crate::util::directories::DataPaths;
|
||||||
|
use crate::util::logger;
|
||||||
|
use crate::scraper::yahoo::{YahooClientPool, QuoteSummaryModule};
|
||||||
|
|
||||||
|
use std::result::Result::Ok;
|
||||||
|
use chrono::{Local, Utc};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||||
|
use tokio::fs::{File, OpenOptions};
|
||||||
|
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||||
|
use futures::stream::{FuturesUnordered, StreamExt};
|
||||||
|
use serde_json::json;
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
|
||||||
|
/// Yahoo enriching data per corporate
|
||||||
|
///
|
||||||
|
/// # Features
|
||||||
|
/// - Graceful shutdown (abort-safe)
|
||||||
|
/// - Task panic isolation (tasks fail independently)
|
||||||
|
/// - Crash-safe persistence (checkpoint + log with fsync)
|
||||||
|
/// - Smart skip logic (only process incomplete data)
|
||||||
|
/// - Uses pending queue instead of retry mechanism
|
||||||
|
/// - Reuses companies_update.log for persistence
|
||||||
|
///
|
||||||
|
/// # Persistence Strategy
|
||||||
|
/// - Checkpoint: companies_yahoo_cleaned.jsonl (atomic state)
|
||||||
|
/// - Log: companies_update.log (append-only updates)
|
||||||
|
/// - On restart: Load checkpoint + replay log
|
||||||
|
/// - Periodic checkpoints (every 50 companies)
|
||||||
|
/// - Batched fsync (every 10 writes or 10 seconds)
|
||||||
Reference in New Issue
Block a user