added integrity dir for set data collection; one state.jsonl
This commit is contained in:
@@ -245,7 +245,7 @@ fn get_fallback_rate(currency: &str) -> f64 {
|
||||
/// - Integrity tracking with content hash validation
|
||||
pub async fn collect_and_save_exchanges(paths: &DataPaths) -> anyhow::Result<usize> {
|
||||
let state_path = paths.data_dir().join("state.jsonl");
|
||||
let manager = StateManager::new(&state_path, &paths.data_dir().to_path_buf())?;
|
||||
let manager = StateManager::new(paths.integrity_dir())?;
|
||||
let step_name = "exchange_collection_complete";
|
||||
|
||||
let output_path = paths.data_dir().join("yahoo_exchanges.json");
|
||||
|
||||
@@ -47,7 +47,7 @@ pub async fn companies_yahoo_cleansed_no_data(paths: &DataPaths) -> Result<usize
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let manager = StateManager::new(&state_path, &data_path.to_path_buf())?;
|
||||
let manager = StateManager::new(paths.integrity_dir())?;
|
||||
let step_name = "yahoo_companies_cleansed_no_data";
|
||||
let content_reference = file_reference(&output_path);
|
||||
|
||||
@@ -179,7 +179,7 @@ pub async fn companies_yahoo_cleansed_low_profile(
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let manager = StateManager::new(&state_path, &data_path.to_path_buf())?;
|
||||
let manager = StateManager::new(paths.integrity_dir())?;
|
||||
let step_name = "yahoo_companies_cleansed_no_data";
|
||||
let content_reference = file_reference(&checkpoint_path);
|
||||
|
||||
|
||||
@@ -81,7 +81,7 @@ pub async fn enrich_companies_with_events(
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let manager = StateManager::new(&state_path, &data_path.to_path_buf())?;
|
||||
let manager = StateManager::new(paths.integrity_dir())?;
|
||||
let step_name = "yahoo_events_enrichment_complete";
|
||||
|
||||
if manager.is_step_valid(step_name).await? {
|
||||
@@ -418,7 +418,7 @@ pub async fn enrich_companies_with_option(
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let manager = StateManager::new(&state_path, &data_path.to_path_buf())?;
|
||||
let manager = StateManager::new(paths.integrity_dir())?;
|
||||
let step_name = "yahoo_option_enrichment_complete";
|
||||
|
||||
if manager.is_step_valid(step_name).await? {
|
||||
@@ -678,7 +678,7 @@ pub async fn enrich_companies_with_chart(
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let manager = StateManager::new(&state_path, &data_path.to_path_buf())?;
|
||||
let manager = StateManager::new(paths.integrity_dir())?;
|
||||
let step_name = "yahoo_chart_enrichment_complete";
|
||||
|
||||
if manager.is_step_valid(step_name).await? {
|
||||
|
||||
@@ -103,7 +103,7 @@ pub async fn update_securities(date_dir: &Path) -> anyhow::Result<()> {
|
||||
|
||||
let dir = DataPaths::new(".")?;
|
||||
let state_path = dir.data_dir().join("state.jsonl");
|
||||
let manager = StateManager::new(&state_path, &dir.data_dir().to_path_buf())?;
|
||||
let manager = StateManager::new(&dir.integrity_dir())?;
|
||||
let step_name = "securities_data_complete";
|
||||
|
||||
let data_dir = dir.data_dir();
|
||||
@@ -1233,7 +1233,7 @@ pub async fn update_lei_mapping(
|
||||
) -> anyhow::Result<bool> {
|
||||
let dir = DataPaths::new(".")?;
|
||||
let state_path = dir.cache_dir().join("state.jsonl");
|
||||
let manager = StateManager::new(&state_path, &dir.cache_dir().to_path_buf())?;
|
||||
let manager = StateManager::new(&dir.integrity_dir())?;
|
||||
let step_name = "lei_figi_mapping_complete";
|
||||
|
||||
let map_cache_dir = dir.cache_gleif_openfigi_map_dir();
|
||||
|
||||
Reference in New Issue
Block a user