migrated checkpoint handling in integrity.rs to ssot principle

This commit is contained in:
2026-01-11 13:05:31 +01:00
parent 0487c2ec49
commit aff340ee2f
15 changed files with 880 additions and 579 deletions

View File

@@ -47,7 +47,7 @@ pub async fn companies_yahoo_cleansed_no_data(paths: &DataPaths) -> Result<usize
return Ok(0);
}
let manager = StateManager::new(&state_path, &data_path.to_path_buf());
let manager = StateManager::new(&state_path, &data_path.to_path_buf())?;
let step_name = "yahoo_companies_cleansed_no_data";
let content_reference = file_reference(&output_path);
@@ -131,7 +131,6 @@ pub async fn companies_yahoo_cleansed_no_data(paths: &DataPaths) -> Result<usize
step_name.to_string(),
content_reference,
DataStage::Data,
vec!["yahoo_companies_cleansed".to_string()], // Dependency
None, // Use default TTL (7 days for Data stage)
).await?;
@@ -180,7 +179,7 @@ pub async fn companies_yahoo_cleansed_low_profile(
return Ok(0);
}
let manager = StateManager::new(&state_path, &data_path.to_path_buf());
let manager = StateManager::new(&state_path, &data_path.to_path_buf())?;
let step_name = "yahoo_companies_cleansed_no_data";
let content_reference = file_reference(&checkpoint_path);
@@ -632,7 +631,6 @@ pub async fn companies_yahoo_cleansed_low_profile(
step_name.to_string(),
content_reference,
DataStage::Data,
vec!["yahoo_companies_cleansed".to_string()], // Dependency
None, // Use default TTL (7 days for Data stage)
).await?;