// src/util/integrity.rs
//! Content integrity and state lifecycle management module with centralized dependencies
//! 
//! Features:
//! - File and directory hashing (SHA-256)
//! - Hash validation against content references
//! - State invalidation based on time or validation failures
//! - 3-stage data lifecycle: cache → data → storage
//! - Inline vs. external hash storage based on size
//! - **Centralized dependency configuration** (Single Source of Truth)
//! - Support for checkpoint groups and hierarchies
//! - Automatic transitive dependency resolution
//! - Cycle detection in dependency graph

use anyhow::{Context, Result, bail};
use chrono::{DateTime, Duration, Utc};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::{HashMap, HashSet};
use std::fs;
use std::io::{BufReader, Read};
use std::path::{Path, PathBuf};
use tokio::fs as async_fs;
use tokio::io::AsyncWriteExt;

// ============================================================================
// CONSTANTS & CONFIGURATION
// ============================================================================

/// Maximum hash size (in bytes) to store inline in state.jsonl
/// Hashes larger than this will be stored in separate files
const INLINE_HASH_THRESHOLD: usize = 1024;

/// Directory for storing external hash files
const HASH_STORAGE_DIR: &str = ".integrity_hashes";

/// File extension for external hash files
const HASH_FILE_EXT: &str = ".hash";

/// Default dependency configuration file name
const DEFAULT_DEPENDENCY_CONFIG: &str = "checkpoint_dependencies.toml";

// ============================================================================
// DEPENDENCY CONFIGURATION (SINGLE SOURCE OF TRUTH)
// ============================================================================

/// Centralized dependency configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DependencyConfig {
    /// Individual checkpoint dependencies
    #[serde(default)]
    pub checkpoints: HashMap<String, CheckpointConfig>,
    
    /// Checkpoint groups (for hierarchical dependencies)
    #[serde(default)]
    pub groups: HashMap<String, GroupConfig>,
}

/// Configuration for a single checkpoint
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CheckpointConfig {
    /// Description of this checkpoint
    #[serde(default)]
    pub description: String,
    
    /// Direct dependencies (checkpoint names this depends on)
    #[serde(default)]
    pub depends_on: Vec<String>,
    
    /// Whether this checkpoint is part of a group
    #[serde(default)]
    pub group: Option<String>,
}

/// Configuration for a checkpoint group
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GroupConfig {
    /// Description of this group
    #[serde(default)]
    pub description: String,
    
    /// Members of this group
    pub members: Vec<String>,
    
    /// Dependencies for all members of this group
    #[serde(default)]
    pub depends_on: Vec<String>,
}

impl DependencyConfig {
    /// Load dependency configuration from TOML file
    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
        let content = fs::read_to_string(path.as_ref())
            .with_context(|| format!("Failed to read dependency config: {}", path.as_ref().display()))?;
        
        let config: DependencyConfig = toml::from_str(&content)
            .context("Failed to parse dependency config")?;
        
        config.validate()?;
        
        Ok(config)
    }
    
    /// Load from default location (dependencies.toml in base_dir)
    pub fn from_default_location<P: AsRef<Path>>(base_dir: P) -> Result<Self> {
        let config_path = base_dir.as_ref().join(DEFAULT_DEPENDENCY_CONFIG);
        
        if !config_path.exists() {
            // Return empty config if file doesn't exist
            return Ok(Self::default());
        }
        
        Self::from_file(config_path)
    }
    
    /// Validate configuration (check for cycles, invalid references)
    pub fn validate(&self) -> Result<()> {
        // Check for cycles
        for checkpoint in self.checkpoints.keys() {
            self.detect_cycle(checkpoint)?;
        }
        
        // Validate group memberships
        for (group_name, group) in &self.groups {
            for member in &group.members {
                if !self.checkpoints.contains_key(member) {
                    bail!("Group '{}' references non-existent checkpoint: {}", group_name, member);
                }
            }
        }
        
        // Validate that checkpoints in groups actually declare the group
        for (checkpoint_name, checkpoint) in &self.checkpoints {
            if let Some(group_name) = &checkpoint.group {
                if let Some(group) = self.groups.get(group_name) {
                    if !group.members.contains(checkpoint_name) {
                        bail!("Checkpoint '{}' claims to be in group '{}' but group doesn't list it", 
                              checkpoint_name, group_name);
                    }
                } else {
                    bail!("Checkpoint '{}' references non-existent group: {}", checkpoint_name, group_name);
                }
            }
        }
        
        Ok(())
    }
    
    /// Detect if there's a cycle in dependencies starting from checkpoint
    fn detect_cycle(&self, start: &str) -> Result<()> {
        let mut visited = HashSet::new();
        let mut stack = HashSet::new();
        
        self.detect_cycle_helper(start, &mut visited, &mut stack)
    }
    
    fn detect_cycle_helper(
        &self,
        checkpoint: &str,
        visited: &mut HashSet<String>,
        stack: &mut HashSet<String>,
    ) -> Result<()> {
        if stack.contains(checkpoint) {
            bail!("Cycle detected in dependency graph at checkpoint: {}", checkpoint);
        }
        
        if visited.contains(checkpoint) {
            return Ok(());
        }
        
        visited.insert(checkpoint.to_string());
        stack.insert(checkpoint.to_string());
        
        // Check direct dependencies
        if let Some(config) = self.checkpoints.get(checkpoint) {
            for dep in &config.depends_on {
                self.detect_cycle_helper(dep, visited, stack)?;
            }
        }
        
        stack.remove(checkpoint);
        Ok(())
    }
    
    /// Get all dependencies for a checkpoint (including transitive and group dependencies)
    pub fn get_all_dependencies(&self, checkpoint: &str) -> Result<Vec<String>> {
        let mut all_deps = Vec::new();
        let mut visited = HashSet::new();
        
        self.collect_dependencies(checkpoint, &mut all_deps, &mut visited)?;
        
        // Remove duplicates while preserving order
        let mut seen = HashSet::new();
        all_deps.retain(|dep| seen.insert(dep.clone()));
        
        Ok(all_deps)
    }
    
    fn collect_dependencies(
        &self,
        checkpoint: &str,
        deps: &mut Vec<String>,
        visited: &mut HashSet<String>,
    ) -> Result<()> {
        if visited.contains(checkpoint) {
            return Ok(());
        }
        visited.insert(checkpoint.to_string());
        
        // Get checkpoint config
        let config = self.checkpoints.get(checkpoint)
            .ok_or_else(|| anyhow::anyhow!("Unknown checkpoint: {}", checkpoint))?;
        
        // Add group dependencies first (if checkpoint is in a group)
        if let Some(group_name) = &config.group {
            if let Some(group) = self.groups.get(group_name) {
                for dep in &group.depends_on {
                    if !visited.contains(dep) {
                        deps.push(dep.clone());
                        self.collect_dependencies(dep, deps, visited)?;
                    }
                }
            }
        }
        
        // Add direct dependencies
        for dep in &config.depends_on {
            if !visited.contains(dep) {
                deps.push(dep.clone());
                self.collect_dependencies(dep, deps, visited)?;
            }
        }
        
        Ok(())
    }
    
    /// Get dependency graph as DOT format (for visualization)
    pub fn to_dot(&self) -> String {
        let mut dot = String::from("digraph Dependencies {\n");
        dot.push_str("  rankdir=LR;\n");
        dot.push_str("  node [shape=box];\n\n");
        
        // Add checkpoints
        for (name, config) in &self.checkpoints {
            let label = if config.description.is_empty() {
                name.clone()
            } else {
                format!("{}\n{}", name, config.description)
            };
            dot.push_str(&format!("  \"{}\" [label=\"{}\"];\n", name, label));
        }
        
        // Add dependencies
        dot.push_str("\n");
        for (name, config) in &self.checkpoints {
            // Add group dependencies
            if let Some(group_name) = &config.group {
                if let Some(group) = self.groups.get(group_name) {
                    for dep in &group.depends_on {
                        dot.push_str(&format!("  \"{}\" -> \"{}\" [label=\"via group {}\"];\n", 
                                            name, dep, group_name));
                    }
                }
            }
            
            // Add direct dependencies
            for dep in &config.depends_on {
                dot.push_str(&format!("  \"{}\" -> \"{}\";\n", name, dep));
            }
        }
        
        dot.push_str("}\n");
        dot
    }
}

impl Default for DependencyConfig {
    fn default() -> Self {
        Self {
            checkpoints: HashMap::new(),
            groups: HashMap::new(),
        }
    }
}

// ============================================================================
// DATA STRUCTURES
// ============================================================================

/// Represents a content reference that can be hashed
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(tag = "type", rename_all = "lowercase")]
pub enum ContentReference {
    /// Single file reference
    File { path: PathBuf },
    
    /// Directory reference (includes all files recursively)
    Directory { 
        path: PathBuf,
        /// Optional: specific files/patterns to include
        include_patterns: Option<Vec<String>>,
        /// Optional: files/patterns to exclude
        exclude_patterns: Option<Vec<String>>,
    },
    
    /// Multiple files/directories combined
    Composite {
        references: Vec<ContentReference>,
    },
}

/// Storage location for hash data
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(tag = "storage", rename_all = "lowercase")]
pub enum HashStorage {
    /// Hash stored directly in state.jsonl
    Inline { hash: String },
    
    /// Hash stored in external file
    External { hash_file: PathBuf },
}

/// Data lifecycle stage
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "lowercase")]
pub enum DataStage {
    /// Temporary/staged data (fast-changing, short-lived)
    Cache,
    
    /// Processed data (intermediate results, medium-lived)
    Data,
    
    /// Final storage (long-term, stable data)
    Storage,
}

impl DataStage {
    /// Get default TTL (time-to-live) for this stage
    pub fn default_ttl(&self) -> Duration {
        match self {
            DataStage::Cache => Duration::hours(24),      // 1 day
            DataStage::Data => Duration::days(7),         // 1 week
            DataStage::Storage => Duration::days(365),    // 1 year
        }
    }
    
    /// Get suggested revalidation interval for this stage
    pub fn revalidation_interval(&self) -> Duration {
        match self {
            DataStage::Cache => Duration::hours(6),       // Every 6 hours
            DataStage::Data => Duration::days(1),         // Daily
            DataStage::Storage => Duration::days(30),     // Monthly
        }
    }
}

/// Enhanced state entry with content integrity tracking
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StateEntry {
    /// Step/function name
    pub step_name: String,
    
    /// Whether this step is completed
    pub completed: bool,
    
    /// Completion timestamp
    pub completed_at: Option<DateTime<Utc>>,
    
    /// Content reference for validation
    pub content_reference: Option<ContentReference>,
    
    /// Hash of the content
    pub content_hash: Option<HashStorage>,
    
    /// Data lifecycle stage
    pub data_stage: Option<DataStage>,
    
    /// Custom TTL override (if None, uses stage default)
    pub ttl_override: Option<Duration>,
    
    /// Last validation timestamp
    pub last_validated_at: Option<DateTime<Utc>>,
    
    /// Validation status
    pub validation_status: ValidationStatus,
    
    /// Dependencies (resolved automatically from config, stored for reference)
    #[serde(default)]
    pub dependencies: Vec<String>,
}

/// Validation status of a state entry
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum ValidationStatus {
    /// Not yet validated
    Unknown,
    
    /// Validated and content matches hash
    Valid,
    
    /// Validation failed (hash mismatch or content missing)
    Invalid { reason: String },
    
    /// Expired (beyond TTL)
    Expired,
    
    /// Invalidated due to dependency failure
    DependencyFailed { failed_dependency: String },
}

// ============================================================================
// HASH COMPUTATION (UNCHANGED FROM ORIGINAL)
// ============================================================================

/// Hash a single file using SHA-256
pub fn hash_file<P: AsRef<Path>>(path: P) -> Result<String> {
    let path = path.as_ref();
    let file = fs::File::open(path)
        .with_context(|| format!("Failed to open file: {}", path.display()))?;
    
    let mut reader = BufReader::new(file);
    let mut hasher = Sha256::new();
    let mut buffer = [0u8; 8192];
    
    loop {
        let bytes_read = reader.read(&mut buffer)?;
        if bytes_read == 0 {
            break;
        }
        hasher.update(&buffer[..bytes_read]);
    }
    
    Ok(format!("{:x}", hasher.finalize()))
}

/// Hash a directory recursively
pub fn hash_directory<P: AsRef<Path>>(
    path: P,
    include_patterns: Option<&[String]>,
    exclude_patterns: Option<&[String]>,
) -> Result<String> {
    let path = path.as_ref();
    
    if !path.is_dir() {
        bail!("Path is not a directory: {}", path.display());
    }
    
    let mut files = Vec::new();
    collect_files_recursive(path, &mut files, include_patterns, exclude_patterns)?;
    
    files.sort();
    
    if files.is_empty() {
        return Ok(String::from("d41d8cd98f00b204e9800998ecf8427e"));
    }
    
    let mut combined_hasher = Sha256::new();
    
    for file_path in files {
        let rel_path = file_path.strip_prefix(path)
            .unwrap_or(&file_path)
            .to_string_lossy();
        combined_hasher.update(rel_path.as_bytes());
        
        let file_hash = hash_file(&file_path)?;
        combined_hasher.update(file_hash.as_bytes());
    }
    
    Ok(format!("{:x}", combined_hasher.finalize()))
}

/// Collect files recursively with pattern filtering
fn collect_files_recursive(
    dir: &Path,
    files: &mut Vec<PathBuf>,
    include_patterns: Option<&[String]>,
    exclude_patterns: Option<&[String]>,
) -> Result<()> {
    if !dir.is_dir() {
        return Ok(());
    }
    
    for entry in fs::read_dir(dir)? {
        let entry = entry?;
        let path = entry.path();
        
        if let Some(name) = path.file_name() {
            if name.to_string_lossy().starts_with('.') {
                continue;
            }
        }
        
        if path.is_dir() {
            collect_files_recursive(&path, files, include_patterns, exclude_patterns)?;
        } else if path.is_file() {
            if should_include_file(&path, include_patterns, exclude_patterns) {
                files.push(path);
            }
        }
    }
    
    Ok(())
}

/// Check if file should be included based on patterns
fn should_include_file(
    path: &Path,
    include_patterns: Option<&[String]>,
    exclude_patterns: Option<&[String]>,
) -> bool {
    let path_str = path.to_string_lossy();
    
    if let Some(excludes) = exclude_patterns {
        for pattern in excludes {
            if glob_match(&path_str, pattern) {
                return false;
            }
        }
    }
    
    if let Some(includes) = include_patterns {
        for pattern in includes {
            if glob_match(&path_str, pattern) {
                return true;
            }
        }
        return false;
    }
    
    true
}

/// Simple glob pattern matching
fn glob_match(path: &str, pattern: &str) -> bool {
    if pattern.contains('*') {
        let parts: Vec<&str> = pattern.split('*').collect();
        if parts.len() == 2 {
            path.contains(parts[0]) && path.ends_with(parts[1])
        } else {
            false
        }
    } else {
        path.ends_with(pattern)
    }
}

/// Hash content based on reference type
pub fn hash_content_reference(reference: &ContentReference) -> Result<String> {
    match reference {
        ContentReference::File { path } => hash_file(path),
        ContentReference::Directory { path, include_patterns, exclude_patterns } => {
            hash_directory(
                path,
                include_patterns.as_deref(),
                exclude_patterns.as_deref(),
            )
        }
        ContentReference::Composite { references } => {
            let mut combined_hasher = Sha256::new();
            for ref_item in references {
                let hash = hash_content_reference(ref_item)?;
                combined_hasher.update(hash.as_bytes());
            }
            Ok(format!("{:x}", combined_hasher.finalize()))
        }
    }
}

/// Determine whether to store hash inline or externally
fn determine_hash_storage(hash: &str, base_dir: &Path) -> HashStorage {
    if hash.len() > INLINE_HASH_THRESHOLD {
        let hash_dir = base_dir.join(HASH_STORAGE_DIR);
        let hash_file = hash_dir.join(format!("{}{}", &hash[..16], HASH_FILE_EXT));
        HashStorage::External { hash_file }
    } else {
        HashStorage::Inline { hash: hash.to_string() }
    }
}

/// Store hash to file if external storage
async fn store_hash(hash: &str, storage: &HashStorage) -> Result<()> {
    if let HashStorage::External { hash_file } = storage {
        if let Some(parent) = hash_file.parent() {
            async_fs::create_dir_all(parent).await?;
        }
        async_fs::write(hash_file, hash.as_bytes()).await?;
    }
    Ok(())
}

/// Load hash from storage
async fn load_hash(storage: &HashStorage) -> Result<String> {
    match storage {
        HashStorage::Inline { hash } => Ok(hash.clone()),
        HashStorage::External { hash_file } => {
            let content = async_fs::read_to_string(hash_file).await?;
            Ok(content.trim().to_string())
        }
    }
}

// ============================================================================
// VALIDATION
// ============================================================================

/// Validate a single state entry
async fn validate_entry(entry: &StateEntry) -> Result<ValidationStatus> {
    if !entry.completed {
        return Ok(ValidationStatus::Unknown);
    }
    
    let content_ref = match &entry.content_reference {
        Some(r) => r,
        None => return Ok(ValidationStatus::Unknown),
    };
    
    let stored_hash_storage = match &entry.content_hash {
        Some(h) => h,
        None => return Ok(ValidationStatus::Unknown),
    };
    
    let stored_hash = load_hash(stored_hash_storage).await?;
    
    let current_hash = match hash_content_reference(content_ref) {
        Ok(h) => h,
        Err(e) => {
            return Ok(ValidationStatus::Invalid {
                reason: format!("Failed to compute hash: {}", e),
            });
        }
    };
    
    if stored_hash != current_hash {
        return Ok(ValidationStatus::Invalid {
            reason: "Hash mismatch".to_string(),
        });
    }
    
    if let Some(stage) = entry.data_stage {
        let ttl = entry.ttl_override.unwrap_or_else(|| stage.default_ttl());
        
        if let Some(completed_at) = entry.completed_at {
            let age = Utc::now() - completed_at;
            if age > ttl {
                return Ok(ValidationStatus::Expired);
            }
        }
    }
    
    Ok(ValidationStatus::Valid)
}

/// Validate all state entries and handle cascade invalidation
pub async fn validate_all_entries(
    entries: &mut HashMap<String, StateEntry>,
) -> Result<ValidationReport> {
    let mut report = ValidationReport::default();
    
    for (name, entry) in entries.iter_mut() {
        let status = validate_entry(entry).await?;
        entry.validation_status = status.clone();
        entry.last_validated_at = Some(Utc::now());
        
        match status {
            ValidationStatus::Valid => report.valid_count += 1,
            ValidationStatus::Invalid { .. } => {
                report.invalid_count += 1;
                report.invalid_entries.push(name.clone());
            }
            ValidationStatus::Expired => {
                report.expired_count += 1;
                report.expired_entries.push(name.clone());
            }
            ValidationStatus::Unknown => report.unknown_count += 1,
            ValidationStatus::DependencyFailed { .. } => {}
        }
    }
    
    let mut invalidated = HashSet::new();
    for name in &report.invalid_entries {
        invalidated.insert(name.clone());
    }
    
    loop {
        let mut newly_invalidated = Vec::new();
        
        for (name, entry) in entries.iter() {
            if invalidated.contains(name) {
                continue;
            }
            
            for dep in &entry.dependencies {
                if invalidated.contains(dep) {
                    newly_invalidated.push((name.clone(), dep.clone()));
                    break;
                }
            }
        }
        
        if newly_invalidated.is_empty() {
            break;
        }
        
        for (name, failed_dep) in newly_invalidated {
            invalidated.insert(name.clone());
            report.cascaded_invalidations.push(name.clone());
            
            if let Some(entry) = entries.get_mut(&name) {
                entry.validation_status = ValidationStatus::DependencyFailed {
                    failed_dependency: failed_dep,
                };
            }
        }
    }
    
    Ok(report)
}

/// Validation report
#[derive(Debug, Default)]
pub struct ValidationReport {
    pub valid_count: usize,
    pub invalid_count: usize,
    pub expired_count: usize,
    pub unknown_count: usize,
    pub invalid_entries: Vec<String>,
    pub expired_entries: Vec<String>,
    pub cascaded_invalidations: Vec<String>,
}

impl ValidationReport {
    pub fn print_summary(&self) {
        println!("=== Validation Report ===");
        println!("Valid:   {}", self.valid_count);
        println!("Invalid: {}", self.invalid_count);
        println!("Expired: {}", self.expired_count);
        println!("Unknown: {}", self.unknown_count);
        
        if !self.invalid_entries.is_empty() {
            println!("\nInvalid entries:");
            for entry in &self.invalid_entries {
                println!("  - {}", entry);
            }
        }
        
        if !self.expired_entries.is_empty() {
            println!("\nExpired entries:");
            for entry in &self.expired_entries {
                println!("  - {}", entry);
            }
        }
        
        if !self.cascaded_invalidations.is_empty() {
            println!("\nCascaded invalidations:");
            for entry in &self.cascaded_invalidations {
                println!("  - {}", entry);
            }
        }
    }
}

// ============================================================================
// STATE MANAGEMENT (UPDATED WITH CENTRALIZED DEPENDENCIES)
// ============================================================================

/// State manager with centralized dependency configuration
pub struct StateManager {
    state_path: PathBuf,
    base_dir: PathBuf,
    dependency_config: DependencyConfig,
}

impl StateManager {
    /// Create new state manager and load dependency configuration
    pub fn new<P: AsRef<Path>>(state_path: P, base_dir: P) -> Result<Self> {
        let base_dir = base_dir.as_ref().to_path_buf();
        let dependency_config = DependencyConfig::from_default_location(&base_dir)?;
        
        Ok(Self {
            state_path: state_path.as_ref().to_path_buf(),
            base_dir,
            dependency_config,
        })
    }
    
    /// Create with explicit dependency configuration
    pub fn with_config<P: AsRef<Path>>(
        state_path: P,
        base_dir: P,
        dependency_config: DependencyConfig,
    ) -> Result<Self> {
        dependency_config.validate()?;
        
        Ok(Self {
            state_path: state_path.as_ref().to_path_buf(),
            base_dir: base_dir.as_ref().to_path_buf(),
            dependency_config,
        })
    }
    
    /// Get the dependency configuration (for inspection/debugging)
    pub fn get_dependency_config(&self) -> &DependencyConfig {
        &self.dependency_config
    }
    
    /// Load all state entries from state.jsonl
    pub async fn load_entries(&self) -> Result<HashMap<String, StateEntry>> {
        let mut entries = HashMap::new();
        
        if !self.state_path.exists() {
            return Ok(entries);
        }
        
        let content = async_fs::read_to_string(&self.state_path).await?;
        
        for line in content.lines() {
            if line.trim().is_empty() {
                continue;
            }
            
            if let Ok(entry) = serde_json::from_str::<StateEntry>(line) {
                entries.insert(entry.step_name.clone(), entry);
            }
        }
        
        Ok(entries)
    }
    
    /// Save all state entries to state.jsonl
    pub async fn save_entries(&self, entries: &HashMap<String, StateEntry>) -> Result<()> {
        if let Some(parent) = self.state_path.parent() {
            async_fs::create_dir_all(parent).await?;
        }
        
        let mut file = async_fs::File::create(&self.state_path).await?;
        
        for entry in entries.values() {
            let line = serde_json::to_string(&entry)? + "\n";
            file.write_all(line.as_bytes()).await?;
        }
        
        file.sync_all().await?;
        
        Ok(())
    }
    
    /// Create or update a state entry with integrity tracking
    /// **UPDATED**: Dependencies are now resolved automatically from config
    pub async fn update_entry(
        &self,
        step_name: String,
        content_reference: ContentReference,
        data_stage: DataStage,
        ttl_override: Option<Duration>,
    ) -> Result<StateEntry> {
        // Resolve dependencies from configuration
        let dependencies = self.dependency_config
            .get_all_dependencies(&step_name)
            .unwrap_or_else(|_| {
                // If checkpoint not in config, no dependencies
                Vec::new()
            });
        
        // Compute hash
        let hash = hash_content_reference(&content_reference)?;
        
        // Determine storage
        let storage = determine_hash_storage(&hash, &self.base_dir);
        
        // Store hash if external
        store_hash(&hash, &storage).await?;
        
        // Create entry
        let entry = StateEntry {
            step_name: step_name.clone(),
            completed: true,
            completed_at: Some(Utc::now()),
            content_reference: Some(content_reference),
            content_hash: Some(storage),
            data_stage: Some(data_stage),
            ttl_override,
            last_validated_at: Some(Utc::now()),
            validation_status: ValidationStatus::Valid,
            dependencies,
        };
        
        // Load existing entries
        let mut entries = self.load_entries().await?;
        
        // Update entry
        entries.insert(step_name, entry.clone());
        
        // Save
        self.save_entries(&entries).await?;
        
        Ok(entry)
    }
    
    /// Check if a step is valid and completed
    pub async fn is_step_valid(&self, step_name: &str) -> Result<bool> {
        let entries = self.load_entries().await?;
        
        if let Some(entry) = entries.get(step_name) {
            let status = validate_entry(entry).await?;
            Ok(matches!(status, ValidationStatus::Valid))
        } else {
            Ok(false)
        }
    }
    
    /// Invalidate a specific entry
    pub async fn invalidate_entry(&self, step_name: &str, reason: String) -> Result<()> {
        let mut entries = self.load_entries().await?;
        
        if let Some(entry) = entries.get_mut(step_name) {
            entry.validation_status = ValidationStatus::Invalid { reason };
            entry.last_validated_at = Some(Utc::now());
        }
        
        self.save_entries(&entries).await?;
        
        Ok(())
    }
    
    /// Run full validation on all entries
    pub async fn validate_all(&self) -> Result<ValidationReport> {
        let mut entries = self.load_entries().await?;
        let report = validate_all_entries(&mut entries).await?;
        self.save_entries(&entries).await?;
        Ok(report)
    }
    
    /// Print dependency graph information
    pub fn print_dependency_graph(&self) {
        println!("=== Dependency Configuration ===");
        println!("\nCheckpoints: {}", self.dependency_config.checkpoints.len());
        println!("Groups: {}", self.dependency_config.groups.len());
        
        println!("\n--- Checkpoints ---");
        for (name, config) in &self.dependency_config.checkpoints {
            println!("{}", name);
            if !config.description.is_empty() {
                println!("  Description: {}", config.description);
            }
            if let Some(group) = &config.group {
                println!("  Group: {}", group);
            }
            if !config.depends_on.is_empty() {
                println!("  Depends on: {}", config.depends_on.join(", "));
            }
            
            // Show resolved dependencies
            if let Ok(all_deps) = self.dependency_config.get_all_dependencies(name) {
                if !all_deps.is_empty() {
                    println!("  Resolved (including transitive): {}", all_deps.join(", "));
                }
            }
            println!();
        }
        
        println!("\n--- Groups ---");
        for (name, group) in &self.dependency_config.groups {
            println!("{}", name);
            if !group.description.is_empty() {
                println!("  Description: {}", group.description);
            }
            println!("  Members: {}", group.members.join(", "));
            if !group.depends_on.is_empty() {
                println!("  Group dependencies: {}", group.depends_on.join(", "));
            }
            println!();
        }
    }
}

// ============================================================================
// HELPER FUNCTIONS
// ============================================================================

/// Create a simple file reference
pub fn file_reference<P: AsRef<Path>>(path: P) -> ContentReference {
    ContentReference::File {
        path: path.as_ref().to_path_buf(),
    }
}

/// Create a directory reference
pub fn directory_reference<P: AsRef<Path>>(
    path: P,
    include_patterns: Option<Vec<String>>,
    exclude_patterns: Option<Vec<String>>,
) -> ContentReference {
    ContentReference::Directory {
        path: path.as_ref().to_path_buf(),
        include_patterns,
        exclude_patterns,
    }
}

/// Create a composite reference
pub fn composite_reference(references: Vec<ContentReference>) -> ContentReference {
    ContentReference::Composite { references }
}