Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c90a276dca | |||
| dc4c23f9d9 | |||
| 3182d57539 | |||
| 8c1ef7f9f6 | |||
| 16020eea50 | |||
| 432a798210 |
@@ -12,7 +12,6 @@
|
|||||||
/// These functions are called from the main agent loop and background tasks. All network operations are asynchronous and robust to transient failures.
|
/// These functions are called from the main agent loop and background tasks. All network operations are asynchronous and robust to transient failures.
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use crate::docker::container;
|
|
||||||
use crate::docker::serverclientcomm::handle_server_message;
|
use crate::docker::serverclientcomm::handle_server_message;
|
||||||
use crate::hardware::HardwareInfo;
|
use crate::hardware::HardwareInfo;
|
||||||
use crate::models::{
|
use crate::models::{
|
||||||
@@ -153,10 +152,45 @@ async fn get_server_id_by_ip(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Broadcasts Docker container information to the monitoring server for service discovery.
|
||||||
|
///
|
||||||
|
/// This function sends the current Docker container configuration to the server
|
||||||
|
/// to register available containers and enable service monitoring. It will
|
||||||
|
/// continuously retry until successful, making it suitable for initial
|
||||||
|
/// registration scenarios.
|
||||||
|
///
|
||||||
|
/// # Arguments
|
||||||
|
///
|
||||||
|
/// * `base_url` - The base URL of the monitoring server API (e.g., "https://monitoring.example.com")
|
||||||
|
/// * `server_id` - The ID of the server to associate the containers with
|
||||||
|
/// * `container_dto` - Mutable reference to Docker container information for broadcast
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
///
|
||||||
|
/// * `Ok(())` - When container information is successfully broadcasted to the server
|
||||||
|
/// * `Err(Box<dyn Error + Send + Sync>)` - If an unrecoverable error occurs (though the function typically retries on transient failures)
|
||||||
|
///
|
||||||
|
/// # Behavior
|
||||||
|
///
|
||||||
|
/// This function operates in a retry loop with the following characteristics:
|
||||||
|
///
|
||||||
|
/// - **Retry Logic**: Attempts broadcast every 10 seconds until successful
|
||||||
|
/// - **Mutation**: Modifies the `container_dto` to set the `server_id` before sending
|
||||||
|
/// - **TLS**: Accepts invalid TLS certificates for development environments
|
||||||
|
/// - **Logging**: Provides detailed console output about broadcast attempts and results
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// This function may return an error in the following cases:
|
||||||
|
///
|
||||||
|
/// * **HTTP Client Creation**: Failed to create HTTP client with TLS configuration
|
||||||
|
/// * **Network Issues**: Persistent connection failures to the backend server
|
||||||
|
/// * **Server Errors**: Backend returns non-success HTTP status codes repeatedly
|
||||||
|
/// * **JSON Serialization**: Cannot serialize container data (should be rare with proper DTOs)
|
||||||
pub async fn broadcast_docker_containers(
|
pub async fn broadcast_docker_containers(
|
||||||
base_url: &str,
|
base_url: &str,
|
||||||
server_id: u16,
|
server_id: u16,
|
||||||
container_dto: &mut DockerRegistrationDto,
|
container_dto: &DockerRegistrationDto,
|
||||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
// First get local IP
|
// First get local IP
|
||||||
println!("Preparing to broadcast docker containers...");
|
println!("Preparing to broadcast docker containers...");
|
||||||
@@ -166,8 +200,8 @@ pub async fn broadcast_docker_containers(
|
|||||||
.build()?;
|
.build()?;
|
||||||
|
|
||||||
// Prepare registration data
|
// Prepare registration data
|
||||||
let container_dto = container_dto;
|
let mut broadcast_data = container_dto.clone();
|
||||||
container_dto.server_id = server_id;
|
broadcast_data.server_id = server_id;
|
||||||
|
|
||||||
// Try to register (will retry on failure)
|
// Try to register (will retry on failure)
|
||||||
loop {
|
loop {
|
||||||
|
|||||||
@@ -129,27 +129,72 @@ impl DockerManager {
|
|||||||
/// Collects Docker metrics for all containers
|
/// Collects Docker metrics for all containers
|
||||||
pub async fn collect_metrics(&self) -> Result<DockerMetricDto, Box<dyn Error + Send + Sync>> {
|
pub async fn collect_metrics(&self) -> Result<DockerMetricDto, Box<dyn Error + Send + Sync>> {
|
||||||
let containers = self.get_containers().await?;
|
let containers = self.get_containers().await?;
|
||||||
let (cpu_stats, net_stats, mem_stats) = stats::get_container_stats(&self.docker).await?;
|
if let Some(first_container) = containers.first() {
|
||||||
|
println!("Debug: Testing stats for container {}", first_container.id);
|
||||||
|
let _ = self.debug_container_stats(&first_container.id).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Get stats with proper error handling
|
||||||
|
let stats_result = stats::get_container_stats(&self.docker).await;
|
||||||
|
let (cpu_stats, net_stats, mem_stats) = match stats_result {
|
||||||
|
Ok(stats) => stats,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("Warning: Failed to get container stats: {}", e);
|
||||||
|
// Return empty stats instead of failing completely
|
||||||
|
(Vec::new(), Vec::new(), Vec::new())
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
println!("Debug: Found {} containers, {} CPU stats, {} network stats, {} memory stats",
|
||||||
|
containers.len(), cpu_stats.len(), net_stats.len(), mem_stats.len());
|
||||||
|
|
||||||
let container_infos_total: Vec<_> = containers
|
let container_infos_total: Vec<_> = containers
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|container| {
|
.map(|container| {
|
||||||
|
// Use short ID for matching (first 12 chars)
|
||||||
|
let container_short_id = if container.id.len() > 12 {
|
||||||
|
&container.id[..12]
|
||||||
|
} else {
|
||||||
|
&container.id
|
||||||
|
};
|
||||||
|
|
||||||
let cpu = cpu_stats
|
let cpu = cpu_stats
|
||||||
.iter()
|
.iter()
|
||||||
.find(|c| c.container_id == Some(container.id.clone()))
|
.find(|c| {
|
||||||
|
c.container_id.as_ref()
|
||||||
|
.map(|id| id.starts_with(container_short_id))
|
||||||
|
.unwrap_or(false)
|
||||||
|
})
|
||||||
.cloned();
|
.cloned();
|
||||||
|
|
||||||
let network = net_stats
|
let network = net_stats
|
||||||
.iter()
|
.iter()
|
||||||
.find(|n| n.container_id == Some(container.id.clone()))
|
.find(|n| {
|
||||||
|
n.container_id.as_ref()
|
||||||
|
.map(|id| id.starts_with(container_short_id))
|
||||||
|
.unwrap_or(false)
|
||||||
|
})
|
||||||
.cloned();
|
.cloned();
|
||||||
|
|
||||||
let ram = mem_stats
|
let ram = mem_stats
|
||||||
.iter()
|
.iter()
|
||||||
.find(|m| m.container_id == Some(container.id.clone()))
|
.find(|m| {
|
||||||
|
m.container_id.as_ref()
|
||||||
|
.map(|id| id.starts_with(container_short_id))
|
||||||
|
.unwrap_or(false)
|
||||||
|
})
|
||||||
.cloned();
|
.cloned();
|
||||||
|
|
||||||
|
// Debug output for this container
|
||||||
|
if cpu.is_none() || network.is_none() || ram.is_none() {
|
||||||
|
println!("Debug: Container {} - CPU: {:?}, Network: {:?}, RAM: {:?}",
|
||||||
|
container_short_id, cpu.is_some(), network.is_some(), ram.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
DockerContainerInfo {
|
DockerContainerInfo {
|
||||||
container: Some(container),
|
container: Some(container),
|
||||||
status: None, // Status can be fetched if needed
|
status: None,
|
||||||
cpu,
|
cpu,
|
||||||
network,
|
network,
|
||||||
ram,
|
ram,
|
||||||
@@ -159,43 +204,57 @@ impl DockerManager {
|
|||||||
|
|
||||||
let container_infos: Vec<DockerCollectMetricDto> = container_infos_total
|
let container_infos: Vec<DockerCollectMetricDto> = container_infos_total
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|info| DockerCollectMetricDto {
|
.filter_map(|info| {
|
||||||
id: Some(info.container.unwrap().id).unwrap_or("".to_string()),
|
let container = match info.container {
|
||||||
cpu: info
|
Some(c) => c,
|
||||||
.cpu
|
None => {
|
||||||
.unwrap()
|
eprintln!("Warning: Container info missing container data, skipping");
|
||||||
.cpu_usage_percent
|
return None;
|
||||||
.map(|load| DockerContainerCpuDto {
|
}
|
||||||
cpu_load: Some(load),
|
};
|
||||||
})
|
|
||||||
.unwrap_or(DockerContainerCpuDto { cpu_load: None }),
|
// Safely handle CPU data with defaults
|
||||||
ram: info
|
let cpu_dto = if let Some(cpu) = info.cpu {
|
||||||
.ram
|
DockerContainerCpuDto {
|
||||||
.unwrap()
|
cpu_load: cpu.cpu_usage_percent,
|
||||||
.memory_usage_percent
|
}
|
||||||
.map(|load| DockerContainerRamDto {
|
} else {
|
||||||
cpu_load: Some(load),
|
DockerContainerCpuDto { cpu_load: None }
|
||||||
})
|
};
|
||||||
.unwrap_or(DockerContainerRamDto { cpu_load: None }),
|
|
||||||
network: DockerContainerNetworkDto {
|
// Safely handle RAM data with defaults
|
||||||
net_in: info
|
let ram_dto = if let Some(ram) = info.ram {
|
||||||
.network
|
DockerContainerRamDto {
|
||||||
.as_ref()
|
ram_load: ram.memory_usage_percent,
|
||||||
.unwrap()
|
}
|
||||||
.rx_bytes
|
} else {
|
||||||
.map(|bytes| bytes as f64)
|
DockerContainerRamDto { ram_load: None }
|
||||||
.or(Some(0.0)),
|
};
|
||||||
net_out: info
|
|
||||||
.network
|
// Safely handle network data with defaults
|
||||||
.unwrap()
|
let network_dto = if let Some(net) = info.network {
|
||||||
.tx_bytes
|
DockerContainerNetworkDto {
|
||||||
.map(|bytes| bytes as f64)
|
net_in: net.rx_bytes.map(|bytes| bytes as f64),
|
||||||
.or(Some(0.0)),
|
net_out: net.tx_bytes.map(|bytes| bytes as f64),
|
||||||
},
|
}
|
||||||
|
} else {
|
||||||
|
DockerContainerNetworkDto {
|
||||||
|
net_in: None,
|
||||||
|
net_out: None,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
Some(DockerCollectMetricDto {
|
||||||
|
id: container.id,
|
||||||
|
cpu: cpu_dto,
|
||||||
|
ram: ram_dto,
|
||||||
|
network: network_dto,
|
||||||
|
})
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let dto = DockerMetricDto {
|
let dto = DockerMetricDto {
|
||||||
server_id: 0,
|
server_id: 0, // This should be set by the caller
|
||||||
containers: serde_json::to_string(&container_infos)?,
|
containers: serde_json::to_string(&container_infos)?,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -207,13 +266,48 @@ impl DockerManager {
|
|||||||
) -> Result<DockerRegistrationDto, Box<dyn Error + Send + Sync>> {
|
) -> Result<DockerRegistrationDto, Box<dyn Error + Send + Sync>> {
|
||||||
let containers = self.get_containers().await?;
|
let containers = self.get_containers().await?;
|
||||||
let dto = DockerRegistrationDto {
|
let dto = DockerRegistrationDto {
|
||||||
server_id: 0,
|
server_id: 0, // This will be set by the caller
|
||||||
//container_count,
|
containers: serde_json::to_string(&containers)
|
||||||
containers: serde_json::to_string(&containers)?,
|
.unwrap_or_else(|_| "[]".to_string()), // Fallback to empty array
|
||||||
};
|
};
|
||||||
|
|
||||||
Ok(dto)
|
Ok(dto)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Debug function to check stats collection for a specific container
|
||||||
|
pub async fn debug_container_stats(
|
||||||
|
&self,
|
||||||
|
container_id: &str
|
||||||
|
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||||
|
println!("=== DEBUG STATS FOR CONTAINER {} ===", container_id);
|
||||||
|
|
||||||
|
let (cpu_info, net_info, mem_info) = stats::get_single_container_stats(&self.docker, container_id).await?;
|
||||||
|
|
||||||
|
println!("CPU Info: {:?}", cpu_info);
|
||||||
|
println!("Network Info: {:?}", net_info);
|
||||||
|
println!("Memory Info: {:?}", mem_info);
|
||||||
|
|
||||||
|
// Also try the individual stats functions
|
||||||
|
println!("--- Individual CPU Stats ---");
|
||||||
|
match stats::cpu::get_single_container_cpu_stats(&self.docker, container_id).await {
|
||||||
|
Ok(cpu) => println!("CPU: {:?}", cpu),
|
||||||
|
Err(e) => println!("CPU Error: {}", e),
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("--- Individual Network Stats ---");
|
||||||
|
match stats::network::get_single_container_network_stats(&self.docker, container_id).await {
|
||||||
|
Ok(net) => println!("Network: {:?}", net),
|
||||||
|
Err(e) => println!("Network Error: {}", e),
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("--- Individual Memory Stats ---");
|
||||||
|
match stats::ram::get_single_container_memory_stats(&self.docker, container_id).await {
|
||||||
|
Ok(mem) => println!("Memory: {:?}", mem),
|
||||||
|
Err(e) => println!("Memory Error: {}", e),
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Keep these as utility functions if needed, but they should use DockerManager internally
|
// Keep these as utility functions if needed, but they should use DockerManager internally
|
||||||
@@ -232,4 +326,4 @@ impl DockerContainer {
|
|||||||
pub fn name(&self) -> &str {
|
pub fn name(&self) -> &str {
|
||||||
&self.name.as_deref().unwrap_or("unknown")
|
&self.name.as_deref().unwrap_or("unknown")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -31,10 +31,8 @@ pub mod hardware;
|
|||||||
pub mod metrics;
|
pub mod metrics;
|
||||||
pub mod models;
|
pub mod models;
|
||||||
|
|
||||||
use bollard::Docker;
|
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::sync::Arc;
|
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
|
|
||||||
/// Awaits a spawned asynchronous task and flattens its nested `Result` type.
|
/// Awaits a spawned asynchronous task and flattens its nested `Result` type.
|
||||||
@@ -153,7 +151,13 @@ async fn main() -> Result<(), Box<dyn Error + Send + Sync>> {
|
|||||||
let docker_manager = docker_manager.as_ref().cloned().unwrap();
|
let docker_manager = docker_manager.as_ref().cloned().unwrap();
|
||||||
async move {
|
async move {
|
||||||
let mut collector = metrics::Collector::new(server_id, ip, docker_manager);
|
let mut collector = metrics::Collector::new(server_id, ip, docker_manager);
|
||||||
collector.run(&server_url).await
|
if let Err(e) = collector.run(&server_url).await {
|
||||||
|
eprintln!("Metrics collection error: {}", e);
|
||||||
|
// Don't panic, just return the error
|
||||||
|
Err(e)
|
||||||
|
} else {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -182,9 +182,6 @@ pub struct Acknowledgment {
|
|||||||
/// - `image`: Docker image name (string)
|
/// - `image`: Docker image name (string)
|
||||||
/// - `Name`: Container name (string)
|
/// - `Name`: Container name (string)
|
||||||
/// - `Status`: Container status ("running", "stopped", etc.)
|
/// - `Status`: Container status ("running", "stopped", etc.)
|
||||||
/// - `_net_in`: Network receive rate in **bytes per second (B/s)**
|
|
||||||
/// - `_net_out`: Network transmit rate in **bytes per second (B/s)**
|
|
||||||
/// - `_cpu_load`: CPU usage as a percentage (**0.0–100.0**)
|
|
||||||
#[derive(Debug, Serialize, Clone)]
|
#[derive(Debug, Serialize, Clone)]
|
||||||
pub struct DockerRegistrationDto {
|
pub struct DockerRegistrationDto {
|
||||||
/// Unique server identifier (integer)
|
/// Unique server identifier (integer)
|
||||||
@@ -238,7 +235,7 @@ pub struct DockerContainerCpuDto {
|
|||||||
|
|
||||||
#[derive(Debug, Serialize, Clone)]
|
#[derive(Debug, Serialize, Clone)]
|
||||||
pub struct DockerContainerRamDto {
|
pub struct DockerContainerRamDto {
|
||||||
pub cpu_load: Option<f64>,
|
pub ram_load: Option<f64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Clone)]
|
#[derive(Debug, Serialize, Clone)]
|
||||||
|
|||||||
Reference in New Issue
Block a user