1use std::collections::{BTreeMap, HashMap};
8
9use serde::Serialize;
10use serde_json::{Number, Value as JsonValue};
11
12use crate::{
13 ONE_GB,
14 ast::{AnalysisTarget, AnalysisVerb, AnalyzeQuery, CollectionTarget, SingularTargetKind},
15 docker::{Container, DockerClient, DockerError, DockerEvent, MetricSample},
16 executor::{ExecutionResult, Row},
17 metrics::{MetricsCollector, MetricsError},
18 storage::{TelemetryError, TelemetryStore},
19};
20
21#[derive(Debug, thiserror::Error)]
25pub enum AnalyzeError {
26 #[error("{0}")]
27 Docker(#[from] DockerError),
28 #[error("{0}")]
29 Metrics(#[from] MetricsError),
30 #[error("{0}")]
31 Telemetry(#[from] TelemetryError),
32 #[error("analysis not supported for this target/verb combination")]
33 Unsupported,
34}
35
36#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)]
40pub enum Severity {
41 Info,
42 Warning,
43 Critical,
44}
45
46impl std::fmt::Display for Severity {
47 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
48 match self {
49 Self::Info => write!(f, "info"),
50 Self::Warning => write!(f, "warning"),
51 Self::Critical => write!(f, "critical"),
52 }
53 }
54}
55
56#[derive(Debug, Clone, Serialize)]
58pub struct Anomaly {
59 pub severity: Severity,
60 pub kind: String,
61 pub container: String,
62 pub message: String,
63 pub evidence: Vec<String>,
65}
66
67impl Anomaly {
68 fn to_row(&self) -> Row {
69 Row {
70 fields: BTreeMap::from([
71 (
72 "severity".to_owned(),
73 JsonValue::String(self.severity.to_string()),
74 ),
75 ("kind".to_owned(), JsonValue::String(self.kind.clone())),
76 (
77 "container".to_owned(),
78 JsonValue::String(self.container.clone()),
79 ),
80 (
81 "message".to_owned(),
82 JsonValue::String(self.message.clone()),
83 ),
84 (
85 "evidence".to_owned(),
86 JsonValue::Array(
87 self.evidence
88 .iter()
89 .map(|e| JsonValue::String(e.clone()))
90 .collect(),
91 ),
92 ),
93 ]),
94 }
95 }
96}
97
98#[derive(Debug, Clone, Serialize)]
100pub struct ContainerExplanation {
101 pub container: String,
102 pub state: String,
103 pub anomalies: Vec<Anomaly>,
104 pub signals: Vec<Signal>,
105}
106
107#[derive(Debug, Clone, Serialize)]
109pub struct Signal {
110 pub name: String,
111 pub value: String,
112 pub status: SignalStatus,
113}
114
115#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)]
117pub enum SignalStatus {
118 Normal,
119 Elevated,
120 Critical,
121}
122
123impl std::fmt::Display for SignalStatus {
124 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
125 match self {
126 Self::Normal => write!(f, "normal"),
127 Self::Elevated => write!(f, "elevated"),
128 Self::Critical => write!(f, "critical"),
129 }
130 }
131}
132
133#[derive(Debug, Clone)]
137pub struct AnalysisThresholds {
138 pub high_cpu_percent: f64,
140 pub critical_cpu_percent: f64,
142 pub memory_pressure_ratio: f64,
144 pub critical_memory_ratio: f64,
146 pub restart_loop_count: u64,
148 pub deployment_error_threshold: u64,
150 pub resource_leak_memory_increase_pct: f64,
152 pub resource_leak_min_samples: u64,
154 pub high_network_bytes: u64,
156 pub critical_network_bytes: u64,
158 pub high_disk_bytes: u64,
160 pub critical_disk_bytes: u64,
162}
163
164impl Default for AnalysisThresholds {
165 fn default() -> Self {
166 Self {
167 high_cpu_percent: 80.0,
168 critical_cpu_percent: 95.0,
169 memory_pressure_ratio: 0.85,
170 critical_memory_ratio: 0.95,
171 restart_loop_count: 3,
172 deployment_error_threshold: 3,
173 resource_leak_memory_increase_pct: 20.0,
174 resource_leak_min_samples: 3,
175 high_network_bytes: 1_048_576, critical_network_bytes: 10_485_760, high_disk_bytes: 10_485_760, critical_disk_bytes: 104_857_600, }
180 }
181}
182
183impl AnalysisThresholds {
184 #[must_use]
186 pub fn from_config(config: &crate::config::DolConfig) -> Self {
187 Self {
188 high_cpu_percent: config.analysis_high_cpu_percent.unwrap_or(80.0),
189 critical_cpu_percent: config.analysis_critical_cpu_percent.unwrap_or(95.0),
190 memory_pressure_ratio: config.analysis_memory_pressure_ratio.unwrap_or(0.85),
191 critical_memory_ratio: config.analysis_critical_memory_ratio.unwrap_or(0.95),
192 restart_loop_count: config.analysis_restart_loop_count.unwrap_or(3),
193 deployment_error_threshold: config.analysis_deployment_error_threshold.unwrap_or(3),
194 resource_leak_memory_increase_pct: config
195 .analysis_leak_memory_increase_pct
196 .unwrap_or(20.0),
197 resource_leak_min_samples: config.analysis_leak_min_samples.unwrap_or(3),
198 high_network_bytes: config.analysis_high_network_bytes.unwrap_or(1_048_576),
199 critical_network_bytes: config.analysis_critical_network_bytes.unwrap_or(10_485_760),
200 high_disk_bytes: config.analysis_high_disk_bytes.unwrap_or(10_485_760),
201 critical_disk_bytes: config.analysis_critical_disk_bytes.unwrap_or(104_857_600),
202 }
203 }
204}
205
206pub async fn execute_analyze<C, M>(
211 query: &AnalyzeQuery,
212 docker: &C,
213 metrics: &M,
214) -> Result<ExecutionResult, AnalyzeError>
215where
216 C: DockerClient + ?Sized,
217 M: MetricsCollector + ?Sized,
218{
219 let cfg = crate::config::DolConfig::load();
220 let thresholds = AnalysisThresholds::from_config(&cfg);
221 execute_analyze_with_thresholds(query, docker, metrics, &thresholds).await
222}
223
224pub async fn execute_analyze_with_thresholds<C, M>(
226 query: &AnalyzeQuery,
227 docker: &C,
228 metrics: &M,
229 thresholds: &AnalysisThresholds,
230) -> Result<ExecutionResult, AnalyzeError>
231where
232 C: DockerClient + ?Sized,
233 M: MetricsCollector + ?Sized,
234{
235 match (&query.target, &query.verb, query.subject.as_deref()) {
236 (
238 AnalysisTarget::Collection(CollectionTarget::Containers),
239 AnalysisVerb::Find,
240 None | Some("anomalies"),
241 ) => find_container_anomalies(docker, metrics, thresholds).await,
242 (
244 AnalysisTarget::Collection(CollectionTarget::Containers),
245 AnalysisVerb::Find,
246 Some("dependencies"),
247 ) => analyze_dependencies(docker).await,
248 (
250 AnalysisTarget::Collection(CollectionTarget::Containers),
251 AnalysisVerb::Find,
252 Some("density"),
253 ) => analyze_density(docker).await,
254 (AnalysisTarget::Collection(CollectionTarget::Containers), AnalysisVerb::Correlate, _) => {
256 correlate_containers(docker, metrics, thresholds).await
257 }
258 (AnalysisTarget::Singular(target), AnalysisVerb::Explain, _)
260 if target.kind == SingularTargetKind::Container =>
261 {
262 explain_container(&target.value, docker, metrics, thresholds).await
263 }
264 (AnalysisTarget::Collection(CollectionTarget::Containers), AnalysisVerb::Explain, _) => {
266 explain_all_containers(docker, metrics, thresholds).await
267 }
268 _ => Err(AnalyzeError::Unsupported),
269 }
270}
271
272pub fn execute_analyze_with_store<S>(
275 query: &AnalyzeQuery,
276 store: &S,
277) -> Result<ExecutionResult, AnalyzeError>
278where
279 S: TelemetryStore + ?Sized,
280{
281 let cfg = crate::config::DolConfig::load();
282 let thresholds = AnalysisThresholds::from_config(&cfg);
283
284 match (&query.target, &query.verb, query.subject.as_deref()) {
285 (
287 AnalysisTarget::Collection(CollectionTarget::Containers),
288 AnalysisVerb::Find,
289 None | Some("anomalies"),
290 ) => find_anomalies_from_store(store, &thresholds),
291 (
293 AnalysisTarget::Collection(CollectionTarget::Containers),
294 AnalysisVerb::Find,
295 Some("leaks"),
296 ) => {
297 let anomalies = detect_resource_leaks(store, &thresholds);
298 if anomalies.is_empty() {
299 Ok(ExecutionResult {
300 rows: vec![Row {
301 fields: BTreeMap::from([
302 ("severity".to_owned(), JsonValue::String("info".to_owned())),
303 (
304 "kind".to_owned(),
305 JsonValue::String("no_leaks_detected".to_owned()),
306 ),
307 ("container".to_owned(), JsonValue::String("*".to_owned())),
308 (
309 "message".to_owned(),
310 JsonValue::String(
311 "No resource leaks detected in stored data".to_owned(),
312 ),
313 ),
314 ("evidence".to_owned(), JsonValue::Array(Vec::new())),
315 ]),
316 }],
317 })
318 } else {
319 Ok(ExecutionResult {
320 rows: anomalies.iter().map(Anomaly::to_row).collect(),
321 })
322 }
323 }
324 (
326 AnalysisTarget::Collection(CollectionTarget::Containers),
327 AnalysisVerb::Find,
328 Some("drift"),
329 ) => {
330 let anomalies = detect_config_drift(store)?;
331 if anomalies.is_empty() {
332 Ok(ExecutionResult {
333 rows: vec![Row {
334 fields: BTreeMap::from([
335 ("severity".to_owned(), JsonValue::String("info".to_owned())),
336 ("kind".to_owned(), JsonValue::String("no_drift".to_owned())),
337 ("container".to_owned(), JsonValue::String("*".to_owned())),
338 (
339 "message".to_owned(),
340 JsonValue::String("No configuration drift detected".to_owned()),
341 ),
342 ("evidence".to_owned(), JsonValue::Array(Vec::new())),
343 ]),
344 }],
345 })
346 } else {
347 Ok(ExecutionResult {
348 rows: anomalies.iter().map(Anomaly::to_row).collect(),
349 })
350 }
351 }
352 _ => Err(AnalyzeError::Unsupported),
353 }
354}
355
356#[must_use]
360pub fn detect_restart_loops(containers: &[Container], threshold: u64) -> Vec<Anomaly> {
361 containers
362 .iter()
363 .filter_map(|c| {
364 let count = c.restart_count.unwrap_or(0);
365 if count >= threshold {
366 let severity = if count >= threshold * 2 {
367 Severity::Critical
368 } else {
369 Severity::Warning
370 };
371 Some(Anomaly {
372 severity,
373 kind: "restart_loop".to_owned(),
374 container: c.name.clone(),
375 message: format!(
376 "Container '{}' has restarted {} times (threshold: {})",
377 c.name, count, threshold
378 ),
379 evidence: vec![
380 format!("restart_count={}", count),
381 format!("state={}", c.state),
382 format!("status={}", c.status),
383 ],
384 })
385 } else {
386 None
387 }
388 })
389 .collect()
390}
391
392#[must_use]
394pub fn detect_high_cpu(
395 samples: &[MetricSample],
396 high_threshold: f64,
397 critical_threshold: f64,
398) -> Vec<Anomaly> {
399 samples
400 .iter()
401 .filter_map(|s| {
402 let cpu = s.cpu_percent?;
403 if cpu >= critical_threshold {
404 Some(Anomaly {
405 severity: Severity::Critical,
406 kind: "high_cpu".to_owned(),
407 container: s.container_name.clone(),
408 message: format!(
409 "Container '{}' CPU at {:.1}% (critical threshold: {:.0}%)",
410 s.container_name, cpu, critical_threshold
411 ),
412 evidence: vec![
413 format!("cpu={:.1}%", cpu),
414 format!("timestamp={}", s.timestamp),
415 ],
416 })
417 } else if cpu >= high_threshold {
418 Some(Anomaly {
419 severity: Severity::Warning,
420 kind: "high_cpu".to_owned(),
421 container: s.container_name.clone(),
422 message: format!(
423 "Container '{}' CPU at {:.1}% (warning threshold: {:.0}%)",
424 s.container_name, cpu, high_threshold
425 ),
426 evidence: vec![
427 format!("cpu={:.1}%", cpu),
428 format!("timestamp={}", s.timestamp),
429 ],
430 })
431 } else {
432 None
433 }
434 })
435 .collect()
436}
437
438#[must_use]
440pub fn detect_memory_pressure(
441 samples: &[MetricSample],
442 pressure_ratio: f64,
443 critical_ratio: f64,
444) -> Vec<Anomaly> {
445 samples
446 .iter()
447 .filter_map(|s| {
448 let usage = s.memory_usage_bytes? as f64;
449 let limit = s.memory_limit_bytes? as f64;
450 if limit <= 0.0 {
451 return None;
452 }
453 let ratio = usage / limit;
454 if ratio >= critical_ratio {
455 Some(Anomaly {
456 severity: Severity::Critical,
457 kind: "memory_pressure".to_owned(),
458 container: s.container_name.clone(),
459 message: format!(
460 "Container '{}' memory at {:.1}% of limit (critical: {:.0}%)",
461 s.container_name,
462 ratio * 100.0,
463 critical_ratio * 100.0
464 ),
465 evidence: vec![
466 format!("memory_usage={}", s.memory_usage_bytes.unwrap_or(0)),
467 format!("memory_limit={}", s.memory_limit_bytes.unwrap_or(0)),
468 format!("ratio={:.2}%", ratio * 100.0),
469 ],
470 })
471 } else if ratio >= pressure_ratio {
472 Some(Anomaly {
473 severity: Severity::Warning,
474 kind: "memory_pressure".to_owned(),
475 container: s.container_name.clone(),
476 message: format!(
477 "Container '{}' memory at {:.1}% of limit (warning: {:.0}%)",
478 s.container_name,
479 ratio * 100.0,
480 pressure_ratio * 100.0
481 ),
482 evidence: vec![
483 format!("memory_usage={}", s.memory_usage_bytes.unwrap_or(0)),
484 format!("memory_limit={}", s.memory_limit_bytes.unwrap_or(0)),
485 format!("ratio={:.2}%", ratio * 100.0),
486 ],
487 })
488 } else {
489 None
490 }
491 })
492 .collect()
493}
494
495#[must_use]
497pub fn detect_deployment_errors(events: &[DockerEvent], threshold: u64) -> Vec<Anomaly> {
498 let mut die_counts: HashMap<String, u64> = HashMap::new();
500 let mut die_times: HashMap<String, Vec<String>> = HashMap::new();
501 for event in events {
502 if event.action == "die" {
503 let container = event
504 .container
505 .as_deref()
506 .unwrap_or(&event.actor_id)
507 .to_owned();
508 *die_counts.entry(container.clone()).or_default() += 1;
509 die_times
510 .entry(container)
511 .or_default()
512 .push(event.time.clone());
513 }
514 }
515
516 die_counts
517 .into_iter()
518 .filter(|(_, count)| *count >= threshold)
519 .map(|(container, count)| {
520 let times = die_times.get(&container).cloned().unwrap_or_default();
521 let severity = if count >= threshold * 2 {
522 Severity::Critical
523 } else {
524 Severity::Warning
525 };
526 Anomaly {
527 severity,
528 kind: "deployment_errors".to_owned(),
529 container: container.clone(),
530 message: format!(
531 "Container '{container}' has died {count} times (threshold: {threshold})"
532 ),
533 evidence: times.into_iter().map(|t| format!("die_at={t}")).collect(),
534 }
535 })
536 .collect()
537}
538
539#[must_use]
541pub fn detect_unhealthy_states(containers: &[Container]) -> Vec<Anomaly> {
542 containers
543 .iter()
544 .filter_map(|c| {
545 let (severity, kind) = match c.state.as_str() {
546 "exited" => (Severity::Warning, "exited_container"),
547 "dead" => (Severity::Critical, "dead_container"),
548 "restarting" => (Severity::Warning, "restarting_container"),
549 _ => return None,
550 };
551 Some(Anomaly {
552 severity,
553 kind: kind.to_owned(),
554 container: c.name.clone(),
555 message: format!(
556 "Container '{}' is in '{}' state: {}",
557 c.name, c.state, c.status
558 ),
559 evidence: vec![
560 format!("state={}", c.state),
561 format!("status={}", c.status),
562 format!("image={}", c.image),
563 ],
564 })
565 })
566 .collect()
567}
568
569#[must_use]
571pub fn detect_high_disk_io(
572 samples: &[MetricSample],
573 high_threshold: u64,
574 critical_threshold: u64,
575) -> Vec<Anomaly> {
576 samples
577 .iter()
578 .filter_map(|s| {
579 let read = s.disk_read_bytes?;
580 let write = s.disk_write_bytes?;
581 let max_bytes = read.max(write);
582
583 if max_bytes >= critical_threshold {
584 Some(Anomaly {
585 severity: Severity::Critical,
586 kind: "high_disk_io".to_owned(),
587 container: s.container_name.clone(),
588 message: format!(
589 "Container '{}' disk I/O at {}/{} (read/write) — critical threshold: {}",
590 s.container_name,
591 format_bytes(read),
592 format_bytes(write),
593 format_bytes(critical_threshold),
594 ),
595 evidence: vec![
596 format!("disk_read={}", read),
597 format!("disk_write={}", write),
598 format!("max_bytes={}", max_bytes),
599 format!("timestamp={}", s.timestamp),
600 ],
601 })
602 } else if max_bytes >= high_threshold {
603 Some(Anomaly {
604 severity: Severity::Warning,
605 kind: "high_disk_io".to_owned(),
606 container: s.container_name.clone(),
607 message: format!(
608 "Container '{}' disk I/O at {}/{} (read/write) — warning threshold: {}",
609 s.container_name,
610 format_bytes(read),
611 format_bytes(write),
612 format_bytes(high_threshold),
613 ),
614 evidence: vec![
615 format!("disk_read={}", read),
616 format!("disk_write={}", write),
617 format!("max_bytes={}", max_bytes),
618 format!("timestamp={}", s.timestamp),
619 ],
620 })
621 } else {
622 None
623 }
624 })
625 .collect()
626}
627
628#[must_use]
630pub fn detect_high_network_io(
631 samples: &[MetricSample],
632 high_threshold: u64,
633 critical_threshold: u64,
634) -> Vec<Anomaly> {
635 samples
636 .iter()
637 .filter_map(|s| {
638 let rx = s.network_rx_bytes?;
639 let tx = s.network_tx_bytes?;
640 let max_bytes = rx.max(tx);
641
642 if max_bytes >= critical_threshold {
643 Some(Anomaly {
644 severity: Severity::Critical,
645 kind: "high_network_io".to_owned(),
646 container: s.container_name.clone(),
647 message: format!(
648 "Container '{}' network I/O at {}/{} (RX/TX) — critical threshold: {}",
649 s.container_name,
650 format_bytes(rx),
651 format_bytes(tx),
652 format_bytes(critical_threshold),
653 ),
654 evidence: vec![
655 format!("network_rx={}", rx),
656 format!("network_tx={}", tx),
657 format!("max_bytes={}", max_bytes),
658 format!("timestamp={}", s.timestamp),
659 ],
660 })
661 } else if max_bytes >= high_threshold {
662 Some(Anomaly {
663 severity: Severity::Warning,
664 kind: "high_network_io".to_owned(),
665 container: s.container_name.clone(),
666 message: format!(
667 "Container '{}' network I/O at {}/{} (RX/TX) — warning threshold: {}",
668 s.container_name,
669 format_bytes(rx),
670 format_bytes(tx),
671 format_bytes(high_threshold),
672 ),
673 evidence: vec![
674 format!("network_rx={}", rx),
675 format!("network_tx={}", tx),
676 format!("max_bytes={}", max_bytes),
677 format!("timestamp={}", s.timestamp),
678 ],
679 })
680 } else {
681 None
682 }
683 })
684 .collect()
685}
686
687pub fn detect_resource_leaks<S>(store: &S, thresholds: &AnalysisThresholds) -> Vec<Anomaly>
692where
693 S: TelemetryStore + ?Sized,
694{
695 let Ok(samples) = store.latest_metrics() else {
696 return Vec::new();
697 };
698
699 let mut by_container: std::collections::BTreeMap<String, Vec<&MetricSample>> =
701 std::collections::BTreeMap::new();
702 for sample in &samples {
703 by_container
704 .entry(sample.container_id.clone())
705 .or_default()
706 .push(sample);
707 }
708
709 let mut anomalies = Vec::new();
710
711 let id_to_name: std::collections::HashMap<String, String> = samples
713 .iter()
714 .map(|s| (s.container_id.clone(), s.container_name.clone()))
715 .collect();
716
717 for (container_id, container_samples) in &by_container {
718 let container_name = id_to_name
719 .get(container_id)
720 .map_or(container_id.as_str(), std::string::String::as_str);
721 if container_samples.len() < thresholds.resource_leak_min_samples as usize {
722 continue;
723 }
724
725 let mut sorted = container_samples.clone();
727 sorted.sort_by_key(|s| s.timestamp.as_str());
728
729 let mem_values: Vec<u64> = sorted.iter().filter_map(|s| s.memory_usage_bytes).collect();
731
732 if mem_values.len() < 2 {
733 continue;
734 }
735
736 let first = mem_values[0];
737 let last = mem_values[mem_values.len() - 1];
738
739 if first == 0 {
740 continue;
741 }
742
743 let increase_pct = ((last as f64 - first as f64) / first as f64) * 100.0;
744
745 if increase_pct >= thresholds.resource_leak_memory_increase_pct {
746 let severity = if increase_pct >= thresholds.resource_leak_memory_increase_pct * 2.0 {
747 Severity::Critical
748 } else {
749 Severity::Warning
750 };
751
752 anomalies.push(Anomaly {
753 severity,
754 kind: "resource_leak".to_owned(),
755 container: container_name.to_string(),
756 message: format!(
757 "Container '{}' memory grew {:.0}% (from {} to {}) over {} samples — possible leak",
758 container_name,
759 increase_pct,
760 format_bytes(first),
761 format_bytes(last),
762 mem_values.len()
763 ),
764 evidence: vec![
765 format!("initial_memory={}", first),
766 format!("latest_memory={}", last),
767 format!("increase_pct={:.1}", increase_pct),
768 format!("samples={}", mem_values.len()),
769 ],
770 });
771 }
772 }
773
774 anomalies
775}
776
777pub async fn analyze_dependencies<C>(docker: &C) -> Result<ExecutionResult, AnalyzeError>
782where
783 C: DockerClient + ?Sized,
784{
785 let containers = docker.list_containers().await?;
786 let networks = docker.list_networks().await.unwrap_or_default();
787 let volumes = docker.list_volumes().await.unwrap_or_default();
788
789 let mut rows = Vec::new();
790
791 let mut by_compose: std::collections::BTreeMap<String, Vec<String>> =
793 std::collections::BTreeMap::new();
794 for c in &containers {
795 let project = c
796 .labels
797 .iter()
798 .find_map(|label| {
799 let parts: Vec<&str> = label.splitn(2, '=').collect();
800 if parts.len() == 2 && parts[0] == "com.docker.compose.project" {
801 Some(parts[1].to_owned())
802 } else {
803 None
804 }
805 })
806 .unwrap_or_else(|| "standalone".to_owned());
807 by_compose.entry(project).or_default().push(c.name.clone());
808 }
809
810 for (project, names) in &by_compose {
811 if names.len() > 1 || project != "standalone" {
812 rows.push(Row {
813 fields: BTreeMap::from([
814 (
815 "dependency".to_owned(),
816 JsonValue::String("compose_project".to_owned()),
817 ),
818 ("key".to_owned(), JsonValue::String(project.clone())),
819 (
820 "containers".to_owned(),
821 JsonValue::Array(
822 names.iter().map(|n| JsonValue::String(n.clone())).collect(),
823 ),
824 ),
825 (
826 "count".to_owned(),
827 JsonValue::Number(Number::from(names.len())),
828 ),
829 ]),
830 });
831 }
832 }
833
834 for n in &networks {
836 if !n.containers.is_empty() {
837 rows.push(Row {
838 fields: BTreeMap::from([
839 (
840 "dependency".to_owned(),
841 JsonValue::String("network".to_owned()),
842 ),
843 ("key".to_owned(), JsonValue::String(n.name.clone())),
844 (
845 "containers".to_owned(),
846 JsonValue::Array(
847 n.containers
848 .iter()
849 .map(|c| JsonValue::String(c.clone()))
850 .collect(),
851 ),
852 ),
853 (
854 "count".to_owned(),
855 JsonValue::Number(Number::from(n.containers.len())),
856 ),
857 ]),
858 });
859 }
860 }
861
862 for v in &volumes {
865 rows.push(Row {
866 fields: BTreeMap::from([
867 (
868 "dependency".to_owned(),
869 JsonValue::String("volume".to_owned()),
870 ),
871 ("key".to_owned(), JsonValue::String(v.name.clone())),
872 ("containers".to_owned(), JsonValue::Array(Vec::new())),
873 ("count".to_owned(), JsonValue::Number(Number::from(0))),
874 ]),
875 });
876 }
877
878 if rows.is_empty() {
879 rows.push(Row {
880 fields: BTreeMap::from([
881 (
882 "dependency".to_owned(),
883 JsonValue::String("none".to_owned()),
884 ),
885 ("key".to_owned(), JsonValue::String("-".to_owned())),
886 ("containers".to_owned(), JsonValue::Array(Vec::new())),
887 ("count".to_owned(), JsonValue::Number(Number::from(0))),
888 ]),
889 });
890 }
891
892 Ok(ExecutionResult { rows })
893}
894
895pub fn detect_config_drift<S>(store: &S) -> Result<Vec<Anomaly>, AnalyzeError>
902where
903 S: TelemetryStore + ?Sized,
904{
905 let mut all = store
907 .all_snapshots()
908 .map_err(|_| AnalyzeError::Unsupported)?;
909
910 if all.is_empty() {
911 return Err(AnalyzeError::Unsupported);
912 }
913
914 let latest = all.pop().expect("at least one snapshot after empty check");
916 let previous = all.pop();
917
918 let mut anomalies = Vec::new();
922
923 match previous {
924 Some(prev) => {
925 for container in &latest.containers {
927 let prev_container = prev.containers.iter().find(|c| c.name == container.name);
928
929 match prev_container {
930 Some(prev_c) => {
931 if prev_c.image != container.image {
933 anomalies.push(Anomaly {
934 severity: Severity::Warning,
935 kind: "config_drift".to_owned(),
936 container: container.name.clone(),
937 message: format!(
938 "Container '{}' image changed from '{}' to '{}'",
939 container.name, prev_c.image, container.image
940 ),
941 evidence: vec![
942 format!("previous_image={}", prev_c.image),
943 format!("current_image={}", container.image),
944 ],
945 });
946 }
947
948 if prev_c.state != container.state {
950 let severity = match container.state.as_str() {
951 "running" => Severity::Info,
952 _ => Severity::Warning,
953 };
954 anomalies.push(Anomaly {
955 severity,
956 kind: "state_change".to_owned(),
957 container: container.name.clone(),
958 message: format!(
959 "Container '{}' state changed from '{}' to '{}'",
960 container.name, prev_c.state, container.state
961 ),
962 evidence: vec![
963 format!("previous_state={}", prev_c.state),
964 format!("current_state={}", container.state),
965 ],
966 });
967 }
968
969 let prev_restarts = prev_c.restart_count.unwrap_or(0);
971 let curr_restarts = container.restart_count.unwrap_or(0);
972 if curr_restarts > prev_restarts {
973 anomalies.push(Anomaly {
974 severity: Severity::Warning,
975 kind: "restart_increase".to_owned(),
976 container: container.name.clone(),
977 message: format!(
978 "Container '{}' restart count increased from {} to {}",
979 container.name, prev_restarts, curr_restarts
980 ),
981 evidence: vec![
982 format!("previous_restarts={}", prev_restarts),
983 format!("current_restarts={}", curr_restarts),
984 ],
985 });
986 }
987
988 let prev_labels: std::collections::BTreeSet<&str> = prev_c
990 .labels
991 .iter()
992 .map(std::string::String::as_str)
993 .collect();
994 let curr_labels: std::collections::BTreeSet<&str> = container
995 .labels
996 .iter()
997 .map(std::string::String::as_str)
998 .collect();
999 let added: Vec<&&str> = curr_labels.difference(&prev_labels).collect();
1000 let removed: Vec<&&str> = prev_labels.difference(&curr_labels).collect();
1001 if !added.is_empty() || !removed.is_empty() {
1002 anomalies.push(Anomaly {
1003 severity: Severity::Info,
1004 kind: "label_drift".to_owned(),
1005 container: container.name.clone(),
1006 message: format!(
1007 "Container '{}' labels changed: added={:?}, removed={:?}",
1008 container.name, added, removed
1009 ),
1010 evidence: vec![
1011 format!("added_labels={:?}", added),
1012 format!("removed_labels={:?}", removed),
1013 ],
1014 });
1015 }
1016 }
1017 None => {
1018 anomalies.push(Anomaly {
1020 severity: Severity::Info,
1021 kind: "container_appeared".to_owned(),
1022 container: container.name.clone(),
1023 message: format!(
1024 "Container '{}' appeared with image={}, state={}",
1025 container.name, container.image, container.state
1026 ),
1027 evidence: vec![
1028 format!("image={}", container.image),
1029 format!("state={}", container.state),
1030 ],
1031 });
1032 }
1033 }
1034 }
1035
1036 for prev_c in &prev.containers {
1038 if !latest.containers.iter().any(|c| c.name == prev_c.name) {
1039 anomalies.push(Anomaly {
1040 severity: Severity::Warning,
1041 kind: "container_disappeared".to_owned(),
1042 container: prev_c.name.clone(),
1043 message: format!(
1044 "Container '{}' was present in previous snapshot but is now gone",
1045 prev_c.name
1046 ),
1047 evidence: vec![
1048 format!("previous_state={}", prev_c.state),
1049 format!("previous_image={}", prev_c.image),
1050 ],
1051 });
1052 }
1053 }
1054 }
1055 None => {
1056 for container in &latest.containers {
1058 anomalies.push(Anomaly {
1059 severity: Severity::Info,
1060 kind: "config_baseline".to_owned(),
1061 container: container.name.clone(),
1062 message: format!(
1063 "Container '{}' baseline — image={}, state={}",
1064 container.name, container.image, container.state
1065 ),
1066 evidence: vec![
1067 format!("image={}", container.image),
1068 format!("state={}", container.state),
1069 format!("restart_count={}", container.restart_count.unwrap_or(0)),
1070 ],
1071 });
1072 }
1073 }
1074 }
1075
1076 Ok(anomalies)
1077}
1078
1079pub async fn analyze_density<C>(docker: &C) -> Result<ExecutionResult, AnalyzeError>
1084where
1085 C: DockerClient + ?Sized,
1086{
1087 let containers = docker.list_containers().await?;
1088 let total = containers.len();
1089 let mut rows = Vec::new();
1090
1091 let mut by_image: std::collections::BTreeMap<String, Vec<String>> =
1093 std::collections::BTreeMap::new();
1094 for c in &containers {
1095 by_image
1096 .entry(c.image.clone())
1097 .or_default()
1098 .push(c.name.clone());
1099 }
1100 for (image, names) in &by_image {
1101 let pct = if total > 0 {
1102 (names.len() as f64 / total as f64) * 100.0
1103 } else {
1104 0.0
1105 };
1106 rows.push(Row {
1107 fields: BTreeMap::from([
1108 (
1109 "dimension".to_owned(),
1110 JsonValue::String("image".to_owned()),
1111 ),
1112 ("value".to_owned(), JsonValue::String(image.clone())),
1113 (
1114 "container_count".to_owned(),
1115 JsonValue::Number(Number::from(names.len())),
1116 ),
1117 (
1118 "density_pct".to_owned(),
1119 JsonValue::Number(
1120 Number::from_f64((pct * 10.0).round() / 10.0)
1121 .unwrap_or_else(|| Number::from(0)),
1122 ),
1123 ),
1124 ]),
1125 });
1126 }
1127
1128 let mut by_state: std::collections::BTreeMap<String, u64> = std::collections::BTreeMap::new();
1130 for c in &containers {
1131 *by_state.entry(c.state.clone()).or_default() += 1;
1132 }
1133 for (state, count) in &by_state {
1134 let pct = if total > 0 {
1135 (*count as f64 / total as f64) * 100.0
1136 } else {
1137 0.0
1138 };
1139 rows.push(Row {
1140 fields: BTreeMap::from([
1141 (
1142 "dimension".to_owned(),
1143 JsonValue::String("state".to_owned()),
1144 ),
1145 ("value".to_owned(), JsonValue::String(state.clone())),
1146 (
1147 "container_count".to_owned(),
1148 JsonValue::Number(Number::from(*count)),
1149 ),
1150 (
1151 "density_pct".to_owned(),
1152 JsonValue::Number(
1153 Number::from_f64((pct * 10.0).round() / 10.0)
1154 .unwrap_or_else(|| Number::from(0)),
1155 ),
1156 ),
1157 ]),
1158 });
1159 }
1160
1161 let mut by_project: std::collections::BTreeMap<String, Vec<String>> =
1163 std::collections::BTreeMap::new();
1164 for c in &containers {
1165 let project = c
1166 .labels
1167 .iter()
1168 .find_map(|label| {
1169 let parts: Vec<&str> = label.splitn(2, '=').collect();
1170 if parts.len() == 2 && parts[0] == "com.docker.compose.project" {
1171 Some(parts[1].to_owned())
1172 } else {
1173 None
1174 }
1175 })
1176 .unwrap_or_else(|| "standalone".to_owned());
1177 by_project.entry(project).or_default().push(c.name.clone());
1178 }
1179 for (project, names) in &by_project {
1180 let pct = if total > 0 {
1181 (names.len() as f64 / total as f64) * 100.0
1182 } else {
1183 0.0
1184 };
1185 rows.push(Row {
1186 fields: BTreeMap::from([
1187 (
1188 "dimension".to_owned(),
1189 JsonValue::String("compose_project".to_owned()),
1190 ),
1191 ("value".to_owned(), JsonValue::String(project.clone())),
1192 (
1193 "container_count".to_owned(),
1194 JsonValue::Number(Number::from(names.len())),
1195 ),
1196 (
1197 "density_pct".to_owned(),
1198 JsonValue::Number(
1199 Number::from_f64((pct * 10.0).round() / 10.0)
1200 .unwrap_or_else(|| Number::from(0)),
1201 ),
1202 ),
1203 ]),
1204 });
1205 }
1206
1207 rows.push(Row {
1209 fields: BTreeMap::from([
1210 (
1211 "dimension".to_owned(),
1212 JsonValue::String("total".to_owned()),
1213 ),
1214 ("value".to_owned(), JsonValue::String("-".to_owned())),
1215 (
1216 "container_count".to_owned(),
1217 JsonValue::Number(Number::from(total)),
1218 ),
1219 (
1220 "density_pct".to_owned(),
1221 JsonValue::Number(Number::from(100)),
1222 ),
1223 ]),
1224 });
1225
1226 Ok(ExecutionResult { rows })
1227}
1228
1229async fn find_container_anomalies<C, M>(
1233 docker: &C,
1234 metrics: &M,
1235 thresholds: &AnalysisThresholds,
1236) -> Result<ExecutionResult, AnalyzeError>
1237where
1238 C: DockerClient + ?Sized,
1239 M: MetricsCollector + ?Sized,
1240{
1241 let containers = docker.list_containers().await?;
1242 let samples = metrics.collect().await?;
1243
1244 let mut anomalies = Vec::new();
1245 anomalies.extend(detect_restart_loops(
1246 &containers,
1247 thresholds.restart_loop_count,
1248 ));
1249 anomalies.extend(detect_high_cpu(
1250 &samples,
1251 thresholds.high_cpu_percent,
1252 thresholds.critical_cpu_percent,
1253 ));
1254 anomalies.extend(detect_memory_pressure(
1255 &samples,
1256 thresholds.memory_pressure_ratio,
1257 thresholds.critical_memory_ratio,
1258 ));
1259 anomalies.extend(detect_unhealthy_states(&containers));
1260 anomalies.extend(detect_high_disk_io(
1261 &samples,
1262 thresholds.high_disk_bytes,
1263 thresholds.critical_disk_bytes,
1264 ));
1265 anomalies.extend(detect_high_network_io(
1266 &samples,
1267 thresholds.high_network_bytes,
1268 thresholds.critical_network_bytes,
1269 ));
1270
1271 anomalies.sort_by_key(|a| std::cmp::Reverse(severity_rank(a.severity)));
1273
1274 if anomalies.is_empty() {
1275 Ok(ExecutionResult {
1276 rows: vec![Row {
1277 fields: BTreeMap::from([
1278 ("severity".to_owned(), JsonValue::String("info".to_owned())),
1279 ("kind".to_owned(), JsonValue::String("healthy".to_owned())),
1280 ("container".to_owned(), JsonValue::String("*".to_owned())),
1281 (
1282 "message".to_owned(),
1283 JsonValue::String("No anomalies detected".to_owned()),
1284 ),
1285 ("evidence".to_owned(), JsonValue::Array(Vec::new())),
1286 ]),
1287 }],
1288 })
1289 } else {
1290 Ok(ExecutionResult {
1291 rows: anomalies.iter().map(Anomaly::to_row).collect(),
1292 })
1293 }
1294}
1295
1296fn find_anomalies_from_store<S>(
1298 store: &S,
1299 thresholds: &AnalysisThresholds,
1300) -> Result<ExecutionResult, AnalyzeError>
1301where
1302 S: TelemetryStore + ?Sized,
1303{
1304 let samples = store.latest_metrics()?;
1305 let events = store.events_between("", "\u{10ffff}")?;
1306
1307 let mut anomalies = Vec::new();
1308 anomalies.extend(detect_high_cpu(
1309 &samples,
1310 thresholds.high_cpu_percent,
1311 thresholds.critical_cpu_percent,
1312 ));
1313 anomalies.extend(detect_memory_pressure(
1314 &samples,
1315 thresholds.memory_pressure_ratio,
1316 thresholds.critical_memory_ratio,
1317 ));
1318 anomalies.extend(detect_deployment_errors(
1319 &events,
1320 thresholds.deployment_error_threshold,
1321 ));
1322 anomalies.extend(detect_high_disk_io(
1323 &samples,
1324 thresholds.high_disk_bytes,
1325 thresholds.critical_disk_bytes,
1326 ));
1327 anomalies.extend(detect_high_network_io(
1328 &samples,
1329 thresholds.high_network_bytes,
1330 thresholds.critical_network_bytes,
1331 ));
1332 anomalies.extend(detect_resource_leaks(store, thresholds));
1333 if let Ok(drift) = detect_config_drift(store) {
1335 anomalies.extend(drift);
1336 }
1337
1338 anomalies.sort_by_key(|a| std::cmp::Reverse(severity_rank(a.severity)));
1339
1340 if anomalies.is_empty() {
1341 Ok(ExecutionResult {
1342 rows: vec![Row {
1343 fields: BTreeMap::from([
1344 ("severity".to_owned(), JsonValue::String("info".to_owned())),
1345 ("kind".to_owned(), JsonValue::String("healthy".to_owned())),
1346 ("container".to_owned(), JsonValue::String("*".to_owned())),
1347 (
1348 "message".to_owned(),
1349 JsonValue::String("No anomalies detected in stored data".to_owned()),
1350 ),
1351 ("evidence".to_owned(), JsonValue::Array(Vec::new())),
1352 ]),
1353 }],
1354 })
1355 } else {
1356 Ok(ExecutionResult {
1357 rows: anomalies.iter().map(Anomaly::to_row).collect(),
1358 })
1359 }
1360}
1361
1362async fn correlate_containers<C, M>(
1364 docker: &C,
1365 _metrics: &M,
1366 _thresholds: &AnalysisThresholds,
1367) -> Result<ExecutionResult, AnalyzeError>
1368where
1369 C: DockerClient + ?Sized,
1370 M: MetricsCollector + ?Sized,
1371{
1372 let containers = docker.list_containers().await?;
1373
1374 let mut by_image: HashMap<String, Vec<String>> = HashMap::new();
1376 for c in &containers {
1377 by_image
1378 .entry(c.image.clone())
1379 .or_default()
1380 .push(c.name.clone());
1381 }
1382
1383 let mut rows = Vec::new();
1384
1385 for (image, names) in &by_image {
1386 if names.len() > 1 {
1387 rows.push(Row {
1388 fields: BTreeMap::from([
1389 (
1390 "correlation".to_owned(),
1391 JsonValue::String("shared_image".to_owned()),
1392 ),
1393 ("key".to_owned(), JsonValue::String(image.clone())),
1394 (
1395 "containers".to_owned(),
1396 JsonValue::Array(
1397 names.iter().map(|n| JsonValue::String(n.clone())).collect(),
1398 ),
1399 ),
1400 (
1401 "count".to_owned(),
1402 JsonValue::Number(Number::from(names.len())),
1403 ),
1404 ]),
1405 });
1406 }
1407 }
1408
1409 let mut by_label: HashMap<String, Vec<String>> = HashMap::new();
1411 for c in &containers {
1412 for label in &c.labels {
1413 by_label
1414 .entry(label.clone())
1415 .or_default()
1416 .push(c.name.clone());
1417 }
1418 }
1419
1420 for (label, names) in &by_label {
1421 if names.len() > 1 {
1422 rows.push(Row {
1423 fields: BTreeMap::from([
1424 (
1425 "correlation".to_owned(),
1426 JsonValue::String("shared_label".to_owned()),
1427 ),
1428 ("key".to_owned(), JsonValue::String(label.clone())),
1429 (
1430 "containers".to_owned(),
1431 JsonValue::Array(
1432 names.iter().map(|n| JsonValue::String(n.clone())).collect(),
1433 ),
1434 ),
1435 (
1436 "count".to_owned(),
1437 JsonValue::Number(Number::from(names.len())),
1438 ),
1439 ]),
1440 });
1441 }
1442 }
1443
1444 if rows.is_empty() {
1445 rows.push(Row {
1446 fields: BTreeMap::from([
1447 (
1448 "correlation".to_owned(),
1449 JsonValue::String("none".to_owned()),
1450 ),
1451 ("key".to_owned(), JsonValue::String("-".to_owned())),
1452 ("containers".to_owned(), JsonValue::Array(Vec::new())),
1453 ("count".to_owned(), JsonValue::Number(Number::from(0))),
1454 ]),
1455 });
1456 }
1457
1458 Ok(ExecutionResult { rows })
1459}
1460
1461async fn explain_container<C, M>(
1463 name: &str,
1464 docker: &C,
1465 metrics: &M,
1466 thresholds: &AnalysisThresholds,
1467) -> Result<ExecutionResult, AnalyzeError>
1468where
1469 C: DockerClient + ?Sized,
1470 M: MetricsCollector + ?Sized,
1471{
1472 let containers = docker.list_containers().await?;
1473 let container = containers
1474 .iter()
1475 .find(|c| c.name == name || c.id == name)
1476 .ok_or(AnalyzeError::Unsupported)?;
1477
1478 let samples = metrics.collect().await?;
1479 let sample = samples
1480 .iter()
1481 .find(|s| s.container_name == name || s.container_id == name);
1482
1483 let explanation = build_explanation(container, sample, thresholds);
1484
1485 Ok(explanation_to_result(&explanation))
1486}
1487
1488async fn explain_all_containers<C, M>(
1490 docker: &C,
1491 metrics: &M,
1492 thresholds: &AnalysisThresholds,
1493) -> Result<ExecutionResult, AnalyzeError>
1494where
1495 C: DockerClient + ?Sized,
1496 M: MetricsCollector + ?Sized,
1497{
1498 let containers = docker.list_containers().await?;
1499 let samples = metrics.collect().await?;
1500 let samples_by_name: HashMap<&str, &MetricSample> = samples
1501 .iter()
1502 .map(|s| (s.container_name.as_str(), s))
1503 .collect();
1504
1505 let mut rows = Vec::new();
1506 for container in &containers {
1507 let sample = samples_by_name.get(container.name.as_str()).copied();
1508 let explanation = build_explanation(container, sample, thresholds);
1509 let result = explanation_to_result(&explanation);
1510 rows.extend(result.rows);
1511 }
1512
1513 if rows.is_empty() {
1514 rows.push(Row {
1515 fields: BTreeMap::from([
1516 ("container".to_owned(), JsonValue::String("-".to_owned())),
1517 ("state".to_owned(), JsonValue::String("-".to_owned())),
1518 ("signal".to_owned(), JsonValue::String("none".to_owned())),
1519 ("value".to_owned(), JsonValue::String("-".to_owned())),
1520 ("status".to_owned(), JsonValue::String("normal".to_owned())),
1521 (
1522 "anomaly_count".to_owned(),
1523 JsonValue::Number(Number::from(0)),
1524 ),
1525 ]),
1526 });
1527 }
1528
1529 Ok(ExecutionResult { rows })
1530}
1531
1532fn build_explanation(
1535 container: &Container,
1536 sample: Option<&MetricSample>,
1537 thresholds: &AnalysisThresholds,
1538) -> ContainerExplanation {
1539 let mut signals = Vec::new();
1540 let mut anomalies = Vec::new();
1541
1542 let state_status = match container.state.as_str() {
1544 "running" => SignalStatus::Normal,
1545 "restarting" => SignalStatus::Elevated,
1546 _ => SignalStatus::Critical,
1547 };
1548 signals.push(Signal {
1549 name: "state".to_owned(),
1550 value: container.state.clone(),
1551 status: state_status,
1552 });
1553
1554 if let Some(count) = container.restart_count {
1556 let status = if count >= thresholds.restart_loop_count * 2 {
1557 SignalStatus::Critical
1558 } else if count >= thresholds.restart_loop_count {
1559 SignalStatus::Elevated
1560 } else {
1561 SignalStatus::Normal
1562 };
1563 signals.push(Signal {
1564 name: "restart_count".to_owned(),
1565 value: count.to_string(),
1566 status,
1567 });
1568
1569 if count >= thresholds.restart_loop_count {
1570 anomalies.extend(detect_restart_loops(
1571 std::slice::from_ref(container),
1572 thresholds.restart_loop_count,
1573 ));
1574 }
1575 }
1576
1577 if let Some(s) = sample {
1579 if let Some(cpu) = s.cpu_percent {
1580 let status = if cpu >= thresholds.critical_cpu_percent {
1581 SignalStatus::Critical
1582 } else if cpu >= thresholds.high_cpu_percent {
1583 SignalStatus::Elevated
1584 } else {
1585 SignalStatus::Normal
1586 };
1587 signals.push(Signal {
1588 name: "cpu".to_owned(),
1589 value: format!("{cpu:.1}%"),
1590 status,
1591 });
1592 }
1593
1594 if let (Some(usage), Some(limit)) = (s.memory_usage_bytes, s.memory_limit_bytes) {
1595 let ratio = if limit > 0 {
1596 usage as f64 / limit as f64
1597 } else {
1598 0.0
1599 };
1600 let status = if ratio >= thresholds.critical_memory_ratio {
1601 SignalStatus::Critical
1602 } else if ratio >= thresholds.memory_pressure_ratio {
1603 SignalStatus::Elevated
1604 } else {
1605 SignalStatus::Normal
1606 };
1607 signals.push(Signal {
1608 name: "memory".to_owned(),
1609 value: format!("{:.1}%", ratio * 100.0),
1610 status,
1611 });
1612 }
1613
1614 if let Some(rx) = s.network_rx_bytes {
1615 signals.push(Signal {
1616 name: "network_rx".to_owned(),
1617 value: format_bytes(rx),
1618 status: SignalStatus::Normal,
1619 });
1620 }
1621
1622 if let Some(tx) = s.network_tx_bytes {
1623 signals.push(Signal {
1624 name: "network_tx".to_owned(),
1625 value: format_bytes(tx),
1626 status: SignalStatus::Normal,
1627 });
1628 }
1629
1630 anomalies.extend(detect_high_cpu(
1632 std::slice::from_ref(s),
1633 thresholds.high_cpu_percent,
1634 thresholds.critical_cpu_percent,
1635 ));
1636 anomalies.extend(detect_memory_pressure(
1637 std::slice::from_ref(s),
1638 thresholds.memory_pressure_ratio,
1639 thresholds.critical_memory_ratio,
1640 ));
1641 }
1642
1643 anomalies.extend(detect_unhealthy_states(std::slice::from_ref(container)));
1645
1646 ContainerExplanation {
1647 container: container.name.clone(),
1648 state: container.state.clone(),
1649 anomalies,
1650 signals,
1651 }
1652}
1653
1654fn explanation_to_result(explanation: &ContainerExplanation) -> ExecutionResult {
1655 let anomaly_count = explanation.anomalies.len();
1656 let rows = explanation
1657 .signals
1658 .iter()
1659 .map(|signal| Row {
1660 fields: BTreeMap::from([
1661 (
1662 "container".to_owned(),
1663 JsonValue::String(explanation.container.clone()),
1664 ),
1665 (
1666 "state".to_owned(),
1667 JsonValue::String(explanation.state.clone()),
1668 ),
1669 ("signal".to_owned(), JsonValue::String(signal.name.clone())),
1670 ("value".to_owned(), JsonValue::String(signal.value.clone())),
1671 (
1672 "status".to_owned(),
1673 JsonValue::String(signal.status.to_string()),
1674 ),
1675 (
1676 "anomaly_count".to_owned(),
1677 JsonValue::Number(Number::from(anomaly_count)),
1678 ),
1679 ]),
1680 })
1681 .collect();
1682
1683 ExecutionResult { rows }
1684}
1685
1686const fn severity_rank(severity: Severity) -> u8 {
1687 match severity {
1688 Severity::Info => 0,
1689 Severity::Warning => 1,
1690 Severity::Critical => 2,
1691 }
1692}
1693
1694fn format_bytes(bytes: u64) -> String {
1695 if bytes >= ONE_GB {
1696 format!("{:.1} GB", bytes as f64 / ONE_GB as f64)
1697 } else if bytes >= 1_048_576 {
1698 format!("{:.1} MB", bytes as f64 / 1_048_576.0)
1699 } else if bytes >= 1024 {
1700 format!("{:.1} KB", bytes as f64 / 1024.0)
1701 } else {
1702 format!("{bytes} B")
1703 }
1704}
1705
1706#[cfg(test)]
1709mod tests {
1710 use super::*;
1711 use crate::docker::Container;
1712
1713 fn rt() -> tokio::runtime::Runtime {
1714 tokio::runtime::Runtime::new().unwrap()
1715 }
1716
1717 fn test_container(name: &str, state: &str, restarts: u64) -> Container {
1718 Container {
1719 id: format!("{name}_id"),
1720 name: name.to_owned(),
1721 image: format!("{name}:latest"),
1722 status: if state == "running" {
1723 "Up 2 minutes".to_owned()
1724 } else {
1725 "Exited (1) 1 minute ago".to_string()
1726 },
1727 state: state.to_owned(),
1728 ports: Vec::new(),
1729 labels: vec!["env=prod".to_owned()],
1730 created_at: None,
1731 started_at: None,
1732 finished_at: None,
1733 restart_count: Some(restarts),
1734 health: None,
1735 }
1736 }
1737
1738 fn test_sample(name: &str, cpu: f64, mem_usage: u64, mem_limit: u64) -> MetricSample {
1739 MetricSample {
1740 container_id: format!("{name}_id"),
1741 container_name: name.to_owned(),
1742 timestamp: "2026-01-01T12:00:00Z".to_owned(),
1743 cpu_percent: Some(cpu),
1744 memory_usage_bytes: Some(mem_usage),
1745 memory_limit_bytes: Some(mem_limit),
1746 network_rx_bytes: Some(1024),
1747 network_tx_bytes: Some(2048),
1748 disk_read_bytes: Some(0),
1749 disk_write_bytes: Some(0),
1750 }
1751 }
1752
1753 fn test_event(time: &str, action: &str, container: &str) -> DockerEvent {
1754 DockerEvent {
1755 time: time.to_owned(),
1756 event_type: "container".to_owned(),
1757 action: action.to_owned(),
1758 actor_id: format!("{container}_id"),
1759 container: Some(container.to_owned()),
1760 image: Some(format!("{container}:latest")),
1761 attributes: vec![("name".to_owned(), container.to_owned())],
1762 }
1763 }
1764
1765 #[test]
1768 fn detects_restart_loop() {
1769 let containers = vec![
1770 test_container("api", "running", 5),
1771 test_container("worker", "running", 0),
1772 ];
1773
1774 let anomalies = detect_restart_loops(&containers, 3);
1775
1776 assert_eq!(anomalies.len(), 1);
1777 assert_eq!(anomalies[0].container, "api");
1778 assert_eq!(anomalies[0].kind, "restart_loop");
1779 assert_eq!(anomalies[0].severity, Severity::Warning);
1780 }
1781
1782 #[test]
1783 fn detects_critical_restart_loop() {
1784 let containers = vec![test_container("api", "running", 10)];
1785 let anomalies = detect_restart_loops(&containers, 3);
1786
1787 assert_eq!(anomalies.len(), 1);
1788 assert_eq!(anomalies[0].severity, Severity::Critical);
1789 }
1790
1791 #[test]
1792 fn no_restart_loop_below_threshold() {
1793 let containers = vec![test_container("api", "running", 2)];
1794 let anomalies = detect_restart_loops(&containers, 3);
1795
1796 assert!(anomalies.is_empty());
1797 }
1798
1799 #[test]
1802 fn detects_high_cpu_warning() {
1803 let samples = vec![test_sample("api", 85.0, 512, 1024)];
1804 let anomalies = detect_high_cpu(&samples, 80.0, 95.0);
1805
1806 assert_eq!(anomalies.len(), 1);
1807 assert_eq!(anomalies[0].severity, Severity::Warning);
1808 assert_eq!(anomalies[0].kind, "high_cpu");
1809 }
1810
1811 #[test]
1812 fn detects_critical_cpu() {
1813 let samples = vec![test_sample("api", 98.0, 512, 1024)];
1814 let anomalies = detect_high_cpu(&samples, 80.0, 95.0);
1815
1816 assert_eq!(anomalies.len(), 1);
1817 assert_eq!(anomalies[0].severity, Severity::Critical);
1818 }
1819
1820 #[test]
1821 fn no_cpu_anomaly_below_threshold() {
1822 let samples = vec![test_sample("api", 50.0, 512, 1024)];
1823 let anomalies = detect_high_cpu(&samples, 80.0, 95.0);
1824
1825 assert!(anomalies.is_empty());
1826 }
1827
1828 #[test]
1831 fn detects_memory_pressure_warning() {
1832 let samples = vec![test_sample("api", 10.0, 900, 1024)]; let anomalies = detect_memory_pressure(&samples, 0.85, 0.95);
1834
1835 assert_eq!(anomalies.len(), 1);
1836 assert_eq!(anomalies[0].severity, Severity::Warning);
1837 assert_eq!(anomalies[0].kind, "memory_pressure");
1838 }
1839
1840 #[test]
1841 fn detects_critical_memory_pressure() {
1842 let samples = vec![test_sample("api", 10.0, 980, 1024)]; let anomalies = detect_memory_pressure(&samples, 0.85, 0.95);
1844
1845 assert_eq!(anomalies.len(), 1);
1846 assert_eq!(anomalies[0].severity, Severity::Critical);
1847 }
1848
1849 #[test]
1850 fn no_memory_pressure_below_threshold() {
1851 let samples = vec![test_sample("api", 10.0, 500, 1024)]; let anomalies = detect_memory_pressure(&samples, 0.85, 0.95);
1853
1854 assert!(anomalies.is_empty());
1855 }
1856
1857 #[test]
1860 fn detects_deployment_errors() {
1861 let events = vec![
1862 test_event("2026-01-01T12:00:00Z", "die", "api"),
1863 test_event("2026-01-01T12:01:00Z", "die", "api"),
1864 test_event("2026-01-01T12:02:00Z", "die", "api"),
1865 test_event("2026-01-01T12:03:00Z", "start", "api"),
1866 ];
1867
1868 let anomalies = detect_deployment_errors(&events, 3);
1869
1870 assert_eq!(anomalies.len(), 1);
1871 assert_eq!(anomalies[0].container, "api");
1872 assert_eq!(anomalies[0].kind, "deployment_errors");
1873 }
1874
1875 #[test]
1876 fn no_deployment_errors_below_threshold() {
1877 let events = vec![
1878 test_event("2026-01-01T12:00:00Z", "die", "api"),
1879 test_event("2026-01-01T12:01:00Z", "start", "api"),
1880 ];
1881
1882 let anomalies = detect_deployment_errors(&events, 3);
1883
1884 assert!(anomalies.is_empty());
1885 }
1886
1887 #[test]
1892 fn detects_high_disk_io_warning() {
1893 let samples = vec![test_sample("api", 10.0, 128, 1024)]; let high_samples = vec![MetricSample {
1895 disk_read_bytes: Some(20_000_000), disk_write_bytes: Some(5_000_000),
1897 ..test_sample("api", 10.0, 128, 1024)
1898 }];
1899
1900 let anomalies = detect_high_disk_io(&high_samples, 10_485_760, 104_857_600);
1901 assert_eq!(anomalies.len(), 1);
1902 assert_eq!(anomalies[0].severity, Severity::Warning);
1903 assert_eq!(anomalies[0].kind, "high_disk_io");
1904 assert_eq!(anomalies[0].container, "api");
1905
1906 let anomalies = detect_high_disk_io(&samples, 10_485_760, 104_857_600);
1908 assert!(anomalies.is_empty());
1909 }
1910
1911 #[test]
1912 fn detects_critical_disk_io() {
1913 let samples = vec![MetricSample {
1914 disk_write_bytes: Some(200_000_000), disk_read_bytes: Some(10_000_000),
1916 ..test_sample("api", 10.0, 128, 1024)
1917 }];
1918
1919 let anomalies = detect_high_disk_io(&samples, 10_485_760, 104_857_600);
1920 assert_eq!(anomalies.len(), 1);
1921 assert_eq!(anomalies[0].severity, Severity::Critical);
1922 assert_eq!(anomalies[0].kind, "high_disk_io");
1923 }
1924
1925 #[test]
1926 fn no_disk_io_anomaly_when_none_supplied() {
1927 let samples = vec![MetricSample {
1928 disk_read_bytes: None,
1929 disk_write_bytes: None,
1930 ..test_sample("api", 10.0, 128, 1024)
1931 }];
1932 let anomalies = detect_high_disk_io(&samples, 10_485_760, 104_857_600);
1933 assert!(anomalies.is_empty());
1934 }
1935
1936 #[test]
1939 fn detects_high_network_io_warning() {
1940 let samples = vec![test_sample("api", 10.0, 128, 1024)]; let high_samples = vec![MetricSample {
1942 network_rx_bytes: Some(5_000_000), network_tx_bytes: Some(2_000_000),
1944 ..test_sample("api", 10.0, 128, 1024)
1945 }];
1946
1947 let anomalies = detect_high_network_io(&high_samples, 1_048_576, 10_485_760);
1948 assert_eq!(anomalies.len(), 1);
1949 assert_eq!(anomalies[0].severity, Severity::Warning);
1950 assert_eq!(anomalies[0].kind, "high_network_io");
1951 assert_eq!(anomalies[0].container, "api");
1952
1953 let anomalies = detect_high_network_io(&samples, 1_048_576, 10_485_760);
1955 assert!(anomalies.is_empty());
1956 }
1957
1958 #[test]
1959 fn detects_critical_network_io() {
1960 let samples = vec![MetricSample {
1961 network_rx_bytes: Some(50_000_000), network_tx_bytes: Some(1_000_000),
1963 ..test_sample("api", 10.0, 128, 1024)
1964 }];
1965
1966 let anomalies = detect_high_network_io(&samples, 1_048_576, 10_485_760);
1967 assert_eq!(anomalies.len(), 1);
1968 assert_eq!(anomalies[0].severity, Severity::Critical);
1969 assert_eq!(anomalies[0].kind, "high_network_io");
1970 }
1971
1972 #[test]
1973 fn no_network_io_anomaly_when_none_supplied() {
1974 let samples = vec![MetricSample {
1975 network_rx_bytes: None,
1976 network_tx_bytes: None,
1977 ..test_sample("api", 10.0, 128, 1024)
1978 }];
1979 let anomalies = detect_high_network_io(&samples, 1_048_576, 10_485_760);
1980 assert!(anomalies.is_empty());
1981 }
1982
1983 #[test]
1984 fn detects_unhealthy_states() {
1985 let containers = vec![
1986 test_container("api", "running", 0),
1987 test_container("worker", "exited", 0),
1988 test_container("db", "dead", 0),
1989 ];
1990
1991 let anomalies = detect_unhealthy_states(&containers);
1992
1993 assert_eq!(anomalies.len(), 2);
1994 let kinds: Vec<&str> = anomalies.iter().map(|a| a.kind.as_str()).collect();
1995 assert!(kinds.contains(&"exited_container"));
1996 assert!(kinds.contains(&"dead_container"));
1997 }
1998
1999 #[test]
2002 fn find_anomalies_returns_multiple_types() {
2003 let docker = crate::docker::MockDockerClient {
2004 containers: vec![
2005 test_container("api", "running", 5),
2006 test_container("worker", "exited", 0),
2007 ],
2008 ..Default::default()
2009 };
2010 let metrics = crate::metrics::MockMetricsCollector {
2011 samples: vec![test_sample("api", 92.0, 950, 1024)],
2012 };
2013
2014 let query = AnalyzeQuery {
2015 target: AnalysisTarget::Collection(CollectionTarget::Containers),
2016 verb: AnalysisVerb::Find,
2017 subject: Some("anomalies".to_owned()),
2018 time: None,
2019 pipeline: Vec::new(),
2020 };
2021
2022 let result = rt()
2023 .block_on(execute_analyze(&query, &docker, &metrics))
2024 .unwrap();
2025
2026 assert!(result.rows.len() >= 3);
2028 let kinds: Vec<String> = result
2029 .rows
2030 .iter()
2031 .filter_map(|r| {
2032 r.fields
2033 .get("kind")
2034 .and_then(|v| v.as_str())
2035 .map(str::to_owned)
2036 })
2037 .collect();
2038 assert!(kinds.contains(&"restart_loop".to_owned()));
2039 assert!(kinds.contains(&"high_cpu".to_owned()));
2040 assert!(kinds.contains(&"exited_container".to_owned()));
2041 }
2042
2043 #[test]
2044 fn find_anomalies_returns_healthy_when_no_issues() {
2045 let docker = crate::docker::MockDockerClient {
2046 containers: vec![test_container("api", "running", 0)],
2047 ..Default::default()
2048 };
2049 let metrics = crate::metrics::MockMetricsCollector {
2050 samples: vec![test_sample("api", 10.0, 128, 1024)],
2051 };
2052
2053 let query = AnalyzeQuery {
2054 target: AnalysisTarget::Collection(CollectionTarget::Containers),
2055 verb: AnalysisVerb::Find,
2056 subject: Some("anomalies".to_owned()),
2057 time: None,
2058 pipeline: Vec::new(),
2059 };
2060
2061 let result = rt()
2062 .block_on(execute_analyze(&query, &docker, &metrics))
2063 .unwrap();
2064
2065 assert_eq!(result.rows.len(), 1);
2066 assert_eq!(
2067 result.rows[0].fields["kind"],
2068 JsonValue::String("healthy".to_owned())
2069 );
2070 }
2071
2072 #[test]
2073 fn explain_container_produces_signals() {
2074 let docker = crate::docker::MockDockerClient {
2075 containers: vec![test_container("api", "running", 5)],
2076 ..Default::default()
2077 };
2078 let metrics = crate::metrics::MockMetricsCollector {
2079 samples: vec![test_sample("api", 85.0, 900, 1024)],
2080 };
2081
2082 let query = AnalyzeQuery {
2083 target: AnalysisTarget::Singular(crate::ast::SingularTarget {
2084 kind: SingularTargetKind::Container,
2085 value: "api".to_owned(),
2086 }),
2087 verb: AnalysisVerb::Explain,
2088 subject: None,
2089 time: None,
2090 pipeline: Vec::new(),
2091 };
2092
2093 let result = rt()
2094 .block_on(execute_analyze(&query, &docker, &metrics))
2095 .unwrap();
2096
2097 assert!(result.rows.len() >= 4);
2099 let signal_names: Vec<String> = result
2100 .rows
2101 .iter()
2102 .filter_map(|r| {
2103 r.fields
2104 .get("signal")
2105 .and_then(|v| v.as_str())
2106 .map(str::to_owned)
2107 })
2108 .collect();
2109 assert!(signal_names.contains(&"state".to_owned()));
2110 assert!(signal_names.contains(&"cpu".to_owned()));
2111 assert!(signal_names.contains(&"memory".to_owned()));
2112 }
2113
2114 #[test]
2115 fn correlate_finds_shared_images() {
2116 let docker = crate::docker::MockDockerClient {
2119 containers: vec![
2120 Container {
2121 image: "api:latest".to_owned(),
2122 ..test_container("api-1", "running", 0)
2123 },
2124 Container {
2125 image: "api:latest".to_owned(),
2126 ..test_container("api-2", "running", 0)
2127 },
2128 ],
2129 ..Default::default()
2130 };
2131 let metrics = crate::metrics::MockMetricsCollector {
2132 samples: Vec::new(),
2133 };
2134
2135 let query = AnalyzeQuery {
2136 target: AnalysisTarget::Collection(CollectionTarget::Containers),
2137 verb: AnalysisVerb::Correlate,
2138 subject: None,
2139 time: None,
2140 pipeline: Vec::new(),
2141 };
2142
2143 let result = rt()
2144 .block_on(execute_analyze(&query, &docker, &metrics))
2145 .unwrap();
2146
2147 let correlations: Vec<String> = result
2149 .rows
2150 .iter()
2151 .filter_map(|r| {
2152 r.fields
2153 .get("correlation")
2154 .and_then(|v| v.as_str())
2155 .map(str::to_owned)
2156 })
2157 .collect();
2158 assert!(correlations.contains(&"shared_image".to_owned()));
2159 }
2160
2161 #[test]
2164 fn detects_resource_leak_with_increasing_memory() {
2165 let mut store = crate::storage::InMemoryTelemetryStore::default();
2166 let thresholds = AnalysisThresholds {
2167 resource_leak_memory_increase_pct: 20.0,
2168 resource_leak_min_samples: 3,
2169 ..AnalysisThresholds::default()
2170 };
2171
2172 for i in 0..5 {
2174 let mem = 100_000_000 + (i * 30_000_000); let sample = MetricSample {
2176 container_id: "leaky_id".to_string(),
2177 container_name: "leaky-container".to_owned(),
2178 timestamp: format!("2026-01-01T12:00:{:02}Z", i),
2179 cpu_percent: Some(50.0),
2180 memory_usage_bytes: Some(mem),
2181 memory_limit_bytes: Some(1_073_741_824),
2182 network_rx_bytes: Some(0),
2183 network_tx_bytes: Some(0),
2184 disk_read_bytes: Some(0),
2185 disk_write_bytes: Some(0),
2186 };
2187 store.write_metric(sample).unwrap();
2188 }
2189
2190 let anomalies = detect_resource_leaks(&store, &thresholds);
2191 assert_eq!(anomalies.len(), 1);
2192 assert_eq!(anomalies[0].kind, "resource_leak");
2193 assert_eq!(anomalies[0].container, "leaky-container");
2194 }
2195
2196 #[test]
2197 fn no_resource_leak_with_stable_memory() {
2198 let mut store = crate::storage::InMemoryTelemetryStore::default();
2199 let thresholds = AnalysisThresholds {
2200 resource_leak_memory_increase_pct: 20.0,
2201 resource_leak_min_samples: 3,
2202 ..AnalysisThresholds::default()
2203 };
2204
2205 for i in 0..3 {
2207 let sample = MetricSample {
2208 container_id: "stable_id".to_string(),
2209 container_name: "stable".to_owned(),
2210 timestamp: format!("2026-01-01T12:00:{:02}Z", i),
2211 cpu_percent: Some(50.0),
2212 memory_usage_bytes: Some(100_000_000),
2213 memory_limit_bytes: Some(1_073_741_824),
2214 network_rx_bytes: Some(0),
2215 network_tx_bytes: Some(0),
2216 disk_read_bytes: Some(0),
2217 disk_write_bytes: Some(0),
2218 };
2219 store.write_metric(sample).unwrap();
2220 }
2221
2222 let anomalies = detect_resource_leaks(&store, &thresholds);
2223 assert!(anomalies.is_empty());
2224 }
2225
2226 #[test]
2227 fn no_resource_leak_with_insufficient_samples() {
2228 let mut store = crate::storage::InMemoryTelemetryStore::default();
2229 let thresholds = AnalysisThresholds {
2230 resource_leak_memory_increase_pct: 20.0,
2231 resource_leak_min_samples: 5,
2232 ..AnalysisThresholds::default()
2233 };
2234
2235 for i in 0..2 {
2237 let sample = MetricSample {
2238 container_id: "few_id".to_string(),
2239 container_name: "few".to_owned(),
2240 timestamp: format!("2026-01-01T12:00:{:02}Z", i),
2241 cpu_percent: Some(50.0),
2242 memory_usage_bytes: Some(100_000_000 + (i * 50_000_000)),
2243 memory_limit_bytes: Some(1_073_741_824),
2244 network_rx_bytes: Some(0),
2245 network_tx_bytes: Some(0),
2246 disk_read_bytes: Some(0),
2247 disk_write_bytes: Some(0),
2248 };
2249 store.write_metric(sample).unwrap();
2250 }
2251
2252 let anomalies = detect_resource_leaks(&store, &thresholds);
2253 assert!(anomalies.is_empty());
2254 }
2255
2256 #[test]
2259 fn finds_dependencies_from_mock() {
2260 let docker = crate::docker::MockDockerClient {
2261 containers: vec![
2262 Container {
2263 image: "api:latest".to_owned(),
2264 labels: vec!["com.docker.compose.project=myapp".to_owned()],
2265 ..test_container("api-1", "running", 0)
2266 },
2267 Container {
2268 image: "api:latest".to_owned(),
2269 labels: vec!["com.docker.compose.project=myapp".to_owned()],
2270 ..test_container("api-2", "running", 0)
2271 },
2272 ],
2273 networks: vec![crate::docker::Network {
2274 id: "net1".to_owned(),
2275 name: "frontend-net".to_owned(),
2276 driver: "bridge".to_owned(),
2277 scope: "local".to_owned(),
2278 containers: vec!["api-1".to_owned()],
2279 labels: Vec::new(),
2280 }],
2281 volumes: vec![crate::docker::Volume {
2282 name: "pgdata".to_owned(),
2283 driver: "local".to_owned(),
2284 mountpoint: Some("/data".to_owned()),
2285 scope: Some("local".to_owned()),
2286 labels: Vec::new(),
2287 }],
2288 ..Default::default()
2289 };
2290
2291 let result = rt().block_on(analyze_dependencies(&docker)).unwrap();
2292
2293 let dep_kinds: Vec<String> = result
2294 .rows
2295 .iter()
2296 .filter_map(|r| {
2297 r.fields
2298 .get("dependency")
2299 .and_then(|v| v.as_str())
2300 .map(str::to_owned)
2301 })
2302 .collect();
2303 assert!(dep_kinds.contains(&"compose_project".to_owned()));
2304 assert!(dep_kinds.contains(&"network".to_owned()));
2305 assert!(dep_kinds.contains(&"volume".to_owned()));
2306 }
2307
2308 #[test]
2309 fn dependencies_empty_returns_none_row() {
2310 let docker = crate::docker::MockDockerClient::default();
2311 let result = rt().block_on(analyze_dependencies(&docker)).unwrap();
2312 assert_eq!(result.rows.len(), 1);
2313 assert_eq!(
2314 result.rows[0].fields["dependency"],
2315 JsonValue::String("none".to_owned())
2316 );
2317 }
2318
2319 #[test]
2322 fn detects_config_drift_from_snapshot() {
2323 let mut store = crate::storage::InMemoryTelemetryStore::default();
2324 store
2325 .write_snapshot(crate::storage::TelemetrySnapshot {
2326 timestamp: "2026-01-01T12:00:00Z".to_owned(),
2327 containers: vec![Container {
2328 id: "abc".to_owned(),
2329 name: "api".to_owned(),
2330 image: "api:latest".to_owned(),
2331 status: "Up 5 minutes".to_owned(),
2332 state: "running".to_owned(),
2333 ports: Vec::new(),
2334 labels: vec!["env=prod".to_owned()],
2335 created_at: None,
2336 started_at: None,
2337 finished_at: None,
2338 restart_count: Some(1),
2339 health: None,
2340 }],
2341 images: Vec::new(),
2342 networks: Vec::new(),
2343 volumes: Vec::new(),
2344 })
2345 .unwrap();
2346
2347 let anomalies = detect_config_drift(&store).unwrap();
2348 assert!(!anomalies.is_empty());
2349 assert_eq!(anomalies[0].kind, "config_baseline");
2350 assert_eq!(anomalies[0].container, "api");
2351 }
2352
2353 #[test]
2354 fn detects_config_drift_between_snapshots() {
2355 let mut store = crate::storage::InMemoryTelemetryStore::default();
2356
2357 store
2359 .write_snapshot(crate::storage::TelemetrySnapshot {
2360 timestamp: "2026-01-01T12:00:00Z".to_owned(),
2361 containers: vec![Container {
2362 id: "abc".to_owned(),
2363 name: "api".to_owned(),
2364 image: "api:v1".to_owned(),
2365 status: "Up 5 minutes".to_owned(),
2366 state: "running".to_owned(),
2367 ports: Vec::new(),
2368 labels: vec!["env=prod".to_owned(), "team=backend".to_owned()],
2369 created_at: None,
2370 started_at: None,
2371 finished_at: None,
2372 restart_count: Some(0),
2373 health: None,
2374 }],
2375 images: Vec::new(),
2376 networks: Vec::new(),
2377 volumes: Vec::new(),
2378 })
2379 .unwrap();
2380
2381 store
2383 .write_snapshot(crate::storage::TelemetrySnapshot {
2384 timestamp: "2026-01-01T13:00:00Z".to_owned(),
2385 containers: vec![Container {
2386 id: "abc".to_owned(),
2387 name: "api".to_owned(),
2388 image: "api:v2".to_owned(),
2389 status: "Exited (1) 1 minute ago".to_owned(),
2390 state: "exited".to_owned(),
2391 ports: Vec::new(),
2392 labels: vec!["env=prod".to_owned(), "team=infra".to_owned()],
2393 created_at: None,
2394 started_at: None,
2395 finished_at: None,
2396 restart_count: Some(3),
2397 health: None,
2398 }],
2399 images: Vec::new(),
2400 networks: Vec::new(),
2401 volumes: Vec::new(),
2402 })
2403 .unwrap();
2404
2405 let anomalies = detect_config_drift(&store).unwrap();
2406 let kinds: Vec<&str> = anomalies.iter().map(|a| a.kind.as_str()).collect();
2407
2408 assert!(
2409 kinds.contains(&"config_drift"),
2410 "should detect image change"
2411 );
2412 assert!(
2413 kinds.contains(&"state_change"),
2414 "should detect state change"
2415 );
2416 assert!(
2417 kinds.contains(&"restart_increase"),
2418 "should detect restart increase"
2419 );
2420 assert!(kinds.contains(&"label_drift"), "should detect label change");
2421 }
2422
2423 #[test]
2424 fn config_drift_returns_empty_when_no_changes() {
2425 let mut store = crate::storage::InMemoryTelemetryStore::default();
2426
2427 let snapshot = crate::storage::TelemetrySnapshot {
2429 timestamp: "2026-01-01T12:00:00Z".to_owned(),
2430 containers: vec![Container {
2431 id: "abc".to_owned(),
2432 name: "api".to_owned(),
2433 image: "api:latest".to_owned(),
2434 status: "Up 5 minutes".to_owned(),
2435 state: "running".to_owned(),
2436 ports: Vec::new(),
2437 labels: vec!["env=prod".to_owned()],
2438 created_at: None,
2439 started_at: None,
2440 finished_at: None,
2441 restart_count: Some(0),
2442 health: None,
2443 }],
2444 images: Vec::new(),
2445 networks: Vec::new(),
2446 volumes: Vec::new(),
2447 };
2448 store.write_snapshot(snapshot.clone()).unwrap();
2449 store.write_snapshot(snapshot).unwrap();
2450
2451 let anomalies = detect_config_drift(&store).unwrap();
2452 assert!(anomalies.is_empty(), "should be empty when no changes");
2453 }
2454
2455 #[test]
2456 fn config_drift_with_empty_store_returns_unsupported() {
2457 let store = crate::storage::InMemoryTelemetryStore::default();
2458 let result = detect_config_drift(&store);
2459 assert!(result.is_err());
2460 }
2461
2462 #[test]
2465 fn analyzes_density_by_image_state_and_project() {
2466 let docker = crate::docker::MockDockerClient {
2467 containers: vec![
2468 Container {
2469 image: "nginx:latest".to_owned(),
2470 labels: vec!["com.docker.compose.project=web".to_owned()],
2471 ..test_container("web-1", "running", 0)
2472 },
2473 Container {
2474 image: "nginx:latest".to_owned(),
2475 labels: vec!["com.docker.compose.project=web".to_owned()],
2476 ..test_container("web-2", "running", 0)
2477 },
2478 Container {
2479 image: "api:latest".to_owned(),
2480 labels: vec![],
2481 ..test_container("api", "exited", 0)
2482 },
2483 ],
2484 ..Default::default()
2485 };
2486
2487 let result = rt().block_on(analyze_density(&docker)).unwrap();
2488
2489 let dimensions: Vec<String> = result
2491 .rows
2492 .iter()
2493 .filter_map(|r| {
2494 r.fields
2495 .get("dimension")
2496 .and_then(|v| v.as_str())
2497 .map(str::to_owned)
2498 })
2499 .collect();
2500 assert!(dimensions.contains(&"image".to_owned()));
2501 assert!(dimensions.contains(&"state".to_owned()));
2502 assert!(dimensions.contains(&"compose_project".to_owned()));
2503 assert!(dimensions.contains(&"total".to_owned()));
2504
2505 let total_row = result
2507 .rows
2508 .iter()
2509 .find(|r| r.fields.get("dimension") == Some(&JsonValue::String("total".to_owned())))
2510 .unwrap();
2511 assert_eq!(
2512 total_row.fields["container_count"],
2513 JsonValue::Number(Number::from(3))
2514 );
2515 }
2516
2517 #[test]
2518 fn density_with_no_containers_returns_total_zero() {
2519 let docker = crate::docker::MockDockerClient::default();
2520 let result = rt().block_on(analyze_density(&docker)).unwrap();
2521 let total_row = result
2522 .rows
2523 .iter()
2524 .find(|r| r.fields.get("dimension") == Some(&JsonValue::String("total".to_owned())))
2525 .unwrap();
2526 assert_eq!(
2527 total_row.fields["container_count"],
2528 JsonValue::Number(Number::from(0))
2529 );
2530 }
2531
2532 #[test]
2535 fn execute_analyze_find_dependencies_from_live_data() {
2536 let docker = crate::docker::MockDockerClient {
2537 containers: vec![test_container("api", "running", 0)],
2538 ..Default::default()
2539 };
2540 let metrics = crate::metrics::MockMetricsCollector {
2541 samples: Vec::new(),
2542 };
2543
2544 let query = AnalyzeQuery {
2545 target: AnalysisTarget::Collection(CollectionTarget::Containers),
2546 verb: AnalysisVerb::Find,
2547 subject: Some("dependencies".to_owned()),
2548 time: None,
2549 pipeline: Vec::new(),
2550 };
2551
2552 let result = rt()
2553 .block_on(execute_analyze(&query, &docker, &metrics))
2554 .unwrap();
2555 assert!(!result.rows.is_empty());
2557 }
2558
2559 #[test]
2560 fn execute_analyze_find_density_from_live_data() {
2561 let docker = crate::docker::MockDockerClient {
2562 containers: vec![test_container("api", "running", 0)],
2563 ..Default::default()
2564 };
2565 let metrics = crate::metrics::MockMetricsCollector {
2566 samples: Vec::new(),
2567 };
2568
2569 let query = AnalyzeQuery {
2570 target: AnalysisTarget::Collection(CollectionTarget::Containers),
2571 verb: AnalysisVerb::Find,
2572 subject: Some("density".to_owned()),
2573 time: None,
2574 pipeline: Vec::new(),
2575 };
2576
2577 let result = rt()
2578 .block_on(execute_analyze(&query, &docker, &metrics))
2579 .unwrap();
2580 let total_row = result
2581 .rows
2582 .iter()
2583 .find(|r| r.fields.get("dimension") == Some(&JsonValue::String("total".to_owned())))
2584 .unwrap();
2585 assert_eq!(
2586 total_row.fields["container_count"],
2587 JsonValue::Number(Number::from(1))
2588 );
2589 }
2590
2591 #[test]
2592 fn execute_analyze_find_leaks_returns_unsupported_from_live() {
2593 let docker = crate::docker::MockDockerClient::default();
2594 let metrics = crate::metrics::MockMetricsCollector {
2595 samples: Vec::new(),
2596 };
2597
2598 let query = AnalyzeQuery {
2599 target: AnalysisTarget::Collection(CollectionTarget::Containers),
2600 verb: AnalysisVerb::Find,
2601 subject: Some("leaks".to_owned()),
2602 time: None,
2603 pipeline: Vec::new(),
2604 };
2605
2606 let result = rt().block_on(execute_analyze(&query, &docker, &metrics));
2607 assert!(result.is_err());
2608 assert!(matches!(result.unwrap_err(), AnalyzeError::Unsupported));
2609 }
2610
2611 #[test]
2612 fn execute_analyze_with_store_find_leaks_detects_pattern() {
2613 let mut store = crate::storage::InMemoryTelemetryStore::default();
2614 for i in 0..5 {
2615 let mem = 100_000_000 + (i * 30_000_000);
2616 store
2617 .write_metric(MetricSample {
2618 container_id: "leaky_id".to_string(),
2619 container_name: "leaky".to_owned(),
2620 timestamp: format!("2026-01-01T12:00:{:02}Z", i),
2621 cpu_percent: Some(50.0),
2622 memory_usage_bytes: Some(mem),
2623 memory_limit_bytes: Some(1_073_741_824),
2624 network_rx_bytes: Some(0),
2625 network_tx_bytes: Some(0),
2626 disk_read_bytes: Some(0),
2627 disk_write_bytes: Some(0),
2628 })
2629 .unwrap();
2630 }
2631
2632 let query = AnalyzeQuery {
2633 target: AnalysisTarget::Collection(CollectionTarget::Containers),
2634 verb: AnalysisVerb::Find,
2635 subject: Some("leaks".to_owned()),
2636 time: None,
2637 pipeline: Vec::new(),
2638 };
2639
2640 let result = execute_analyze_with_store(&query, &store).unwrap();
2641 let kinds: Vec<String> = result
2642 .rows
2643 .iter()
2644 .filter_map(|r| {
2645 r.fields
2646 .get("kind")
2647 .and_then(|v| v.as_str())
2648 .map(str::to_owned)
2649 })
2650 .collect();
2651 assert!(kinds.contains(&"resource_leak".to_owned()));
2652 }
2653
2654 #[test]
2655 fn execute_analyze_with_store_find_drift_from_snapshot() {
2656 let mut store = crate::storage::InMemoryTelemetryStore::default();
2657 store
2658 .write_snapshot(crate::storage::TelemetrySnapshot {
2659 timestamp: "2026-01-01T12:00:00Z".to_owned(),
2660 containers: vec![Container {
2661 id: "abc".to_owned(),
2662 name: "api".to_owned(),
2663 image: "api:latest".to_owned(),
2664 status: "Up 5m".to_owned(),
2665 state: "running".to_owned(),
2666 ports: Vec::new(),
2667 labels: vec!["env=prod".to_owned()],
2668 created_at: None,
2669 started_at: None,
2670 finished_at: None,
2671 restart_count: Some(0),
2672 health: None,
2673 }],
2674 images: Vec::new(),
2675 networks: Vec::new(),
2676 volumes: Vec::new(),
2677 })
2678 .unwrap();
2679
2680 let query = AnalyzeQuery {
2681 target: AnalysisTarget::Collection(CollectionTarget::Containers),
2682 verb: AnalysisVerb::Find,
2683 subject: Some("drift".to_owned()),
2684 time: None,
2685 pipeline: Vec::new(),
2686 };
2687
2688 let result = execute_analyze_with_store(&query, &store).unwrap();
2689 let kinds: Vec<String> = result
2690 .rows
2691 .iter()
2692 .filter_map(|r| {
2693 r.fields
2694 .get("kind")
2695 .and_then(|v| v.as_str())
2696 .map(str::to_owned)
2697 })
2698 .collect();
2699 assert!(kinds.contains(&"config_baseline".to_owned()));
2700 }
2701
2702 #[test]
2705 fn find_anomalies_from_store_includes_leaks_and_drift() {
2706 let mut store = crate::storage::InMemoryTelemetryStore::default();
2707
2708 for i in 0..5 {
2710 let mem = 100_000_000 + (i * 30_000_000);
2711 store
2712 .write_metric(MetricSample {
2713 container_id: "leaky_id".to_string(),
2714 container_name: "leaky".to_owned(),
2715 timestamp: format!("2026-01-01T12:00:{:02}Z", i),
2716 cpu_percent: Some(50.0),
2717 memory_usage_bytes: Some(mem),
2718 memory_limit_bytes: Some(1_073_741_824),
2719 network_rx_bytes: Some(0),
2720 network_tx_bytes: Some(0),
2721 disk_read_bytes: Some(0),
2722 disk_write_bytes: Some(0),
2723 })
2724 .unwrap();
2725 }
2726
2727 store
2729 .write_snapshot(crate::storage::TelemetrySnapshot {
2730 timestamp: "2026-01-01T12:00:00Z".to_owned(),
2731 containers: vec![Container {
2732 id: "leaky_id".to_owned(),
2733 name: "leaky".to_owned(),
2734 image: "leaky:latest".to_owned(),
2735 status: "Up 5m".to_owned(),
2736 state: "running".to_owned(),
2737 ports: Vec::new(),
2738 labels: vec!["env=prod".to_owned()],
2739 created_at: None,
2740 started_at: None,
2741 finished_at: None,
2742 restart_count: Some(0),
2743 health: None,
2744 }],
2745 images: Vec::new(),
2746 networks: Vec::new(),
2747 volumes: Vec::new(),
2748 })
2749 .unwrap();
2750
2751 let query = AnalyzeQuery {
2752 target: AnalysisTarget::Collection(CollectionTarget::Containers),
2753 verb: AnalysisVerb::Find,
2754 subject: Some("anomalies".to_owned()),
2755 time: None,
2756 pipeline: Vec::new(),
2757 };
2758
2759 let result = execute_analyze_with_store(&query, &store).unwrap();
2760
2761 let kinds: Vec<String> = result
2762 .rows
2763 .iter()
2764 .filter_map(|r| {
2765 r.fields
2766 .get("kind")
2767 .and_then(|v| v.as_str())
2768 .map(str::to_owned)
2769 })
2770 .collect();
2771 assert!(
2772 kinds.contains(&"resource_leak".to_owned()),
2773 "find_anomalies_from_store should detect resource leaks: {kinds:?}"
2774 );
2775 assert!(
2776 kinds.contains(&"config_baseline".to_owned()),
2777 "find_anomalies_from_store should detect config drift baseline: {kinds:?}"
2778 );
2779 }
2780
2781 #[test]
2782 fn find_anomalies_from_store_detects_deployment_errors() {
2783 let mut store = crate::storage::InMemoryTelemetryStore::default();
2784
2785 store
2786 .write_event(test_event("2026-01-01T12:00:00Z", "die", "api"))
2787 .unwrap();
2788 store
2789 .write_event(test_event("2026-01-01T12:01:00Z", "die", "api"))
2790 .unwrap();
2791 store
2792 .write_event(test_event("2026-01-01T12:02:00Z", "die", "api"))
2793 .unwrap();
2794
2795 let query = AnalyzeQuery {
2796 target: AnalysisTarget::Collection(CollectionTarget::Containers),
2797 verb: AnalysisVerb::Find,
2798 subject: Some("anomalies".to_owned()),
2799 time: None,
2800 pipeline: Vec::new(),
2801 };
2802
2803 let result = execute_analyze_with_store(&query, &store).unwrap();
2804
2805 let kinds: Vec<String> = result
2806 .rows
2807 .iter()
2808 .filter_map(|r| {
2809 r.fields
2810 .get("kind")
2811 .and_then(|v| v.as_str())
2812 .map(str::to_owned)
2813 })
2814 .collect();
2815 assert!(kinds.contains(&"deployment_errors".to_owned()));
2816 }
2817}