Skip to main content

dol/
analyze.rs

1//! Deterministic analysis and insight engine.
2//!
3//! Provides anomaly detection, pattern recognition, and diagnostic summaries
4//! for Docker containers based on metrics, events, and state snapshots.
5//! All analyses are fully deterministic — no AI/LLM calls are made.
6
7use std::collections::{BTreeMap, HashMap};
8
9use serde::Serialize;
10use serde_json::{Number, Value as JsonValue};
11
12use crate::{
13    ONE_GB,
14    ast::{AnalysisTarget, AnalysisVerb, AnalyzeQuery, CollectionTarget, SingularTargetKind},
15    docker::{Container, DockerClient, DockerError, DockerEvent, MetricSample},
16    executor::{ExecutionResult, Row},
17    metrics::{MetricsCollector, MetricsError},
18    storage::{TelemetryError, TelemetryStore},
19};
20
21// ── Public error type ──────────────────────────────────────────────────────
22
23/// Errors that can occur during analysis.
24#[derive(Debug, thiserror::Error)]
25pub enum AnalyzeError {
26    #[error("{0}")]
27    Docker(#[from] DockerError),
28    #[error("{0}")]
29    Metrics(#[from] MetricsError),
30    #[error("{0}")]
31    Telemetry(#[from] TelemetryError),
32    #[error("analysis not supported for this target/verb combination")]
33    Unsupported,
34}
35
36// ── Anomaly / Insight types ────────────────────────────────────────────────
37
38/// Severity level for a detected anomaly.
39#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)]
40pub enum Severity {
41    Info,
42    Warning,
43    Critical,
44}
45
46impl std::fmt::Display for Severity {
47    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
48        match self {
49            Self::Info => write!(f, "info"),
50            Self::Warning => write!(f, "warning"),
51            Self::Critical => write!(f, "critical"),
52        }
53    }
54}
55
56/// A single detected anomaly or insight.
57#[derive(Debug, Clone, Serialize)]
58pub struct Anomaly {
59    pub severity: Severity,
60    pub kind: String,
61    pub container: String,
62    pub message: String,
63    /// Supporting evidence: metric values, event timestamps, etc.
64    pub evidence: Vec<String>,
65}
66
67impl Anomaly {
68    fn to_row(&self) -> Row {
69        Row {
70            fields: BTreeMap::from([
71                (
72                    "severity".to_owned(),
73                    JsonValue::String(self.severity.to_string()),
74                ),
75                ("kind".to_owned(), JsonValue::String(self.kind.clone())),
76                (
77                    "container".to_owned(),
78                    JsonValue::String(self.container.clone()),
79                ),
80                (
81                    "message".to_owned(),
82                    JsonValue::String(self.message.clone()),
83                ),
84                (
85                    "evidence".to_owned(),
86                    JsonValue::Array(
87                        self.evidence
88                            .iter()
89                            .map(|e| JsonValue::String(e.clone()))
90                            .collect(),
91                    ),
92                ),
93            ]),
94        }
95    }
96}
97
98/// Summary of a single container's health signals.
99#[derive(Debug, Clone, Serialize)]
100pub struct ContainerExplanation {
101    pub container: String,
102    pub state: String,
103    pub anomalies: Vec<Anomaly>,
104    pub signals: Vec<Signal>,
105}
106
107/// A diagnostic signal extracted from telemetry data.
108#[derive(Debug, Clone, Serialize)]
109pub struct Signal {
110    pub name: String,
111    pub value: String,
112    pub status: SignalStatus,
113}
114
115/// Status classification for a signal.
116#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)]
117pub enum SignalStatus {
118    Normal,
119    Elevated,
120    Critical,
121}
122
123impl std::fmt::Display for SignalStatus {
124    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
125        match self {
126            Self::Normal => write!(f, "normal"),
127            Self::Elevated => write!(f, "elevated"),
128            Self::Critical => write!(f, "critical"),
129        }
130    }
131}
132
133// ── Thresholds ─────────────────────────────────────────────────────────────
134
135/// Configurable thresholds for anomaly detection.
136#[derive(Debug, Clone)]
137pub struct AnalysisThresholds {
138    /// CPU percentage above which a container is considered high CPU.
139    pub high_cpu_percent: f64,
140    /// CPU percentage above which a container is critical.
141    pub critical_cpu_percent: f64,
142    /// Memory usage ratio (usage/limit) above which a container is under pressure.
143    pub memory_pressure_ratio: f64,
144    /// Memory usage ratio above which a container is critical.
145    pub critical_memory_ratio: f64,
146    /// Number of restarts indicating a restart loop.
147    pub restart_loop_count: u64,
148    /// Number of "die" events in recent history that indicate deployment failure.
149    pub deployment_error_threshold: u64,
150    /// Memory usage increase percentage indicating a resource leak (e.g., 20 = 20% increase).
151    pub resource_leak_memory_increase_pct: f64,
152    /// Minimum number of metric samples needed to detect a trend.
153    pub resource_leak_min_samples: u64,
154    /// Network I/O bytes above which a warning is issued (cumulative per sample).
155    pub high_network_bytes: u64,
156    /// Network I/O bytes above which a critical alert is issued.
157    pub critical_network_bytes: u64,
158    /// Disk I/O bytes above which a warning is issued (cumulative per sample).
159    pub high_disk_bytes: u64,
160    /// Disk I/O bytes above which a critical alert is issued.
161    pub critical_disk_bytes: u64,
162}
163
164impl Default for AnalysisThresholds {
165    fn default() -> Self {
166        Self {
167            high_cpu_percent: 80.0,
168            critical_cpu_percent: 95.0,
169            memory_pressure_ratio: 0.85,
170            critical_memory_ratio: 0.95,
171            restart_loop_count: 3,
172            deployment_error_threshold: 3,
173            resource_leak_memory_increase_pct: 20.0,
174            resource_leak_min_samples: 3,
175            high_network_bytes: 1_048_576,      // 1 MB
176            critical_network_bytes: 10_485_760, // 10 MB
177            high_disk_bytes: 10_485_760,        // 10 MB
178            critical_disk_bytes: 104_857_600,   // 100 MB
179        }
180    }
181}
182
183impl AnalysisThresholds {
184    /// Build thresholds from a [`DolConfig`], falling back to defaults for unset fields.
185    #[must_use]
186    pub fn from_config(config: &crate::config::DolConfig) -> Self {
187        Self {
188            high_cpu_percent: config.analysis_high_cpu_percent.unwrap_or(80.0),
189            critical_cpu_percent: config.analysis_critical_cpu_percent.unwrap_or(95.0),
190            memory_pressure_ratio: config.analysis_memory_pressure_ratio.unwrap_or(0.85),
191            critical_memory_ratio: config.analysis_critical_memory_ratio.unwrap_or(0.95),
192            restart_loop_count: config.analysis_restart_loop_count.unwrap_or(3),
193            deployment_error_threshold: config.analysis_deployment_error_threshold.unwrap_or(3),
194            resource_leak_memory_increase_pct: config
195                .analysis_leak_memory_increase_pct
196                .unwrap_or(20.0),
197            resource_leak_min_samples: config.analysis_leak_min_samples.unwrap_or(3),
198            high_network_bytes: config.analysis_high_network_bytes.unwrap_or(1_048_576),
199            critical_network_bytes: config.analysis_critical_network_bytes.unwrap_or(10_485_760),
200            high_disk_bytes: config.analysis_high_disk_bytes.unwrap_or(10_485_760),
201            critical_disk_bytes: config.analysis_critical_disk_bytes.unwrap_or(104_857_600),
202        }
203    }
204}
205
206// ── Main analysis dispatcher ───────────────────────────────────────────────
207
208/// Execute an analysis query using live Docker data + metrics.
209/// Thresholds are loaded from the user's config file (if available), falling back to defaults.
210pub async fn execute_analyze<C, M>(
211    query: &AnalyzeQuery,
212    docker: &C,
213    metrics: &M,
214) -> Result<ExecutionResult, AnalyzeError>
215where
216    C: DockerClient + ?Sized,
217    M: MetricsCollector + ?Sized,
218{
219    let cfg = crate::config::DolConfig::load();
220    let thresholds = AnalysisThresholds::from_config(&cfg);
221    execute_analyze_with_thresholds(query, docker, metrics, &thresholds).await
222}
223
224/// Execute an analysis query with custom thresholds.
225pub async fn execute_analyze_with_thresholds<C, M>(
226    query: &AnalyzeQuery,
227    docker: &C,
228    metrics: &M,
229    thresholds: &AnalysisThresholds,
230) -> Result<ExecutionResult, AnalyzeError>
231where
232    C: DockerClient + ?Sized,
233    M: MetricsCollector + ?Sized,
234{
235    match (&query.target, &query.verb, query.subject.as_deref()) {
236        // analyze containers find anomalies (default)
237        (
238            AnalysisTarget::Collection(CollectionTarget::Containers),
239            AnalysisVerb::Find,
240            None | Some("anomalies"),
241        ) => find_container_anomalies(docker, metrics, thresholds).await,
242        // analyze containers find dependencies
243        (
244            AnalysisTarget::Collection(CollectionTarget::Containers),
245            AnalysisVerb::Find,
246            Some("dependencies"),
247        ) => analyze_dependencies(docker).await,
248        // analyze containers find density
249        (
250            AnalysisTarget::Collection(CollectionTarget::Containers),
251            AnalysisVerb::Find,
252            Some("density"),
253        ) => analyze_density(docker).await,
254        // analyze containers correlate
255        (AnalysisTarget::Collection(CollectionTarget::Containers), AnalysisVerb::Correlate, _) => {
256            correlate_containers(docker, metrics, thresholds).await
257        }
258        // analyze container <name> explain
259        (AnalysisTarget::Singular(target), AnalysisVerb::Explain, _)
260            if target.kind == SingularTargetKind::Container =>
261        {
262            explain_container(&target.value, docker, metrics, thresholds).await
263        }
264        // analyze containers explain (explain all)
265        (AnalysisTarget::Collection(CollectionTarget::Containers), AnalysisVerb::Explain, _) => {
266            explain_all_containers(docker, metrics, thresholds).await
267        }
268        _ => Err(AnalyzeError::Unsupported),
269    }
270}
271
272/// Execute an analysis query using historical data from a telemetry store.
273/// Thresholds are loaded from the user's config file (if available), falling back to defaults.
274pub fn execute_analyze_with_store<S>(
275    query: &AnalyzeQuery,
276    store: &S,
277) -> Result<ExecutionResult, AnalyzeError>
278where
279    S: TelemetryStore + ?Sized,
280{
281    let cfg = crate::config::DolConfig::load();
282    let thresholds = AnalysisThresholds::from_config(&cfg);
283
284    match (&query.target, &query.verb, query.subject.as_deref()) {
285        // analyze containers find anomalies (default, from store)
286        (
287            AnalysisTarget::Collection(CollectionTarget::Containers),
288            AnalysisVerb::Find,
289            None | Some("anomalies"),
290        ) => find_anomalies_from_store(store, &thresholds),
291        // analyze containers find leaks (from store)
292        (
293            AnalysisTarget::Collection(CollectionTarget::Containers),
294            AnalysisVerb::Find,
295            Some("leaks"),
296        ) => {
297            let anomalies = detect_resource_leaks(store, &thresholds);
298            if anomalies.is_empty() {
299                Ok(ExecutionResult {
300                    rows: vec![Row {
301                        fields: BTreeMap::from([
302                            ("severity".to_owned(), JsonValue::String("info".to_owned())),
303                            (
304                                "kind".to_owned(),
305                                JsonValue::String("no_leaks_detected".to_owned()),
306                            ),
307                            ("container".to_owned(), JsonValue::String("*".to_owned())),
308                            (
309                                "message".to_owned(),
310                                JsonValue::String(
311                                    "No resource leaks detected in stored data".to_owned(),
312                                ),
313                            ),
314                            ("evidence".to_owned(), JsonValue::Array(Vec::new())),
315                        ]),
316                    }],
317                })
318            } else {
319                Ok(ExecutionResult {
320                    rows: anomalies.iter().map(Anomaly::to_row).collect(),
321                })
322            }
323        }
324        // analyze containers find drift (from store)
325        (
326            AnalysisTarget::Collection(CollectionTarget::Containers),
327            AnalysisVerb::Find,
328            Some("drift"),
329        ) => {
330            let anomalies = detect_config_drift(store)?;
331            if anomalies.is_empty() {
332                Ok(ExecutionResult {
333                    rows: vec![Row {
334                        fields: BTreeMap::from([
335                            ("severity".to_owned(), JsonValue::String("info".to_owned())),
336                            ("kind".to_owned(), JsonValue::String("no_drift".to_owned())),
337                            ("container".to_owned(), JsonValue::String("*".to_owned())),
338                            (
339                                "message".to_owned(),
340                                JsonValue::String("No configuration drift detected".to_owned()),
341                            ),
342                            ("evidence".to_owned(), JsonValue::Array(Vec::new())),
343                        ]),
344                    }],
345                })
346            } else {
347                Ok(ExecutionResult {
348                    rows: anomalies.iter().map(Anomaly::to_row).collect(),
349                })
350            }
351        }
352        _ => Err(AnalyzeError::Unsupported),
353    }
354}
355
356// ── Detectors ──────────────────────────────────────────────────────────────
357
358/// Detect restart loops: containers with `restart_count` >= threshold.
359#[must_use]
360pub fn detect_restart_loops(containers: &[Container], threshold: u64) -> Vec<Anomaly> {
361    containers
362        .iter()
363        .filter_map(|c| {
364            let count = c.restart_count.unwrap_or(0);
365            if count >= threshold {
366                let severity = if count >= threshold * 2 {
367                    Severity::Critical
368                } else {
369                    Severity::Warning
370                };
371                Some(Anomaly {
372                    severity,
373                    kind: "restart_loop".to_owned(),
374                    container: c.name.clone(),
375                    message: format!(
376                        "Container '{}' has restarted {} times (threshold: {})",
377                        c.name, count, threshold
378                    ),
379                    evidence: vec![
380                        format!("restart_count={}", count),
381                        format!("state={}", c.state),
382                        format!("status={}", c.status),
383                    ],
384                })
385            } else {
386                None
387            }
388        })
389        .collect()
390}
391
392/// Detect high CPU usage anomalies.
393#[must_use]
394pub fn detect_high_cpu(
395    samples: &[MetricSample],
396    high_threshold: f64,
397    critical_threshold: f64,
398) -> Vec<Anomaly> {
399    samples
400        .iter()
401        .filter_map(|s| {
402            let cpu = s.cpu_percent?;
403            if cpu >= critical_threshold {
404                Some(Anomaly {
405                    severity: Severity::Critical,
406                    kind: "high_cpu".to_owned(),
407                    container: s.container_name.clone(),
408                    message: format!(
409                        "Container '{}' CPU at {:.1}% (critical threshold: {:.0}%)",
410                        s.container_name, cpu, critical_threshold
411                    ),
412                    evidence: vec![
413                        format!("cpu={:.1}%", cpu),
414                        format!("timestamp={}", s.timestamp),
415                    ],
416                })
417            } else if cpu >= high_threshold {
418                Some(Anomaly {
419                    severity: Severity::Warning,
420                    kind: "high_cpu".to_owned(),
421                    container: s.container_name.clone(),
422                    message: format!(
423                        "Container '{}' CPU at {:.1}% (warning threshold: {:.0}%)",
424                        s.container_name, cpu, high_threshold
425                    ),
426                    evidence: vec![
427                        format!("cpu={:.1}%", cpu),
428                        format!("timestamp={}", s.timestamp),
429                    ],
430                })
431            } else {
432                None
433            }
434        })
435        .collect()
436}
437
438/// Detect memory pressure: containers with memory usage/limit ratio above threshold.
439#[must_use]
440pub fn detect_memory_pressure(
441    samples: &[MetricSample],
442    pressure_ratio: f64,
443    critical_ratio: f64,
444) -> Vec<Anomaly> {
445    samples
446        .iter()
447        .filter_map(|s| {
448            let usage = s.memory_usage_bytes? as f64;
449            let limit = s.memory_limit_bytes? as f64;
450            if limit <= 0.0 {
451                return None;
452            }
453            let ratio = usage / limit;
454            if ratio >= critical_ratio {
455                Some(Anomaly {
456                    severity: Severity::Critical,
457                    kind: "memory_pressure".to_owned(),
458                    container: s.container_name.clone(),
459                    message: format!(
460                        "Container '{}' memory at {:.1}% of limit (critical: {:.0}%)",
461                        s.container_name,
462                        ratio * 100.0,
463                        critical_ratio * 100.0
464                    ),
465                    evidence: vec![
466                        format!("memory_usage={}", s.memory_usage_bytes.unwrap_or(0)),
467                        format!("memory_limit={}", s.memory_limit_bytes.unwrap_or(0)),
468                        format!("ratio={:.2}%", ratio * 100.0),
469                    ],
470                })
471            } else if ratio >= pressure_ratio {
472                Some(Anomaly {
473                    severity: Severity::Warning,
474                    kind: "memory_pressure".to_owned(),
475                    container: s.container_name.clone(),
476                    message: format!(
477                        "Container '{}' memory at {:.1}% of limit (warning: {:.0}%)",
478                        s.container_name,
479                        ratio * 100.0,
480                        pressure_ratio * 100.0
481                    ),
482                    evidence: vec![
483                        format!("memory_usage={}", s.memory_usage_bytes.unwrap_or(0)),
484                        format!("memory_limit={}", s.memory_limit_bytes.unwrap_or(0)),
485                        format!("ratio={:.2}%", ratio * 100.0),
486                    ],
487                })
488            } else {
489                None
490            }
491        })
492        .collect()
493}
494
495/// Detect deployment-related error spikes from historical events.
496#[must_use]
497pub fn detect_deployment_errors(events: &[DockerEvent], threshold: u64) -> Vec<Anomaly> {
498    // Count "die" events per container.
499    let mut die_counts: HashMap<String, u64> = HashMap::new();
500    let mut die_times: HashMap<String, Vec<String>> = HashMap::new();
501    for event in events {
502        if event.action == "die" {
503            let container = event
504                .container
505                .as_deref()
506                .unwrap_or(&event.actor_id)
507                .to_owned();
508            *die_counts.entry(container.clone()).or_default() += 1;
509            die_times
510                .entry(container)
511                .or_default()
512                .push(event.time.clone());
513        }
514    }
515
516    die_counts
517        .into_iter()
518        .filter(|(_, count)| *count >= threshold)
519        .map(|(container, count)| {
520            let times = die_times.get(&container).cloned().unwrap_or_default();
521            let severity = if count >= threshold * 2 {
522                Severity::Critical
523            } else {
524                Severity::Warning
525            };
526            Anomaly {
527                severity,
528                kind: "deployment_errors".to_owned(),
529                container: container.clone(),
530                message: format!(
531                    "Container '{container}' has died {count} times (threshold: {threshold})"
532                ),
533                evidence: times.into_iter().map(|t| format!("die_at={t}")).collect(),
534            }
535        })
536        .collect()
537}
538
539/// Detect containers in unhealthy states (exited, dead, restarting).
540#[must_use]
541pub fn detect_unhealthy_states(containers: &[Container]) -> Vec<Anomaly> {
542    containers
543        .iter()
544        .filter_map(|c| {
545            let (severity, kind) = match c.state.as_str() {
546                "exited" => (Severity::Warning, "exited_container"),
547                "dead" => (Severity::Critical, "dead_container"),
548                "restarting" => (Severity::Warning, "restarting_container"),
549                _ => return None,
550            };
551            Some(Anomaly {
552                severity,
553                kind: kind.to_owned(),
554                container: c.name.clone(),
555                message: format!(
556                    "Container '{}' is in '{}' state: {}",
557                    c.name, c.state, c.status
558                ),
559                evidence: vec![
560                    format!("state={}", c.state),
561                    format!("status={}", c.status),
562                    format!("image={}", c.image),
563                ],
564            })
565        })
566        .collect()
567}
568
569/// Detect high disk I/O (read/write) above thresholds.
570#[must_use]
571pub fn detect_high_disk_io(
572    samples: &[MetricSample],
573    high_threshold: u64,
574    critical_threshold: u64,
575) -> Vec<Anomaly> {
576    samples
577        .iter()
578        .filter_map(|s| {
579            let read = s.disk_read_bytes?;
580            let write = s.disk_write_bytes?;
581            let max_bytes = read.max(write);
582
583            if max_bytes >= critical_threshold {
584                Some(Anomaly {
585                    severity: Severity::Critical,
586                    kind: "high_disk_io".to_owned(),
587                    container: s.container_name.clone(),
588                    message: format!(
589                        "Container '{}' disk I/O at {}/{} (read/write) — critical threshold: {}",
590                        s.container_name,
591                        format_bytes(read),
592                        format_bytes(write),
593                        format_bytes(critical_threshold),
594                    ),
595                    evidence: vec![
596                        format!("disk_read={}", read),
597                        format!("disk_write={}", write),
598                        format!("max_bytes={}", max_bytes),
599                        format!("timestamp={}", s.timestamp),
600                    ],
601                })
602            } else if max_bytes >= high_threshold {
603                Some(Anomaly {
604                    severity: Severity::Warning,
605                    kind: "high_disk_io".to_owned(),
606                    container: s.container_name.clone(),
607                    message: format!(
608                        "Container '{}' disk I/O at {}/{} (read/write) — warning threshold: {}",
609                        s.container_name,
610                        format_bytes(read),
611                        format_bytes(write),
612                        format_bytes(high_threshold),
613                    ),
614                    evidence: vec![
615                        format!("disk_read={}", read),
616                        format!("disk_write={}", write),
617                        format!("max_bytes={}", max_bytes),
618                        format!("timestamp={}", s.timestamp),
619                    ],
620                })
621            } else {
622                None
623            }
624        })
625        .collect()
626}
627
628/// Detect high network I/O (RX/TX) above thresholds.
629#[must_use]
630pub fn detect_high_network_io(
631    samples: &[MetricSample],
632    high_threshold: u64,
633    critical_threshold: u64,
634) -> Vec<Anomaly> {
635    samples
636        .iter()
637        .filter_map(|s| {
638            let rx = s.network_rx_bytes?;
639            let tx = s.network_tx_bytes?;
640            let max_bytes = rx.max(tx);
641
642            if max_bytes >= critical_threshold {
643                Some(Anomaly {
644                    severity: Severity::Critical,
645                    kind: "high_network_io".to_owned(),
646                    container: s.container_name.clone(),
647                    message: format!(
648                        "Container '{}' network I/O at {}/{} (RX/TX) — critical threshold: {}",
649                        s.container_name,
650                        format_bytes(rx),
651                        format_bytes(tx),
652                        format_bytes(critical_threshold),
653                    ),
654                    evidence: vec![
655                        format!("network_rx={}", rx),
656                        format!("network_tx={}", tx),
657                        format!("max_bytes={}", max_bytes),
658                        format!("timestamp={}", s.timestamp),
659                    ],
660                })
661            } else if max_bytes >= high_threshold {
662                Some(Anomaly {
663                    severity: Severity::Warning,
664                    kind: "high_network_io".to_owned(),
665                    container: s.container_name.clone(),
666                    message: format!(
667                        "Container '{}' network I/O at {}/{} (RX/TX) — warning threshold: {}",
668                        s.container_name,
669                        format_bytes(rx),
670                        format_bytes(tx),
671                        format_bytes(high_threshold),
672                    ),
673                    evidence: vec![
674                        format!("network_rx={}", rx),
675                        format!("network_tx={}", tx),
676                        format!("max_bytes={}", max_bytes),
677                        format!("timestamp={}", s.timestamp),
678                    ],
679                })
680            } else {
681                None
682            }
683        })
684        .collect()
685}
686
687// ── New analysis: Resource Leak Detection ────────────────────────────
688
689/// Detect containers whose memory usage is trending upward over time,
690/// indicating a possible memory leak.
691pub fn detect_resource_leaks<S>(store: &S, thresholds: &AnalysisThresholds) -> Vec<Anomaly>
692where
693    S: TelemetryStore + ?Sized,
694{
695    let Ok(samples) = store.latest_metrics() else {
696        return Vec::new();
697    };
698
699    // Group samples by container ID and sort by timestamp.
700    let mut by_container: std::collections::BTreeMap<String, Vec<&MetricSample>> =
701        std::collections::BTreeMap::new();
702    for sample in &samples {
703        by_container
704            .entry(sample.container_id.clone())
705            .or_default()
706            .push(sample);
707    }
708
709    let mut anomalies = Vec::new();
710
711    // Build a reverse lookup from container_id -> name for display.
712    let id_to_name: std::collections::HashMap<String, String> = samples
713        .iter()
714        .map(|s| (s.container_id.clone(), s.container_name.clone()))
715        .collect();
716
717    for (container_id, container_samples) in &by_container {
718        let container_name = id_to_name
719            .get(container_id)
720            .map_or(container_id.as_str(), std::string::String::as_str);
721        if container_samples.len() < thresholds.resource_leak_min_samples as usize {
722            continue;
723        }
724
725        // Sort by timestamp (chronological order).
726        let mut sorted = container_samples.clone();
727        sorted.sort_by_key(|s| s.timestamp.as_str());
728
729        // Collect memory readings.
730        let mem_values: Vec<u64> = sorted.iter().filter_map(|s| s.memory_usage_bytes).collect();
731
732        if mem_values.len() < 2 {
733            continue;
734        }
735
736        let first = mem_values[0];
737        let last = mem_values[mem_values.len() - 1];
738
739        if first == 0 {
740            continue;
741        }
742
743        let increase_pct = ((last as f64 - first as f64) / first as f64) * 100.0;
744
745        if increase_pct >= thresholds.resource_leak_memory_increase_pct {
746            let severity = if increase_pct >= thresholds.resource_leak_memory_increase_pct * 2.0 {
747                Severity::Critical
748            } else {
749                Severity::Warning
750            };
751
752            anomalies.push(Anomaly {
753                severity,
754                kind: "resource_leak".to_owned(),
755                container: container_name.to_string(),
756                message: format!(
757                    "Container '{}' memory grew {:.0}% (from {} to {}) over {} samples — possible leak",
758                    container_name,
759                    increase_pct,
760                    format_bytes(first),
761                    format_bytes(last),
762                    mem_values.len()
763                ),
764                evidence: vec![
765                    format!("initial_memory={}", first),
766                    format!("latest_memory={}", last),
767                    format!("increase_pct={:.1}", increase_pct),
768                    format!("samples={}", mem_values.len()),
769                ],
770            });
771        }
772    }
773
774    anomalies
775}
776
777// ── New analysis: Dependency Analysis ─────────────────────────────────
778
779/// Analyze container dependencies: which containers share networks,
780/// compose projects, and images.
781pub async fn analyze_dependencies<C>(docker: &C) -> Result<ExecutionResult, AnalyzeError>
782where
783    C: DockerClient + ?Sized,
784{
785    let containers = docker.list_containers().await?;
786    let networks = docker.list_networks().await.unwrap_or_default();
787    let volumes = docker.list_volumes().await.unwrap_or_default();
788
789    let mut rows = Vec::new();
790
791    // 1. Compose project groupings (from labels).
792    let mut by_compose: std::collections::BTreeMap<String, Vec<String>> =
793        std::collections::BTreeMap::new();
794    for c in &containers {
795        let project = c
796            .labels
797            .iter()
798            .find_map(|label| {
799                let parts: Vec<&str> = label.splitn(2, '=').collect();
800                if parts.len() == 2 && parts[0] == "com.docker.compose.project" {
801                    Some(parts[1].to_owned())
802                } else {
803                    None
804                }
805            })
806            .unwrap_or_else(|| "standalone".to_owned());
807        by_compose.entry(project).or_default().push(c.name.clone());
808    }
809
810    for (project, names) in &by_compose {
811        if names.len() > 1 || project != "standalone" {
812            rows.push(Row {
813                fields: BTreeMap::from([
814                    (
815                        "dependency".to_owned(),
816                        JsonValue::String("compose_project".to_owned()),
817                    ),
818                    ("key".to_owned(), JsonValue::String(project.clone())),
819                    (
820                        "containers".to_owned(),
821                        JsonValue::Array(
822                            names.iter().map(|n| JsonValue::String(n.clone())).collect(),
823                        ),
824                    ),
825                    (
826                        "count".to_owned(),
827                        JsonValue::Number(Number::from(names.len())),
828                    ),
829                ]),
830            });
831        }
832    }
833
834    // 2. Network dependencies.
835    for n in &networks {
836        if !n.containers.is_empty() {
837            rows.push(Row {
838                fields: BTreeMap::from([
839                    (
840                        "dependency".to_owned(),
841                        JsonValue::String("network".to_owned()),
842                    ),
843                    ("key".to_owned(), JsonValue::String(n.name.clone())),
844                    (
845                        "containers".to_owned(),
846                        JsonValue::Array(
847                            n.containers
848                                .iter()
849                                .map(|c| JsonValue::String(c.clone()))
850                                .collect(),
851                        ),
852                    ),
853                    (
854                        "count".to_owned(),
855                        JsonValue::Number(Number::from(n.containers.len())),
856                    ),
857                ]),
858            });
859        }
860    }
861
862    // 3. Volume dependencies (by compose project labels).
863    // Volumes don't have container lists, so we note them by driver.
864    for v in &volumes {
865        rows.push(Row {
866            fields: BTreeMap::from([
867                (
868                    "dependency".to_owned(),
869                    JsonValue::String("volume".to_owned()),
870                ),
871                ("key".to_owned(), JsonValue::String(v.name.clone())),
872                ("containers".to_owned(), JsonValue::Array(Vec::new())),
873                ("count".to_owned(), JsonValue::Number(Number::from(0))),
874            ]),
875        });
876    }
877
878    if rows.is_empty() {
879        rows.push(Row {
880            fields: BTreeMap::from([
881                (
882                    "dependency".to_owned(),
883                    JsonValue::String("none".to_owned()),
884                ),
885                ("key".to_owned(), JsonValue::String("-".to_owned())),
886                ("containers".to_owned(), JsonValue::Array(Vec::new())),
887                ("count".to_owned(), JsonValue::Number(Number::from(0))),
888            ]),
889        });
890    }
891
892    Ok(ExecutionResult { rows })
893}
894
895// ── New analysis: Config Drift Detection ───────────────────────────────
896
897/// Detect configuration drift between historical snapshots.
898///
899/// Compares the two most recent snapshots in the store and emits anomalies
900/// for containers whose image, state, or labels have changed.
901pub fn detect_config_drift<S>(store: &S) -> Result<Vec<Anomaly>, AnalyzeError>
902where
903    S: TelemetryStore + ?Sized,
904{
905    // Get all snapshots to find the two most recent.
906    let mut all = store
907        .all_snapshots()
908        .map_err(|_| AnalyzeError::Unsupported)?;
909
910    if all.is_empty() {
911        return Err(AnalyzeError::Unsupported);
912    }
913
914    // Snapshots are sorted by timestamp ascending.
915    let latest = all.pop().expect("at least one snapshot after empty check");
916    let previous = all.pop();
917
918    // If there's only one snapshot, emit baselines.
919    // Otherwise, compare the two most recent.
920
921    let mut anomalies = Vec::new();
922
923    match previous {
924        Some(prev) => {
925            // We have two snapshots — compare them.
926            for container in &latest.containers {
927                let prev_container = prev.containers.iter().find(|c| c.name == container.name);
928
929                match prev_container {
930                    Some(prev_c) => {
931                        // Check for image drift.
932                        if prev_c.image != container.image {
933                            anomalies.push(Anomaly {
934                                severity: Severity::Warning,
935                                kind: "config_drift".to_owned(),
936                                container: container.name.clone(),
937                                message: format!(
938                                    "Container '{}' image changed from '{}' to '{}'",
939                                    container.name, prev_c.image, container.image
940                                ),
941                                evidence: vec![
942                                    format!("previous_image={}", prev_c.image),
943                                    format!("current_image={}", container.image),
944                                ],
945                            });
946                        }
947
948                        // Check for state drift.
949                        if prev_c.state != container.state {
950                            let severity = match container.state.as_str() {
951                                "running" => Severity::Info,
952                                _ => Severity::Warning,
953                            };
954                            anomalies.push(Anomaly {
955                                severity,
956                                kind: "state_change".to_owned(),
957                                container: container.name.clone(),
958                                message: format!(
959                                    "Container '{}' state changed from '{}' to '{}'",
960                                    container.name, prev_c.state, container.state
961                                ),
962                                evidence: vec![
963                                    format!("previous_state={}", prev_c.state),
964                                    format!("current_state={}", container.state),
965                                ],
966                            });
967                        }
968
969                        // Check for restart count drift.
970                        let prev_restarts = prev_c.restart_count.unwrap_or(0);
971                        let curr_restarts = container.restart_count.unwrap_or(0);
972                        if curr_restarts > prev_restarts {
973                            anomalies.push(Anomaly {
974                                severity: Severity::Warning,
975                                kind: "restart_increase".to_owned(),
976                                container: container.name.clone(),
977                                message: format!(
978                                    "Container '{}' restart count increased from {} to {}",
979                                    container.name, prev_restarts, curr_restarts
980                                ),
981                                evidence: vec![
982                                    format!("previous_restarts={}", prev_restarts),
983                                    format!("current_restarts={}", curr_restarts),
984                                ],
985                            });
986                        }
987
988                        // Check for label drift.
989                        let prev_labels: std::collections::BTreeSet<&str> = prev_c
990                            .labels
991                            .iter()
992                            .map(std::string::String::as_str)
993                            .collect();
994                        let curr_labels: std::collections::BTreeSet<&str> = container
995                            .labels
996                            .iter()
997                            .map(std::string::String::as_str)
998                            .collect();
999                        let added: Vec<&&str> = curr_labels.difference(&prev_labels).collect();
1000                        let removed: Vec<&&str> = prev_labels.difference(&curr_labels).collect();
1001                        if !added.is_empty() || !removed.is_empty() {
1002                            anomalies.push(Anomaly {
1003                                severity: Severity::Info,
1004                                kind: "label_drift".to_owned(),
1005                                container: container.name.clone(),
1006                                message: format!(
1007                                    "Container '{}' labels changed: added={:?}, removed={:?}",
1008                                    container.name, added, removed
1009                                ),
1010                                evidence: vec![
1011                                    format!("added_labels={:?}", added),
1012                                    format!("removed_labels={:?}", removed),
1013                                ],
1014                            });
1015                        }
1016                    }
1017                    None => {
1018                        // New container appeared.
1019                        anomalies.push(Anomaly {
1020                            severity: Severity::Info,
1021                            kind: "container_appeared".to_owned(),
1022                            container: container.name.clone(),
1023                            message: format!(
1024                                "Container '{}' appeared with image={}, state={}",
1025                                container.name, container.image, container.state
1026                            ),
1027                            evidence: vec![
1028                                format!("image={}", container.image),
1029                                format!("state={}", container.state),
1030                            ],
1031                        });
1032                    }
1033                }
1034            }
1035
1036            // Check for containers that disappeared.
1037            for prev_c in &prev.containers {
1038                if !latest.containers.iter().any(|c| c.name == prev_c.name) {
1039                    anomalies.push(Anomaly {
1040                        severity: Severity::Warning,
1041                        kind: "container_disappeared".to_owned(),
1042                        container: prev_c.name.clone(),
1043                        message: format!(
1044                            "Container '{}' was present in previous snapshot but is now gone",
1045                            prev_c.name
1046                        ),
1047                        evidence: vec![
1048                            format!("previous_state={}", prev_c.state),
1049                            format!("previous_image={}", prev_c.image),
1050                        ],
1051                    });
1052                }
1053            }
1054        }
1055        None => {
1056            // Only one snapshot — report it as baseline.
1057            for container in &latest.containers {
1058                anomalies.push(Anomaly {
1059                    severity: Severity::Info,
1060                    kind: "config_baseline".to_owned(),
1061                    container: container.name.clone(),
1062                    message: format!(
1063                        "Container '{}' baseline — image={}, state={}",
1064                        container.name, container.image, container.state
1065                    ),
1066                    evidence: vec![
1067                        format!("image={}", container.image),
1068                        format!("state={}", container.state),
1069                        format!("restart_count={}", container.restart_count.unwrap_or(0)),
1070                    ],
1071                });
1072            }
1073        }
1074    }
1075
1076    Ok(anomalies)
1077}
1078
1079// ── New analysis: Density Analysis ─────────────────────────────────────
1080
1081/// Analyze container density / distribution across images, states,
1082/// and compose projects.
1083pub async fn analyze_density<C>(docker: &C) -> Result<ExecutionResult, AnalyzeError>
1084where
1085    C: DockerClient + ?Sized,
1086{
1087    let containers = docker.list_containers().await?;
1088    let total = containers.len();
1089    let mut rows = Vec::new();
1090
1091    // 1. By image.
1092    let mut by_image: std::collections::BTreeMap<String, Vec<String>> =
1093        std::collections::BTreeMap::new();
1094    for c in &containers {
1095        by_image
1096            .entry(c.image.clone())
1097            .or_default()
1098            .push(c.name.clone());
1099    }
1100    for (image, names) in &by_image {
1101        let pct = if total > 0 {
1102            (names.len() as f64 / total as f64) * 100.0
1103        } else {
1104            0.0
1105        };
1106        rows.push(Row {
1107            fields: BTreeMap::from([
1108                (
1109                    "dimension".to_owned(),
1110                    JsonValue::String("image".to_owned()),
1111                ),
1112                ("value".to_owned(), JsonValue::String(image.clone())),
1113                (
1114                    "container_count".to_owned(),
1115                    JsonValue::Number(Number::from(names.len())),
1116                ),
1117                (
1118                    "density_pct".to_owned(),
1119                    JsonValue::Number(
1120                        Number::from_f64((pct * 10.0).round() / 10.0)
1121                            .unwrap_or_else(|| Number::from(0)),
1122                    ),
1123                ),
1124            ]),
1125        });
1126    }
1127
1128    // 2. By state.
1129    let mut by_state: std::collections::BTreeMap<String, u64> = std::collections::BTreeMap::new();
1130    for c in &containers {
1131        *by_state.entry(c.state.clone()).or_default() += 1;
1132    }
1133    for (state, count) in &by_state {
1134        let pct = if total > 0 {
1135            (*count as f64 / total as f64) * 100.0
1136        } else {
1137            0.0
1138        };
1139        rows.push(Row {
1140            fields: BTreeMap::from([
1141                (
1142                    "dimension".to_owned(),
1143                    JsonValue::String("state".to_owned()),
1144                ),
1145                ("value".to_owned(), JsonValue::String(state.clone())),
1146                (
1147                    "container_count".to_owned(),
1148                    JsonValue::Number(Number::from(*count)),
1149                ),
1150                (
1151                    "density_pct".to_owned(),
1152                    JsonValue::Number(
1153                        Number::from_f64((pct * 10.0).round() / 10.0)
1154                            .unwrap_or_else(|| Number::from(0)),
1155                    ),
1156                ),
1157            ]),
1158        });
1159    }
1160
1161    // 3. By compose project.
1162    let mut by_project: std::collections::BTreeMap<String, Vec<String>> =
1163        std::collections::BTreeMap::new();
1164    for c in &containers {
1165        let project = c
1166            .labels
1167            .iter()
1168            .find_map(|label| {
1169                let parts: Vec<&str> = label.splitn(2, '=').collect();
1170                if parts.len() == 2 && parts[0] == "com.docker.compose.project" {
1171                    Some(parts[1].to_owned())
1172                } else {
1173                    None
1174                }
1175            })
1176            .unwrap_or_else(|| "standalone".to_owned());
1177        by_project.entry(project).or_default().push(c.name.clone());
1178    }
1179    for (project, names) in &by_project {
1180        let pct = if total > 0 {
1181            (names.len() as f64 / total as f64) * 100.0
1182        } else {
1183            0.0
1184        };
1185        rows.push(Row {
1186            fields: BTreeMap::from([
1187                (
1188                    "dimension".to_owned(),
1189                    JsonValue::String("compose_project".to_owned()),
1190                ),
1191                ("value".to_owned(), JsonValue::String(project.clone())),
1192                (
1193                    "container_count".to_owned(),
1194                    JsonValue::Number(Number::from(names.len())),
1195                ),
1196                (
1197                    "density_pct".to_owned(),
1198                    JsonValue::Number(
1199                        Number::from_f64((pct * 10.0).round() / 10.0)
1200                            .unwrap_or_else(|| Number::from(0)),
1201                    ),
1202                ),
1203            ]),
1204        });
1205    }
1206
1207    // Total summary.
1208    rows.push(Row {
1209        fields: BTreeMap::from([
1210            (
1211                "dimension".to_owned(),
1212                JsonValue::String("total".to_owned()),
1213            ),
1214            ("value".to_owned(), JsonValue::String("-".to_owned())),
1215            (
1216                "container_count".to_owned(),
1217                JsonValue::Number(Number::from(total)),
1218            ),
1219            (
1220                "density_pct".to_owned(),
1221                JsonValue::Number(Number::from(100)),
1222            ),
1223        ]),
1224    });
1225
1226    Ok(ExecutionResult { rows })
1227}
1228
1229// ── Composite analysis functions ───────────────────────────────────────────
1230
1231/// `analyze containers find anomalies` — runs all detectors on live data.
1232async fn find_container_anomalies<C, M>(
1233    docker: &C,
1234    metrics: &M,
1235    thresholds: &AnalysisThresholds,
1236) -> Result<ExecutionResult, AnalyzeError>
1237where
1238    C: DockerClient + ?Sized,
1239    M: MetricsCollector + ?Sized,
1240{
1241    let containers = docker.list_containers().await?;
1242    let samples = metrics.collect().await?;
1243
1244    let mut anomalies = Vec::new();
1245    anomalies.extend(detect_restart_loops(
1246        &containers,
1247        thresholds.restart_loop_count,
1248    ));
1249    anomalies.extend(detect_high_cpu(
1250        &samples,
1251        thresholds.high_cpu_percent,
1252        thresholds.critical_cpu_percent,
1253    ));
1254    anomalies.extend(detect_memory_pressure(
1255        &samples,
1256        thresholds.memory_pressure_ratio,
1257        thresholds.critical_memory_ratio,
1258    ));
1259    anomalies.extend(detect_unhealthy_states(&containers));
1260    anomalies.extend(detect_high_disk_io(
1261        &samples,
1262        thresholds.high_disk_bytes,
1263        thresholds.critical_disk_bytes,
1264    ));
1265    anomalies.extend(detect_high_network_io(
1266        &samples,
1267        thresholds.high_network_bytes,
1268        thresholds.critical_network_bytes,
1269    ));
1270
1271    // Sort by severity (Critical first).
1272    anomalies.sort_by_key(|a| std::cmp::Reverse(severity_rank(a.severity)));
1273
1274    if anomalies.is_empty() {
1275        Ok(ExecutionResult {
1276            rows: vec![Row {
1277                fields: BTreeMap::from([
1278                    ("severity".to_owned(), JsonValue::String("info".to_owned())),
1279                    ("kind".to_owned(), JsonValue::String("healthy".to_owned())),
1280                    ("container".to_owned(), JsonValue::String("*".to_owned())),
1281                    (
1282                        "message".to_owned(),
1283                        JsonValue::String("No anomalies detected".to_owned()),
1284                    ),
1285                    ("evidence".to_owned(), JsonValue::Array(Vec::new())),
1286                ]),
1287            }],
1288        })
1289    } else {
1290        Ok(ExecutionResult {
1291            rows: anomalies.iter().map(Anomaly::to_row).collect(),
1292        })
1293    }
1294}
1295
1296/// `analyze containers find anomalies` using historical store data.
1297fn find_anomalies_from_store<S>(
1298    store: &S,
1299    thresholds: &AnalysisThresholds,
1300) -> Result<ExecutionResult, AnalyzeError>
1301where
1302    S: TelemetryStore + ?Sized,
1303{
1304    let samples = store.latest_metrics()?;
1305    let events = store.events_between("", "\u{10ffff}")?;
1306
1307    let mut anomalies = Vec::new();
1308    anomalies.extend(detect_high_cpu(
1309        &samples,
1310        thresholds.high_cpu_percent,
1311        thresholds.critical_cpu_percent,
1312    ));
1313    anomalies.extend(detect_memory_pressure(
1314        &samples,
1315        thresholds.memory_pressure_ratio,
1316        thresholds.critical_memory_ratio,
1317    ));
1318    anomalies.extend(detect_deployment_errors(
1319        &events,
1320        thresholds.deployment_error_threshold,
1321    ));
1322    anomalies.extend(detect_high_disk_io(
1323        &samples,
1324        thresholds.high_disk_bytes,
1325        thresholds.critical_disk_bytes,
1326    ));
1327    anomalies.extend(detect_high_network_io(
1328        &samples,
1329        thresholds.high_network_bytes,
1330        thresholds.critical_network_bytes,
1331    ));
1332    anomalies.extend(detect_resource_leaks(store, thresholds));
1333    // Config drift requires snapshots — skip silently if unavailable
1334    if let Ok(drift) = detect_config_drift(store) {
1335        anomalies.extend(drift);
1336    }
1337
1338    anomalies.sort_by_key(|a| std::cmp::Reverse(severity_rank(a.severity)));
1339
1340    if anomalies.is_empty() {
1341        Ok(ExecutionResult {
1342            rows: vec![Row {
1343                fields: BTreeMap::from([
1344                    ("severity".to_owned(), JsonValue::String("info".to_owned())),
1345                    ("kind".to_owned(), JsonValue::String("healthy".to_owned())),
1346                    ("container".to_owned(), JsonValue::String("*".to_owned())),
1347                    (
1348                        "message".to_owned(),
1349                        JsonValue::String("No anomalies detected in stored data".to_owned()),
1350                    ),
1351                    ("evidence".to_owned(), JsonValue::Array(Vec::new())),
1352                ]),
1353            }],
1354        })
1355    } else {
1356        Ok(ExecutionResult {
1357            rows: anomalies.iter().map(Anomaly::to_row).collect(),
1358        })
1359    }
1360}
1361
1362/// `analyze containers correlate` — find containers sharing images/networks.
1363async fn correlate_containers<C, M>(
1364    docker: &C,
1365    _metrics: &M,
1366    _thresholds: &AnalysisThresholds,
1367) -> Result<ExecutionResult, AnalyzeError>
1368where
1369    C: DockerClient + ?Sized,
1370    M: MetricsCollector + ?Sized,
1371{
1372    let containers = docker.list_containers().await?;
1373
1374    // Group containers by image.
1375    let mut by_image: HashMap<String, Vec<String>> = HashMap::new();
1376    for c in &containers {
1377        by_image
1378            .entry(c.image.clone())
1379            .or_default()
1380            .push(c.name.clone());
1381    }
1382
1383    let mut rows = Vec::new();
1384
1385    for (image, names) in &by_image {
1386        if names.len() > 1 {
1387            rows.push(Row {
1388                fields: BTreeMap::from([
1389                    (
1390                        "correlation".to_owned(),
1391                        JsonValue::String("shared_image".to_owned()),
1392                    ),
1393                    ("key".to_owned(), JsonValue::String(image.clone())),
1394                    (
1395                        "containers".to_owned(),
1396                        JsonValue::Array(
1397                            names.iter().map(|n| JsonValue::String(n.clone())).collect(),
1398                        ),
1399                    ),
1400                    (
1401                        "count".to_owned(),
1402                        JsonValue::Number(Number::from(names.len())),
1403                    ),
1404                ]),
1405            });
1406        }
1407    }
1408
1409    // Group by shared labels.
1410    let mut by_label: HashMap<String, Vec<String>> = HashMap::new();
1411    for c in &containers {
1412        for label in &c.labels {
1413            by_label
1414                .entry(label.clone())
1415                .or_default()
1416                .push(c.name.clone());
1417        }
1418    }
1419
1420    for (label, names) in &by_label {
1421        if names.len() > 1 {
1422            rows.push(Row {
1423                fields: BTreeMap::from([
1424                    (
1425                        "correlation".to_owned(),
1426                        JsonValue::String("shared_label".to_owned()),
1427                    ),
1428                    ("key".to_owned(), JsonValue::String(label.clone())),
1429                    (
1430                        "containers".to_owned(),
1431                        JsonValue::Array(
1432                            names.iter().map(|n| JsonValue::String(n.clone())).collect(),
1433                        ),
1434                    ),
1435                    (
1436                        "count".to_owned(),
1437                        JsonValue::Number(Number::from(names.len())),
1438                    ),
1439                ]),
1440            });
1441        }
1442    }
1443
1444    if rows.is_empty() {
1445        rows.push(Row {
1446            fields: BTreeMap::from([
1447                (
1448                    "correlation".to_owned(),
1449                    JsonValue::String("none".to_owned()),
1450                ),
1451                ("key".to_owned(), JsonValue::String("-".to_owned())),
1452                ("containers".to_owned(), JsonValue::Array(Vec::new())),
1453                ("count".to_owned(), JsonValue::Number(Number::from(0))),
1454            ]),
1455        });
1456    }
1457
1458    Ok(ExecutionResult { rows })
1459}
1460
1461/// `explain container <name>` — produce a diagnostic summary for one container.
1462async fn explain_container<C, M>(
1463    name: &str,
1464    docker: &C,
1465    metrics: &M,
1466    thresholds: &AnalysisThresholds,
1467) -> Result<ExecutionResult, AnalyzeError>
1468where
1469    C: DockerClient + ?Sized,
1470    M: MetricsCollector + ?Sized,
1471{
1472    let containers = docker.list_containers().await?;
1473    let container = containers
1474        .iter()
1475        .find(|c| c.name == name || c.id == name)
1476        .ok_or(AnalyzeError::Unsupported)?;
1477
1478    let samples = metrics.collect().await?;
1479    let sample = samples
1480        .iter()
1481        .find(|s| s.container_name == name || s.container_id == name);
1482
1483    let explanation = build_explanation(container, sample, thresholds);
1484
1485    Ok(explanation_to_result(&explanation))
1486}
1487
1488/// `analyze containers explain` — produce diagnostic summaries for all containers.
1489async fn explain_all_containers<C, M>(
1490    docker: &C,
1491    metrics: &M,
1492    thresholds: &AnalysisThresholds,
1493) -> Result<ExecutionResult, AnalyzeError>
1494where
1495    C: DockerClient + ?Sized,
1496    M: MetricsCollector + ?Sized,
1497{
1498    let containers = docker.list_containers().await?;
1499    let samples = metrics.collect().await?;
1500    let samples_by_name: HashMap<&str, &MetricSample> = samples
1501        .iter()
1502        .map(|s| (s.container_name.as_str(), s))
1503        .collect();
1504
1505    let mut rows = Vec::new();
1506    for container in &containers {
1507        let sample = samples_by_name.get(container.name.as_str()).copied();
1508        let explanation = build_explanation(container, sample, thresholds);
1509        let result = explanation_to_result(&explanation);
1510        rows.extend(result.rows);
1511    }
1512
1513    if rows.is_empty() {
1514        rows.push(Row {
1515            fields: BTreeMap::from([
1516                ("container".to_owned(), JsonValue::String("-".to_owned())),
1517                ("state".to_owned(), JsonValue::String("-".to_owned())),
1518                ("signal".to_owned(), JsonValue::String("none".to_owned())),
1519                ("value".to_owned(), JsonValue::String("-".to_owned())),
1520                ("status".to_owned(), JsonValue::String("normal".to_owned())),
1521                (
1522                    "anomaly_count".to_owned(),
1523                    JsonValue::Number(Number::from(0)),
1524                ),
1525            ]),
1526        });
1527    }
1528
1529    Ok(ExecutionResult { rows })
1530}
1531
1532// ── Helpers ────────────────────────────────────────────────────────────────
1533
1534fn build_explanation(
1535    container: &Container,
1536    sample: Option<&MetricSample>,
1537    thresholds: &AnalysisThresholds,
1538) -> ContainerExplanation {
1539    let mut signals = Vec::new();
1540    let mut anomalies = Vec::new();
1541
1542    // State signal.
1543    let state_status = match container.state.as_str() {
1544        "running" => SignalStatus::Normal,
1545        "restarting" => SignalStatus::Elevated,
1546        _ => SignalStatus::Critical,
1547    };
1548    signals.push(Signal {
1549        name: "state".to_owned(),
1550        value: container.state.clone(),
1551        status: state_status,
1552    });
1553
1554    // Restart count signal.
1555    if let Some(count) = container.restart_count {
1556        let status = if count >= thresholds.restart_loop_count * 2 {
1557            SignalStatus::Critical
1558        } else if count >= thresholds.restart_loop_count {
1559            SignalStatus::Elevated
1560        } else {
1561            SignalStatus::Normal
1562        };
1563        signals.push(Signal {
1564            name: "restart_count".to_owned(),
1565            value: count.to_string(),
1566            status,
1567        });
1568
1569        if count >= thresholds.restart_loop_count {
1570            anomalies.extend(detect_restart_loops(
1571                std::slice::from_ref(container),
1572                thresholds.restart_loop_count,
1573            ));
1574        }
1575    }
1576
1577    // Metric signals.
1578    if let Some(s) = sample {
1579        if let Some(cpu) = s.cpu_percent {
1580            let status = if cpu >= thresholds.critical_cpu_percent {
1581                SignalStatus::Critical
1582            } else if cpu >= thresholds.high_cpu_percent {
1583                SignalStatus::Elevated
1584            } else {
1585                SignalStatus::Normal
1586            };
1587            signals.push(Signal {
1588                name: "cpu".to_owned(),
1589                value: format!("{cpu:.1}%"),
1590                status,
1591            });
1592        }
1593
1594        if let (Some(usage), Some(limit)) = (s.memory_usage_bytes, s.memory_limit_bytes) {
1595            let ratio = if limit > 0 {
1596                usage as f64 / limit as f64
1597            } else {
1598                0.0
1599            };
1600            let status = if ratio >= thresholds.critical_memory_ratio {
1601                SignalStatus::Critical
1602            } else if ratio >= thresholds.memory_pressure_ratio {
1603                SignalStatus::Elevated
1604            } else {
1605                SignalStatus::Normal
1606            };
1607            signals.push(Signal {
1608                name: "memory".to_owned(),
1609                value: format!("{:.1}%", ratio * 100.0),
1610                status,
1611            });
1612        }
1613
1614        if let Some(rx) = s.network_rx_bytes {
1615            signals.push(Signal {
1616                name: "network_rx".to_owned(),
1617                value: format_bytes(rx),
1618                status: SignalStatus::Normal,
1619            });
1620        }
1621
1622        if let Some(tx) = s.network_tx_bytes {
1623            signals.push(Signal {
1624                name: "network_tx".to_owned(),
1625                value: format_bytes(tx),
1626                status: SignalStatus::Normal,
1627            });
1628        }
1629
1630        // Also run metric detectors for anomalies.
1631        anomalies.extend(detect_high_cpu(
1632            std::slice::from_ref(s),
1633            thresholds.high_cpu_percent,
1634            thresholds.critical_cpu_percent,
1635        ));
1636        anomalies.extend(detect_memory_pressure(
1637            std::slice::from_ref(s),
1638            thresholds.memory_pressure_ratio,
1639            thresholds.critical_memory_ratio,
1640        ));
1641    }
1642
1643    // Unhealthy state.
1644    anomalies.extend(detect_unhealthy_states(std::slice::from_ref(container)));
1645
1646    ContainerExplanation {
1647        container: container.name.clone(),
1648        state: container.state.clone(),
1649        anomalies,
1650        signals,
1651    }
1652}
1653
1654fn explanation_to_result(explanation: &ContainerExplanation) -> ExecutionResult {
1655    let anomaly_count = explanation.anomalies.len();
1656    let rows = explanation
1657        .signals
1658        .iter()
1659        .map(|signal| Row {
1660            fields: BTreeMap::from([
1661                (
1662                    "container".to_owned(),
1663                    JsonValue::String(explanation.container.clone()),
1664                ),
1665                (
1666                    "state".to_owned(),
1667                    JsonValue::String(explanation.state.clone()),
1668                ),
1669                ("signal".to_owned(), JsonValue::String(signal.name.clone())),
1670                ("value".to_owned(), JsonValue::String(signal.value.clone())),
1671                (
1672                    "status".to_owned(),
1673                    JsonValue::String(signal.status.to_string()),
1674                ),
1675                (
1676                    "anomaly_count".to_owned(),
1677                    JsonValue::Number(Number::from(anomaly_count)),
1678                ),
1679            ]),
1680        })
1681        .collect();
1682
1683    ExecutionResult { rows }
1684}
1685
1686const fn severity_rank(severity: Severity) -> u8 {
1687    match severity {
1688        Severity::Info => 0,
1689        Severity::Warning => 1,
1690        Severity::Critical => 2,
1691    }
1692}
1693
1694fn format_bytes(bytes: u64) -> String {
1695    if bytes >= ONE_GB {
1696        format!("{:.1} GB", bytes as f64 / ONE_GB as f64)
1697    } else if bytes >= 1_048_576 {
1698        format!("{:.1} MB", bytes as f64 / 1_048_576.0)
1699    } else if bytes >= 1024 {
1700        format!("{:.1} KB", bytes as f64 / 1024.0)
1701    } else {
1702        format!("{bytes} B")
1703    }
1704}
1705
1706// ── Tests ──────────────────────────────────────────────────────────────────
1707
1708#[cfg(test)]
1709mod tests {
1710    use super::*;
1711    use crate::docker::Container;
1712
1713    fn rt() -> tokio::runtime::Runtime {
1714        tokio::runtime::Runtime::new().unwrap()
1715    }
1716
1717    fn test_container(name: &str, state: &str, restarts: u64) -> Container {
1718        Container {
1719            id: format!("{name}_id"),
1720            name: name.to_owned(),
1721            image: format!("{name}:latest"),
1722            status: if state == "running" {
1723                "Up 2 minutes".to_owned()
1724            } else {
1725                "Exited (1) 1 minute ago".to_string()
1726            },
1727            state: state.to_owned(),
1728            ports: Vec::new(),
1729            labels: vec!["env=prod".to_owned()],
1730            created_at: None,
1731            started_at: None,
1732            finished_at: None,
1733            restart_count: Some(restarts),
1734            health: None,
1735        }
1736    }
1737
1738    fn test_sample(name: &str, cpu: f64, mem_usage: u64, mem_limit: u64) -> MetricSample {
1739        MetricSample {
1740            container_id: format!("{name}_id"),
1741            container_name: name.to_owned(),
1742            timestamp: "2026-01-01T12:00:00Z".to_owned(),
1743            cpu_percent: Some(cpu),
1744            memory_usage_bytes: Some(mem_usage),
1745            memory_limit_bytes: Some(mem_limit),
1746            network_rx_bytes: Some(1024),
1747            network_tx_bytes: Some(2048),
1748            disk_read_bytes: Some(0),
1749            disk_write_bytes: Some(0),
1750        }
1751    }
1752
1753    fn test_event(time: &str, action: &str, container: &str) -> DockerEvent {
1754        DockerEvent {
1755            time: time.to_owned(),
1756            event_type: "container".to_owned(),
1757            action: action.to_owned(),
1758            actor_id: format!("{container}_id"),
1759            container: Some(container.to_owned()),
1760            image: Some(format!("{container}:latest")),
1761            attributes: vec![("name".to_owned(), container.to_owned())],
1762        }
1763    }
1764
1765    // ── Restart loop tests ─────────────────────────────────────────────
1766
1767    #[test]
1768    fn detects_restart_loop() {
1769        let containers = vec![
1770            test_container("api", "running", 5),
1771            test_container("worker", "running", 0),
1772        ];
1773
1774        let anomalies = detect_restart_loops(&containers, 3);
1775
1776        assert_eq!(anomalies.len(), 1);
1777        assert_eq!(anomalies[0].container, "api");
1778        assert_eq!(anomalies[0].kind, "restart_loop");
1779        assert_eq!(anomalies[0].severity, Severity::Warning);
1780    }
1781
1782    #[test]
1783    fn detects_critical_restart_loop() {
1784        let containers = vec![test_container("api", "running", 10)];
1785        let anomalies = detect_restart_loops(&containers, 3);
1786
1787        assert_eq!(anomalies.len(), 1);
1788        assert_eq!(anomalies[0].severity, Severity::Critical);
1789    }
1790
1791    #[test]
1792    fn no_restart_loop_below_threshold() {
1793        let containers = vec![test_container("api", "running", 2)];
1794        let anomalies = detect_restart_loops(&containers, 3);
1795
1796        assert!(anomalies.is_empty());
1797    }
1798
1799    // ── High CPU tests ─────────────────────────────────────────────────
1800
1801    #[test]
1802    fn detects_high_cpu_warning() {
1803        let samples = vec![test_sample("api", 85.0, 512, 1024)];
1804        let anomalies = detect_high_cpu(&samples, 80.0, 95.0);
1805
1806        assert_eq!(anomalies.len(), 1);
1807        assert_eq!(anomalies[0].severity, Severity::Warning);
1808        assert_eq!(anomalies[0].kind, "high_cpu");
1809    }
1810
1811    #[test]
1812    fn detects_critical_cpu() {
1813        let samples = vec![test_sample("api", 98.0, 512, 1024)];
1814        let anomalies = detect_high_cpu(&samples, 80.0, 95.0);
1815
1816        assert_eq!(anomalies.len(), 1);
1817        assert_eq!(anomalies[0].severity, Severity::Critical);
1818    }
1819
1820    #[test]
1821    fn no_cpu_anomaly_below_threshold() {
1822        let samples = vec![test_sample("api", 50.0, 512, 1024)];
1823        let anomalies = detect_high_cpu(&samples, 80.0, 95.0);
1824
1825        assert!(anomalies.is_empty());
1826    }
1827
1828    // ── Memory pressure tests ──────────────────────────────────────────
1829
1830    #[test]
1831    fn detects_memory_pressure_warning() {
1832        let samples = vec![test_sample("api", 10.0, 900, 1024)]; // ~87.9%
1833        let anomalies = detect_memory_pressure(&samples, 0.85, 0.95);
1834
1835        assert_eq!(anomalies.len(), 1);
1836        assert_eq!(anomalies[0].severity, Severity::Warning);
1837        assert_eq!(anomalies[0].kind, "memory_pressure");
1838    }
1839
1840    #[test]
1841    fn detects_critical_memory_pressure() {
1842        let samples = vec![test_sample("api", 10.0, 980, 1024)]; // ~95.7%
1843        let anomalies = detect_memory_pressure(&samples, 0.85, 0.95);
1844
1845        assert_eq!(anomalies.len(), 1);
1846        assert_eq!(anomalies[0].severity, Severity::Critical);
1847    }
1848
1849    #[test]
1850    fn no_memory_pressure_below_threshold() {
1851        let samples = vec![test_sample("api", 10.0, 500, 1024)]; // ~48.8%
1852        let anomalies = detect_memory_pressure(&samples, 0.85, 0.95);
1853
1854        assert!(anomalies.is_empty());
1855    }
1856
1857    // ── Deployment errors tests ────────────────────────────────────────
1858
1859    #[test]
1860    fn detects_deployment_errors() {
1861        let events = vec![
1862            test_event("2026-01-01T12:00:00Z", "die", "api"),
1863            test_event("2026-01-01T12:01:00Z", "die", "api"),
1864            test_event("2026-01-01T12:02:00Z", "die", "api"),
1865            test_event("2026-01-01T12:03:00Z", "start", "api"),
1866        ];
1867
1868        let anomalies = detect_deployment_errors(&events, 3);
1869
1870        assert_eq!(anomalies.len(), 1);
1871        assert_eq!(anomalies[0].container, "api");
1872        assert_eq!(anomalies[0].kind, "deployment_errors");
1873    }
1874
1875    #[test]
1876    fn no_deployment_errors_below_threshold() {
1877        let events = vec![
1878            test_event("2026-01-01T12:00:00Z", "die", "api"),
1879            test_event("2026-01-01T12:01:00Z", "start", "api"),
1880        ];
1881
1882        let anomalies = detect_deployment_errors(&events, 3);
1883
1884        assert!(anomalies.is_empty());
1885    }
1886
1887    // ── Unhealthy state tests ──────────────────────────────────────────
1888
1889    // ── Disk I/O tests ─────────────────────────────────────────────
1890
1891    #[test]
1892    fn detects_high_disk_io_warning() {
1893        let samples = vec![test_sample("api", 10.0, 128, 1024)]; // default disk=0, below threshold
1894        let high_samples = vec![MetricSample {
1895            disk_read_bytes: Some(20_000_000), // ~19 MB, above 10 MB warning
1896            disk_write_bytes: Some(5_000_000),
1897            ..test_sample("api", 10.0, 128, 1024)
1898        }];
1899
1900        let anomalies = detect_high_disk_io(&high_samples, 10_485_760, 104_857_600);
1901        assert_eq!(anomalies.len(), 1);
1902        assert_eq!(anomalies[0].severity, Severity::Warning);
1903        assert_eq!(anomalies[0].kind, "high_disk_io");
1904        assert_eq!(anomalies[0].container, "api");
1905
1906        // Low I/O should produce nothing
1907        let anomalies = detect_high_disk_io(&samples, 10_485_760, 104_857_600);
1908        assert!(anomalies.is_empty());
1909    }
1910
1911    #[test]
1912    fn detects_critical_disk_io() {
1913        let samples = vec![MetricSample {
1914            disk_write_bytes: Some(200_000_000), // ~191 MB, above 100 MB critical
1915            disk_read_bytes: Some(10_000_000),
1916            ..test_sample("api", 10.0, 128, 1024)
1917        }];
1918
1919        let anomalies = detect_high_disk_io(&samples, 10_485_760, 104_857_600);
1920        assert_eq!(anomalies.len(), 1);
1921        assert_eq!(anomalies[0].severity, Severity::Critical);
1922        assert_eq!(anomalies[0].kind, "high_disk_io");
1923    }
1924
1925    #[test]
1926    fn no_disk_io_anomaly_when_none_supplied() {
1927        let samples = vec![MetricSample {
1928            disk_read_bytes: None,
1929            disk_write_bytes: None,
1930            ..test_sample("api", 10.0, 128, 1024)
1931        }];
1932        let anomalies = detect_high_disk_io(&samples, 10_485_760, 104_857_600);
1933        assert!(anomalies.is_empty());
1934    }
1935
1936    // ── Network I/O tests ─────────────────────────────────────────────
1937
1938    #[test]
1939    fn detects_high_network_io_warning() {
1940        let samples = vec![test_sample("api", 10.0, 128, 1024)]; // default net=1024/2048, below threshold
1941        let high_samples = vec![MetricSample {
1942            network_rx_bytes: Some(5_000_000), // ~4.8 MB, above 1 MB warning
1943            network_tx_bytes: Some(2_000_000),
1944            ..test_sample("api", 10.0, 128, 1024)
1945        }];
1946
1947        let anomalies = detect_high_network_io(&high_samples, 1_048_576, 10_485_760);
1948        assert_eq!(anomalies.len(), 1);
1949        assert_eq!(anomalies[0].severity, Severity::Warning);
1950        assert_eq!(anomalies[0].kind, "high_network_io");
1951        assert_eq!(anomalies[0].container, "api");
1952
1953        // Low I/O should produce nothing
1954        let anomalies = detect_high_network_io(&samples, 1_048_576, 10_485_760);
1955        assert!(anomalies.is_empty());
1956    }
1957
1958    #[test]
1959    fn detects_critical_network_io() {
1960        let samples = vec![MetricSample {
1961            network_rx_bytes: Some(50_000_000), // ~47.7 MB, above 10 MB critical
1962            network_tx_bytes: Some(1_000_000),
1963            ..test_sample("api", 10.0, 128, 1024)
1964        }];
1965
1966        let anomalies = detect_high_network_io(&samples, 1_048_576, 10_485_760);
1967        assert_eq!(anomalies.len(), 1);
1968        assert_eq!(anomalies[0].severity, Severity::Critical);
1969        assert_eq!(anomalies[0].kind, "high_network_io");
1970    }
1971
1972    #[test]
1973    fn no_network_io_anomaly_when_none_supplied() {
1974        let samples = vec![MetricSample {
1975            network_rx_bytes: None,
1976            network_tx_bytes: None,
1977            ..test_sample("api", 10.0, 128, 1024)
1978        }];
1979        let anomalies = detect_high_network_io(&samples, 1_048_576, 10_485_760);
1980        assert!(anomalies.is_empty());
1981    }
1982
1983    #[test]
1984    fn detects_unhealthy_states() {
1985        let containers = vec![
1986            test_container("api", "running", 0),
1987            test_container("worker", "exited", 0),
1988            test_container("db", "dead", 0),
1989        ];
1990
1991        let anomalies = detect_unhealthy_states(&containers);
1992
1993        assert_eq!(anomalies.len(), 2);
1994        let kinds: Vec<&str> = anomalies.iter().map(|a| a.kind.as_str()).collect();
1995        assert!(kinds.contains(&"exited_container"));
1996        assert!(kinds.contains(&"dead_container"));
1997    }
1998
1999    // ── Integration tests ──────────────────────────────────────────────
2000
2001    #[test]
2002    fn find_anomalies_returns_multiple_types() {
2003        let docker = crate::docker::MockDockerClient {
2004            containers: vec![
2005                test_container("api", "running", 5),
2006                test_container("worker", "exited", 0),
2007            ],
2008            ..Default::default()
2009        };
2010        let metrics = crate::metrics::MockMetricsCollector {
2011            samples: vec![test_sample("api", 92.0, 950, 1024)],
2012        };
2013
2014        let query = AnalyzeQuery {
2015            target: AnalysisTarget::Collection(CollectionTarget::Containers),
2016            verb: AnalysisVerb::Find,
2017            subject: Some("anomalies".to_owned()),
2018            time: None,
2019            pipeline: Vec::new(),
2020        };
2021
2022        let result = rt()
2023            .block_on(execute_analyze(&query, &docker, &metrics))
2024            .unwrap();
2025
2026        // Should find: restart_loop + high_cpu + memory_pressure + exited_container
2027        assert!(result.rows.len() >= 3);
2028        let kinds: Vec<String> = result
2029            .rows
2030            .iter()
2031            .filter_map(|r| {
2032                r.fields
2033                    .get("kind")
2034                    .and_then(|v| v.as_str())
2035                    .map(str::to_owned)
2036            })
2037            .collect();
2038        assert!(kinds.contains(&"restart_loop".to_owned()));
2039        assert!(kinds.contains(&"high_cpu".to_owned()));
2040        assert!(kinds.contains(&"exited_container".to_owned()));
2041    }
2042
2043    #[test]
2044    fn find_anomalies_returns_healthy_when_no_issues() {
2045        let docker = crate::docker::MockDockerClient {
2046            containers: vec![test_container("api", "running", 0)],
2047            ..Default::default()
2048        };
2049        let metrics = crate::metrics::MockMetricsCollector {
2050            samples: vec![test_sample("api", 10.0, 128, 1024)],
2051        };
2052
2053        let query = AnalyzeQuery {
2054            target: AnalysisTarget::Collection(CollectionTarget::Containers),
2055            verb: AnalysisVerb::Find,
2056            subject: Some("anomalies".to_owned()),
2057            time: None,
2058            pipeline: Vec::new(),
2059        };
2060
2061        let result = rt()
2062            .block_on(execute_analyze(&query, &docker, &metrics))
2063            .unwrap();
2064
2065        assert_eq!(result.rows.len(), 1);
2066        assert_eq!(
2067            result.rows[0].fields["kind"],
2068            JsonValue::String("healthy".to_owned())
2069        );
2070    }
2071
2072    #[test]
2073    fn explain_container_produces_signals() {
2074        let docker = crate::docker::MockDockerClient {
2075            containers: vec![test_container("api", "running", 5)],
2076            ..Default::default()
2077        };
2078        let metrics = crate::metrics::MockMetricsCollector {
2079            samples: vec![test_sample("api", 85.0, 900, 1024)],
2080        };
2081
2082        let query = AnalyzeQuery {
2083            target: AnalysisTarget::Singular(crate::ast::SingularTarget {
2084                kind: SingularTargetKind::Container,
2085                value: "api".to_owned(),
2086            }),
2087            verb: AnalysisVerb::Explain,
2088            subject: None,
2089            time: None,
2090            pipeline: Vec::new(),
2091        };
2092
2093        let result = rt()
2094            .block_on(execute_analyze(&query, &docker, &metrics))
2095            .unwrap();
2096
2097        // Should have signals for state, restart_count, cpu, memory, network_rx, network_tx
2098        assert!(result.rows.len() >= 4);
2099        let signal_names: Vec<String> = result
2100            .rows
2101            .iter()
2102            .filter_map(|r| {
2103                r.fields
2104                    .get("signal")
2105                    .and_then(|v| v.as_str())
2106                    .map(str::to_owned)
2107            })
2108            .collect();
2109        assert!(signal_names.contains(&"state".to_owned()));
2110        assert!(signal_names.contains(&"cpu".to_owned()));
2111        assert!(signal_names.contains(&"memory".to_owned()));
2112    }
2113
2114    #[test]
2115    fn correlate_finds_shared_images() {
2116        // Create containers with the same image to test correlation.
2117        // so create containers with same image.
2118        let docker = crate::docker::MockDockerClient {
2119            containers: vec![
2120                Container {
2121                    image: "api:latest".to_owned(),
2122                    ..test_container("api-1", "running", 0)
2123                },
2124                Container {
2125                    image: "api:latest".to_owned(),
2126                    ..test_container("api-2", "running", 0)
2127                },
2128            ],
2129            ..Default::default()
2130        };
2131        let metrics = crate::metrics::MockMetricsCollector {
2132            samples: Vec::new(),
2133        };
2134
2135        let query = AnalyzeQuery {
2136            target: AnalysisTarget::Collection(CollectionTarget::Containers),
2137            verb: AnalysisVerb::Correlate,
2138            subject: None,
2139            time: None,
2140            pipeline: Vec::new(),
2141        };
2142
2143        let result = rt()
2144            .block_on(execute_analyze(&query, &docker, &metrics))
2145            .unwrap();
2146
2147        // Should find shared_image and shared_label correlations
2148        let correlations: Vec<String> = result
2149            .rows
2150            .iter()
2151            .filter_map(|r| {
2152                r.fields
2153                    .get("correlation")
2154                    .and_then(|v| v.as_str())
2155                    .map(str::to_owned)
2156            })
2157            .collect();
2158        assert!(correlations.contains(&"shared_image".to_owned()));
2159    }
2160
2161    // ── Resource Leak tests ────────────────────────────────────────
2162
2163    #[test]
2164    fn detects_resource_leak_with_increasing_memory() {
2165        let mut store = crate::storage::InMemoryTelemetryStore::default();
2166        let thresholds = AnalysisThresholds {
2167            resource_leak_memory_increase_pct: 20.0,
2168            resource_leak_min_samples: 3,
2169            ..AnalysisThresholds::default()
2170        };
2171
2172        // Write 5 samples with increasing memory (leak pattern)
2173        for i in 0..5 {
2174            let mem = 100_000_000 + (i * 30_000_000); // 100M, 130M, 160M, 190M, 220M
2175            let sample = MetricSample {
2176                container_id: "leaky_id".to_string(),
2177                container_name: "leaky-container".to_owned(),
2178                timestamp: format!("2026-01-01T12:00:{:02}Z", i),
2179                cpu_percent: Some(50.0),
2180                memory_usage_bytes: Some(mem),
2181                memory_limit_bytes: Some(1_073_741_824),
2182                network_rx_bytes: Some(0),
2183                network_tx_bytes: Some(0),
2184                disk_read_bytes: Some(0),
2185                disk_write_bytes: Some(0),
2186            };
2187            store.write_metric(sample).unwrap();
2188        }
2189
2190        let anomalies = detect_resource_leaks(&store, &thresholds);
2191        assert_eq!(anomalies.len(), 1);
2192        assert_eq!(anomalies[0].kind, "resource_leak");
2193        assert_eq!(anomalies[0].container, "leaky-container");
2194    }
2195
2196    #[test]
2197    fn no_resource_leak_with_stable_memory() {
2198        let mut store = crate::storage::InMemoryTelemetryStore::default();
2199        let thresholds = AnalysisThresholds {
2200            resource_leak_memory_increase_pct: 20.0,
2201            resource_leak_min_samples: 3,
2202            ..AnalysisThresholds::default()
2203        };
2204
2205        // Write 3 samples with stable memory
2206        for i in 0..3 {
2207            let sample = MetricSample {
2208                container_id: "stable_id".to_string(),
2209                container_name: "stable".to_owned(),
2210                timestamp: format!("2026-01-01T12:00:{:02}Z", i),
2211                cpu_percent: Some(50.0),
2212                memory_usage_bytes: Some(100_000_000),
2213                memory_limit_bytes: Some(1_073_741_824),
2214                network_rx_bytes: Some(0),
2215                network_tx_bytes: Some(0),
2216                disk_read_bytes: Some(0),
2217                disk_write_bytes: Some(0),
2218            };
2219            store.write_metric(sample).unwrap();
2220        }
2221
2222        let anomalies = detect_resource_leaks(&store, &thresholds);
2223        assert!(anomalies.is_empty());
2224    }
2225
2226    #[test]
2227    fn no_resource_leak_with_insufficient_samples() {
2228        let mut store = crate::storage::InMemoryTelemetryStore::default();
2229        let thresholds = AnalysisThresholds {
2230            resource_leak_memory_increase_pct: 20.0,
2231            resource_leak_min_samples: 5,
2232            ..AnalysisThresholds::default()
2233        };
2234
2235        // Only 2 samples — below threshold
2236        for i in 0..2 {
2237            let sample = MetricSample {
2238                container_id: "few_id".to_string(),
2239                container_name: "few".to_owned(),
2240                timestamp: format!("2026-01-01T12:00:{:02}Z", i),
2241                cpu_percent: Some(50.0),
2242                memory_usage_bytes: Some(100_000_000 + (i * 50_000_000)),
2243                memory_limit_bytes: Some(1_073_741_824),
2244                network_rx_bytes: Some(0),
2245                network_tx_bytes: Some(0),
2246                disk_read_bytes: Some(0),
2247                disk_write_bytes: Some(0),
2248            };
2249            store.write_metric(sample).unwrap();
2250        }
2251
2252        let anomalies = detect_resource_leaks(&store, &thresholds);
2253        assert!(anomalies.is_empty());
2254    }
2255
2256    // ── Dependency tests ────────────────────────────────────────────
2257
2258    #[test]
2259    fn finds_dependencies_from_mock() {
2260        let docker = crate::docker::MockDockerClient {
2261            containers: vec![
2262                Container {
2263                    image: "api:latest".to_owned(),
2264                    labels: vec!["com.docker.compose.project=myapp".to_owned()],
2265                    ..test_container("api-1", "running", 0)
2266                },
2267                Container {
2268                    image: "api:latest".to_owned(),
2269                    labels: vec!["com.docker.compose.project=myapp".to_owned()],
2270                    ..test_container("api-2", "running", 0)
2271                },
2272            ],
2273            networks: vec![crate::docker::Network {
2274                id: "net1".to_owned(),
2275                name: "frontend-net".to_owned(),
2276                driver: "bridge".to_owned(),
2277                scope: "local".to_owned(),
2278                containers: vec!["api-1".to_owned()],
2279                labels: Vec::new(),
2280            }],
2281            volumes: vec![crate::docker::Volume {
2282                name: "pgdata".to_owned(),
2283                driver: "local".to_owned(),
2284                mountpoint: Some("/data".to_owned()),
2285                scope: Some("local".to_owned()),
2286                labels: Vec::new(),
2287            }],
2288            ..Default::default()
2289        };
2290
2291        let result = rt().block_on(analyze_dependencies(&docker)).unwrap();
2292
2293        let dep_kinds: Vec<String> = result
2294            .rows
2295            .iter()
2296            .filter_map(|r| {
2297                r.fields
2298                    .get("dependency")
2299                    .and_then(|v| v.as_str())
2300                    .map(str::to_owned)
2301            })
2302            .collect();
2303        assert!(dep_kinds.contains(&"compose_project".to_owned()));
2304        assert!(dep_kinds.contains(&"network".to_owned()));
2305        assert!(dep_kinds.contains(&"volume".to_owned()));
2306    }
2307
2308    #[test]
2309    fn dependencies_empty_returns_none_row() {
2310        let docker = crate::docker::MockDockerClient::default();
2311        let result = rt().block_on(analyze_dependencies(&docker)).unwrap();
2312        assert_eq!(result.rows.len(), 1);
2313        assert_eq!(
2314            result.rows[0].fields["dependency"],
2315            JsonValue::String("none".to_owned())
2316        );
2317    }
2318
2319    // ── Config Drift tests ──────────────────────────────────────────
2320
2321    #[test]
2322    fn detects_config_drift_from_snapshot() {
2323        let mut store = crate::storage::InMemoryTelemetryStore::default();
2324        store
2325            .write_snapshot(crate::storage::TelemetrySnapshot {
2326                timestamp: "2026-01-01T12:00:00Z".to_owned(),
2327                containers: vec![Container {
2328                    id: "abc".to_owned(),
2329                    name: "api".to_owned(),
2330                    image: "api:latest".to_owned(),
2331                    status: "Up 5 minutes".to_owned(),
2332                    state: "running".to_owned(),
2333                    ports: Vec::new(),
2334                    labels: vec!["env=prod".to_owned()],
2335                    created_at: None,
2336                    started_at: None,
2337                    finished_at: None,
2338                    restart_count: Some(1),
2339                    health: None,
2340                }],
2341                images: Vec::new(),
2342                networks: Vec::new(),
2343                volumes: Vec::new(),
2344            })
2345            .unwrap();
2346
2347        let anomalies = detect_config_drift(&store).unwrap();
2348        assert!(!anomalies.is_empty());
2349        assert_eq!(anomalies[0].kind, "config_baseline");
2350        assert_eq!(anomalies[0].container, "api");
2351    }
2352
2353    #[test]
2354    fn detects_config_drift_between_snapshots() {
2355        let mut store = crate::storage::InMemoryTelemetryStore::default();
2356
2357        // Snapshot A: api v1, running
2358        store
2359            .write_snapshot(crate::storage::TelemetrySnapshot {
2360                timestamp: "2026-01-01T12:00:00Z".to_owned(),
2361                containers: vec![Container {
2362                    id: "abc".to_owned(),
2363                    name: "api".to_owned(),
2364                    image: "api:v1".to_owned(),
2365                    status: "Up 5 minutes".to_owned(),
2366                    state: "running".to_owned(),
2367                    ports: Vec::new(),
2368                    labels: vec!["env=prod".to_owned(), "team=backend".to_owned()],
2369                    created_at: None,
2370                    started_at: None,
2371                    finished_at: None,
2372                    restart_count: Some(0),
2373                    health: None,
2374                }],
2375                images: Vec::new(),
2376                networks: Vec::new(),
2377                volumes: Vec::new(),
2378            })
2379            .unwrap();
2380
2381        // Snapshot B: api v2, exited, new label, increased restarts
2382        store
2383            .write_snapshot(crate::storage::TelemetrySnapshot {
2384                timestamp: "2026-01-01T13:00:00Z".to_owned(),
2385                containers: vec![Container {
2386                    id: "abc".to_owned(),
2387                    name: "api".to_owned(),
2388                    image: "api:v2".to_owned(),
2389                    status: "Exited (1) 1 minute ago".to_owned(),
2390                    state: "exited".to_owned(),
2391                    ports: Vec::new(),
2392                    labels: vec!["env=prod".to_owned(), "team=infra".to_owned()],
2393                    created_at: None,
2394                    started_at: None,
2395                    finished_at: None,
2396                    restart_count: Some(3),
2397                    health: None,
2398                }],
2399                images: Vec::new(),
2400                networks: Vec::new(),
2401                volumes: Vec::new(),
2402            })
2403            .unwrap();
2404
2405        let anomalies = detect_config_drift(&store).unwrap();
2406        let kinds: Vec<&str> = anomalies.iter().map(|a| a.kind.as_str()).collect();
2407
2408        assert!(
2409            kinds.contains(&"config_drift"),
2410            "should detect image change"
2411        );
2412        assert!(
2413            kinds.contains(&"state_change"),
2414            "should detect state change"
2415        );
2416        assert!(
2417            kinds.contains(&"restart_increase"),
2418            "should detect restart increase"
2419        );
2420        assert!(kinds.contains(&"label_drift"), "should detect label change");
2421    }
2422
2423    #[test]
2424    fn config_drift_returns_empty_when_no_changes() {
2425        let mut store = crate::storage::InMemoryTelemetryStore::default();
2426
2427        // Two identical snapshots
2428        let snapshot = crate::storage::TelemetrySnapshot {
2429            timestamp: "2026-01-01T12:00:00Z".to_owned(),
2430            containers: vec![Container {
2431                id: "abc".to_owned(),
2432                name: "api".to_owned(),
2433                image: "api:latest".to_owned(),
2434                status: "Up 5 minutes".to_owned(),
2435                state: "running".to_owned(),
2436                ports: Vec::new(),
2437                labels: vec!["env=prod".to_owned()],
2438                created_at: None,
2439                started_at: None,
2440                finished_at: None,
2441                restart_count: Some(0),
2442                health: None,
2443            }],
2444            images: Vec::new(),
2445            networks: Vec::new(),
2446            volumes: Vec::new(),
2447        };
2448        store.write_snapshot(snapshot.clone()).unwrap();
2449        store.write_snapshot(snapshot).unwrap();
2450
2451        let anomalies = detect_config_drift(&store).unwrap();
2452        assert!(anomalies.is_empty(), "should be empty when no changes");
2453    }
2454
2455    #[test]
2456    fn config_drift_with_empty_store_returns_unsupported() {
2457        let store = crate::storage::InMemoryTelemetryStore::default();
2458        let result = detect_config_drift(&store);
2459        assert!(result.is_err());
2460    }
2461
2462    // ── Density tests ───────────────────────────────────────────────
2463
2464    #[test]
2465    fn analyzes_density_by_image_state_and_project() {
2466        let docker = crate::docker::MockDockerClient {
2467            containers: vec![
2468                Container {
2469                    image: "nginx:latest".to_owned(),
2470                    labels: vec!["com.docker.compose.project=web".to_owned()],
2471                    ..test_container("web-1", "running", 0)
2472                },
2473                Container {
2474                    image: "nginx:latest".to_owned(),
2475                    labels: vec!["com.docker.compose.project=web".to_owned()],
2476                    ..test_container("web-2", "running", 0)
2477                },
2478                Container {
2479                    image: "api:latest".to_owned(),
2480                    labels: vec![],
2481                    ..test_container("api", "exited", 0)
2482                },
2483            ],
2484            ..Default::default()
2485        };
2486
2487        let result = rt().block_on(analyze_density(&docker)).unwrap();
2488
2489        // Should have image rows (nginx, api) + state rows (running, exited) + project rows (web, standalone) + total
2490        let dimensions: Vec<String> = result
2491            .rows
2492            .iter()
2493            .filter_map(|r| {
2494                r.fields
2495                    .get("dimension")
2496                    .and_then(|v| v.as_str())
2497                    .map(str::to_owned)
2498            })
2499            .collect();
2500        assert!(dimensions.contains(&"image".to_owned()));
2501        assert!(dimensions.contains(&"state".to_owned()));
2502        assert!(dimensions.contains(&"compose_project".to_owned()));
2503        assert!(dimensions.contains(&"total".to_owned()));
2504
2505        // Verify total
2506        let total_row = result
2507            .rows
2508            .iter()
2509            .find(|r| r.fields.get("dimension") == Some(&JsonValue::String("total".to_owned())))
2510            .unwrap();
2511        assert_eq!(
2512            total_row.fields["container_count"],
2513            JsonValue::Number(Number::from(3))
2514        );
2515    }
2516
2517    #[test]
2518    fn density_with_no_containers_returns_total_zero() {
2519        let docker = crate::docker::MockDockerClient::default();
2520        let result = rt().block_on(analyze_density(&docker)).unwrap();
2521        let total_row = result
2522            .rows
2523            .iter()
2524            .find(|r| r.fields.get("dimension") == Some(&JsonValue::String("total".to_owned())))
2525            .unwrap();
2526        assert_eq!(
2527            total_row.fields["container_count"],
2528            JsonValue::Number(Number::from(0))
2529        );
2530    }
2531
2532    // ── Integration: new subjects from dispatcher ────────────────────
2533
2534    #[test]
2535    fn execute_analyze_find_dependencies_from_live_data() {
2536        let docker = crate::docker::MockDockerClient {
2537            containers: vec![test_container("api", "running", 0)],
2538            ..Default::default()
2539        };
2540        let metrics = crate::metrics::MockMetricsCollector {
2541            samples: Vec::new(),
2542        };
2543
2544        let query = AnalyzeQuery {
2545            target: AnalysisTarget::Collection(CollectionTarget::Containers),
2546            verb: AnalysisVerb::Find,
2547            subject: Some("dependencies".to_owned()),
2548            time: None,
2549            pipeline: Vec::new(),
2550        };
2551
2552        let result = rt()
2553            .block_on(execute_analyze(&query, &docker, &metrics))
2554            .unwrap();
2555        // Should have at least the "none" row
2556        assert!(!result.rows.is_empty());
2557    }
2558
2559    #[test]
2560    fn execute_analyze_find_density_from_live_data() {
2561        let docker = crate::docker::MockDockerClient {
2562            containers: vec![test_container("api", "running", 0)],
2563            ..Default::default()
2564        };
2565        let metrics = crate::metrics::MockMetricsCollector {
2566            samples: Vec::new(),
2567        };
2568
2569        let query = AnalyzeQuery {
2570            target: AnalysisTarget::Collection(CollectionTarget::Containers),
2571            verb: AnalysisVerb::Find,
2572            subject: Some("density".to_owned()),
2573            time: None,
2574            pipeline: Vec::new(),
2575        };
2576
2577        let result = rt()
2578            .block_on(execute_analyze(&query, &docker, &metrics))
2579            .unwrap();
2580        let total_row = result
2581            .rows
2582            .iter()
2583            .find(|r| r.fields.get("dimension") == Some(&JsonValue::String("total".to_owned())))
2584            .unwrap();
2585        assert_eq!(
2586            total_row.fields["container_count"],
2587            JsonValue::Number(Number::from(1))
2588        );
2589    }
2590
2591    #[test]
2592    fn execute_analyze_find_leaks_returns_unsupported_from_live() {
2593        let docker = crate::docker::MockDockerClient::default();
2594        let metrics = crate::metrics::MockMetricsCollector {
2595            samples: Vec::new(),
2596        };
2597
2598        let query = AnalyzeQuery {
2599            target: AnalysisTarget::Collection(CollectionTarget::Containers),
2600            verb: AnalysisVerb::Find,
2601            subject: Some("leaks".to_owned()),
2602            time: None,
2603            pipeline: Vec::new(),
2604        };
2605
2606        let result = rt().block_on(execute_analyze(&query, &docker, &metrics));
2607        assert!(result.is_err());
2608        assert!(matches!(result.unwrap_err(), AnalyzeError::Unsupported));
2609    }
2610
2611    #[test]
2612    fn execute_analyze_with_store_find_leaks_detects_pattern() {
2613        let mut store = crate::storage::InMemoryTelemetryStore::default();
2614        for i in 0..5 {
2615            let mem = 100_000_000 + (i * 30_000_000);
2616            store
2617                .write_metric(MetricSample {
2618                    container_id: "leaky_id".to_string(),
2619                    container_name: "leaky".to_owned(),
2620                    timestamp: format!("2026-01-01T12:00:{:02}Z", i),
2621                    cpu_percent: Some(50.0),
2622                    memory_usage_bytes: Some(mem),
2623                    memory_limit_bytes: Some(1_073_741_824),
2624                    network_rx_bytes: Some(0),
2625                    network_tx_bytes: Some(0),
2626                    disk_read_bytes: Some(0),
2627                    disk_write_bytes: Some(0),
2628                })
2629                .unwrap();
2630        }
2631
2632        let query = AnalyzeQuery {
2633            target: AnalysisTarget::Collection(CollectionTarget::Containers),
2634            verb: AnalysisVerb::Find,
2635            subject: Some("leaks".to_owned()),
2636            time: None,
2637            pipeline: Vec::new(),
2638        };
2639
2640        let result = execute_analyze_with_store(&query, &store).unwrap();
2641        let kinds: Vec<String> = result
2642            .rows
2643            .iter()
2644            .filter_map(|r| {
2645                r.fields
2646                    .get("kind")
2647                    .and_then(|v| v.as_str())
2648                    .map(str::to_owned)
2649            })
2650            .collect();
2651        assert!(kinds.contains(&"resource_leak".to_owned()));
2652    }
2653
2654    #[test]
2655    fn execute_analyze_with_store_find_drift_from_snapshot() {
2656        let mut store = crate::storage::InMemoryTelemetryStore::default();
2657        store
2658            .write_snapshot(crate::storage::TelemetrySnapshot {
2659                timestamp: "2026-01-01T12:00:00Z".to_owned(),
2660                containers: vec![Container {
2661                    id: "abc".to_owned(),
2662                    name: "api".to_owned(),
2663                    image: "api:latest".to_owned(),
2664                    status: "Up 5m".to_owned(),
2665                    state: "running".to_owned(),
2666                    ports: Vec::new(),
2667                    labels: vec!["env=prod".to_owned()],
2668                    created_at: None,
2669                    started_at: None,
2670                    finished_at: None,
2671                    restart_count: Some(0),
2672                    health: None,
2673                }],
2674                images: Vec::new(),
2675                networks: Vec::new(),
2676                volumes: Vec::new(),
2677            })
2678            .unwrap();
2679
2680        let query = AnalyzeQuery {
2681            target: AnalysisTarget::Collection(CollectionTarget::Containers),
2682            verb: AnalysisVerb::Find,
2683            subject: Some("drift".to_owned()),
2684            time: None,
2685            pipeline: Vec::new(),
2686        };
2687
2688        let result = execute_analyze_with_store(&query, &store).unwrap();
2689        let kinds: Vec<String> = result
2690            .rows
2691            .iter()
2692            .filter_map(|r| {
2693                r.fields
2694                    .get("kind")
2695                    .and_then(|v| v.as_str())
2696                    .map(str::to_owned)
2697            })
2698            .collect();
2699        assert!(kinds.contains(&"config_baseline".to_owned()));
2700    }
2701
2702    // ── Pre-existing tests ───────────────────────────────────────────
2703
2704    #[test]
2705    fn find_anomalies_from_store_includes_leaks_and_drift() {
2706        let mut store = crate::storage::InMemoryTelemetryStore::default();
2707
2708        // Write metrics with increasing memory (resource leak pattern)
2709        for i in 0..5 {
2710            let mem = 100_000_000 + (i * 30_000_000);
2711            store
2712                .write_metric(MetricSample {
2713                    container_id: "leaky_id".to_string(),
2714                    container_name: "leaky".to_owned(),
2715                    timestamp: format!("2026-01-01T12:00:{:02}Z", i),
2716                    cpu_percent: Some(50.0),
2717                    memory_usage_bytes: Some(mem),
2718                    memory_limit_bytes: Some(1_073_741_824),
2719                    network_rx_bytes: Some(0),
2720                    network_tx_bytes: Some(0),
2721                    disk_read_bytes: Some(0),
2722                    disk_write_bytes: Some(0),
2723                })
2724                .unwrap();
2725        }
2726
2727        // Write a snapshot (for config drift baseline)
2728        store
2729            .write_snapshot(crate::storage::TelemetrySnapshot {
2730                timestamp: "2026-01-01T12:00:00Z".to_owned(),
2731                containers: vec![Container {
2732                    id: "leaky_id".to_owned(),
2733                    name: "leaky".to_owned(),
2734                    image: "leaky:latest".to_owned(),
2735                    status: "Up 5m".to_owned(),
2736                    state: "running".to_owned(),
2737                    ports: Vec::new(),
2738                    labels: vec!["env=prod".to_owned()],
2739                    created_at: None,
2740                    started_at: None,
2741                    finished_at: None,
2742                    restart_count: Some(0),
2743                    health: None,
2744                }],
2745                images: Vec::new(),
2746                networks: Vec::new(),
2747                volumes: Vec::new(),
2748            })
2749            .unwrap();
2750
2751        let query = AnalyzeQuery {
2752            target: AnalysisTarget::Collection(CollectionTarget::Containers),
2753            verb: AnalysisVerb::Find,
2754            subject: Some("anomalies".to_owned()),
2755            time: None,
2756            pipeline: Vec::new(),
2757        };
2758
2759        let result = execute_analyze_with_store(&query, &store).unwrap();
2760
2761        let kinds: Vec<String> = result
2762            .rows
2763            .iter()
2764            .filter_map(|r| {
2765                r.fields
2766                    .get("kind")
2767                    .and_then(|v| v.as_str())
2768                    .map(str::to_owned)
2769            })
2770            .collect();
2771        assert!(
2772            kinds.contains(&"resource_leak".to_owned()),
2773            "find_anomalies_from_store should detect resource leaks: {kinds:?}"
2774        );
2775        assert!(
2776            kinds.contains(&"config_baseline".to_owned()),
2777            "find_anomalies_from_store should detect config drift baseline: {kinds:?}"
2778        );
2779    }
2780
2781    #[test]
2782    fn find_anomalies_from_store_detects_deployment_errors() {
2783        let mut store = crate::storage::InMemoryTelemetryStore::default();
2784
2785        store
2786            .write_event(test_event("2026-01-01T12:00:00Z", "die", "api"))
2787            .unwrap();
2788        store
2789            .write_event(test_event("2026-01-01T12:01:00Z", "die", "api"))
2790            .unwrap();
2791        store
2792            .write_event(test_event("2026-01-01T12:02:00Z", "die", "api"))
2793            .unwrap();
2794
2795        let query = AnalyzeQuery {
2796            target: AnalysisTarget::Collection(CollectionTarget::Containers),
2797            verb: AnalysisVerb::Find,
2798            subject: Some("anomalies".to_owned()),
2799            time: None,
2800            pipeline: Vec::new(),
2801        };
2802
2803        let result = execute_analyze_with_store(&query, &store).unwrap();
2804
2805        let kinds: Vec<String> = result
2806            .rows
2807            .iter()
2808            .filter_map(|r| {
2809                r.fields
2810                    .get("kind")
2811                    .and_then(|v| v.as_str())
2812                    .map(str::to_owned)
2813            })
2814            .collect();
2815        assert!(kinds.contains(&"deployment_errors".to_owned()));
2816    }
2817}