fix(evict): heal index after out-of-band deletes + add richer monitoring metrics

README.md

@@ -126,6 +126,20 @@ linked. Cached objects share the `--cache-max-mb` budget and LRU eviction with t
 git mirrors. No configuration is needed: LFS is served on the same endpoints the git
 client already routes through the proxy.
 
+### Cache management
+
+The cache is self-managing. Mirrors are created on first request, and with
+`--cache-max-mb` set, least-recently-used idle ones are evicted in the background
+once the total exceeds the cap - that cap is the intended way to reclaim disk. There
+is no endpoint to delete a specific mirror, and none is needed: an evicted or missing
+mirror is transparently re-cloned on its next request, so eviction is a cache
+concern, never a correctness one.
+
+Deleting a mirror directory by hand works but is not the intended lever. The proxy
+assumes it owns the cache root, so a manual deletion reaches the cache-size metrics
+only once the background task reconciles it (within a minute) or the next restart
+rescans. Prefer `--cache-max-mb`.
+
 ## Benchmark
 
 For a fleet of ephemeral clients fetching the same content, only the first fetch
@@ -227,8 +241,11 @@ Endpoints: `/healthz`, `/readyz`, `/metrics` (see [Metrics](#metrics)).
 | `upstream_ops_total`        | counter   | `op`, `result`, `repo` | Upstream git operations (initial mirror clone, refresh fetch, want fetch)     |
 | `cache_bytes`               | gauge     | -                      | Total size of the on-disk mirror cache, in bytes                              |
 | `cache_mirrors`             | gauge     | -                      | Number of cached mirrors on disk                                              |
+| `cache_max_bytes`           | gauge     | -                      | Configured on-disk cache cap in bytes (`0` = unlimited)                        |
+| `fetch_ttl_seconds`         | gauge     | -                      | Configured upstream fetch TTL in seconds (`0` = always fetch)                  |
 | `evictions_total`           | counter   | -                      | Idle mirrors evicted to keep the cache under `--cache-max-mb`                 |
 | `lfs_objects_total`         | counter   | `result`               | Cached git-LFS object lookups                                                 |
+| `cache_outcome_total`       | counter   | `outcome`              | info/refs operations by outcome: served locally vs. fetched from upstream     |
 | `upstream_duration_seconds` | histogram | `op`, `repo`           | Wall-clock duration of successful upstream clones/fetches                     |
 | `serve_duration_seconds`    | histogram | `kind`, `repo`         | Time to serve a client, measured to the end of the response                   |
 
@@ -245,6 +262,9 @@ Label values:
 - `op`: `clone` (first mirror of a repo), `fetch` (refresh of an existing
   mirror), `want_fetch` (on-demand fetch of a bare SHA the mirror lacks, e.g. a
   PR merge commit).
+- `outcome` on `cache_outcome_total`: `cloned` (first mirror of a repo),
+  `fetched` (refresh from upstream), `cached` (served from the local mirror with
+  no upstream call). Recorded once per info/refs advertisement.
 - `repo`: the served repo path (e.g. `group/project.git`). To keep cardinality
   bounded, the real path is emitted only on success; every failure (failed auth,
   malformed path, upstream error) uses a `-` sentinel, so a flood of distinct
@@ -255,8 +275,13 @@ Notes:
 
 - `cache_bytes` and `cache_mirrors` are maintained only when a cache cap is
   configured (`--cache-max-mb`); without a cap they stay `0`.
+- `cache_max_bytes` and `fetch_ttl_seconds` echo the configured flags as static
+  gauges, set once at startup, so a dashboard can read cache utilization as
+  `cache_bytes / cache_max_bytes` without hardcoding the cap.
 - `lfs_objects_total` has no `repo` label: LFS objects are content-addressed
   and shared across repos.
+- `cache_outcome_total` is the local-vs-upstream signal: the share of operations
+  served without touching upstream is `cached / sum without (outcome) (...)`.
 - The histograms are observed only on success and share buckets from 50ms to
   300s, sized for anything from a cached ref advertisement to a large clone
   over a slow WAN.

src/evict.rs

@@ -18,16 +18,22 @@
 //! behaviour.
 
 use std::collections::HashSet;
+use std::io::ErrorKind;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex, MutexGuard};
-use std::time::SystemTime;
+use std::time::{Duration, SystemTime};
 
 use lru::LruCache;
 use tokio::sync::{Notify, watch};
 
 use crate::git::GitCache;
 use crate::metrics::Metrics;
 
+/// How often the background task reconciles the in-memory index against disk,
+/// dropping entries whose mirror an operator deleted out of band. Only a
+/// coarse resync of the cache-size gauges, so a slow cadence is plenty.
+const RECONCILE_INTERVAL: Duration = Duration::from_secs(60);
+
 struct Inner {
     /// Mirror name -> size, kept in access order (front = most-recently-used, back =
     /// least). The `lru` crate does the O(1) promote-on-use and tail eviction.
@@ -178,6 +184,54 @@ impl CacheIndex {
         victims
     }
 
+    /// Drop tracked entries whose on-disk path has vanished, correcting the total
+    /// and gauges. The index normally assumes the proxy owns the cache dir and keeps
+    /// the total by incremental bookkeeping; this is the one path that tolerates an
+    /// out-of-band deletion (an operator removing a mirror by hand), so the gauges do
+    /// not drift until the next restart rescans. Runs on a timer, off the request
+    /// path; the stat sweep is blocking, so it is done on the blocking pool.
+    async fn reconcile_missing(&self) {
+        let tracked: Vec<(String, PathBuf)> = {
+            let inner = self.lock();
+            inner
+                .cache
+                .iter()
+                .map(|(name, _)| (name.clone(), self.cache_dir(name)))
+                .collect()
+        };
+        if tracked.is_empty() {
+            return;
+        }
+        // Treat only a definitive "not found" as gone; a transient stat error must
+        // not evict a live entry from the index.
+        let missing = tokio::task::spawn_blocking(move || {
+            tracked
+                .into_iter()
+                .filter(|(_, dir)| {
+                    matches!(std::fs::symlink_metadata(dir), Err(e) if e.kind() == ErrorKind::NotFound)
+                })
+                .map(|(name, _)| name)
+                .collect::<Vec<_>>()
+        })
+        .await
+        .unwrap_or_default();
+        if missing.is_empty() {
+            return;
+        }
+        let mut inner = self.lock();
+        for name in missing {
+            // An entry re-touched since the snapshot (a concurrent re-clone marks it
+            // dirty) is tracked again and pending measurement, so leave it be.
+            if inner.dirty.contains(&name) {
+                continue;
+            }
+            if let Some(size) = inner.cache.pop(&name) {
+                inner.total -= size;
+            }
+        }
+        self.set_gauges(&inner);
+    }
+
     fn lock(&self) -> MutexGuard<'_, Inner> {
         self.state.lock().expect("cache index lock")
     }
@@ -188,21 +242,28 @@ impl CacheIndex {
 }
 
 /// Background maintenance task. Measures changed mirrors and evicts the LRU tail
-/// whenever the index signals work, and exits cleanly when `shutdown` fires (or its
+/// whenever the index signals work, periodically reconciles the index against disk
+/// to drop out-of-band deletions, and exits cleanly when `shutdown` fires (or its
 /// sender drops).
 pub async fn run(
     cache: Arc<GitCache>,
     index: Arc<CacheIndex>,
     mut shutdown: watch::Receiver<bool>,
 ) {
+    let mut reconcile = tokio::time::interval(RECONCILE_INTERVAL);
+    // A stall must not unleash a burst of catch-up ticks; one resync suffices.
+    reconcile.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+    // The startup scan just read disk, so skip the interval's immediate first tick.
+    reconcile.tick().await;
     loop {
         // Run first: covers the startup-over-cap permit and any signal that arrived
         // while the previous pass ran (a stored `notify_one` permit makes the next
         // wait return immediately, so no work is missed).
         maintain(&cache, &index).await;
         tokio::select! {
-            biased; // prefer shutdown over another pass when both are ready
+            biased; // prefer shutdown over work/reconcile when several are ready
             _ = shutdown.changed() => break,
+            _ = reconcile.tick() => index.reconcile_missing().await,
             _ = index.work.notified() => {}
         }
     }
@@ -605,6 +666,60 @@ mod tests {
         assert!(metrics.gather().contains("gitcacheproxy_evictions_total 1"));
     }
 
+    #[tokio::test]
+    async fn reconcile_drops_mirrors_deleted_out_of_band() {
+        let tmp = tempfile::tempdir().unwrap();
+        let root = tmp.path();
+        make_mirror(&root.join("keep.git"), 1000, None);
+        make_mirror(&root.join("gone.git"), 2000, None);
+        let metrics = Arc::new(Metrics::new());
+        let idx = CacheIndex::new(root.to_path_buf(), u64::MAX, metrics.clone());
+        let (before, count) = idx.totals();
+        assert_eq!(count, 2);
+
+        // An operator deletes a mirror by hand; nothing signals the index.
+        std::fs::remove_dir_all(root.join("gone.git")).unwrap();
+        assert_eq!(idx.totals(), (before, 2), "index is stale until reconcile");
+
+        idx.reconcile_missing().await;
+        // HEAD (21 bytes) + the 1000-byte pack file.
+        assert_eq!(
+            idx.totals(),
+            (1021, 1),
+            "the vanished mirror is dropped and the total corrected"
+        );
+        assert!(metrics.gather().contains("gitcacheproxy_cache_mirrors 1"));
+    }
+
+    #[tokio::test]
+    async fn reconcile_is_a_noop_when_nothing_is_missing() {
+        let tmp = tempfile::tempdir().unwrap();
+        let root = tmp.path();
+        // An empty index returns before any disk work.
+        let empty = CacheIndex::new(root.to_path_buf(), u64::MAX, Arc::new(Metrics::new()));
+        empty.reconcile_missing().await;
+        assert_eq!(empty.totals(), (0, 0));
+
+        // A populated index whose mirrors all still exist drops nothing.
+        make_mirror(&root.join("present.git"), 100, None);
+        let idx = CacheIndex::new(root.to_path_buf(), u64::MAX, Arc::new(Metrics::new()));
+        let before = idx.totals();
+        idx.reconcile_missing().await;
+        assert_eq!(idx.totals(), before, "present mirrors are left tracked");
+    }
+
+    #[tokio::test]
+    async fn reconcile_keeps_a_dirty_entry_pending_measurement() {
+        let tmp = tempfile::tempdir().unwrap();
+        let idx = CacheIndex::new(tmp.path().to_path_buf(), u64::MAX, Arc::new(Metrics::new()));
+        // Tracked and dirty but not yet on disk: a mirror deleted and re-cloned,
+        // re-tracked before the reclone has finished writing its dir. Reconcile must
+        // not drop it out from under the pending measurement.
+        idx.mark_changed("recloning.git");
+        idx.reconcile_missing().await;
+        assert_eq!(idx.totals(), (0, 1));
+    }
+
     fn dummy_cfg() -> GitConfig {
         // Eviction never shells out to git, so the binary is irrelevant.
         GitConfig {

src/main.rs

@@ -59,6 +59,10 @@ async fn main() -> Result<()> {
     };
 
     let metrics = Arc::new(metrics::Metrics::new());
+    metrics.set_config(
+        cfg.cache_max_mb.saturating_mul(1024 * 1024),
+        cfg.fetch_ttl_seconds,
+    );
 
     // Bound the cache on disk only when a cap is set; with `0` no index is built
     // and the eviction machinery stays inert, preserving the zero-overhead

src/metrics.rs

@@ -16,6 +16,8 @@ use prometheus::{
     TextEncoder,
 };
 
+use crate::git::CacheOutcome;
+
 /// Histogram buckets, in seconds, for git operation latency: from a fast cached
 /// advertisement (tens of ms) to a large clone over a slow WAN (minutes).
 const DURATION_BUCKETS: &[f64] = &[
@@ -137,12 +139,24 @@ pub struct Metrics {
     cache_bytes: IntGauge,
     /// `cache_mirrors` - number of cached mirrors (same caveat as `cache_bytes`).
     cache_mirrors: IntGauge,
+    /// `cache_max_bytes` - the configured on-disk cache cap in bytes (`0` =
+    /// unlimited). Static, set once at startup; pairs with `cache_bytes` so a
+    /// dashboard can show utilization without hardcoding the cap.
+    cache_max_bytes: IntGauge,
+    /// `fetch_ttl_seconds` - the configured upstream fetch TTL in seconds (`0` =
+    /// always fetch). Static, set once at startup.
+    fetch_ttl_seconds: IntGauge,
     /// `evictions_total` - idle mirrors evicted to keep the cache under the cap.
     evictions: IntCounter,
     /// `lfs_objects_total{result}` - cached git-LFS object lookups; result = hit
     /// (served from disk) | miss (fetched from upstream, then cached) | error. No
     /// `repo` label: objects are content-addressed and shared across repos.
     lfs_objects: IntCounterVec,
+    /// `cache_outcome_total{outcome}` - info/refs operations by outcome: cloned
+    /// (first mirror of a repo) | fetched (refresh from upstream) | cached (served
+    /// from the local mirror, no upstream call). Recorded once per advertisement,
+    /// so the local-hit ratio is `cached / sum without (outcome) (...)`.
+    cache_outcome: IntCounterVec,
     /// `upstream_duration_seconds{op, repo}` - clone/fetch wall-clock, observed
     /// only on success (same bounded-`repo` discipline as the counters).
     upstream_duration: HistogramVec,
@@ -178,6 +192,16 @@ impl Metrics {
             "Number of cached mirrors on disk",
         )
         .expect("valid metric");
+        let cache_max_bytes = IntGauge::new(
+            "gitcacheproxy_cache_max_bytes",
+            "Configured on-disk cache cap in bytes (0 = unlimited)",
+        )
+        .expect("valid metric");
+        let fetch_ttl_seconds = IntGauge::new(
+            "gitcacheproxy_fetch_ttl_seconds",
+            "Configured upstream fetch TTL in seconds (0 = always fetch)",
+        )
+        .expect("valid metric");
         let evictions = IntCounter::new(
             "gitcacheproxy_evictions_total",
             "Idle mirrors evicted to keep the cache under the configured cap",
@@ -191,6 +215,14 @@ impl Metrics {
             &["result"],
         )
         .expect("valid metric");
+        let cache_outcome = IntCounterVec::new(
+            Opts::new(
+                "gitcacheproxy_cache_outcome_total",
+                "info/refs operations by cache outcome (cloned/fetched/cached)",
+            ),
+            &["outcome"],
+        )
+        .expect("valid metric");
         let upstream_duration = HistogramVec::new(
             HistogramOpts::new(
                 "gitcacheproxy_upstream_duration_seconds",
@@ -221,12 +253,21 @@ impl Metrics {
         registry
             .register(Box::new(cache_mirrors.clone()))
             .expect("register cache_mirrors");
+        registry
+            .register(Box::new(cache_max_bytes.clone()))
+            .expect("register cache_max_bytes");
+        registry
+            .register(Box::new(fetch_ttl_seconds.clone()))
+            .expect("register fetch_ttl_seconds");
         registry
             .register(Box::new(evictions.clone()))
             .expect("register evictions");
         registry
             .register(Box::new(lfs_objects.clone()))
             .expect("register lfs_objects");
+        registry
+            .register(Box::new(cache_outcome.clone()))
+            .expect("register cache_outcome");
         registry
             .register(Box::new(upstream_duration.clone()))
             .expect("register upstream_duration");
@@ -239,8 +280,11 @@ impl Metrics {
             upstream,
             cache_bytes,
             cache_mirrors,
+            cache_max_bytes,
+            fetch_ttl_seconds,
             evictions,
             lfs_objects,
+            cache_outcome,
             upstream_duration,
             serve_duration,
         }
@@ -269,6 +313,12 @@ impl Metrics {
         self.cache_mirrors.set(mirrors as i64);
     }
 
+    /// Publish the static config gauges. Called once at startup.
+    pub fn set_config(&self, cache_max_bytes: u64, fetch_ttl_seconds: u64) {
+        self.cache_max_bytes.set(cache_max_bytes as i64);
+        self.fetch_ttl_seconds.set(fetch_ttl_seconds as i64);
+    }
+
     /// Record one evicted mirror.
     pub fn record_eviction(&self) {
         self.evictions.inc();
@@ -279,6 +329,17 @@ impl Metrics {
         self.lfs_objects.with_label_values(&[result.as_str()]).inc();
     }
 
+    /// Record the outcome of an info/refs operation (cloned / fetched / cached) -
+    /// the local-vs-upstream signal for one clone/fetch.
+    pub fn record_cache_outcome(&self, outcome: CacheOutcome) {
+        let label = match outcome {
+            CacheOutcome::Cloned => "cloned",
+            CacheOutcome::Fetched => "fetched",
+            CacheOutcome::Cached => "cached",
+        };
+        self.cache_outcome.with_label_values(&[label]).inc();
+    }
+
     /// Observe an upstream op's duration. Call only on success with the real repo,
     /// matching the counters' bounded-`repo` cardinality discipline.
     pub fn observe_upstream(&self, op: UpstreamOp, repo: &str, seconds: f64) {
@@ -322,19 +383,28 @@ mod tests {
         m.record_upstream(UpstreamOp::Fetch, Status::Ok, "group/foo.git");
         m.record_upstream(UpstreamOp::Clone, Status::Error, "group/bar.git");
         m.set_cache_size(2048, 3);
+        m.set_config(1024 * 1024, 10);
         m.record_eviction();
         m.record_eviction();
         m.record_lfs(LfsResult::Hit);
         m.record_lfs(LfsResult::Miss);
+        m.record_cache_outcome(CacheOutcome::Cached);
+        m.record_cache_outcome(CacheOutcome::Fetched);
+        m.record_cache_outcome(CacheOutcome::Cloned);
         m.observe_upstream(UpstreamOp::Clone, "group/foo.git", 1.5);
         m.observe_serve(ServeKind::UploadPack, "group/foo.git", 2.0);
 
         let out = m.gather();
         assert!(out.contains("gitcacheproxy_cache_bytes 2048"));
         assert!(out.contains("gitcacheproxy_cache_mirrors 3"));
+        assert!(out.contains("gitcacheproxy_cache_max_bytes 1048576"));
+        assert!(out.contains("gitcacheproxy_fetch_ttl_seconds 10"));
         assert!(out.contains("gitcacheproxy_evictions_total 2"));
         assert!(out.contains(r#"gitcacheproxy_lfs_objects_total{result="hit"} 1"#));
         assert!(out.contains(r#"gitcacheproxy_lfs_objects_total{result="miss"} 1"#));
+        assert!(out.contains(r#"gitcacheproxy_cache_outcome_total{outcome="cached"} 1"#));
+        assert!(out.contains(r#"gitcacheproxy_cache_outcome_total{outcome="fetched"} 1"#));
+        assert!(out.contains(r#"gitcacheproxy_cache_outcome_total{outcome="cloned"} 1"#));
         assert!(out.contains(
             r#"gitcacheproxy_upstream_duration_seconds_count{op="clone",repo="group/foo.git"} 1"#
         ));