From e362f958cce1d5dc8f10eb66d50e7565066ef8ae Mon Sep 17 00:00:00 2001 From: Roland Groza Date: Sat, 31 Oct 2026 13:46:55 +0900 Subject: [PATCH 1/4] fix(evict): heal index after out-of-band deletes The cache-size gauges come from an in-memory index that assumes the proxy owns the cache dir, so a mirror deleted by hand left them stale until restart. The eviction task now reconciles the index against disk on a timer, dropping entries whose path has vanished and refreshing the gauges. Closes #59. Assisted-by: Claude:claude-opus-4-8 --- src/evict.rs | 121 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 118 insertions(+), 3 deletions(-) diff --git a/src/evict.rs b/src/evict.rs index d6ef4b4..239de85 100644 --- a/src/evict.rs +++ b/src/evict.rs @@ -18,9 +18,10 @@ //! behaviour. use std::collections::HashSet; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::SystemTime; +use std::time::{Duration, SystemTime}; use lru::LruCache; use tokio::sync::{Notify, watch}; @@ -28,6 +29,11 @@ use tokio::sync::{Notify, watch}; use crate::git::GitCache; use crate::metrics::Metrics; +/// How often the background task reconciles the in-memory index against disk, +/// dropping entries whose mirror an operator deleted out of band. Only a +/// coarse resync of the cache-size gauges, so a slow cadence is plenty. +const RECONCILE_INTERVAL: Duration = Duration::from_secs(60); + struct Inner { /// Mirror name -> size, kept in access order (front = most-recently-used, back = /// least). The `lru` crate does the O(1) promote-on-use and tail eviction. @@ -178,6 +184,54 @@ impl CacheIndex { victims } + /// Drop tracked entries whose on-disk path has vanished, correcting the total + /// and gauges. The index normally assumes the proxy owns the cache dir and keeps + /// the total by incremental bookkeeping; this is the one path that tolerates an + /// out-of-band deletion (an operator removing a mirror by hand), so the gauges do + /// not drift until the next restart rescans. Runs on a timer, off the request + /// path; the stat sweep is blocking, so it is done on the blocking pool. + async fn reconcile_missing(&self) { + let tracked: Vec<(String, PathBuf)> = { + let inner = self.lock(); + inner + .cache + .iter() + .map(|(name, _)| (name.clone(), self.cache_dir(name))) + .collect() + }; + if tracked.is_empty() { + return; + } + // Treat only a definitive "not found" as gone; a transient stat error must + // not evict a live entry from the index. + let missing = tokio::task::spawn_blocking(move || { + tracked + .into_iter() + .filter(|(_, dir)| { + matches!(std::fs::symlink_metadata(dir), Err(e) if e.kind() == ErrorKind::NotFound) + }) + .map(|(name, _)| name) + .collect::>() + }) + .await + .unwrap_or_default(); + if missing.is_empty() { + return; + } + let mut inner = self.lock(); + for name in missing { + // An entry re-touched since the snapshot (a concurrent re-clone marks it + // dirty) is tracked again and pending measurement, so leave it be. + if inner.dirty.contains(&name) { + continue; + } + if let Some(size) = inner.cache.pop(&name) { + inner.total -= size; + } + } + self.set_gauges(&inner); + } + fn lock(&self) -> MutexGuard<'_, Inner> { self.state.lock().expect("cache index lock") } @@ -188,21 +242,28 @@ impl CacheIndex { } /// Background maintenance task. Measures changed mirrors and evicts the LRU tail -/// whenever the index signals work, and exits cleanly when `shutdown` fires (or its +/// whenever the index signals work, periodically reconciles the index against disk +/// to drop out-of-band deletions, and exits cleanly when `shutdown` fires (or its /// sender drops). pub async fn run( cache: Arc, index: Arc, mut shutdown: watch::Receiver, ) { + let mut reconcile = tokio::time::interval(RECONCILE_INTERVAL); + // A stall must not unleash a burst of catch-up ticks; one resync suffices. + reconcile.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + // The startup scan just read disk, so skip the interval's immediate first tick. + reconcile.tick().await; loop { // Run first: covers the startup-over-cap permit and any signal that arrived // while the previous pass ran (a stored `notify_one` permit makes the next // wait return immediately, so no work is missed). maintain(&cache, &index).await; tokio::select! { - biased; // prefer shutdown over another pass when both are ready + biased; // prefer shutdown over work/reconcile when several are ready _ = shutdown.changed() => break, + _ = reconcile.tick() => index.reconcile_missing().await, _ = index.work.notified() => {} } } @@ -605,6 +666,60 @@ mod tests { assert!(metrics.gather().contains("gitcacheproxy_evictions_total 1")); } + #[tokio::test] + async fn reconcile_drops_mirrors_deleted_out_of_band() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + make_mirror(&root.join("keep.git"), 1000, None); + make_mirror(&root.join("gone.git"), 2000, None); + let metrics = Arc::new(Metrics::new()); + let idx = CacheIndex::new(root.to_path_buf(), u64::MAX, metrics.clone()); + let (before, count) = idx.totals(); + assert_eq!(count, 2); + + // An operator deletes a mirror by hand; nothing signals the index. + std::fs::remove_dir_all(root.join("gone.git")).unwrap(); + assert_eq!(idx.totals(), (before, 2), "index is stale until reconcile"); + + idx.reconcile_missing().await; + // HEAD (21 bytes) + the 1000-byte pack file. + assert_eq!( + idx.totals(), + (1021, 1), + "the vanished mirror is dropped and the total corrected" + ); + assert!(metrics.gather().contains("gitcacheproxy_cache_mirrors 1")); + } + + #[tokio::test] + async fn reconcile_is_a_noop_when_nothing_is_missing() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + // An empty index returns before any disk work. + let empty = CacheIndex::new(root.to_path_buf(), u64::MAX, Arc::new(Metrics::new())); + empty.reconcile_missing().await; + assert_eq!(empty.totals(), (0, 0)); + + // A populated index whose mirrors all still exist drops nothing. + make_mirror(&root.join("present.git"), 100, None); + let idx = CacheIndex::new(root.to_path_buf(), u64::MAX, Arc::new(Metrics::new())); + let before = idx.totals(); + idx.reconcile_missing().await; + assert_eq!(idx.totals(), before, "present mirrors are left tracked"); + } + + #[tokio::test] + async fn reconcile_keeps_a_dirty_entry_pending_measurement() { + let tmp = tempfile::tempdir().unwrap(); + let idx = CacheIndex::new(tmp.path().to_path_buf(), u64::MAX, Arc::new(Metrics::new())); + // Tracked and dirty but not yet on disk: a mirror deleted and re-cloned, + // re-tracked before the reclone has finished writing its dir. Reconcile must + // not drop it out from under the pending measurement. + idx.mark_changed("recloning.git"); + idx.reconcile_missing().await; + assert_eq!(idx.totals(), (0, 1)); + } + fn dummy_cfg() -> GitConfig { // Eviction never shells out to git, so the binary is irrelevant. GitConfig { From 04f7ca0426e186d6fed67acb883f94ab0d5dc69d Mon Sep 17 00:00:00 2001 From: Roland Groza Date: Sat, 31 Oct 2026 13:46:56 +0900 Subject: [PATCH 2/4] docs(readme): explain cache management and deletion Document that the cache is self-managing and that `--cache-max-mb` is the intended way to reclaim disk, since manual mirror deletion is tolerated but not the supported lever (see #59). Assisted-by: Claude:claude-opus-4-8 --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 0a67225..2e9fc8f 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,20 @@ linked. Cached objects share the `--cache-max-mb` budget and LRU eviction with t git mirrors. No configuration is needed: LFS is served on the same endpoints the git client already routes through the proxy. +### Cache management + +The cache is self-managing. Mirrors are created on first request, and with +`--cache-max-mb` set, least-recently-used idle ones are evicted in the background +once the total exceeds the cap - that cap is the intended way to reclaim disk. There +is no endpoint to delete a specific mirror, and none is needed: an evicted or missing +mirror is transparently re-cloned on its next request, so eviction is a cache +concern, never a correctness one. + +Deleting a mirror directory by hand works but is not the intended lever. The proxy +assumes it owns the cache root, so a manual deletion reaches the cache-size metrics +only once the background task reconciles it (within a minute) or the next restart +rescans. Prefer `--cache-max-mb`. + ## Benchmark For a fleet of ephemeral clients fetching the same content, only the first fetch From f88a0b4cfcb7654f25380f73657b878912beb905 Mon Sep 17 00:00:00 2001 From: Roland Groza Date: Sat, 31 Oct 2026 16:19:47 +0900 Subject: [PATCH 3/4] feat(metrics): expose cache cap and TTL gauges Expose the configured cache cap (cache_max_bytes) and fetch TTL (fetch_ttl_seconds) as static gauges set at startup, so a dashboard can compute cache utilization without hardcoding the cap (see #57). Assisted-by: Claude:claude-opus-4-8 --- README.md | 5 +++++ src/main.rs | 4 ++++ src/metrics.rs | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/README.md b/README.md index 2e9fc8f..474a5f5 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,8 @@ Endpoints: `/healthz`, `/readyz`, `/metrics` (see [Metrics](#metrics)). | `upstream_ops_total` | counter | `op`, `result`, `repo` | Upstream git operations (initial mirror clone, refresh fetch, want fetch) | | `cache_bytes` | gauge | - | Total size of the on-disk mirror cache, in bytes | | `cache_mirrors` | gauge | - | Number of cached mirrors on disk | +| `cache_max_bytes` | gauge | - | Configured on-disk cache cap in bytes (`0` = unlimited) | +| `fetch_ttl_seconds` | gauge | - | Configured upstream fetch TTL in seconds (`0` = always fetch) | | `evictions_total` | counter | - | Idle mirrors evicted to keep the cache under `--cache-max-mb` | | `lfs_objects_total` | counter | `result` | Cached git-LFS object lookups | | `upstream_duration_seconds` | histogram | `op`, `repo` | Wall-clock duration of successful upstream clones/fetches | @@ -269,6 +271,9 @@ Notes: - `cache_bytes` and `cache_mirrors` are maintained only when a cache cap is configured (`--cache-max-mb`); without a cap they stay `0`. +- `cache_max_bytes` and `fetch_ttl_seconds` echo the configured flags as static + gauges, set once at startup, so a dashboard can read cache utilization as + `cache_bytes / cache_max_bytes` without hardcoding the cap. - `lfs_objects_total` has no `repo` label: LFS objects are content-addressed and shared across repos. - The histograms are observed only on success and share buckets from 50ms to diff --git a/src/main.rs b/src/main.rs index 21ea510..6dd9223 100644 --- a/src/main.rs +++ b/src/main.rs @@ -59,6 +59,10 @@ async fn main() -> Result<()> { }; let metrics = Arc::new(metrics::Metrics::new()); + metrics.set_config( + cfg.cache_max_mb.saturating_mul(1024 * 1024), + cfg.fetch_ttl_seconds, + ); // Bound the cache on disk only when a cap is set; with `0` no index is built // and the eviction machinery stays inert, preserving the zero-overhead diff --git a/src/metrics.rs b/src/metrics.rs index 1ea516e..ebfa51f 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -137,6 +137,13 @@ pub struct Metrics { cache_bytes: IntGauge, /// `cache_mirrors` - number of cached mirrors (same caveat as `cache_bytes`). cache_mirrors: IntGauge, + /// `cache_max_bytes` - the configured on-disk cache cap in bytes (`0` = + /// unlimited). Static, set once at startup; pairs with `cache_bytes` so a + /// dashboard can show utilization without hardcoding the cap. + cache_max_bytes: IntGauge, + /// `fetch_ttl_seconds` - the configured upstream fetch TTL in seconds (`0` = + /// always fetch). Static, set once at startup. + fetch_ttl_seconds: IntGauge, /// `evictions_total` - idle mirrors evicted to keep the cache under the cap. evictions: IntCounter, /// `lfs_objects_total{result}` - cached git-LFS object lookups; result = hit @@ -178,6 +185,16 @@ impl Metrics { "Number of cached mirrors on disk", ) .expect("valid metric"); + let cache_max_bytes = IntGauge::new( + "gitcacheproxy_cache_max_bytes", + "Configured on-disk cache cap in bytes (0 = unlimited)", + ) + .expect("valid metric"); + let fetch_ttl_seconds = IntGauge::new( + "gitcacheproxy_fetch_ttl_seconds", + "Configured upstream fetch TTL in seconds (0 = always fetch)", + ) + .expect("valid metric"); let evictions = IntCounter::new( "gitcacheproxy_evictions_total", "Idle mirrors evicted to keep the cache under the configured cap", @@ -221,6 +238,12 @@ impl Metrics { registry .register(Box::new(cache_mirrors.clone())) .expect("register cache_mirrors"); + registry + .register(Box::new(cache_max_bytes.clone())) + .expect("register cache_max_bytes"); + registry + .register(Box::new(fetch_ttl_seconds.clone())) + .expect("register fetch_ttl_seconds"); registry .register(Box::new(evictions.clone())) .expect("register evictions"); @@ -239,6 +262,8 @@ impl Metrics { upstream, cache_bytes, cache_mirrors, + cache_max_bytes, + fetch_ttl_seconds, evictions, lfs_objects, upstream_duration, @@ -269,6 +294,12 @@ impl Metrics { self.cache_mirrors.set(mirrors as i64); } + /// Publish the static config gauges. Called once at startup. + pub fn set_config(&self, cache_max_bytes: u64, fetch_ttl_seconds: u64) { + self.cache_max_bytes.set(cache_max_bytes as i64); + self.fetch_ttl_seconds.set(fetch_ttl_seconds as i64); + } + /// Record one evicted mirror. pub fn record_eviction(&self) { self.evictions.inc(); @@ -322,6 +353,7 @@ mod tests { m.record_upstream(UpstreamOp::Fetch, Status::Ok, "group/foo.git"); m.record_upstream(UpstreamOp::Clone, Status::Error, "group/bar.git"); m.set_cache_size(2048, 3); + m.set_config(1024 * 1024, 10); m.record_eviction(); m.record_eviction(); m.record_lfs(LfsResult::Hit); @@ -332,6 +364,8 @@ mod tests { let out = m.gather(); assert!(out.contains("gitcacheproxy_cache_bytes 2048")); assert!(out.contains("gitcacheproxy_cache_mirrors 3")); + assert!(out.contains("gitcacheproxy_cache_max_bytes 1048576")); + assert!(out.contains("gitcacheproxy_fetch_ttl_seconds 10")); assert!(out.contains("gitcacheproxy_evictions_total 2")); assert!(out.contains(r#"gitcacheproxy_lfs_objects_total{result="hit"} 1"#)); assert!(out.contains(r#"gitcacheproxy_lfs_objects_total{result="miss"} 1"#)); From 0374173c62596923205ba1e779b0d38410fd8797 Mon Sep 17 00:00:00 2001 From: Roland Groza Date: Sat, 31 Oct 2026 16:25:26 +0900 Subject: [PATCH 4/4] feat(metrics): count info/refs cache outcomes Add cache_outcome_total{outcome=cloned|fetched|cached}, recorded once per info/refs advertisement where the clone/fetch/TTL decision is made. It is the local-vs-upstream signal: the share served without touching upstream is cached / sum without (outcome) (...). The outcome was already computed by ensure_fresh and discarded; this wires it up. Closes #57. Assisted-by: Claude:claude-opus-4-8 --- README.md | 6 ++++++ src/metrics.rs | 36 ++++++++++++++++++++++++++++++++++++ src/server.rs | 17 ++++++++++++----- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 474a5f5..0465269 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,7 @@ Endpoints: `/healthz`, `/readyz`, `/metrics` (see [Metrics](#metrics)). | `fetch_ttl_seconds` | gauge | - | Configured upstream fetch TTL in seconds (`0` = always fetch) | | `evictions_total` | counter | - | Idle mirrors evicted to keep the cache under `--cache-max-mb` | | `lfs_objects_total` | counter | `result` | Cached git-LFS object lookups | +| `cache_outcome_total` | counter | `outcome` | info/refs operations by outcome: served locally vs. fetched from upstream | | `upstream_duration_seconds` | histogram | `op`, `repo` | Wall-clock duration of successful upstream clones/fetches | | `serve_duration_seconds` | histogram | `kind`, `repo` | Time to serve a client, measured to the end of the response | @@ -261,6 +262,9 @@ Label values: - `op`: `clone` (first mirror of a repo), `fetch` (refresh of an existing mirror), `want_fetch` (on-demand fetch of a bare SHA the mirror lacks, e.g. a PR merge commit). +- `outcome` on `cache_outcome_total`: `cloned` (first mirror of a repo), + `fetched` (refresh from upstream), `cached` (served from the local mirror with + no upstream call). Recorded once per info/refs advertisement. - `repo`: the served repo path (e.g. `group/project.git`). To keep cardinality bounded, the real path is emitted only on success; every failure (failed auth, malformed path, upstream error) uses a `-` sentinel, so a flood of distinct @@ -276,6 +280,8 @@ Notes: `cache_bytes / cache_max_bytes` without hardcoding the cap. - `lfs_objects_total` has no `repo` label: LFS objects are content-addressed and shared across repos. +- `cache_outcome_total` is the local-vs-upstream signal: the share of operations + served without touching upstream is `cached / sum without (outcome) (...)`. - The histograms are observed only on success and share buckets from 50ms to 300s, sized for anything from a cached ref advertisement to a large clone over a slow WAN. diff --git a/src/metrics.rs b/src/metrics.rs index ebfa51f..83b3dc9 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -16,6 +16,8 @@ use prometheus::{ TextEncoder, }; +use crate::git::CacheOutcome; + /// Histogram buckets, in seconds, for git operation latency: from a fast cached /// advertisement (tens of ms) to a large clone over a slow WAN (minutes). const DURATION_BUCKETS: &[f64] = &[ @@ -150,6 +152,11 @@ pub struct Metrics { /// (served from disk) | miss (fetched from upstream, then cached) | error. No /// `repo` label: objects are content-addressed and shared across repos. lfs_objects: IntCounterVec, + /// `cache_outcome_total{outcome}` - info/refs operations by outcome: cloned + /// (first mirror of a repo) | fetched (refresh from upstream) | cached (served + /// from the local mirror, no upstream call). Recorded once per advertisement, + /// so the local-hit ratio is `cached / sum without (outcome) (...)`. + cache_outcome: IntCounterVec, /// `upstream_duration_seconds{op, repo}` - clone/fetch wall-clock, observed /// only on success (same bounded-`repo` discipline as the counters). upstream_duration: HistogramVec, @@ -208,6 +215,14 @@ impl Metrics { &["result"], ) .expect("valid metric"); + let cache_outcome = IntCounterVec::new( + Opts::new( + "gitcacheproxy_cache_outcome_total", + "info/refs operations by cache outcome (cloned/fetched/cached)", + ), + &["outcome"], + ) + .expect("valid metric"); let upstream_duration = HistogramVec::new( HistogramOpts::new( "gitcacheproxy_upstream_duration_seconds", @@ -250,6 +265,9 @@ impl Metrics { registry .register(Box::new(lfs_objects.clone())) .expect("register lfs_objects"); + registry + .register(Box::new(cache_outcome.clone())) + .expect("register cache_outcome"); registry .register(Box::new(upstream_duration.clone())) .expect("register upstream_duration"); @@ -266,6 +284,7 @@ impl Metrics { fetch_ttl_seconds, evictions, lfs_objects, + cache_outcome, upstream_duration, serve_duration, } @@ -310,6 +329,17 @@ impl Metrics { self.lfs_objects.with_label_values(&[result.as_str()]).inc(); } + /// Record the outcome of an info/refs operation (cloned / fetched / cached) - + /// the local-vs-upstream signal for one clone/fetch. + pub fn record_cache_outcome(&self, outcome: CacheOutcome) { + let label = match outcome { + CacheOutcome::Cloned => "cloned", + CacheOutcome::Fetched => "fetched", + CacheOutcome::Cached => "cached", + }; + self.cache_outcome.with_label_values(&[label]).inc(); + } + /// Observe an upstream op's duration. Call only on success with the real repo, /// matching the counters' bounded-`repo` cardinality discipline. pub fn observe_upstream(&self, op: UpstreamOp, repo: &str, seconds: f64) { @@ -358,6 +388,9 @@ mod tests { m.record_eviction(); m.record_lfs(LfsResult::Hit); m.record_lfs(LfsResult::Miss); + m.record_cache_outcome(CacheOutcome::Cached); + m.record_cache_outcome(CacheOutcome::Fetched); + m.record_cache_outcome(CacheOutcome::Cloned); m.observe_upstream(UpstreamOp::Clone, "group/foo.git", 1.5); m.observe_serve(ServeKind::UploadPack, "group/foo.git", 2.0); @@ -369,6 +402,9 @@ mod tests { assert!(out.contains("gitcacheproxy_evictions_total 2")); assert!(out.contains(r#"gitcacheproxy_lfs_objects_total{result="hit"} 1"#)); assert!(out.contains(r#"gitcacheproxy_lfs_objects_total{result="miss"} 1"#)); + assert!(out.contains(r#"gitcacheproxy_cache_outcome_total{outcome="cached"} 1"#)); + assert!(out.contains(r#"gitcacheproxy_cache_outcome_total{outcome="fetched"} 1"#)); + assert!(out.contains(r#"gitcacheproxy_cache_outcome_total{outcome="cloned"} 1"#)); assert!(out.contains( r#"gitcacheproxy_upstream_duration_seconds_count{op="clone",repo="group/foo.git"} 1"# )); diff --git a/src/server.rs b/src/server.rs index 704c532..403733a 100644 --- a/src/server.rs +++ b/src/server.rs @@ -336,11 +336,18 @@ async fn info_refs(st: AppState, path: &str, git_protocol: Option<&str>) -> Resp // here we only account for the client request. The `repo` label is emitted only // once a request is served; failures use `-` so a flood of distinct but doomed // repo paths cannot inflate label cardinality (see `metrics`). - if let Err(e) = st.cache.ensure_fresh(&repo, true, &[]).await { - st.metrics - .record_request(RequestKind::InfoRefs, Status::UpstreamError, "-"); - tracing::warn!(repo = %name, error = %e, "ensure_fresh failed"); - return err(StatusCode::BAD_GATEWAY, "upstream fetch failed"); + // Record the cache outcome here, not on the upload-pack path: info/refs is + // where the clone/fetch/TTL decision is made, so it is the per-operation + // local-vs-upstream signal (upload-pack runs with `want_fetch=false` and is + // almost always `Cached`, which would skew the ratio). + match st.cache.ensure_fresh(&repo, true, &[]).await { + Ok(outcome) => st.metrics.record_cache_outcome(outcome), + Err(e) => { + st.metrics + .record_request(RequestKind::InfoRefs, Status::UpstreamError, "-"); + tracing::warn!(repo = %name, error = %e, "ensure_fresh failed"); + return err(StatusCode::BAD_GATEWAY, "upstream fetch failed"); + } } match st.cache.advertise_refs(&repo, git_protocol).await {