diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d45517e0..c4b7776e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,8 +43,11 @@ jobs: with: go-version-file: go.mod + - name: Install golangci-lint + run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.13.1 + - name: golangci-lint - run: go tool golangci-lint run ./... + run: golangci-lint run ./... helm: name: Helm chart @@ -54,7 +57,7 @@ jobs: with: persist-credentials: false - - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 with: version: v3.18.6 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bc064b6f..b7ac4703 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -26,12 +26,12 @@ jobs: persist-credentials: false - name: Set up QEMU - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: platforms: linux/amd64,linux/arm64 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Log in to the Container registry uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f @@ -50,7 +50,7 @@ jobs: - name: Build and push Docker image id: build - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc with: context: . platforms: linux/amd64,linux/arm64 @@ -112,7 +112,7 @@ jobs: persist-credentials: false ref: ${{ github.sha }} - - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 with: version: v3.18.6 diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index 38c42c3c..4112a992 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -22,7 +22,7 @@ jobs: go-version-file: go.mod - name: Install swag - run: go install github.com/swaggo/swag/cmd/swag@latest + run: go install github.com/swaggo/swag/cmd/swag@v1.16.6 - name: Generate swagger run: go generate ./internal/server diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index ac587dd4..1f6df799 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -5,14 +5,22 @@ on: branches: - main paths: - - '.github/workflows/**' + - '.github/**' + - '**/action.yml' + - '**/action.yaml' + - 'zizmor.yml' + - 'zizmor.yaml' pull_request: - branches: - - main paths: - - '.github/workflows/**' + - '.github/**' + - '**/action.yml' + - '**/action.yaml' + - 'zizmor.yml' + - 'zizmor.yaml' workflow_dispatch: +permissions: {} + jobs: zizmor: runs-on: ubuntu-latest @@ -26,4 +34,4 @@ jobs: persist-credentials: false - name: Run zizmor - uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 + uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 68a6acf9..2c697283 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -160,6 +160,7 @@ mux.Handle("/newregistry/", http.StripPrefix("/newregistry", newHandler.Routes() - Keep functions short and focused - Write tests for new functionality - Document exported types and functions +- A new Prometheus metric needs a tile on `/ui/analytics` and an entry in `metricSurface` (`internal/server/analytics_coverage_test.go`). The page is meant to be a complete view of `/metrics`, so `TestEveryMetricIsSurfaced` fails until both exist. ## Testing diff --git a/Dockerfile b/Dockerfile index 4b0c5d26..de784e82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ COPY . . ARG TARGETARCH RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /proxy ./cmd/proxy -FROM alpine:3.24.1 +FROM alpine:3.24.2 RUN apk add --no-cache ca-certificates diff --git a/README.md b/README.md index 7d374bd1..70de9d79 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,13 @@ Or use environment variable: npm_config_registry=http://localhost:8080/npm/ npm install ``` +`npm audit`, `pnpm audit`, `yarn npm audit` and `npm audit signatures` work +through the proxy: the audit and signing-key endpoints are passed through to the +configured upstream registry, with upstream authentication applied. Advisories +therefore come from upstream's database, not from the proxy's own vulnerability +data, and versions withheld by [cooldown](#version-cooldown) are not excluded +from the report. + ### Cargo Create or edit `~/.cargo/config.toml`: @@ -494,6 +501,29 @@ upstream: debian: "http://archive.ubuntu.com/ubuntu" ``` +A release's security updates are served by a separate archive, so additional +archives are configured under `upstream.debian_repositories` and served at +`/debian/{name}/`: + +```yaml +upstream: + debian: "http://deb.debian.org/debian" + debian_repositories: + security: "https://security.debian.org/debian-security" +``` + +``` +deb http://localhost:8080/debian trixie main +deb http://localhost:8080/debian/security trixie-security main +``` + +`upstream.debian` keeps serving `/debian/pool/…` and `/debian/dists/…` +unchanged. See [Debian archives](docs/configuration.md#debian-archives). + +The version denylist blocks recognized `.deb` downloads but leaves signed APT +indexes unchanged. To keep APT from selecting a denied version, configure +[client-side version pinning](docs/configuration.md#apt-version-pinning). + ### RPM / Yum / DNF Configure yum/dnf to use the proxy in `/etc/yum.repos.d/proxy.repo`: @@ -860,6 +890,8 @@ Recently cached: | `GET /stats` | Cache statistics (JSON) | | `GET /metrics` | Prometheus metrics | | `GET /npm/*` | npm registry protocol | +| `POST /npm/-/npm/v1/security/*` | npm/pnpm/Yarn audit endpoints, passed through to upstream | +| `GET /npm/-/npm/v1/keys` | npm registry signing keys, passed through to upstream | | `GET /cargo/*` | Cargo sparse index protocol | | `GET /gem/*` | RubyGems protocol | | `GET /go/*` | Go module proxy protocol | @@ -880,7 +912,8 @@ Recently cached: | `GET /v2/homebrew/core/*` | Homebrew core bottle manifests and blobs from GHCR | | `GET /apk/{repository}/*` | Alpine APK repository protocol | | `GET /generic/{name}/*` | Generic HTTP download proxy (GitHub release assets, mise/aqua) | -| `GET /debian/*` | Debian/APT repository protocol | +| `GET /debian/*` | Debian/APT repository protocol (main archive) | +| `GET /debian/{repository}/*` | Debian/APT repository protocol (named archive, e.g. security) | | `GET /rpm/*` | RPM/Yum repository protocol | ### Mirror API @@ -1078,6 +1111,7 @@ Response: The proxy serves a web UI under `/ui`. No separate frontend build is needed -- templates and assets are embedded in the binary. `GET /` redirects to `/ui/`. The UI is mounted under its own prefix so a reverse proxy can apply different access rules to it than to the package endpoints (for example, requiring auth for `PathPrefix(/ui)` while leaving `/npm`, `/pypi` etc. open to build machines). - **Dashboard** (`/ui/`) -- cache stats, popular packages, recently cached artifacts, and vulnerability overview. +- **Analytics** (`/ui/analytics`) -- accumulated download size as a ring broken down by ecosystem with the total in the middle, the cache size, artifact, package and version counts, a per-ecosystem table, the vulnerability overview, and a Runtime card mirroring every counter `/metrics` exposes. See [Analytics](#analytics). - **Install guide** (`/ui/install`) -- per-ecosystem configuration instructions, so you don't have to look them up here. - **Package browser** (`/ui/packages`) -- browse all cached packages with filtering by ecosystem and sorting by hits, size, name, or vulnerability count. - **Search** (`/ui/search?q=...`) -- search cached packages by name. @@ -1106,8 +1140,19 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre | `proxy_health_probe_failures_total` | counter | `step` | Storage health probe failures by failing step (`write`, `size`, `read`, `verify`, `delete`). | | `proxy_circuit_breaker_state` | gauge | `registry` | Artifact-fetch circuit breaker state per upstream registry (0 closed, 2 open). Published once that registry's breaker has tripped. | | `proxy_circuit_breaker_trips_total` | counter | `registry` | Circuit breaker trips per upstream registry. | +| `proxy_ecosystem_downloaded_bytes` | gauge | `ecosystem` | Accumulated bytes served from cache: cache hits multiplied by the artifact size they served. | +| `proxy_ecosystem_artifact_downloads` | gauge | `ecosystem` | Accumulated artifact downloads served from cache. | +| `proxy_ecosystem_cache_size_bytes` | gauge | `ecosystem` | Size of cached artifacts per ecosystem. | +| `proxy_ecosystem_cached_artifacts` | gauge | `ecosystem` | Number of cached artifacts per ecosystem. | +| `proxy_ecosystem_packages` | gauge | `ecosystem` | Known packages per ecosystem. | +| `proxy_ecosystem_versions` | gauge | `ecosystem` | Known package versions per ecosystem. | +| `proxy_response_bytes_total` | counter | `ecosystem` | Response body bytes written to clients. Route-labelled, see the label caveat below. | +| `proxy_client_requests_total` | counter | `client` | Requests by client tool, from the User-Agent. | +| `proxy_client_response_bytes_total` | counter | `client` | Response bytes by client tool. | -Cache size and artifact count are refreshed every 60 seconds. Circuit breaker state is read from the fetcher on each scrape of `/metrics` and each `/health` request, so `proxy_circuit_breaker_trips_total` counts the trips visible between those reads — a breaker that opens and recovers entirely between two scrapes is not counted. The remaining metrics update on each request. +The `ecosystem` label on `proxy_requests_total` and `proxy_request_duration_seconds` is the mounted route a request arrived on, not the ecosystem recorded against the package it served: `/gem` reports as `rubygems`, `/go` as `golang`, `/composer` as `packagist`, `/apk` as `alpine` and `/v2` as `oci`. Anything outside a package route -- the UI, `/health`, `/metrics`, `/stats` -- reports as `other`, and so did `/apk`, `/helm`, `/homebrew`, `/generic` and `/swift` before they were listed; traffic on those five routes now appears under its own name instead. + +Cache size, artifact count and the per-ecosystem gauges are refreshed every 60 seconds, from a single pass over the database. Circuit breaker state is read from the fetcher on each scrape of `/metrics` and each `/health` request, so `proxy_circuit_breaker_trips_total` counts the trips visible between those reads — a breaker that opens and recovers entirely between two scrapes is not counted. The remaining metrics update on each request. The breaker metrics carry one series per upstream host, but only for hosts whose breaker has tripped at least once since startup. A breaker is created per host the proxy fetches artifacts from, and for some ecosystems that host comes from upstream metadata rather than from configuration (composer takes it from a package's `dist.url`, helm from the chart URLs in `index.yaml`), so publishing every host would let upstream content grow the series count for the lifetime of the process. Once a host has tripped it keeps reporting, so a recovery still shows up as a transition to 0 rather than as a series that vanishes. `/health` is not a persistent time series and lists every breaker, tripped or not. @@ -1115,6 +1160,68 @@ The `registry` label is the host of the URL the artifact was fetched from. Becau Alert on `proxy_circuit_breaker_state == 2` sustained for more than a few minutes: while a breaker is open, artifact downloads for that upstream fail with HTTP 502 on every cache miss, and only a single probe request per backoff interval reaches the upstream. Cached artifacts keep serving, and so does metadata for the same ecosystem (metadata does not go through the circuit breaker), so installs fail in a way that looks like a partial upstream outage. +#### Accumulated download size + +`proxy_ecosystem_downloaded_bytes` is, for every cached artifact, the number of times it was served multiplied by its size. It answers "how much traffic has this proxy actually carried", which is the number that matters when sizing egress or justifying the cache. Two properties are worth knowing before alerting on it. + +**It counts cache hits, not upstream fetches.** The request that first pulls an artifact through the proxy is a miss and is not counted; only later hits are. So the accumulated total is also the upstream bandwidth the cache has saved, not the total bytes the proxy has ever sent. + +**Eviction removes history.** Evicting an artifact clears its size, so its past hits drop out of the total. That is why these are gauges rather than counters, and why the figure can step downwards. Chart them with `max_over_time` rather than `increase`, and read a drop after an eviction sweep as expected rather than as data loss. + +Nothing at the schema level ties `artifacts.version_purl` to a version row, so a cached artifact can end up with no ecosystem to attribute it to. Those are reported under the ecosystem `unattributed` rather than dropped, which keeps the per-ecosystem figures adding up to `proxy_cache_size_bytes` and `proxy_cached_artifacts_total`. A non-zero `unattributed` means the database holds artifact rows whose version or package rows have gone missing. + +The `ecosystem` label on these six is taken from the package record and normalized, so aliases collapse: a database carrying both `gem` and `rubygems` rows -- the proxy writes the former, git-pkgs the latter -- reports one `rubygems` series with the two summed. + +### Analytics + +`/ui/analytics` reports the accumulated download size as a ring broken down by ecosystem, the cache figures from the dashboard, a per-ecosystem table, the vulnerability overview, and a **Runtime** card covering every remaining metric `/metrics` exposes. + +The ring shows at most six slices, because part-to-whole stops being readable past that. When more ecosystems are active the smallest are folded into a single "Other" slice; the table below lists every one of them, so nothing is hidden, only summarised. + +#### No history is kept + +The proxy stores no time series. The page reads the database and the in-process metric registry at request time and reports current state; there is nowhere for it to read yesterday's figures from, and nothing is written for tomorrow. That splits the figures in two, and the page says which is which. + +**Database-derived figures survive a restart.** Download volume, cache size and the package, version and artifact counts come from the `artifacts`, `packages` and `versions` tables, so they are as durable as the database. + +**Registry-derived figures do not.** Everything in the Runtime card -- request counts and latencies, cache hit rate, upstream and storage errors, circuit breaker state, scan results -- lives only in this process's Prometheus registry and starts from zero on restart. A small number there next to a large one above just means the proxy started recently. + +For history, trends and alerting, scrape `/metrics` with Prometheus. That is the intended split: the UI answers "what is true now", Prometheus answers "what happened". + +The same figures are available as JSON from `GET /stats`, which reports `downloaded_bytes`, `downloads` and an `ecosystems` array carrying the per-ecosystem breakdown, served from the same 60-second snapshot the page and the gauges read. When the aggregation fails with no snapshot to fall back on, the response carries `stats_unavailable: true` rather than passing zeros off as a count -- the endpoint keeps answering with the artifact count and cache size either way. + +#### Three ecosystem label sets + +`ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which. + +**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`. + +**From the request path.** `proxy_requests_total`, `proxy_request_duration_seconds` and `proxy_response_bytes_total`. The names mostly coincide with the normalized ones -- these also report `rubygems`, `packagist` and `golang` -- but the Debian route reports `debian` where the package record says `deb`, and any path that is not a package endpoint reports `other`, which corresponds to no ecosystem at all. + +**From the handler's own name.** `proxy_upstream_fetch_duration_seconds` and `proxy_upstream_errors_total`, which report `composer`, `gem` and `go` where the other two sets report `packagist`, `rubygems` and `golang`. These are published series and are deliberately left as they are; renaming them would break existing queries and alerts. + +### Request sources + +Package managers do not say who invoked them. A request from `pip` or `go` carries a `Host`, an `Accept` and a `User-Agent` -- no `Referer`, no originating URL, nothing naming a repository, pipeline or job. Whatever identity you want has to come from something on the wire, so the proxy attributes requests by the two things always present. + +**Address** -- the TCP peer, or the leftmost `X-Forwarded-For` entry when `trust_forwarded_for` is enabled. Enable that only behind a load balancer or ingress that sets the header; any client can send it, so in front of one it lets a caller forge its own attribution and, by cycling synthetic addresses, fill the table and push every genuine caller into the overflow row. The totals stay correct; the attribution is what is lost. + +**Client** -- the tool, taken from the leading User-Agent token: `pip`, `npm`, `go`, `docker`, `apt`, `curl` and so on. Anything unrecognised reports as `other`. + +Set `ui_request_sources: true` and both appear on `/ui/analytics` under **Runtime -> Request sources**, as a table of the busiest callers by bytes downloaded plus a per-tool breakdown. The table is in-memory and process-lifetime, like the rest of that card. It tracks 200 callers, evicting the least recently seen once full, and summarises everything it is not showing individually -- both evicted callers and those ranked below the display limit -- in a single "other callers" row, so the rows always add up to the totals above them. + +**The flag defaults to off because the page is not authenticated.** `/ui` carries no auth of its own -- it is mounted under its own prefix so a reverse proxy *can* gate it separately, as [Behind a Reverse Proxy](#behind-a-reverse-proxy) describes, but nothing makes you -- and until now it exposed only package data. The sources table changes what is on offer: anyone who can reach the proxy can read the addresses of your build fleet, which tool each runs, and how much each pulled. Turn it on once `/ui` is gated, or leave it off and read the same detail from the access log. + +**What this can and cannot tell you.** How much an address gives you depends entirely on your network. A fleet of build machines with stable addresses attributes cleanly. Containerised CI usually does not: with Docker or Kubernetes executors every job gets an ephemeral address, and egress is commonly NAT'd behind one gateway, so you get runner-node or gateway granularity, not per-project. If you need per-project attribution the caller has to send something naming itself -- a basic-auth username, or a per-project base URL -- which the proxy does not currently read. Say so and it can be added. + +**Why addresses are not Prometheus labels.** Client tool names are exported as `proxy_client_requests_total{client}` because they come from a closed set. Addresses are not exported at all: the caller set is unbounded and outside the proxy's control, and every new address would create a time series that lives forever in your TSDB. The same goes for anything job-scoped -- a pipeline ID must never become a label. Per-address and per-request detail belongs in the access log, which records `remote_ip`, `user_agent`, `client`, `ecosystem` and `bytes` on every line as JSONL, ready for `jq`, Loki or whatever you ship logs to. + +### Grafana dashboard + +A ready-made dashboard lives at [`deploy/grafana/git-pkgs-proxy.json`](deploy/grafana/git-pkgs-proxy.json). Import it via **Dashboards -> New -> Import** and pick your Prometheus data source when prompted; it has no hardcoded data source UID. + +It carries three ecosystem filters rather than one, because `ecosystem` means three different things across `/metrics` -- see the label sets above. **Ecosystem** filters the database-derived gauges, **Route** the request-path counters, and **Upstream** the two upstream fetch metrics. All three are query variables, so they populate from whatever labels your proxy is actually reporting; a panel is on the one its metric belongs to, and the panel descriptions say which. + ### Health Check `/health` returns a structured JSON report of subsystem health. HTTP 200 if all checks pass; 503 if any fail. @@ -1306,4 +1413,4 @@ go test ./... ## License -GPL-3.0-or-later +[GPL-3.0-or-later](LICENSE). diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index c943a4d4..5b625631 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -106,11 +106,13 @@ import ( "log/slog" "os" "os/signal" + "runtime/debug" "strings" "syscall" "github.com/git-pkgs/proxy/internal/config" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/handler" "github.com/git-pkgs/proxy/internal/mirror" "github.com/git-pkgs/proxy/internal/server" @@ -128,6 +130,15 @@ var ( Commit = "unknown" ) +func init() { + if Version != "dev" { + return + } + if bi, ok := debug.ReadBuildInfo(); ok && bi.Main.Version != "" && bi.Main.Version != "(devel)" { + Version = bi.Main.Version + } +} + func main() { if len(os.Args) > 1 { switch os.Args[1] { @@ -468,6 +479,10 @@ func runMirror() { fmt.Fprintf(os.Stderr, "invalid configuration: %v\n", err) os.Exit(1) } + if !cfg.Storage.CacheArtifacts { + fmt.Fprintf(os.Stderr, "error: mirror is not available with storage.cache_artifacts: false: mirrored artifacts would never be served\n") + os.Exit(1) + } logger := setupLogger("info", "text") @@ -507,6 +522,12 @@ func runMirror() { fetcher := fetch.NewFetcher() resolver := fetch.NewResolver() proxy := handler.NewProxy(db, store, fetcher, resolver, logger) + proxy.Denylist, err = denylist.New(cfg.Denylist.Packages) + if err != nil { + _ = db.Close() + fmt.Fprintf(os.Stderr, "invalid denylist: %v\n", err) + os.Exit(1) //nolint:gocritic // db closed above + } proxy.CacheMetadata = true // mirror always caches metadata proxy.MetadataTTL = cfg.ParseMetadataTTL() proxy.MetadataMaxSize = cfg.ParseMetadataMaxSize() diff --git a/config.example.yaml b/config.example.yaml index 82a617e3..49b81efd 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -14,12 +14,34 @@ base_url: "http://localhost:8080" # Set to "0" to disable the timeout. Default: "30s". # http_timeout: "30s" +# Memory for rewritten npm and Composer metadata, so each upstream document is +# rewritten once rather than on every request. Set to "0" to rewrite on every +# request. Default: "256MB". +# metadata_rewrite_cache_size: "256MB" + # Public URL where the web UI is reached. Defaults to base_url when unset. # Set this separately when the UI is served on a different hostname than the # package endpoints — for example, the UI on a public domain behind auth while # build machines hit a Docker network alias for the package endpoints. # ui_base_url: "https://proxy.example.com/ui" +# Attribute requests to the leftmost X-Forwarded-For entry rather than the TCP +# peer address, in the structured log, the access log and the request-source +# table on /ui/analytics. +# +# Enable this only when the proxy sits behind a load balancer or ingress that +# sets the header. Any client can send it: behind one it is the only way to see +# past the hop, in front of one it lets a caller forge its own address. +# trust_forwarded_for: false + +# Show the request-source table on /ui/analytics: caller addresses, the tool +# each ran and how much each pulled. +# +# Off by default. The proxy has no authentication of its own, so leave this off +# unless /ui is gated by a reverse proxy; anyone who can reach the page can +# otherwise read the addresses of your build fleet. +# ui_request_sources: false + # Artifact storage configuration storage: # Storage backend URL @@ -52,6 +74,12 @@ storage: # Empty or "0" means unlimited max_size: "" + # Store fetched artifacts. Set to false to stream every download from + # upstream without storing it; metadata filtering, cooldown and the + # denylist still apply. Useful when another cache sits in front of the + # proxy. false is incompatible with direct_serve, scanning and mirror_api. + cache_artifacts: true + # Redirect cached artifact downloads to presigned storage URLs (HTTP 302) # instead of streaming through the proxy. Only effective for S3, GCS, and Azure. # Leave disabled if clients reach the proxy through an authenticating gateway, @@ -81,6 +109,10 @@ database: # Example: "postgres://user:password@localhost:5432/proxy?sslmode=disable" url: "" + # How often cache hit counts and last-access times are written, batched in + # one transaction. "0" writes each hit as it happens. + hit_flush_interval: "1s" + # Logging configuration log: # Minimum log level: "debug", "info", "warn", "error" @@ -174,6 +206,11 @@ upstream: # Debian/APT repository URL (used by /debian endpoint) debian: "http://deb.debian.org/debian" + # Additional Debian/APT archives, served at /debian/{name}/. + # The names "pool" and "dists" are reserved for the main archive's own paths. + debian_repositories: + security: "https://security.debian.org/debian-security" + # RPM repository URL (used by /rpm endpoint) rpm: "https://dl.fedoraproject.org/pub/fedora/linux" @@ -295,6 +332,20 @@ cooldown: # "pkg:npm/lodash": "0" # "pkg:npm/@babel/core": "14d" + # Per-package glob overrides, after exact packages and before ecosystems. + # package_patterns: + # "pkg:npm/@example/*": "0" + +# Exact versions to deny, independently of cooldown and scanning. +# Metadata filtering: npm, PyPI and Cargo. Shared artifact downloads, including +# cache hits, are blocked with 403; signed APT metadata is left unchanged. +# Use versioned PURLs without qualifiers/subpaths. Restart after changes. +# denylist: +# packages: +# - "pkg:pypi/requests@2.31.0" +# - "pkg:cargo/some-crate@1.2.3" +# - "pkg:npm/%40scope/example@4.5.6" + # Pre-cache artifact scanning. When enabled, every artifact is staged into # storage and scanned by the configured scanners before it is committed to # the cache and served to clients. Scanners never receive artifact bytes diff --git a/deploy/grafana/git-pkgs-proxy.json b/deploy/grafana/git-pkgs-proxy.json new file mode 100644 index 00000000..e875469b --- /dev/null +++ b/deploy/grafana/git-pkgs-proxy.json @@ -0,0 +1,2935 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Download volume, cache composition and upstream health for the git-pkgs proxy. Accumulated download size is cache hits multiplied by artifact size, in total and per ecosystem.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Total bytes served to clients from cache: for each artifact, its size multiplied by the number of times it was served. This is traffic carried, not storage used, so it is normally far larger than the cache on disk. Artifacts evicted from the cache stop contributing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Accumulated download size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Bytes the cached artifacts currently occupy in storage \u2014 what the cache costs to keep. Not an average, and not the same as accumulated download size, which is what the cache has served.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(proxy_cache_size_bytes{job=~\"$job\"})", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Cache size on disk", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Accumulated download size divided by the bytes currently cached: how many times over the cache has served what it stores. This is the number that makes the two size figures beside it comparable, and it is the clearest single measure of what the cache is worth.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "suffix:\u00d7" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 10, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"}) / sum(proxy_ecosystem_cache_size_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Served per byte cached", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of distinct artifacts held in the cache. A small count with a large accumulated download size just means a few artifacts are being served repeatedly, which is the cache doing its job.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 15, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(proxy_cached_artifacts_total{job=~\"$job\"})", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Cached artifacts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Share of artifact lookups served from cache over the dashboard's time range. Reads as No data when nothing was requested in that range \u2014 a range with no traffic has no hit ratio, which is not the same as a hit ratio of zero.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds", + "fixedColor": "text" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__range]))\n/\n(\n sum(increase(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__range]))\n +\n sum(increase(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__range]))\n)", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Cache hit ratio", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 7, + "panels": [], + "title": "Download volume", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Accumulated bytes served from cache, per ecosystem, largest first. One measure across categories, so a single hue carries magnitude.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 8, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "left", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [], + "fields": "/^Value$/", + "values": true + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "text", + "legend": { + "showLegend": false + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "A", + "legendFormat": "{{ecosystem}}", + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": {}, + "renameByName": {} + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Value" + } + ] + } + } + ], + "title": "Download size by ecosystem", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "How the accumulated total has grown, stacked by ecosystem. A step down means artifacts were evicted, which removes their historical hits from the total.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 18, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Accumulated download size over time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Every per-ecosystem figure the charts leave to a tooltip.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Downloaded" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge", + "valueDisplayMode": "text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Downloaded" + } + ] + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_downloaded_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "A", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_artifact_downloads{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "B", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_cache_size_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "C", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_cached_artifacts{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "D", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_packages{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "E", + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_versions{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": false, + "instant": true, + "refId": "F", + "format": "table" + } + ], + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "ecosystem", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true + }, + "indexByName": {}, + "renameByName": { + "ecosystem": "Ecosystem", + "Value #A": "Downloaded", + "Value #B": "Downloads", + "Value #C": "Cache size", + "Value #D": "Artifacts", + "Value #E": "Packages", + "Value #F": "Versions" + } + } + } + ], + "title": "Per-ecosystem breakdown", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 11, + "panels": [], + "title": "Traffic and cache", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Responses per second, by package ecosystem. The ecosystem label on this metric is derived from the request path, so it uses route names (rubygems, packagist, debian, other) rather than the package-ecosystem names the cache metrics use. It is filtered by the Route variable, not Ecosystem.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (rate(proxy_requests_total{job=~\"$job\", ecosystem=~\"$route\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Request rate by route", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Artifact lookups per second that were served from cache versus fetched upstream.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(proxy_cache_hits_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "hits" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(proxy_cache_misses_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "B", + "legendFormat": "misses" + } + ], + "title": "Cache hits and misses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "95th percentile time to serve a proxy request, by ecosystem. The ecosystem label on this metric is derived from the request path, so it uses route names (rubygems, packagist, debian, other) rather than the package-ecosystem names the cache metrics use. It is filtered by the Route variable, not Ecosystem.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, ecosystem) (rate(proxy_request_duration_seconds_bucket{job=~\"$job\", ecosystem=~\"$route\"}[$__rate_interval])))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Request duration p95", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "95th percentile time spent fetching an artifact from the upstream registry, by ecosystem. Only cache misses reach an upstream. The ecosystem label on this metric is the handler's own name, which differs from both the cache metrics (composer vs packagist, gem vs rubygems, go vs golang) and the route names, so it is filtered by the Upstream variable.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, ecosystem) (rate(proxy_upstream_fetch_duration_seconds_bucket{job=~\"$job\", ecosystem=~\"$upstream\"}[$__rate_interval])))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Upstream fetch duration p95", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 16, + "panels": [], + "title": "Reliability", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Upstreams whose artifact-fetch breaker is open. While a breaker is open, cache misses for that host return 502 without contacting the upstream. Alert on this being above zero for more than a few minutes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 45 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(proxy_circuit_breaker_state{job=~\"$job\"} == 2) or vector(0)", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Circuit breakers open", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Requests currently in flight.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "text" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 45 + }, + "id": 18, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(proxy_active_requests{job=~\"$job\"})", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Active requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Circuit breaker trips per upstream registry. A breaker is created per host the proxy fetches artifacts from, and only reports once it has tripped at least once, so an empty panel means no upstream has failed repeatedly.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 45 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (registry) (increase(proxy_circuit_breaker_trips_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "{{registry}}", + "range": true, + "instant": false, + "refId": "A" + } + ], + "title": "Circuit breaker trips", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Upstream fetch failures per second, by ecosystem and error type. The ecosystem label on this metric is the handler's own name, which differs from both the cache metrics (composer vs packagist, gem vs rubygems, go vs golang) and the route names, so it is filtered by the Upstream variable.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 45 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem, error_type) (rate(proxy_upstream_errors_total{job=~\"$job\", ecosystem=~\"$upstream\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}} \u00b7 {{error_type}}" + } + ], + "title": "Upstream errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Storage read/write failures, cached artifacts that failed hash verification on read, and storage health probe failures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 45 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation) (rate(proxy_storage_errors_total{job=~\"$job\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "storage \u00b7 {{operation}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (rate(proxy_integrity_failures_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "B", + "legendFormat": "integrity \u00b7 {{ecosystem}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (step) (rate(proxy_health_probe_failures_total{job=~\"$job\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "C", + "legendFormat": "health probe \u00b7 {{step}}" + } + ], + "title": "Storage and integrity failures", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 22, + "panels": [], + "title": "Storage and scanning", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "95th percentile storage read/write latency, by operation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(proxy_storage_operation_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{operation}}" + } + ], + "title": "Storage operation latency p95", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Storage operations per second, by operation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation) (rate(proxy_storage_operation_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{operation}}" + } + ], + "title": "Storage operation rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Completed pre-cache scans per second, by scanner. Empty when scanning is not configured.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 61 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (scanner) (rate(proxy_scan_duration_seconds_count{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{scanner}}" + } + ], + "title": "Scan rate by scanner", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "95th percentile pre-cache scan duration, by scanner. A block-mode scanner's latency is on the critical path of a cache miss.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 61 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, scanner) (rate(proxy_scan_duration_seconds_bucket{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval])))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{scanner}}" + } + ], + "title": "Scan duration p95", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Artifacts a block-mode scanner refused, and scan calls that failed, timed out or were cancelled. A rising error rate means artifacts are being admitted without a verdict.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 61 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem, scanner) (rate(proxy_scan_blocked_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "blocked \u00b7 {{ecosystem}} \u00b7 {{scanner}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (scanner, error_type) (rate(proxy_scan_errors_total{job=~\"$job\", ecosystem=~\"$ecosystem\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "B", + "legendFormat": "error \u00b7 {{scanner}} \u00b7 {{error_type}}" + } + ], + "title": "Artifacts blocked and scan errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 28, + "panels": [], + "title": "Cache composition", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Bytes held in the cache per ecosystem. A drop is an eviction sweep.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 70 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_cache_size_bytes{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Cache size by ecosystem", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Artifacts held in the cache per ecosystem.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_cached_artifacts{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Cached artifacts by ecosystem", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Package and version rows known per ecosystem. These grow as metadata is fetched, independently of what is cached.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 78 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_packages{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "packages \u00b7 {{ecosystem}}" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (proxy_ecosystem_versions{job=~\"$job\", ecosystem=~\"$ecosystem\"})", + "range": true, + "instant": false, + "refId": "B", + "legendFormat": "versions \u00b7 {{ecosystem}}" + } + ], + "title": "Known packages and versions", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 32, + "panels": [], + "title": "Request sources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Response body bytes per second written to clients. The ecosystem label on this counter comes from the request path, not the package record, so it is filtered by the Route variable and reports route names (debian, and other for the UI and health checks). Distinct from proxy_ecosystem_downloaded_bytes, which is derived from the database as cache hits times artifact size.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 18, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 87 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (ecosystem) (rate(proxy_response_bytes_total{job=~\"$job\", ecosystem=~\"$route\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{ecosystem}}" + } + ], + "title": "Bytes served by route", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Response bytes per second by client tool, identified from the User-Agent. The label set is closed \u2014 an unrecognised User-Agent reports as 'other' \u2014 so a caller cannot create new time series. Caller addresses are deliberately not exported as labels; see the access log or the proxy's own analytics page for per-address detail.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 18, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 87 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (client) (rate(proxy_client_response_bytes_total{job=~\"$job\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{client}}" + } + ], + "title": "Bytes served by client", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Requests per second by client tool. Useful for spotting a runaway CI job or a tool that is not honouring caches.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 96 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (client) (rate(proxy_client_requests_total{job=~\"$job\"}[$__rate_interval]))", + "range": true, + "instant": false, + "refId": "A", + "legendFormat": "{{client}}" + } + ], + "title": "Requests by client", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "git-pkgs", + "proxy", + "cache" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(proxy_cache_size_bytes, job)", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(proxy_cache_size_bytes, job)", + "refId": "job" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(proxy_ecosystem_downloaded_bytes, ecosystem)", + "hide": 0, + "includeAll": true, + "label": "Ecosystem", + "multi": true, + "name": "ecosystem", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(proxy_ecosystem_downloaded_bytes, ecosystem)", + "refId": "ecosystem" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(proxy_requests_total, ecosystem)", + "description": "Route names taken from the request path. These differ from the package-ecosystem names used by the cache metrics.", + "hide": 0, + "includeAll": true, + "label": "Route", + "multi": true, + "name": "route", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(proxy_requests_total, ecosystem)", + "refId": "route" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(proxy_upstream_fetch_duration_seconds_count, ecosystem)", + "description": "Ecosystem names as the upstream fetch metrics report them, taken from the handler rather than the package record: composer, gem and go where the cache metrics say packagist, rubygems and golang.", + "hide": 0, + "includeAll": true, + "label": "Upstream", + "multi": true, + "name": "upstream", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(proxy_upstream_fetch_duration_seconds_count, ecosystem)", + "refId": "upstream" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "git-pkgs proxy", + "uid": "git-pkgs-proxy", + "version": 1, + "weekStart": "" +} diff --git a/docs/architecture.md b/docs/architecture.md index 9e656ef4..0d37a60a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -62,6 +62,14 @@ Metadata is not cached - always fetched fresh. This ensures clients see new vers - Return reader to handler - Handler streams file to client +Artifact handlers honor single byte ranges when the storage reader supports +efficient seeking (currently the local filesystem reader). They advertise +`Accept-Ranges: bytes`, return `206` or `416` as appropriate, and respect +`If-Range`. Malformed and multi-range requests fall back to the full response. +Non-seekable storage readers do not advertise range support, and direct-storage +redirects rely on the destination's capabilities. Range responses cannot verify +the full artifact digest because they read only part of the artifact. + ``` ┌────────┐ GET /npm/lodash/-/lodash-4.17.21.tgz ┌─────────────┐ │ Client │ ──────────────────────────────────────▶│ NPMHandler │ @@ -146,6 +154,11 @@ artifacts ( ) -- indexes: (version_purl, filename) unique, storage_path, last_accessed_at +pending_deletes ( + path TEXT NOT NULL PRIMARY KEY, -- storage path no record points at + queued_at DATETIME NOT NULL +) + vulnerabilities ( id INTEGER PRIMARY KEY, vuln_id TEXT NOT NULL, -- e.g. CVE-2021-1234 @@ -209,10 +222,10 @@ type Storage interface { ``` **Filesystem implementation:** -- Stores files in nested directories: `{ecosystem}/{name}/{version}/{filename}` +- Stores files in nested directories: `{ecosystem}/{name}/{version}/{fetch id}/{filename}`, a new id per fetch, so fetches of one artifact never overwrite or delete each other's object (artifacts cached before this sit at `{ecosystem}/{name}/{version}/{filename}`) - Atomic writes using temp file + rename - Computes SHA256 hash during write -- Cleans up empty parent directories on delete +- Removes a fetch's directory once its object is deleted **Path structure:** @@ -221,15 +234,18 @@ cache/artifacts/ ├── npm/ │ ├── lodash/ │ │ └── 4.17.21/ -│ │ └── lodash-4.17.21.tgz +│ │ └── 3f9a0c1d2e4b5a67/ +│ │ └── lodash-4.17.21.tgz │ └── @babel/ │ └── core/ │ └── 7.23.0/ -│ └── core-7.23.0.tgz +│ └── 8c2e41f09a7d3b15/ +│ └── core-7.23.0.tgz └── cargo/ └── serde/ └── 1.0.193/ - └── serde-1.0.193.crate + └── d05b7e9c14a2f863/ + └── serde-1.0.193.crate ``` ### `internal/upstream` @@ -341,6 +357,8 @@ Eviction can be implemented as: 2. When over limit, get LRU artifacts 3. Delete from storage and clear database records +An object a record stops pointing at, because a refetch replaced it or its entry was discarded, is not deleted at once: a request that read the record may still be opening it. Its path goes into `pending_deletes`, and a background loop deletes it after a grace period of at least an hour, or `direct_serve_ttl` if longer, so signed URLs to it stay valid. This runs whether or not `max_size` is set. + ## Design Decisions **Why SQLite?** diff --git a/docs/configuration.md b/docs/configuration.md index 6d27d7f8..8261ebae 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -19,6 +19,8 @@ See `config.example.yaml` in the repository root for a complete example. | `listen` | `PROXY_LISTEN` | `-listen` | `:8080` | Address to listen on | | `base_url` | `PROXY_BASE_URL` | `-base-url` | `http://localhost:8080` | Public URL package managers use to reach this proxy | | `ui_base_url` | `PROXY_UI_URL` | - | (defaults to `base_url`) | Public URL where the web UI is reached. Set separately when the UI lives behind a different hostname than package endpoints (e.g. public domain vs Docker network alias). Used for canonical/og:url tags and the install guide banner. The proxy still serves package endpoints on the same listener, so any reverse proxy fronting the UI publicly should restrict the public route to `PathPrefix(/ui)` to avoid exposing package endpoints. | +| `trust_forwarded_for` | `PROXY_TRUST_FORWARDED_FOR` | - | `false` | Attribute requests to the leftmost `X-Forwarded-For` entry instead of the TCP peer address, in the structured log, the access log and the request-source table on `/ui/analytics`. | +| `ui_request_sources` | `PROXY_UI_REQUEST_SOURCES` | - | `false` | Show the request-source table on `/ui/analytics`, which reports caller addresses, the tool each ran and how much each pulled. | ## Storage @@ -43,6 +45,22 @@ storage: | `storage.url` | `PROXY_STORAGE_URL` | `-storage-url` | Storage URL (file:// or s3://) | | `storage.path` | `PROXY_STORAGE_PATH` | `-storage-path` | Local path (deprecated, use url) | | `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB") | +| `storage.cache_artifacts` | `PROXY_STORAGE_CACHE_ARTIFACTS` | - | Store fetched artifacts (default: true); `false` streams them from upstream | + +`storage.max_size` counts cached artifacts only. An artifact replaced by a refetch stays in storage for at least an hour, or `storage.direct_serve_ttl` if longer, so requests already reading it can finish, and storage use can exceed the limit by what was replaced in that time. + +### Serving artifacts without storing them + +With `storage.cache_artifacts: false` the proxy streams every artifact download from upstream to the client and stores no artifacts. Metadata is still filtered and cached, so cooldown and the denylist apply as usual, and the database still holds the publish times cooldown needs. Use it when another caching layer, such as an Artifactory remote repository, sits in front of the proxy and caching artifacts twice only costs storage. + +```yaml +storage: + cache_artifacts: false +``` + +Every download is a fresh upstream fetch, and concurrent requests for the same artifact are not combined. Artifacts with a digest known up front (OCI blobs, Swift archives, Helm charts) are verified while streaming: the response is sent chunked, and on a mismatch the connection is aborted before the response completes so the client never receives a tampered artifact as a good one. The same happens when the upstream connection fails mid-download. + +`cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve` or `mirror_api`, which all need stored artifacts, and the `mirror` command refuses to run with it. ### Amazon S3 @@ -95,6 +113,21 @@ database: |--------|-------------|------|-------------| | `database.url` | `PROXY_DATABASE_URL` | `-database-url` | PostgreSQL connection URL | +### Hit counts + +Every cache hit updates the artifact's hit count and last-access time, which the stats pages and LRU eviction use. Rather than writing each hit as its own transaction, the proxy counts hits in memory and writes them together every `hit_flush_interval`. SQLite allows one writer at a time, and the proxy uses a single SQLite connection, so with a write per hit, concurrent downloads queue behind each other. + +```yaml +database: + hit_flush_interval: "1s" +``` + +| Config | Environment | Flag | Description | +|--------|-------------|------|-------------| +| `database.hit_flush_interval` | `PROXY_DATABASE_HIT_FLUSH_INTERVAL` | - | How often batched hits are written (default `1s`). `0` writes each hit as it happens. | + +Hits not yet written are lost if the process is killed; a normal shutdown writes them. + ## Logging ```yaml @@ -121,6 +154,10 @@ access_log: |--------|-------------|------|-------------| | `access_log.path` | `PROXY_ACCESS_LOG_PATH` | `-access-log` | File to append JSONL records to; empty disables the log | +Each client request record carries `remote_addr` (the TCP peer, host and port), `remote_ip` (the address the request is attributed to), `user_agent`, `client` (the tool name derived from the User-Agent), `ecosystem`, and `bytes` (the response body size written to the client). The structured log carries the same `client`, `remote` and `remote_ip` fields. + +`remote_ip` follows the top-level `trust_forwarded_for` setting. Enable that only when the proxy sits behind a load balancer or ingress that sets the header: any client can send `X-Forwarded-For`, so behind such a hop it is the only way to see the real caller, but in front of one it lets a caller choose what address it is logged as. It can also choose a fresh one per request, which fills the bounded source table on `/ui/analytics` and evicts the genuine callers from it. `remote_addr` is unaffected and always records the TCP peer. + The parent directory must exist and be writable when the proxy starts. A newly created log file is readable and writable only by the proxy process owner. A request that receives a rate limit response from an upstream can produce records like these: @@ -211,6 +248,13 @@ repository's `index.yaml` so chart archives are downloaded through the proxy. Chart archives are retained only when their SHA-256 digest matches the digest listed in the index. Relative and absolute chart URLs are both supported. +HTTP indexes can also contain `oci://` chart references. References matching the +host and port of a configured `upstream.oci` registry or `upstream.oci_default` +are rewritten through the OCI proxy, with named registries preferred. Unmatched +references are left unchanged, so Helm contacts those registries directly. +Automatic rewriting requires both the OCI upstream URL and `base_url` to be +root URLs without a path prefix. Repository paths, tags, and digests are preserved. + Generic HTTP upstreams proxy plain downloads from fixed base URLs: ```yaml @@ -265,6 +309,35 @@ verification keeps working; `.apk` packages use the shared artifact cache. When `upstream.apk` is empty, a single repository named `alpine` pointing at the official mirror is available; configuring any entry replaces that default. +### Debian archives + +```yaml +upstream: + + # The main APT archive, served at /debian/. + debian: "http://deb.debian.org/debian" + + # Additional APT archives, served at /debian/{name}/. + debian_repositories: + security: "https://security.debian.org/debian-security" +``` + +A Debian release is served by more than one archive: security updates live on +a separate host from the main archive, so `upstream.debian` alone cannot serve +a complete suite set. Requests to `/debian/{name}/…` mirror the upstream +layout, e.g. `/debian/security/dists/trixie-security/InRelease`. + +`upstream.debian_repositories` is additive. `/debian/pool/…` and +`/debian/dists/…` continue to address `upstream.debian` with unchanged cache +identities, so existing deployments and their warm caches are unaffected. A +repository name shadows the main archive's root path of the same name; `pool` +and `dists` are refused at config load for that reason, and other root paths +an archive may serve (`indices`, `project`, `doc`, `tools`) are not, so avoid +those names unless the shadowing is intended. + +This field has no environment variable, as with the other named upstream maps. +`PROXY_UPSTREAM_DEBIAN` still sets `upstream.debian`. + ## Authentication Configure authentication for private upstream registries. The same authentication-aware client is used for metadata and artifact downloads, and credentials can reference environment variables using `${VAR_NAME}` syntax. @@ -385,6 +458,8 @@ cooldown: packages: "pkg:npm/lodash": "0" "pkg:npm/@babel/core": "14d" + package_patterns: + "pkg:npm/@example/*": "0" ``` | Config | Environment | Description | @@ -392,17 +467,110 @@ cooldown: | `cooldown.default` | `PROXY_COOLDOWN_DEFAULT` | Global default cooldown | | `cooldown.ecosystems` | - | Per-ecosystem overrides | | `cooldown.packages` | - | Per-package overrides (keyed by PURL) | +| `cooldown.package_patterns` | - | Per-package glob overrides (keyed by PURL glob) | Durations support days (`7d`), hours (`48h`), and minutes (`30m`). Set to `0` to disable. Package PURL keys are normalized to canonical form before matching, so `pkg:npm/@babel/core` and `pkg:npm/%40babel/core` are equivalent, as are `pkg:pypi/Django` and `pkg:pypi/django`. If both forms configure the same package, the canonical entry wins. -Resolution order: package override, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted packages. +`package_patterns` matches canonical, versionless PURLs using `*` for zero or more characters and `?` for one character. Neither wildcard crosses `/` separators. Character classes (`[...]`) and backslash escapes are rejected at startup. For example, `"pkg:npm/@example/*"` matches packages under the `@example` npm scope. Patterns accept `@` as an alias for `%40`; other characters must use their canonical PURL form. Equivalent patterns with different durations are rejected at startup. Equal durations, such as `1d` and `24h`, are accepted. + +Exact `packages` entries take precedence over patterns. When several patterns match, longer patterns win after excluding `*` and `?` from the length. Ties use lexical order of the normalized patterns. + +Resolution order: exact package override, then package pattern, then ecosystem override, then global default. This lets you set a conservative default while exempting trusted package families. Currently supported for npm, PyPI, pub.dev, Composer, Cargo, NuGet, Conda, RubyGems, and Hex. These ecosystems include publish timestamps in their metadata. Note: Hex cooldown requires disabling registry signature verification since the proxy re-encodes the protobuf payload without the original signature. Set `HEX_NO_VERIFY_REPO_ORIGIN=1` or configure your repo with `no_verify: true`. +## Version denylist + +Block exact package versions independently of cooldown and artifact scanning: + +```yaml +denylist: + packages: + - "pkg:pypi/requests@2.31.0" + - "pkg:cargo/some-crate@1.2.3" + - "pkg:npm/%40scope/example@4.5.6" +``` + +Entries must be versioned PURLs without qualifiers or subpaths. Package names +are canonicalized (including PyPI case/separator normalization and npm scopes); +versions match exactly, not by range or glob. Invalid entries prevent startup. +An empty list disables the policy. Restart the proxy after changing the list. + +Denied versions are removed from npm packuments, Cargo sparse indexes, and PyPI +simple HTML/JSON and release metadata, including responses served from cached +metadata during an upstream outage. No publication timestamp or enabled cooldown +is required. npm tags pointing at denied versions are removed; `latest` is moved +to the highest remaining stable version when available. Other tags are not +retargeted. PyPI version-specific JSON endpoints return 404 for denied versions. + +The shared artifact download pipeline returns 403 for denied package/version +identities before reading storage, issuing a signed redirect, or fetching bytes. +This also applies to mirror downloads and recognized Debian package downloads. +Existing cached artifacts are retained, so removing a denial restores access. +Already-issued storage URLs and clients' local caches cannot be revoked by this +configuration. + +Metadata filtering currently covers npm, PyPI, and Cargo. Other ecosystems' +metadata is unchanged; in particular, APT indexes cannot be rewritten without +regenerating their checksums and signatures. Pass-through resources without a +resolved package/version identity are not covered. This is an explicit operator +policy, not automatic CVE detection or cache re-scanning. Exact pins or dependency +constraints may still make an install fail when no allowed version can satisfy it. + +### APT version pinning + +For Debian and Ubuntu clients, combine the proxy's download denylist with +client-side APT preferences. The proxy blocks recognized denied `.deb` downloads +with 403, but does not remove versions from APT indexes. Without a client-side +pin, APT can select a denied version and then fail when downloading it. + +On each client, add a record to `/etc/apt/preferences.d/proxy-denylist.pref`: + +```text +Package: somepkg +Pin: version 1.2.3-1 +Pin-Priority: -1 +``` + +Replace `somepkg` with the binary package name and `1.2.3-1` with its full APT +version, including the epoch if present. A negative priority excludes that +version from normal candidate selection. For multi-architecture clients, use +`Package: somepkg:any` to cover all architectures. Separate additional records +with blank lines. See [apt_preferences(5)](https://manpages.debian.org/stable/apt/apt_preferences.5.en.html). + +Check the effective priorities and simulate installation before deploying the +preferences broadly: + +```bash +apt-cache policy somepkg +apt-get --simulate install somepkg +``` + +Confirm that the denied version has priority `-1` and is not selected. Existing +specific pins can take precedence, so check for conflicts. APT can select +another candidate only if one is available and satisfies dependency constraints; +a Debian suite commonly offers only one version. Pinning does not remove an +already-installed package or automatically downgrade it. + +These preferences are managed on clients, not distributed by the proxy. Keep +the corresponding proxy denylist entries as a separate download guard. + +APT indexes are left unchanged to preserve upstream authentication: `Release` +metadata contains index checksums and is signed by the repository. Filtering +would require regenerating metadata and signing it with a key clients trust, +making the proxy a repository authority rather than a transparent cache. Do not +disable signature verification to work around this limitation. See +[apt-secure(8)](https://manpages.debian.org/stable/apt/apt-secure.8.en.html). + +If centrally curated APT indexes are required, use a repository manager such as +[aptly](https://www.aptly.info/) or [reprepro](https://salsa.debian.org/debian/reprepro) +as the upstream behind the proxy. Filtered APT repository generation and signing +are outside the proxy's current scope. + ## Artifact Scanning Cooldown only ever looks at a version's *publish timestamp* — it never inspects the actual bytes of an artifact. Artifact scanning runs after a fetched artifact is staged into storage but before it becomes visible from cache, so an external scanner (trivy, ClamAV, Wiz, or any custom service) can block a bad verdict from ever reaching a client. @@ -538,6 +706,16 @@ metadata_max_size: "100MB" # default Or via environment variable: `PROXY_METADATA_MAX_SIZE=250MB`. +### Rewritten metadata cache + +The npm and Composer handlers rewrite every metadata document they serve, so that download URLs point at the proxy. That means decoding the whole document and encoding it again, and Composer's minified documents are also expanded, which together cost milliseconds per typical package and far more for very large ones. The proxy keeps rewritten documents in memory, so each distinct upstream document is rewritten once and requests arriving while it is being rewritten wait for that rewrite. New bytes from upstream are rewritten again. With version cooldown enabled the cache is bypassed, because cooldown filtering depends on the current time. + +```yaml +metadata_rewrite_cache_size: "256MB" # default; "0" rewrites on every request +``` + +Or via environment variable: `PROXY_METADATA_REWRITE_CACHE_SIZE=1GB`. + ## Upstream HTTP timeout Protocol handlers use a shared HTTP client for upstream requests such as metadata fetches and pass-through file downloads. `http_timeout` sets that client's per-request timeout. Raise it if slow upstreams or large metadata responses cause `context deadline exceeded` errors. @@ -548,7 +726,7 @@ http_timeout: "30s" # default Or via environment variable: `PROXY_HTTP_TIMEOUT=2m`. -Set to `"0"` to disable the timeout entirely (requests then rely only on the server's write timeout). +Set to `"0"` to disable the timeout entirely (requests then rely only on the server's write timeout). Independently of this setting, the shared transport gives up on an upstream that has not sent response headers within 60 seconds. ## Mirror API diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index cc88b4c8..dd50bf92 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -559,6 +559,35 @@ const docTemplate = `{ } } }, + "server.EcosystemStatsEntry": { + "type": "object", + "properties": { + "cache_size_bytes": { + "type": "integer" + }, + "cached_artifacts": { + "type": "integer" + }, + "downloaded": { + "type": "string" + }, + "downloaded_bytes": { + "type": "integer" + }, + "downloads": { + "type": "integer" + }, + "ecosystem": { + "type": "string" + }, + "packages": { + "type": "integer" + }, + "versions": { + "type": "integer" + } + } + }, "server.ErrorResponse": { "type": "object", "properties": { @@ -806,6 +835,26 @@ const docTemplate = `{ "database_path": { "type": "string" }, + "downloaded": { + "type": "string" + }, + "downloaded_bytes": { + "description": "DownloadedBytes is the accumulated download volume across every\necosystem: cache hits multiplied by the artifact size they served.", + "type": "integer" + }, + "downloads": { + "type": "integer" + }, + "ecosystems": { + "type": "array", + "items": { + "$ref": "#/definitions/server.EcosystemStatsEntry" + } + }, + "stats_unavailable": { + "description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.", + "type": "boolean" + }, "storage_url": { "type": "string" }, diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 5db91660..50007b6e 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -552,6 +552,35 @@ } } }, + "server.EcosystemStatsEntry": { + "type": "object", + "properties": { + "cache_size_bytes": { + "type": "integer" + }, + "cached_artifacts": { + "type": "integer" + }, + "downloaded": { + "type": "string" + }, + "downloaded_bytes": { + "type": "integer" + }, + "downloads": { + "type": "integer" + }, + "ecosystem": { + "type": "string" + }, + "packages": { + "type": "integer" + }, + "versions": { + "type": "integer" + } + } + }, "server.ErrorResponse": { "type": "object", "properties": { @@ -799,6 +828,26 @@ "database_path": { "type": "string" }, + "downloaded": { + "type": "string" + }, + "downloaded_bytes": { + "description": "DownloadedBytes is the accumulated download volume across every\necosystem: cache hits multiplied by the artifact size they served.", + "type": "integer" + }, + "downloads": { + "type": "integer" + }, + "ecosystems": { + "type": "array", + "items": { + "$ref": "#/definitions/server.EcosystemStatsEntry" + } + }, + "stats_unavailable": { + "description": "StatsUnavailable distinguishes a proxy that has served nothing from one\nwhose aggregation failed with no snapshot to fall back on. Without it\nboth report zeros and an empty array.", + "type": "boolean" + }, "storage_url": { "type": "string" }, diff --git a/go.mod b/go.mod index bcb9cd00..0fd1e1b0 100644 --- a/go.mod +++ b/go.mod @@ -5,326 +5,129 @@ go 1.26.7 require ( github.com/BurntSushi/toml v1.6.0 github.com/CycloneDX/cyclonedx-go v0.12.0 - github.com/aws/aws-sdk-go-v2/config v1.32.40 - github.com/aws/aws-sdk-go-v2/service/ecr v1.61.0 - github.com/git-pkgs/archives v0.7.0 + github.com/Masterminds/semver/v3 v3.5.0 + github.com/aws/aws-sdk-go-v2/config v1.33.5 + github.com/aws/aws-sdk-go-v2/service/ecr v1.66.0 + github.com/git-pkgs/archives v0.8.1 github.com/git-pkgs/artifacts v0.2.1 github.com/git-pkgs/cooldown v0.2.0 - github.com/git-pkgs/enrichment v0.7.1 + github.com/git-pkgs/enrichment v0.7.2 github.com/git-pkgs/gcs v0.1.0 github.com/git-pkgs/integrity v0.1.1 - github.com/git-pkgs/magic v0.3.1 - github.com/git-pkgs/purl v0.1.20 - github.com/git-pkgs/registries v0.9.1 - github.com/git-pkgs/spdx v0.3.1 - github.com/git-pkgs/vers v0.7.0 - github.com/git-pkgs/vulns v0.2.3 + github.com/git-pkgs/magic v0.5.0 + github.com/git-pkgs/purl v0.1.21 + github.com/git-pkgs/registries v0.9.3 + github.com/git-pkgs/spdx v0.3.3 + github.com/git-pkgs/vers v0.7.2 + github.com/git-pkgs/vulns v0.2.4 github.com/go-chi/chi/v5 v5.3.2 github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.12.3 github.com/opencontainers/go-digest v1.0.0 github.com/prometheus/client_golang v1.24.1 - github.com/prometheus/client_model v0.6.2 + github.com/prometheus/client_model v0.6.3 github.com/spdx/tools-golang v0.5.7 github.com/swaggo/swag v1.16.6 gocloud.dev v0.46.0 - golang.org/x/sync v0.22.0 + golang.org/x/net v0.59.0 + golang.org/x/sync v0.23.0 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.57.0 + modernc.org/sqlite v1.59.0 ) require ( - 4d63.com/gocheckcompilerdirectives v1.4.0 // indirect - 4d63.com/gochecknoglobals v0.2.2 // indirect - charm.land/lipgloss/v2 v2.0.6 // indirect cloud.google.com/go/auth v0.21.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - codeberg.org/chavacava/garif v0.2.0 // indirect - codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect - dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect - dev.gaijin.team/go/exhaustruct/v5 v5.0.3 // indirect - dev.gaijin.team/go/golib v0.8.1 // indirect - github.com/4meepo/tagalign v1.4.3 // indirect - github.com/Abirdcfly/dupword v0.1.8 // indirect - github.com/AdminBenni/iota-mixing v1.0.0 // indirect - github.com/AlwxSin/noinlineerr v1.0.6 // indirect - github.com/Antonboom/errname v1.1.2 // indirect - github.com/Antonboom/nilnil v1.1.2 // indirect - github.com/Antonboom/testifylint v1.6.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 // indirect - github.com/ClickHouse/clickhouse-go-linter v1.2.1 // indirect - github.com/Djarvur/go-err113 v0.1.1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect - github.com/Masterminds/semver/v3 v3.5.0 // indirect - github.com/MirrexOne/unqueryvet v1.5.4 // indirect - github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/alecthomas/chroma/v2 v2.27.0 // indirect - github.com/alecthomas/go-check-sumtype v0.3.1 // indirect - github.com/alexkohler/nakedret/v2 v2.0.6 // indirect - github.com/alexkohler/prealloc v1.1.0 // indirect - github.com/alfatraining/structtag v1.0.0 // indirect - github.com/alingse/asasalint v0.0.11 // indirect - github.com/alingse/nilnesserr v0.2.0 // indirect github.com/anchore/go-struct-converter v0.1.0 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect - github.com/ashanbrown/forbidigo/v2 v2.3.1 // indirect - github.com/ashanbrown/makezero/v2 v2.2.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.44.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.47.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.19.39 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.34.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.46.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.0 // indirect github.com/aws/smithy-go v1.28.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bkielbasa/cyclop v1.2.3 // indirect - github.com/blizzy78/varnamelen v0.8.0 // indirect - github.com/bombsimon/wsl/v4 v4.7.0 // indirect - github.com/bombsimon/wsl/v5 v5.9.0 // indirect - github.com/breml/bidichk v0.3.3 // indirect - github.com/breml/errchkjson v0.4.1 // indirect - github.com/butuzov/ireturn v0.4.1 // indirect - github.com/butuzov/mirror v1.3.3 // indirect - github.com/catenacyber/perfsprint v0.10.1 // indirect - github.com/ccojocar/zxcvbn-go v1.0.4 // indirect github.com/cenk/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/charithe/durationcheck v0.0.11 // indirect - github.com/charmbracelet/colorprofile v0.4.3 // indirect - github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 // indirect - github.com/charmbracelet/x/ansi v0.11.8 // indirect - github.com/charmbracelet/x/term v0.2.2 // indirect - github.com/charmbracelet/x/termios v0.1.1 // indirect - github.com/charmbracelet/x/windows v0.2.2 // indirect - github.com/ckaznocha/intrange v0.3.1 // indirect - github.com/clipperhouse/displaywidth v0.11.0 // indirect - github.com/clipperhouse/uax29/v2 v2.7.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect - github.com/curioswitch/go-reassign v0.3.0 // indirect - github.com/daixiang0/gci v0.13.7 // indirect - github.com/dave/dst v0.27.3 // indirect - github.com/denis-tingaikin/go-header v0.5.0 // indirect - github.com/dlclark/regexp2/v2 v2.2.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ecosyste-ms/ecosystems-go v0.4.0 // indirect - github.com/ettle/strcase v0.2.0 // indirect + github.com/ecosyste-ms/ecosystems-go v0.5.0 // indirect github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect - github.com/fatih/color v1.19.0 // indirect - github.com/fatih/structtag v1.2.0 // indirect - github.com/firefart/nonamedreturns v1.0.8 // indirect - github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/fzipp/gocyclo v0.6.0 // indirect - github.com/ghostiam/protogetter v0.3.21 // indirect - github.com/git-pkgs/packageurl-go v0.3.1 // indirect - github.com/git-pkgs/pom v0.1.7 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/git-pkgs/pom v0.1.8 // indirect github.com/github/go-spdx/v2 v2.7.0 // indirect - github.com/go-critic/go-critic v0.14.4 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/spec v0.20.4 // indirect github.com/go-openapi/swag v0.19.15 // indirect - github.com/go-toolsmith/astcast v1.1.0 // indirect - github.com/go-toolsmith/astcopy v1.1.0 // indirect - github.com/go-toolsmith/astequal v1.2.0 // indirect - github.com/go-toolsmith/astfmt v1.1.0 // indirect - github.com/go-toolsmith/astp v1.1.0 // indirect - github.com/go-toolsmith/strparse v1.1.0 // indirect - github.com/go-toolsmith/typep v1.1.0 // indirect - github.com/go-viper/mapstructure/v2 v2.5.0 // indirect - github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect - github.com/gobwas/glob v0.2.3 // indirect - github.com/godoc-lint/godoc-lint v0.11.2 // indirect - github.com/gofrs/flock v0.13.0 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect - github.com/golangci/asciicheck v0.5.0 // indirect - github.com/golangci/dupl v0.0.0-20260401084720-c99c5cf5c202 // indirect - github.com/golangci/go-printf-func-name v0.1.1 // indirect - github.com/golangci/gofmt v0.0.0-20260820135601-e84e05053792 // indirect - github.com/golangci/golangci-lint/v2 v2.13.1 // indirect - github.com/golangci/golines v0.15.0 // indirect - github.com/golangci/misspell v0.8.0 // indirect - github.com/golangci/plugin-module-register v0.1.2 // indirect - github.com/golangci/revgrep v0.8.0 // indirect - github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba // indirect - github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect - github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect - github.com/google/go-cmp v0.7.0 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/uuid v1.6.0 // indirect github.com/google/wire v0.7.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.18 // indirect github.com/googleapis/gax-go/v2 v2.23.0 // indirect - github.com/gordonklaus/ineffassign v0.2.0 // indirect - github.com/gostaticanalysis/analysisutil v0.7.1 // indirect - github.com/gostaticanalysis/comment v1.5.0 // indirect - github.com/gostaticanalysis/forcetypeassert v0.2.0 // indirect - github.com/gostaticanalysis/nilerr v0.1.2 // indirect - github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect - github.com/hashicorp/go-version v1.9.0 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/hexops/gotextdiff v1.0.3 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jgautheron/goconst v1.11.0 // indirect - github.com/jjti/go-spancheck v0.6.5 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/julz/importas v0.2.0 // indirect - github.com/karamaru-alpha/copyloopvar v1.2.2 // indirect - github.com/kisielk/errcheck v1.20.0 // indirect - github.com/kkHAIKE/contextcheck v1.1.6 // indirect - github.com/klauspost/compress v1.19.2 // indirect - github.com/kulti/thelper v0.7.1 // indirect - github.com/kunwardeep/paralleltest v1.0.15 // indirect + github.com/klauspost/compress v1.20.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/lasiar/canonicalheader v1.1.2 // indirect - github.com/ldez/exptostd v0.4.5 // indirect - github.com/ldez/gomoddirectives v0.9.0 // indirect - github.com/ldez/grignotin v0.10.1 // indirect - github.com/ldez/structtags v0.6.1 // indirect - github.com/ldez/tagliatelle v0.7.2 // indirect - github.com/ldez/usetesting v0.5.0 // indirect - github.com/leonklingele/grouper v1.1.2 // indirect - github.com/lucasb-eyer/go-colorful v1.4.1 // indirect - github.com/macabu/inamedparam v0.2.0 // indirect - github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/manuelarte/embeddedstructfieldcheck v0.4.0 // indirect - github.com/manuelarte/funcorder v0.6.0 // indirect - github.com/maratori/testableexamples v1.0.1 // indirect - github.com/maratori/testpackage v1.1.2 // indirect - github.com/matoous/godox v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.24 // indirect - github.com/mattn/go-runewidth v0.0.24 // indirect - github.com/mgechev/revive v1.15.0 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moricho/tparallel v0.3.2 // indirect - github.com/muesli/cancelreader v0.2.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nakabonne/nestif v0.3.1 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/nishanths/exhaustive v0.12.0 // indirect - github.com/nishanths/predeclared v0.2.2 // indirect - github.com/nunnatsa/ginkgolinter v0.24.0 // indirect github.com/oapi-codegen/nullable v1.2.0 // indirect - github.com/oapi-codegen/runtime v1.6.0 // indirect + github.com/oapi-codegen/runtime v1.7.0 // indirect github.com/package-url/packageurl-go v0.1.7 // indirect github.com/pandatix/go-cvss v0.6.4 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.4.3 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/quasilyte/go-ruleguard v0.4.5 // indirect - github.com/quasilyte/go-ruleguard/dsl v0.3.23 // indirect - github.com/quasilyte/gogrep v0.5.0 // indirect - github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect - github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect - github.com/raeperd/recvcheck v0.3.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.16.0 // indirect github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 // indirect github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/ryancurrah/gomodguard v1.4.1 // indirect - github.com/ryancurrah/gomodguard/v2 v2.1.3 // indirect - github.com/ryanrolds/sqlclosecheck v0.6.0 // indirect - github.com/sanposhiho/wastedassign/v2 v2.1.0 // indirect - github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect - github.com/sashamelentyev/interfacebloat v1.1.0 // indirect - github.com/sashamelentyev/usestdlibvars v1.29.0 // indirect - github.com/securego/gosec/v2 v2.28.0 // indirect - github.com/sirupsen/logrus v1.10.1 // indirect - github.com/sivchari/containedctx v1.0.3 // indirect - github.com/sonatard/noctx v0.5.1 // indirect - github.com/sourcegraph/go-diff v0.8.0 // indirect - github.com/spf13/afero v1.15.0 // indirect - github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/cobra v1.10.2 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.10 // indirect - github.com/spf13/viper v1.12.0 // indirect - github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect - github.com/stbenjam/no-sprintf-host-port v0.3.1 // indirect - github.com/stretchr/objx v0.5.3 // indirect - github.com/stretchr/testify v1.12.1 // indirect - github.com/subosito/gotenv v1.4.1 // indirect - github.com/tetafro/godot v1.5.6 // indirect - github.com/timakin/bodyclose v0.0.0-20260129054331-73d1f95b84b4 // indirect - github.com/timonwong/loggercheck v0.11.0 // indirect - github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect - github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect github.com/ulikunitz/xz v0.5.16 // indirect - github.com/ultraware/funlen v0.2.0 // indirect - github.com/ultraware/whitespace v0.2.0 // indirect - github.com/urfave/cli/v2 v2.3.0 // indirect - github.com/uudashr/gocognit v1.2.1 // indirect - github.com/uudashr/iface v1.5.0 // indirect - github.com/xen0n/gosmopolitan v1.3.0 // indirect - github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - github.com/yagipy/maintidx v1.0.0 // indirect - github.com/yeya24/promlinter v0.3.0 // indirect - github.com/ykadowak/zerologlint v0.1.5 // indirect - gitlab.com/bosi/decorder v0.4.2 // indirect - go-simpler.org/musttag v0.14.0 // indirect - go-simpler.org/sloglint v0.12.0 // indirect - go.augendre.info/arangolint v0.4.0 // indirect - go.augendre.info/fatcontext v0.10.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.44.0 // indirect - go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/otel/sdk v1.44.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect - go.opentelemetry.io/otel/trace v1.44.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.1 // indirect - go.yaml.in/yaml/v2 v2.4.4 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/sdk v1.45.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/crypto v0.55.0 // indirect - golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect - golang.org/x/exp/typeparams v0.0.0-20260811152304-ee035b5b010f // indirect - golang.org/x/mod v0.40.0 // indirect - golang.org/x/net v0.58.0 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/mod v0.41.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.41.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect golang.org/x/tools v0.49.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/api v0.288.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect - google.golang.org/grpc v1.83.1 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + google.golang.org/grpc v1.83.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - honnef.co/go/tools v0.8.0 // indirect - modernc.org/libc v1.74.4 // indirect + modernc.org/libc v1.75.7 // indirect modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.11.0 // indirect - mvdan.cc/gofumpt v0.11.0 // indirect - mvdan.cc/unparam v0.0.0-20260818115549-3f964bcb5673 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect + modernc.org/memory v1.12.1 // indirect ) - -tool github.com/golangci/golangci-lint/v2/cmd/golangci-lint diff --git a/go.sum b/go.sum index 5ef1906b..782b6139 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,5 @@ -4d63.com/gocheckcompilerdirectives v1.4.0 h1:ZLq62rbGWVmQhiZ8kuNVIT/M09xCSTdJz9K3xOdT/CY= -4d63.com/gocheckcompilerdirectives v1.4.0/go.mod h1:9ZOAiMOjqC/nRwci2fcUXVHUNLG/cH6r6rhUh+jTFtQ= -4d63.com/gochecknoglobals v0.2.2 h1:H1vdnwnMaZdQW/N+NrkT1SZMTBmcwHe9Vq8lJcYYTtU= -4d63.com/gochecknoglobals v0.2.2/go.mod h1:lLxwTQjL5eIesRbvnzIP3jZtG140FnTdz+AlMa+ogt0= cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= -charm.land/lipgloss/v2 v2.0.6 h1:EaGKeuA8FvF+v2BT5VmZd2LoYLaMZJXA5n34th8nCIQ= -charm.land/lipgloss/v2 v2.0.6/go.mod h1:ipDDJNSGa1hlwDtSfW1s2/xR8Vdhbut4PXh2zEKZd0Q= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= cloud.google.com/go/auth v0.21.0 h1:g/QwYfYb2Ai6HH8oomAOyBaIHLbscZ4+T/F/f5JZHkE= @@ -20,33 +14,9 @@ cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhO cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= cloud.google.com/go/storage v1.61.3 h1:VS//ZfBuPGDvakfD9xyPW1RGF1Vy3BWUoVZXgW1KMOg= cloud.google.com/go/storage v1.61.3/go.mod h1:JtqK8BBB7TWv0HVGHubtUdzYYrakOQIsMLffZ2Z/HWk= -codeberg.org/chavacava/garif v0.2.0 h1:F0tVjhYbuOCnvNcU3YSpO6b3Waw6Bimy4K0mM8y6MfY= -codeberg.org/chavacava/garif v0.2.0/go.mod h1:P2BPbVbT4QcvLZrORc2T29szK3xEOlnl0GiPTJmEqBQ= -codeberg.org/polyfloyd/go-errorlint v1.9.0 h1:VkdEEmA1VBpH6ecQoMR4LdphVI3fA4RrCh2an7YmodI= -codeberg.org/polyfloyd/go-errorlint v1.9.0/go.mod h1:GPRRu2LzVijNn4YkrZYJfatQIdS+TrcK8rL5Xs24qw8= -dev.gaijin.team/go/exhaustruct/v4 v4.0.0 h1:873r7aNneqoBB3IaFIzhvt2RFYTuHgmMjoKfwODoI1Y= -dev.gaijin.team/go/exhaustruct/v4 v4.0.0/go.mod h1:aZ/k2o4Y05aMJtiux15x8iXaumE88YdiB0Ai4fXOzPI= -dev.gaijin.team/go/exhaustruct/v5 v5.0.3 h1:yOeA7DNjlT8y4yfmN6nWWYYggA13N523YAj9/TXbuTM= -dev.gaijin.team/go/exhaustruct/v5 v5.0.3/go.mod h1:KwtBsX8nHHH1YxhxkpiBq6bfsmw5WnazWpNvJPHgY9Y= -dev.gaijin.team/go/golib v0.8.1 h1:JYju4x9BSo+QD/AYeHULVDcvEhiFg8wOi6pT0IaZF5E= -dev.gaijin.team/go/golib v0.8.1/go.mod h1:c5fu7t1RSGMxSQgcUYO1sODbzsYnOCXJLmHeNG1Eb+0= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= -github.com/4meepo/tagalign v1.4.3 h1:Bnu7jGWwbfpAie2vyl63Zup5KuRv21olsPIha53BJr8= -github.com/4meepo/tagalign v1.4.3/go.mod h1:00WwRjiuSbrRJnSVeGWPLp2epS5Q/l4UEy0apLLS37c= -github.com/Abirdcfly/dupword v0.1.8 h1:SrhcUuGsROBuChFxHALRYzyyPODWn9zwghmzPvD9Cd8= -github.com/Abirdcfly/dupword v0.1.8/go.mod h1:XZrhVnI7YGpsTiWZANSQaBJ4QpL/Tq5vIEdKJJAs9WI= -github.com/AdminBenni/iota-mixing v1.0.0 h1:Os6lpjG2dp/AE5fYBPAA1zfa2qMdCAWwPMCgpwKq7wo= -github.com/AdminBenni/iota-mixing v1.0.0/go.mod h1:i4+tpAaB+qMVIV9OK3m4/DAynOd5bQFaOu+2AhtBCNY= -github.com/AlwxSin/noinlineerr v1.0.6 h1:KAvuxunTe9QxvqrFB7nZTdb/7Wzas4AvifslTnG0Ld8= -github.com/AlwxSin/noinlineerr v1.0.6/go.mod h1:+QgkkoYrMH7RHvcdxdlI7vYYEdgeoFOVjU9sUhw/rQc= -github.com/Antonboom/errname v1.1.2 h1:dxwONZJua3VB8Xh/VaCjqAcqF645sWWv7xj26zy7tdQ= -github.com/Antonboom/errname v1.1.2/go.mod h1:YeZIpgLMxT+SNkruGgYkLhzq/9vs3fsolTZegKaKDZI= -github.com/Antonboom/nilnil v1.1.2 h1:aNlFuJhaEseXe4fHO3xbjXlSeEiQVYa2lEkWD2s2hAY= -github.com/Antonboom/nilnil v1.1.2/go.mod h1:0ynwvphOLmAuMwTNDyBnDZmSwZoDpcFXmUHmzoHH2WA= -github.com/Antonboom/testifylint v1.6.4 h1:gs9fUEy+egzxkEbq9P4cpcMB6/G0DYdMeiFS87UiqmQ= -github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9qpVy7nXGbxDD4= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= @@ -63,15 +33,10 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0 h1:4iB+IesclUXdP0ICgAabvq2FYLXrJWKx1fJQ+GxSo3Y= github.com/AzureAD/microsoft-authentication-library-for-go v1.7.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/ClickHouse/clickhouse-go-linter v1.2.1 h1:zGEKIyd5YL08ieWG/LOUmlau2DxbxPVOfAeo+4Jz3ck= -github.com/ClickHouse/clickhouse-go-linter v1.2.1/go.mod h1:pLorS7ffPTfuUV9M0SJgfHA/h/WQPQUk2FWG9x74cQ4= github.com/CycloneDX/cyclonedx-go v0.12.0 h1:/7Jum36UA6V043tQZ/fE3jf+Nf9gn/qxUFfd7QReMy8= github.com/CycloneDX/cyclonedx-go v0.12.0/go.mod h1:V2577HhxDDCDLYfkm55WJrz16nHTfyQZwcWUBSG7Z28= -github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao4g= -github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= @@ -82,222 +47,119 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/MirrexOne/unqueryvet v1.5.4 h1:38QOxShO7JmMWT+eCdDMbcUgGCOeJphVkzzRgyLJgsQ= -github.com/MirrexOne/unqueryvet v1.5.4/go.mod h1:fs9Zq6eh1LRIhsDIsxf9PONVUjYdFHdtkHIgZdJnyPU= -github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= -github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= -github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= -github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= -github.com/alecthomas/chroma/v2 v2.27.0 h1:FodwmyOBgJULFYmDqibcp9pvfDLWdtPRh9v/r5BXYZs= -github.com/alecthomas/chroma/v2 v2.27.0/go.mod h1:NjJ3ciIgrqBNeIkWZ4e46nseoLDslxU1LmfCoL+wcY8= -github.com/alecthomas/go-check-sumtype v0.3.1 h1:u9aUvbGINJxLVXiFvHUlPEaD7VDULsrxJb4Aq31NLkU= -github.com/alecthomas/go-check-sumtype v0.3.1/go.mod h1:A8TSiN3UPRw3laIgWEUOHHLPa6/r9MtoigdlP5h3K/E= -github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= -github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/alexkohler/nakedret/v2 v2.0.6 h1:ME3Qef1/KIKr3kWX3nti3hhgNxw6aqN5pZmQiFSsuzQ= -github.com/alexkohler/nakedret/v2 v2.0.6/go.mod h1:l3RKju/IzOMQHmsEvXwkqMDzHHvurNQfAgE1eVmT40Q= -github.com/alexkohler/prealloc v1.1.0 h1:cKGRBqlXw5iyQGLYhrXrDlcHxugXpTq4tQ5c91wkf8M= -github.com/alexkohler/prealloc v1.1.0/go.mod h1:fT39Jge3bQrfA7nPMDngUfvUbQGQeJyGQnR+913SCig= -github.com/alfatraining/structtag v1.0.0 h1:2qmcUqNcCoyVJ0up879K614L9PazjBSFruTB0GOFjCc= -github.com/alfatraining/structtag v1.0.0/go.mod h1:p3Xi5SwzTi+Ryj64DqjLWz7XurHxbGsq6y3ubePJPus= -github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw= -github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I= -github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEWd/w= -github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9dedlWa0OhD30= github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= -github.com/ashanbrown/forbidigo/v2 v2.3.1 h1:KAZijvQ7zeIBKbhikT4jCm0TLYXC4u78bTiLh/8JROI= -github.com/ashanbrown/forbidigo/v2 v2.3.1/go.mod h1:2QDkLTzU6TV937eFROamXrW92M3paehdae4HCDCOZCM= -github.com/ashanbrown/makezero/v2 v2.2.1 h1:A7uU8dgB1PA9aelTxHMfHIQ8Qev8AB3JLxJUBUsejqM= -github.com/ashanbrown/makezero/v2 v2.2.1/go.mod h1:aEGT/9q3S8DHeE57C88z2a6xydvgx8J5hgXIGWgo0MY= -github.com/aws/aws-sdk-go-v2 v1.44.0 h1:4IbaHhtzy+4h37z4JQyO9a2QsiCml3CNYHtq5hIHigo= -github.com/aws/aws-sdk-go-v2 v1.44.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2 v1.47.0 h1:0jsHallhJCeaU0Ko48c/3FK1ctOQ7NpzggxriJOQ8MQ= +github.com/aws/aws-sdk-go-v2 v1.47.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94= -github.com/aws/aws-sdk-go-v2/config v1.32.40 h1:lAVC9gMmKusmqDRe32dPtgKl/BWvJmMJoWELKHCAObw= -github.com/aws/aws-sdk-go-v2/config v1.32.40/go.mod h1:8xOJLbe/hOj1g4PVsfJYV7O2byq+UGET1onDdUgbwqc= -github.com/aws/aws-sdk-go-v2/credentials v1.19.39 h1:XOg8LC3Kgnsa3WiPQjc7Bi8k5IBN92cPYfIV9XMFss0= -github.com/aws/aws-sdk-go-v2/credentials v1.19.39/go.mod h1:GonTDBQ+mTpCVNwaHjj0PagspfrYYMEqOx7FehoEP/I= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40 h1:r5aGipEVgI9aT/tAGjdrPbDQvIAKdTrS3rUPQtG4Rmo= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.40/go.mod h1:vOD3CnPxAdkL6MWZeROkZsTlskklMFfgVFkHzx/oZpY= +github.com/aws/aws-sdk-go-v2/config v1.33.5 h1:UA1dmokBFOLFoOyVBhO6HjM6edy0MIk5AZSkJVcksQw= +github.com/aws/aws-sdk-go-v2/config v1.33.5/go.mod h1:Dop8axzz0xx38GExIYWXdeyc8QQ7Cr+nPsxpD/LYy4U= +github.com/aws/aws-sdk-go-v2/credentials v1.20.5 h1:wklUVvHMc9xTQ3rcp49/ISpiMnhbCicJcA6n6S8m7J8= +github.com/aws/aws-sdk-go-v2/credentials v1.20.5/go.mod h1:fyEdrn6ccLFOkoK84j5bQyGTxp9zPt5l2XMhxf4DVZs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 h1:AM4hHjww+PSFtt6E+UrBrPlZkWsePCLEt9AjkfQX+yM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0/go.mod h1:3x/yXezeQjpOvBb4jEMxrS8SXvpdvJ5abv6l5c1gWM8= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3 h1:w5OoDiMN6x53ROmiIImGzmVcxXv2q1GXY+aKV4WAJYM= github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.2.3/go.mod h1:dAhgYp776bX3LuWvnSCFwQEjNs6fuFg7YXIy5PXcP3Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40 h1:UIXlbijuB2XK1Kr57fo8iIxCuaSHJzwZ1uo+2tbEYIk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.40/go.mod h1:wcEsL6jscjZjVUinb0Q5qD/GXOG1yT3GNfmT9HuDwzU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40 h1:xLQVRDs2NddDmK9BEyh5KSlJ1Gpy5/GIJXrV6WcVGAE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.40/go.mod h1:XRXnpFVFGLaEVK+olDdFIM1vNa04ETW452oFGEPUxAo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41 h1:nv/ILuCY0yXACzMQwvtt/HbqDDjemZiI0AeDbxGQlnU= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.41/go.mod h1:dzvOSpxaPqQ3j0xS6Lc1vyVuWW0RBj7s/QqYpzu3Q/0= -github.com/aws/aws-sdk-go-v2/service/ecr v1.61.0 h1:H+odOoYtvBGmUvwLgjq6MN8hmBYBny5R4FosoO+j8NU= -github.com/aws/aws-sdk-go-v2/service/ecr v1.61.0/go.mod h1:HrP4KYHFcZzSEjQNb6yDrWuEes3MeE62PYoDnaff+/0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 h1:Hp/VgjP0BysR3OgLlR057Vz2LcbbVnoWeJ+3qWiS/fY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3/go.mod h1:nwGV5qw7F1IZPgxCvA/ph8N2TAuz+BkRG/bXn808qMA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 h1:MUaM4f+kj1ZIBPZfUS8cxP1GKXXZtHJjAthy93AN7SM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3/go.mod h1:6YmVmEVRI5ZZzRjCSsb9SryKH0hAlMRdgA7kG9aDvBU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 h1:fuSCw4Z2qfRCztMPO3GXJNSiEp6Wee+WOLwrHHUMy9c= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3/go.mod h1:6SxcHheD1pPR5+kWm1wGvjlL/YqUsh267sAfEmN4K7A= +github.com/aws/aws-sdk-go-v2/service/ecr v1.66.0 h1:9i19IigAYxnAxTUQcsxkiIXeeytHllTOetWBhr1DTQs= +github.com/aws/aws-sdk-go-v2/service/ecr v1.66.0/go.mod h1:iSlv4VibruxMhWfiKrNbn97Yjhe2855EFrgUitSESfM= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18 h1:W/EyPFl9A5rXrtoilfwHYEvzHER+K4SpBPtMXi24Mos= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.18/go.mod h1:UG50K+pvd/uy6xExbobg0rjqFBFZe6I3l75EPDZw4tg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40 h1:gr3Fw1cxZXNCdeo/lQ7isHEHzvHVM7z75qb2zW9aMjw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.40/go.mod h1:8z/9CmfnQhiuXD7Ykbcg4a/whSWsniE0ODSx9uwVzfk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 h1:bON1rJf67TSTDCKg816AAIE4xSTtoo9tl0XRkO72R+I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3/go.mod h1:c5BBpjJcQXpfeq9iASyVKA3T6vX6B6LEXY4mL/gklDY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDbiit/GcBE2K4IUpMZymaA0kOz3xK978= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM= github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298= github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0= -github.com/aws/aws-sdk-go-v2/service/signin v1.6.0 h1:agcr0j8YeFEzdXNo17Rg9MbbjLRjrimabwNtji4e+lU= -github.com/aws/aws-sdk-go-v2/service/signin v1.6.0/go.mod h1:qU5PxgQ4JiUOOMotzfO3+5oUda5W+8JDVKyLQqlrJik= -github.com/aws/aws-sdk-go-v2/service/sso v1.34.0 h1:FxaN8/sn61DTXNI6Gt678tFJUY8iUsCchm6Y/F/RjaA= -github.com/aws/aws-sdk-go-v2/service/sso v1.34.0/go.mod h1:vu4OY6s8LJtT8BtYG2LD6BGSZMptkYn3o5hvCPB22jc= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0 h1:crWKPeGYTBTuBxQ3p73kjfJvt4brUIsr+Fuypko8FxY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.39.0/go.mod h1:HjjZVhaBz0JBR/kbWKThmNDhFKS7y6EURuk493tJk9Y= -github.com/aws/aws-sdk-go-v2/service/sts v1.46.0 h1:IZ63JdogSNNjex/jsODNv7jGDcO/xJYd9FsgyfCsp1g= -github.com/aws/aws-sdk-go-v2/service/sts v1.46.0/go.mod h1:I+rwAf3spG5dITBaAo3xXRowk8kiOhtU1kYxfvCTC44= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 h1:ZD5qFpWcaOKdTuhBi431pIDkCgrMkMlMT6jlpSPoIRI= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.0/go.mod h1:8Nuuf+tR346PjJ3MvZPh9pekbLiLQFWJhzMXfwy7alA= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 h1:JGeeBcMlhg1xtOXYpeCaTQBZObtXMPQCUqBcmr65NRA= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.0/go.mod h1:XwteswG9EOMRFm73UT0t+MbTwyLxMrEXkU6e+v92Lzo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 h1:obhahQXDEdVEv8y5bTKXR30LVaxYe1kyYM0L7l2Iq+k= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0/go.mod h1:6twZZ/aXHNy1vXUO8koUbp++MYzMASkOgEBdkbJYmO0= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.0 h1:Zpnqa6XtrNzXZnwbdCqHOXpXhMsa01ql/pcRQ1sb4hk= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.0/go.mod h1:/8JRcdTt//hG0Q4BTmGbuOplT7ABe+5rdtqUHqXvYIM= github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bkielbasa/cyclop v1.2.3 h1:faIVMIGDIANuGPWH031CZJTi2ymOQBULs9H21HSMa5w= -github.com/bkielbasa/cyclop v1.2.3/go.mod h1:kHTwA9Q0uZqOADdupvcFJQtp/ksSnytRMe8ztxG8Fuo= -github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= -github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= -github.com/bombsimon/wsl/v4 v4.7.0 h1:1Ilm9JBPRczjyUs6hvOPKvd7VL1Q++PL8M0SXBDf+jQ= -github.com/bombsimon/wsl/v4 v4.7.0/go.mod h1:uV/+6BkffuzSAVYD+yGyld1AChO7/EuLrCF/8xTiapg= -github.com/bombsimon/wsl/v5 v5.9.0 h1:WCrgZ7RQnZO5oEwbVTlYgBdU3wL294kR1BSWV8vTfsU= -github.com/bombsimon/wsl/v5 v5.9.0/go.mod h1:kjo4HiAV5FDkHC8/uzJq9mBffEEd6WT/nvN7DoMovDM= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= -github.com/breml/bidichk v0.3.3 h1:WSM67ztRusf1sMoqH6/c4OBCUlRVTKq+CbSeo0R17sE= -github.com/breml/bidichk v0.3.3/go.mod h1:ISbsut8OnjB367j5NseXEGGgO/th206dVa427kR8YTE= -github.com/breml/errchkjson v0.4.1 h1:keFSS8D7A2T0haP9kzZTi7o26r7kE3vymjZNeNDRDwg= -github.com/breml/errchkjson v0.4.1/go.mod h1:a23OvR6Qvcl7DG/Z4o0el6BRAjKnaReoPQFciAl9U3s= -github.com/butuzov/ireturn v0.4.1 h1:vWb3NO4t77iku/sjCQ/2pHTQeOmxEhjIriJqRLg1Y+I= -github.com/butuzov/ireturn v0.4.1/go.mod h1:q+DXKzTDV5guNuXLnIab9fKXizTn2miZHLhxH7V/GB4= -github.com/butuzov/mirror v1.3.3 h1:v0RsWBhfFc1RQqE/f3sHpSttKDtodFn0gFmtYyD4/hA= -github.com/butuzov/mirror v1.3.3/go.mod h1:h9BzzwYnTiHO0GzgvaTqIg7VSsOUhdIv51cHFFBmX1w= -github.com/catenacyber/perfsprint v0.10.1 h1:u7Riei30bk46XsG8nknMhKLXG9BcXz3+3tl/WpKm0PQ= -github.com/catenacyber/perfsprint v0.10.1/go.mod h1:DJTGsi/Zufpuus6XPGJyKOTMELe347o6akPvWG9Zcsc= -github.com/ccojocar/zxcvbn-go v1.0.4 h1:FWnCIRMXPj43ukfX000kvBZvV6raSxakYr1nzyNrUcc= -github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das= github.com/cenk/backoff v2.2.1+incompatible h1:djdFT7f4gF2ttuzRKPbMOWgZajgesItGLwG5FTQKmmE= github.com/cenk/backoff v2.2.1+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/charithe/durationcheck v0.0.11 h1:g1/EX1eIiKS57NTWsYtHDZ/APfeXKhye1DidBcABctk= -github.com/charithe/durationcheck v0.0.11/go.mod h1:x5iZaixRNl8ctbM+3B2RrPG5t856TxRyVQEnbIEM2X4= -github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= -github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= -github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 h1:rdnVWKgJpTVXKuKuJyxDJ+NFJdUaUqGvyGy61OcvlbA= -github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886/go.mod h1:nAw0d9PhFp1qdzi2xhQU5YOu5sVpDIHWlaW2Uz/bCro= -github.com/charmbracelet/x/ansi v0.11.8 h1:JMFwp0CgDC2+jcOB162HH5k7I3FVbgFSMMYg7dSPBQQ= -github.com/charmbracelet/x/ansi v0.11.8/go.mod h1:ZNN+3mXny/516oTQPLMPIBeSINvNJJQ8uQXDgbeJxY0= -github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= -github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= -github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY= -github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= -github.com/charmbracelet/x/windows v0.2.2 h1:IofanmuvaxnKHuV04sC0eBy/smG6kIKrWG2/jYn2GuM= -github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k= -github.com/ckaznocha/intrange v0.3.1 h1:j1onQyXvHUsPWujDH6WIjhyH26gkRt/txNlV7LspvJs= -github.com/ckaznocha/intrange v0.3.1/go.mod h1:QVepyz1AkUoFQkpEqksSYpNpUo3c5W7nWh/s6SHIJJk= -github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= -github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= -github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= -github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= -github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/curioswitch/go-reassign v0.3.0 h1:dh3kpQHuADL3cobV/sSGETA8DOv457dwl+fbBAhrQPs= -github.com/curioswitch/go-reassign v0.3.0/go.mod h1:nApPCCTtqLJN/s8HfItCcKV0jIPwluBOvZP+dsJGA88= -github.com/daixiang0/gci v0.13.7 h1:+0bG5eK9vlI08J+J/NWGbWPTNiXPG4WhNLJOkSxWITQ= -github.com/daixiang0/gci v0.13.7/go.mod h1:812WVN6JLFY9S6Tv76twqmNqevN0pa3SX3nih0brVzQ= -github.com/dave/dst v0.27.3 h1:P1HPoMza3cMEquVf9kKy8yXsFirry4zEnWOdYPOoIzY= -github.com/dave/dst v0.27.3/go.mod h1:jHh6EOibnHgcUW3WjKHisiooEkYwqpHLBSX1iOBhEyc= -github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo= -github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8= -github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= -github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8= -github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0= -github.com/dlclark/regexp2/v2 v2.2.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ecosyste-ms/ecosystems-go v0.4.0 h1:5A+zF+XWT8sYYYjlc2/tI1SmiDGzbHLyT9CapVc5dGA= -github.com/ecosyste-ms/ecosystems-go v0.4.0/go.mod h1:FVswCrp3DQkur1HjVqfDF/gYrDSEmiFflntcB1G0DbA= +github.com/ecosyste-ms/ecosystems-go v0.5.0 h1:qZqgar3Do9RrtW327OXlsrGKiRvNdeX8CmXb1aX4Njo= +github.com/ecosyste-ms/ecosystems-go v0.5.0/go.mod h1:IhsYBZfxkofxya0O1O2DlhDt281utWJ6VQO0mPVb99I= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= -github.com/ettle/strcase v0.2.0 h1:fGNiVF21fHXpX1niBgk0aROov1LagYsOwV/xqKDKR/Q= -github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= -github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= -github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= -github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= -github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= -github.com/firefart/nonamedreturns v1.0.8 h1:iB32Dl17zJl1zlVEj/WlUWgx0HiRyQ85OUw1WHa4/II= -github.com/firefart/nonamedreturns v1.0.8/go.mod h1:vxFNvm5AfP/8rgAKFzYmnqx0yp1HjrYsErZ9pHPTznA= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= -github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= -github.com/ghostiam/protogetter v0.3.21 h1:EeWTGvL/Eyosp653hiWb6Byx4b69iJC4/E+za7vQHoI= -github.com/ghostiam/protogetter v0.3.21/go.mod h1:iAKSpyoHwYzay+OpjoWgwzRtPFthEfuUvmlomTThck0= -github.com/git-pkgs/archives v0.7.0 h1:cRQEKK1N7LMabzXxExwmoAtRvxjuIAkzBxEZmYfH040= -github.com/git-pkgs/archives v0.7.0/go.mod h1:LH7LSbREEaRlxtLwG2PC7evfhlWNgUB/DMBFr6+KsAA= +github.com/git-pkgs/archives v0.8.1 h1:IcT1bVsAec8SqJ4m9vTnD644KiadQZ4ERrWnm/30t3o= +github.com/git-pkgs/archives v0.8.1/go.mod h1:gIyPXPQr5e+YrMqA5KQ7x6pLrFXHNF1+CjdedNJdGUU= github.com/git-pkgs/artifacts v0.2.1 h1:VwdxR4yTDaqBZ34h0slxQBVyr2Xfa+QGOKCKviMx2xk= github.com/git-pkgs/artifacts v0.2.1/go.mod h1:Otosgq52pXT5UNN7lh6s/lszpWKVDO8XrOjN1M70/IA= github.com/git-pkgs/cooldown v0.2.0 h1:0MWPHtkzZgvCR0wdiQeyvMea/dxgw9tParH1zzaFopc= github.com/git-pkgs/cooldown v0.2.0/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E= -github.com/git-pkgs/enrichment v0.7.1 h1:8PRYE7gaB8y4M5wnRw/ymNDk0uOHtbQ8CEE7hF09Bv0= -github.com/git-pkgs/enrichment v0.7.1/go.mod h1:QYLG8MtVWPqZojnq7KBKK/lllBWSjLnqvJCWvzGuNwU= +github.com/git-pkgs/enrichment v0.7.2 h1:flD/M89HxkOegUi6tC8EOspFiIMsl7m0o576IQKOp48= +github.com/git-pkgs/enrichment v0.7.2/go.mod h1:xGyc/iz7u1UnbrOHk8lIX5ZaNF3mKFDEZc3VqBJ/LOk= github.com/git-pkgs/gcs v0.1.0 h1:E3awGtsO0xZyHT9FUfEwMHjMkRxw41Bh+c7lgTmPvBo= github.com/git-pkgs/gcs v0.1.0/go.mod h1:bdkCFD66ryaWnU8MBhokVA3WkJfyAEchm9qec5woBpE= github.com/git-pkgs/integrity v0.1.1 h1:nHQ7SktOiGM1dOb5BFnkdtttG/6FCgE6r5ru6QnsGts= github.com/git-pkgs/integrity v0.1.1/go.mod h1:hxu24lcd230377hCF28JQW7sGcCbuNLqo/0ULeb+F1Q= -github.com/git-pkgs/magic v0.3.1 h1:UzjFRyEwJITA/JgznjmIM4VwuBszD2K4Q8XHgkgL+DM= -github.com/git-pkgs/magic v0.3.1/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= -github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= -github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= -github.com/git-pkgs/pom v0.1.7 h1:4yKdtw6eyShtjul6bcZdyz7yLQ+jdrYeYkKbskDGi4c= -github.com/git-pkgs/pom v0.1.7/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk= -github.com/git-pkgs/purl v0.1.20 h1:a4qzvUy5mBZ2GGjOQNW2h/ocFqjTjOiTDMv2ONtivmM= -github.com/git-pkgs/purl v0.1.20/go.mod h1:hthV5mp+Q67HpQ9+LnRLLmsReu5ooyQ5EaJsCGrA8yE= -github.com/git-pkgs/registries v0.9.1 h1:z5GVFfLHWGoVEawppqXTaE2Y6RADkUbTPYctEs3BZ4M= -github.com/git-pkgs/registries v0.9.1/go.mod h1:5rmFrC76K3zmOAJTTQPcYy0vV2i7MRByM1OQiQdGzl4= -github.com/git-pkgs/spdx v0.3.1 h1:58JPY5X9pYpXvnzzZIgehItlBykeOOw52pNc4OBcS+c= -github.com/git-pkgs/spdx v0.3.1/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto= -github.com/git-pkgs/vers v0.7.0 h1:7PD2DKFB8jTDIfiyWXbqYW54iVuNkFCBgXHgxOTdr8A= -github.com/git-pkgs/vers v0.7.0/go.mod h1:ofLiBpPNkQmC0LB1k0zmN1gCv7Hi3MiZx8WtmWZ4A9A= -github.com/git-pkgs/vulns v0.2.3 h1:G8icINpR9WFgtwp+4mSdgO4THStiQvi6QuHl83J7iOs= -github.com/git-pkgs/vulns v0.2.3/go.mod h1:+z7pZMjctLmUxMsq+tZbciD6xAAoejljDosC6n2YXps= +github.com/git-pkgs/magic v0.5.0 h1:Pm+4fkHR+K6kg9pwJM4hnwWUuBlkYdLmF1yIAI8lU9Q= +github.com/git-pkgs/magic v0.5.0/go.mod h1:SXOqcsNmbmpZjJZHEEWnwxprbsFvpwWgTAZrgXp4Jm4= +github.com/git-pkgs/pom v0.1.8 h1:mXCuWw6XkrKKZb6IurRZyIznYLPhV5AYg5ZO38Uj/l8= +github.com/git-pkgs/pom v0.1.8/go.mod h1:sQBeYRzjCUn77fcABPn6I/a1aBHFxm55jTmkNxYtGtw= +github.com/git-pkgs/purl v0.1.21 h1:PDZUX43lxIiXPn2eHcsNiiiwQo+HajntZ/4TwSvCGlw= +github.com/git-pkgs/purl v0.1.21/go.mod h1:4dOmris48ehjiKmWBqSmMHguUQht48BLeXW+hQUnuj0= +github.com/git-pkgs/registries v0.9.3 h1:nXIP//6kVqLKqYXOWhEMoWUkJPDAZYVLRUYfHz3jM2Y= +github.com/git-pkgs/registries v0.9.3/go.mod h1:BpXlfRCfu4S5bTzm7u7igppVu94mXgll/PENe0AjKmI= +github.com/git-pkgs/spdx v0.3.3 h1:xXlUoZzjDSkwxC6mWHfaW+kc8a0+LH6d+gGHuZWftqI= +github.com/git-pkgs/spdx v0.3.3/go.mod h1:n9rAicgw+Wqtfursju6oKaAL6cUCTiIS926mb1KWoKU= +github.com/git-pkgs/vers v0.7.2 h1:Ctex1O+Ai/FONmyDaw37EolBXg0og6UdxbCjjYbeRHs= +github.com/git-pkgs/vers v0.7.2/go.mod h1:ofLiBpPNkQmC0LB1k0zmN1gCv7Hi3MiZx8WtmWZ4A9A= +github.com/git-pkgs/vulns v0.2.4 h1:QnsQg9sWUfM8ZzA7A/1LSl8DJ/0/6Yrm7vfRwkpyXUY= +github.com/git-pkgs/vulns v0.2.4/go.mod h1:csd+UWvmYWVVAkka7FNbyz840/1R3QXS7LwD2Irsf4Q= github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ= github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0= github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY= github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= -github.com/go-critic/go-critic v0.14.4 h1:dSX4C3pWSeuMVxvQh6yG8U0ReSf3YOmKi4nwX5q7n/8= -github.com/go-critic/go-critic v0.14.4/go.mod h1:xwntfW6SYAd7h1OqDzmN6hBX/JxsEKl5up/Y2bsxgVQ= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -310,73 +172,13 @@ github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474= -github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= -github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= -github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8= -github.com/go-toolsmith/astcast v1.1.0/go.mod h1:qdcuFWeGGS2xX5bLM/c3U9lewg7+Zu4mr+xPwZIB4ZU= -github.com/go-toolsmith/astcopy v1.1.0 h1:YGwBN0WM+ekI/6SS6+52zLDEf8Yvp3n2seZITCUBt5s= -github.com/go-toolsmith/astcopy v1.1.0/go.mod h1:hXM6gan18VA1T/daUEHCFcYiW8Ai1tIwIzHY6srfEAw= -github.com/go-toolsmith/astequal v1.0.3/go.mod h1:9Ai4UglvtR+4up+bAD4+hCj7iTo4m/OXVTSLnCyTAx4= -github.com/go-toolsmith/astequal v1.1.0/go.mod h1:sedf7VIdCL22LD8qIvv7Nn9MuWJruQA/ysswh64lffQ= -github.com/go-toolsmith/astequal v1.2.0 h1:3Fs3CYZ1k9Vo4FzFhwwewC3CHISHDnVUPC4x0bI2+Cw= -github.com/go-toolsmith/astequal v1.2.0/go.mod h1:c8NZ3+kSFtFY/8lPso4v8LuJjdJiUFVnSuU3s0qrrDY= -github.com/go-toolsmith/astfmt v1.1.0 h1:iJVPDPp6/7AaeLJEruMsBUlOYCmvg0MoCfJprsOmcco= -github.com/go-toolsmith/astfmt v1.1.0/go.mod h1:OrcLlRwu0CuiIBp/8b5PYF9ktGVZUjlNMV634mhwuQ4= -github.com/go-toolsmith/astp v1.1.0 h1:dXPuCl6u2llURjdPLLDxJeZInAeZ0/eZwFJmqZMnpQA= -github.com/go-toolsmith/astp v1.1.0/go.mod h1:0T1xFGz9hicKs8Z5MfAqSUitoUYS30pDMsRVIDHs8CA= -github.com/go-toolsmith/pkgload v1.2.2 h1:0CtmHq/02QhxcF7E9N5LIFcYFsMR5rdovfqTtRKkgIk= -github.com/go-toolsmith/pkgload v1.2.2/go.mod h1:R2hxLNRKuAsiXCo2i5J6ZQPhnPMOVtU+f0arbFPWCus= -github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8= -github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw= -github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= -github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUNHus= -github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig= -github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= -github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/go-xmlfmt/xmlfmt v1.1.3 h1:t8Ey3Uy7jDSEisW2K3somuMKIpzktkWptA0iFCnRUWY= -github.com/go-xmlfmt/xmlfmt v1.1.3/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/godoc-lint/godoc-lint v0.11.2 h1:Bp0FkJWoSdNsBikdNgIcgtaoo+xz6I/Y9s5WSBQUeeM= -github.com/godoc-lint/godoc-lint v0.11.2/go.mod h1:iVpGdL1JCikNH2gGeAn3Hh+AgN5Gx/I/cxV+91L41jo= -github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= -github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golangci/asciicheck v0.5.0 h1:jczN/BorERZwK8oiFBOGvlGPknhvq0bjnysTj4nUfo0= -github.com/golangci/asciicheck v0.5.0/go.mod h1:5RMNAInbNFw2krqN6ibBxN/zfRFa9S6tA1nPdM0l8qQ= -github.com/golangci/dupl v0.0.0-20260401084720-c99c5cf5c202 h1:CbTB8KpqnViI6lIXxp03Oclc4VFHi3K4BWC1TacsZ+A= -github.com/golangci/dupl v0.0.0-20260401084720-c99c5cf5c202/go.mod h1:NUw9Zr2Sy7+HxzdjIULge71wI6yEg1lWQr7Evcu8K0E= -github.com/golangci/go-printf-func-name v0.1.1 h1:hIYTFJqAGp1iwoIfsNTpoq1xZAarogrvjO9AfiW3B4U= -github.com/golangci/go-printf-func-name v0.1.1/go.mod h1:Es64MpWEZbh0UBtTAICOZiB+miW53w/K9Or/4QogJss= -github.com/golangci/gofmt v0.0.0-20260820135601-e84e05053792 h1:WL8YKrt3UbOBqSRU7GpP5BTtQTMWtVtj+mfPijgZeIg= -github.com/golangci/gofmt v0.0.0-20260820135601-e84e05053792/go.mod h1:te5hX0dW4C5r6YbXs+6ysNr8Q5UTmdIqGbb+mlFiYmA= -github.com/golangci/golangci-lint/v2 v2.13.1 h1:RuM4OcluM4xFQcGuRE6R7jA33pqxK/W1EsBxpugdZjg= -github.com/golangci/golangci-lint/v2 v2.13.1/go.mod h1:HwX7mDzqHbcSxlhrTygjX1GJbAfQ3sJAqOx41qQlhDE= -github.com/golangci/golines v0.15.0 h1:Qnph25g8Y1c5fdo1X7GaRDGgnMHgnxh4Gk4VfPTtRx0= -github.com/golangci/golines v0.15.0/go.mod h1:AZjXd23tbHMpowhtnGlj9KCNsysj72aeZVVHnVcZx10= -github.com/golangci/misspell v0.8.0 h1:qvxQhiE2/5z+BVRo1kwYA8yGz+lOlu5Jfvtx2b04Jbg= -github.com/golangci/misspell v0.8.0/go.mod h1:WZyyI2P3hxPY2UVHs3cS8YcllAeyfquQcKfdeE9AFVg= -github.com/golangci/plugin-module-register v0.1.2 h1:e5WM6PO6NIAEcij3B053CohVp3HIYbzSuP53UAYgOpg= -github.com/golangci/plugin-module-register v0.1.2/go.mod h1:1+QGTsKBvAIvPvoY/os+G5eoqxWn70HYDm2uvUyGuVw= -github.com/golangci/revgrep v0.8.0 h1:EZBctwbVd0aMeRnNUsFogoyayvKHyxlV3CdUA46FX2s= -github.com/golangci/revgrep v0.8.0/go.mod h1:U4R/s9dlXZsg8uJmaR1GrloUr14D7qDl8gi2iPXJH8k= -github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba h1:lqtcnSMDuuJdu/LrKWi5RJzpSNLOJXYe/nzQutTI5kg= -github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba/go.mod h1:sCBNcpRmhJCtbFGz49+IM3ETTFf7QdJ30AeYCd43NKk= -github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e h1:ai0EfmVYE2bRA5htgAG9r7s3tHsfjIhN98WshBTJ9jM= -github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e/go.mod h1:Vrn4B5oR9qRwM+f54koyeH3yzphlecwERs0el27Fr/s= -github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e h1:gD6P7NEo7Eqtt0ssnqSJNNndxe69DOQ24A5h7+i3KpM= -github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e/go.mod h1:h+wZwLjUTJnm/P2rwlbJdRPZXOzaT36/FwnPnY2inzc= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV4uIlFds5weTmxbo= @@ -397,56 +199,17 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.18 h1:hvVi34VucdrV1IIsiW github.com/googleapis/enterprise-certificate-proxy v0.3.18/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= -github.com/gordonklaus/ineffassign v0.2.0 h1:Uths4KnmwxNJNzq87fwQQDDnbNb7De00VOk9Nu0TySs= -github.com/gordonklaus/ineffassign v0.2.0/go.mod h1:TIpymnagPSexySzs7F9FnO1XFTy8IT3a59vmZp5Y9Lw= -github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk= -github.com/gostaticanalysis/analysisutil v0.7.1/go.mod h1:v21E3hY37WKMGSnbsw2S/ojApNWb6C1//mXO48CXbVc= -github.com/gostaticanalysis/comment v1.4.2/go.mod h1:KLUTGDv6HOCotCH8h2erHKmpci2ZoR8VPu34YA2uzdM= -github.com/gostaticanalysis/comment v1.5.0 h1:X82FLl+TswsUMpMh17srGRuKaaXprTaytmEpgnKIDu8= -github.com/gostaticanalysis/comment v1.5.0/go.mod h1:V6eb3gpCv9GNVqb6amXzEUX3jXLVK/AdA+IrAMSqvEc= -github.com/gostaticanalysis/forcetypeassert v0.2.0 h1:uSnWrrUEYDr86OCxWa4/Tp2jeYDlogZiZHzGkWFefTk= -github.com/gostaticanalysis/forcetypeassert v0.2.0/go.mod h1:M5iPavzE9pPqWyeiVXSFghQjljW1+l/Uke3PXHS6ILY= -github.com/gostaticanalysis/nilerr v0.1.2 h1:S6nk8a9N8g062nsx63kUkF6AzbHGw7zzyHMcpu52xQU= -github.com/gostaticanalysis/nilerr v0.1.2/go.mod h1:A19UHhoY3y8ahoL7YKz6sdjDtduwTSI4CsymaC2htPA= -github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= -github.com/gostaticanalysis/testutil v0.5.0 h1:Dq4wT1DdTwTGCQQv3rl3IvD5Ld0E6HiY+3Zh0sUGqw8= -github.com/gostaticanalysis/testutil v0.5.0/go.mod h1:OLQSbuM6zw2EvCcXTz1lVq5unyoNft372msDY0nY5Hs= -github.com/hashicorp/go-immutable-radix/v2 v2.1.0 h1:CUW5RYIcysz+D3B+l1mDeXrQ7fUvGGCwJfdASSzbrfo= -github.com/hashicorp/go-immutable-radix/v2 v2.1.0/go.mod h1:hgdqLXA4f6NIjRVisM1TJ9aOJVNRqKZj+xDGF6m7PBw= -github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= -github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= -github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= -github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jgautheron/goconst v1.11.0 h1:KgN90z5qXt5f0Uzf3cWXev3hfMMFUyNeKdpkSBRvLDk= -github.com/jgautheron/goconst v1.11.0/go.mod h1:0p+wv1lFOiUr0IlNNT1nrm6+8DB8u2sU6KHGzFRXHDc= -github.com/jjti/go-spancheck v0.6.5 h1:lmi7pKxa37oKYIMScialXUK6hP3iY5F1gu+mLBPgYB8= -github.com/jjti/go-spancheck v0.6.5/go.mod h1:aEogkeatBrbYsyW6y5TgDfihCulDYciL1B7rG2vSsrU= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= -github.com/julz/importas v0.2.0 h1:y+MJN/UdL63QbFJHws9BVC5RpA2iq0kpjrFajTGivjQ= -github.com/julz/importas v0.2.0/go.mod h1:pThlt589EnCYtMnmhmRYY/qn9lCf/frPOK+WMx3xiJY= -github.com/karamaru-alpha/copyloopvar v1.2.2 h1:yfNQvP9YaGQR7VaWLYcfZUlRP2eo2vhExWKxD/fP6q0= -github.com/karamaru-alpha/copyloopvar v1.2.2/go.mod h1:oY4rGZqZ879JkJMtX3RRkcXRkmUvH0x35ykgaKgsgJY= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/kisielk/errcheck v1.20.0 h1:9rwHBNKzd4wkDWcROy3DvFGNqEPlkxBg305rvk7HabI= -github.com/kisielk/errcheck v1.20.0/go.mod h1:O+f80MKNwX8Oor2jwgpeQ9An7uJm+hRSgT+h22knRJU= -github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= -github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= -github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= -github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -454,110 +217,35 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kulti/thelper v0.7.1 h1:fI8QITAoFVLx+y+vSyuLBP+rcVIB8jKooNSCT2EiI98= -github.com/kulti/thelper v0.7.1/go.mod h1:NsMjfQEy6sd+9Kfw8kCP61W1I0nerGSYSFnGaxQkcbs= -github.com/kunwardeep/paralleltest v1.0.15 h1:ZMk4Qt306tHIgKISHWFJAO1IDQJLc6uDyJMLyncOb6w= -github.com/kunwardeep/paralleltest v1.0.15/go.mod h1:di4moFqtfz3ToSKxhNjhOZL+696QtJGCFe132CbBLGk= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lasiar/canonicalheader v1.1.2 h1:vZ5uqwvDbyJCnMhmFYimgMZnJMjwljN5VGY0VKbMXb4= -github.com/lasiar/canonicalheader v1.1.2/go.mod h1:qJCeLFS0G/QlLQ506T+Fk/fWMa2VmBUiEI2cuMK4djI= -github.com/ldez/exptostd v0.4.5 h1:kv2ZGUVI6VwRfp/+bcQ6Nbx0ghFWcGIKInkG/oFn1aQ= -github.com/ldez/exptostd v0.4.5/go.mod h1:QRjHRMXJrCTIm9WxVNH6VW7oN7KrGSht69bIRwvdFsM= -github.com/ldez/gomoddirectives v0.9.0 h1:2YV/EX7nVlWL4jySusYTzBKHuE3D2fgcRsQuMa3yIoo= -github.com/ldez/gomoddirectives v0.9.0/go.mod h1:DdZzfm9MdXCjn2/UGYXCFfo+tzrp2Ib4iD2Q0kIJkwE= -github.com/ldez/grignotin v0.10.1 h1:keYi9rYsgbvqAZGI1liek5c+jv9UUjbvdj3Tbn5fn4o= -github.com/ldez/grignotin v0.10.1/go.mod h1:UlDbXFCARrXbWGNGP3S5vsysNXAPhnSuBufpTEbwOas= -github.com/ldez/structtags v0.6.1 h1:bUooFLbXx41tW8SvkfwfFkkjPYvFFs59AAMgVg6DUBk= -github.com/ldez/structtags v0.6.1/go.mod h1:YDxVSgDy/MON6ariaxLF2X09bh19qL7MtGBN5MrvbdY= -github.com/ldez/tagliatelle v0.7.2 h1:KuOlL70/fu9paxuxbeqlicJnCspCRjH0x8FW+NfgYUk= -github.com/ldez/tagliatelle v0.7.2/go.mod h1:PtGgm163ZplJfZMZ2sf5nhUT170rSuPgBimoyYtdaSI= -github.com/ldez/usetesting v0.5.0 h1:3/QtzZObBKLy1F4F8jLuKJiKBjjVFi1IavpoWbmqLwc= -github.com/ldez/usetesting v0.5.0/go.mod h1:Spnb4Qppf8JTuRgblLrEWb7IE6rDmUpGvxY3iRrzvDQ= -github.com/leonklingele/grouper v1.1.2 h1:o1ARBDLOmmasUaNDesWqWCIFH3u7hoFlM84YrjT3mIY= -github.com/leonklingele/grouper v1.1.2/go.mod h1:6D0M/HVkhs2yRKRFZUoGjeDy7EZTfFBE9gl4kjmIGkA= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= -github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss= -github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/macabu/inamedparam v0.2.0 h1:VyPYpOc10nkhI2qeNUdh3Zket4fcZjEWe35poddBCpE= -github.com/macabu/inamedparam v0.2.0/go.mod h1:+Pee9/YfGe5LJ62pYXqB89lJ+0k5bsR8Wgz/C0Zlq3U= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/manuelarte/embeddedstructfieldcheck v0.4.0 h1:3mAIyaGRtjK6EO9E73JlXLtiy7ha80b2ZVGyacxgfww= -github.com/manuelarte/embeddedstructfieldcheck v0.4.0/go.mod h1:z8dFSyXqp+fC6NLDSljRJeNQJJDWnY7RoWFzV3PC6UM= -github.com/manuelarte/funcorder v0.6.0 h1:0hBngc4fa1IgNiI65A7sFGkMvoMCc878RjqB5V7rWP0= -github.com/manuelarte/funcorder v0.6.0/go.mod h1:id3NDhXdQBmeqXH7eVC6Z89xS6JxvZ8kF9xUxpArU/g= -github.com/maratori/testableexamples v1.0.1 h1:HfOQXs+XgfeRBJ+Wz0XfH+FHnoY9TVqL6Fcevpzy4q8= -github.com/maratori/testableexamples v1.0.1/go.mod h1:XE2F/nQs7B9N08JgyRmdGjYVGqxWwClLPCGSQhXQSrQ= -github.com/maratori/testpackage v1.1.2 h1:ffDSh+AgqluCLMXhM19f/cpvQAKygKAJXFl9aUjmbqs= -github.com/maratori/testpackage v1.1.2/go.mod h1:8F24GdVDFW5Ew43Et02jamrVMNXLUNaOynhDssITGfc= -github.com/matoous/godox v1.1.0 h1:W5mqwbyWrwZv6OQ5Z1a/DHGMOvXYCBP3+Ht7KMoJhq4= -github.com/matoous/godox v1.1.0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= -github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= -github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= -github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= -github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= -github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= -github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/mgechev/revive v1.15.0 h1:vJ0HzSBzfNyPbHKolgiFjHxLek9KUijhqh42yGoqZ8Q= -github.com/mgechev/revive v1.15.0/go.mod h1:LlAKO3QQe9OJ0pVZzI2GPa8CbXGZ/9lNpCGvK4T/a8A= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moricho/tparallel v0.3.2 h1:odr8aZVFA3NZrNybggMkYO3rgPRcqjeQUlBBFVxKHTI= -github.com/moricho/tparallel v0.3.2/go.mod h1:OQ+K3b4Ln3l2TZveGCywybl68glfLEwFGqvnjok8b+U= -github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= -github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U= -github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhKRf3Swg= -github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs= -github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= -github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= -github.com/nunnatsa/ginkgolinter v0.24.0 h1:Mp0EagluLFP98JatP6nqp/gGEoljNG97uf9AcxcBVy8= -github.com/nunnatsa/ginkgolinter v0.24.0/go.mod h1:2ZMRuzX6+3XXyY6UZOwb6n+MCocVGbkIsDBC4vuWz5c= github.com/oapi-codegen/nullable v1.2.0 h1:VflFkDW980KhBPiFF7nWSyjg+r4Obqj8lXipV0UkP5w= github.com/oapi-codegen/nullable v1.2.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= -github.com/oapi-codegen/runtime v1.6.0 h1:7Xx+GlueD6nRuyKoCPzL434Jfi3BetbiJOrzCHp/VPU= -github.com/oapi-codegen/runtime v1.6.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= -github.com/onsi/ginkgo/v2 v2.32.0 h1:Hw7s2pVrQo/8Yz5N77qdnpHaoc+c6cC9WIV1Jce+J6E= -github.com/onsi/ginkgo/v2 v2.32.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= -github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= -github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= +github.com/oapi-codegen/runtime v1.7.0 h1:t7358VYPvNbWJ9gdAkIK/smVeHpBf6yp8VTsaZsb/7k= +github.com/oapi-codegen/runtime v1.7.0/go.mod h1:GwV7hC2hviaMzj+ITfHVRESK5J2W/GefVwIND/bMGvU= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= -github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= -github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= -github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= -github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= -github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= -github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/package-url/packageurl-go v0.1.7 h1:iFWg6tzAjLA6F/qX3M5nZaiMHJgc+p2zxVyr/fY+sZY= github.com/package-url/packageurl-go v0.1.7/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= github.com/pandatix/go-cvss v0.6.4 h1:9w2RCO/Q4UTiJyEgpCHRiVc6CfrsFEnkoX+OtATqKio= github.com/pandatix/go-cvss v0.6.4/go.mod h1:/ukvQnYlrKl3o/DVp7/GO2UZyZheuo/maOK0U1nBEhQ= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= -github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea h1:sKwxy1H95npauwu8vtF95vG/syrL0p8fSZo/XlDg5gk= github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea/go.mod h1:1VcHEd3ro4QMoHfiNl/j7Jkln9+KQuorp0PItHMJYNg= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -569,161 +257,42 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= -github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= -github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= +github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= -github.com/quasilyte/go-ruleguard v0.4.5 h1:AGY0tiOT5hJX9BTdx/xBdoCubQUAE2grkqY2lSwvZcA= -github.com/quasilyte/go-ruleguard v0.4.5/go.mod h1:Vl05zJ538vcEEwu16V/Hdu7IYZWyKSwIy4c88Ro1kRE= -github.com/quasilyte/go-ruleguard/dsl v0.3.23 h1:lxjt5B6ZCiBeeNO8/oQsegE6fLeCzuMRoVWSkXC4uvY= -github.com/quasilyte/go-ruleguard/dsl v0.3.23/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= -github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo= -github.com/quasilyte/gogrep v0.5.0/go.mod h1:Cm9lpz9NZjEoL1tgZ2OgeUKPIxL1meE7eo60Z6Sk+Ng= -github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl980XxGFEZSS6KlBGIV0diGdySzxATTWoqaU= -github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= -github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs= -github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ= -github.com/raeperd/recvcheck v0.3.0 h1:PM+XYvyxIj3bo+kobJfFTdTuU3Lmfu96mKDbyHDbRt8= -github.com/raeperd/recvcheck v0.3.0/go.mod h1:PZNwG+HztFYMH2ZPq0Hu3QgkV2yiA6VrtNz9c1fXWJo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= -github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529 h1:18kd+8ZUlt/ARXhljq+14TwAoKa61q6dX8jtwOf6DH8= github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/rubyist/circuitbreaker v2.2.1+incompatible h1:KUKd/pV8Geg77+8LNDwdow6rVCAYOp8+kHUyFvL6Mhk= github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryancurrah/gomodguard v1.4.1 h1:eWC8eUMNZ/wM/PWuZBv7JxxqT5fiIKSIyTvjb7Elr+g= -github.com/ryancurrah/gomodguard v1.4.1/go.mod h1:qnMJwV1hX9m+YJseXEBhd2s90+1Xn6x9dLz11ualI1I= -github.com/ryancurrah/gomodguard/v2 v2.1.3 h1:E7sz3PJwE9Ba1reVxSpF6XLCPJZ74Kfw/LabTNM4GIA= -github.com/ryancurrah/gomodguard/v2 v2.1.3/go.mod h1:CQicdLGatWMxLX53JzoBjYlsNZhHbmLv2AVa0s2aivU= -github.com/ryanrolds/sqlclosecheck v0.6.0 h1:pEyL9okISdg1F1SEpJNlrEotkTGerv5BMk7U4AG0eVg= -github.com/ryanrolds/sqlclosecheck v0.6.0/go.mod h1:xyX16hsDaCMXHrMJ3JMzGf5OpDfHTOTTQrT7HOFUmeU= -github.com/sanposhiho/wastedassign/v2 v2.1.0 h1:crurBF7fJKIORrV85u9UUpePDYGWnwvv3+A96WvwXT0= -github.com/sanposhiho/wastedassign/v2 v2.1.0/go.mod h1:+oSmSC+9bQ+VUAxA66nBb0Z7N8CK7mscKTDYC6aIek4= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tMEOsumirXcOJqAw= -github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ= -github.com/sashamelentyev/usestdlibvars v1.29.0 h1:8J0MoRrw4/NAXtjQqTHrbW9NN+3iMf7Knkq057v4XOQ= -github.com/sashamelentyev/usestdlibvars v1.29.0/go.mod h1:8PpnjHMk5VdeWlVb4wCdrB8PNbLqZ3wBZTZWkrpZZL8= -github.com/securego/gosec/v2 v2.28.0 h1:ZsSdiDb0AtTpLFVol5z91gbMei9ZiLEPG/pZjZujp7c= -github.com/securego/gosec/v2 v2.28.0/go.mod h1:lb4/9AHe+lJy/kjWmWRWWsEipvbwGKuxf+tY1Pmjdnk= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.10.1 h1:xi4336Zh11WpU14fXR6I67V3yaTPQYwRx2WEtHbRg4Q= -github.com/sirupsen/logrus v1.10.1/go.mod h1:vsQHnG7xzNsxk3NrwboUiWPnIC3dmbjcGPykD7+tiHk= -github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+Wwfd0XE= -github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4= -github.com/sonatard/noctx v0.5.1 h1:wklWg9c9ZYugOAk7qG4yP4PBrlQsmSLPTvW1K4PRQMs= -github.com/sonatard/noctx v0.5.1/go.mod h1:64XdbzFb18XL4LporKXp8poqZtPKbCrqQ402CV+kJas= -github.com/sourcegraph/go-diff v0.8.0 h1:ipIyu4cTsLbIrln4l0qtHA3r0a7gyK4ntKjtQytHhvY= -github.com/sourcegraph/go-diff v0.8.0/go.mod h1:hWlcO7Al+UZStZAP8rBumHpCK5ZHQ5BXsMls8p4+F5E= github.com/spdx/tools-golang v0.5.7 h1:+sWcKGnhwp3vLdMqPcLdA6QK679vd86cK9hQWH3AwCg= github.com/spdx/tools-golang v0.5.7/go.mod h1:jg7w0LOpoNAw6OxKEzCoqPC2GCTj45LyTlVmXubDsYw= -github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= -github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= -github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= -github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= -github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= -github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0= -github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= -github.com/stbenjam/no-sprintf-host-port v0.3.1 h1:AyX7+dxI4IdLBPtDbsGAyqiTSLpCP9hWRrXQDU4Cm/g= -github.com/stbenjam/no-sprintf-host-port v0.3.1/go.mod h1:ODbZesTCHMVKthBHskvUUexdcNHAQRXk9NpSsL8p/HQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= -github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= -github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= -github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= -github.com/tenntenn/modver v1.0.1 h1:2klLppGhDgzJrScMpkj9Ujy3rXPUspSjAcev9tSEBgA= -github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0= -github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpRQGxTSkNYKJ51yaw6ChIqO+Je8UqsTKN/cDag= -github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY= github.com/terminalstatic/go-xsd-validate v0.1.8 h1:UVrTCy1j3DhwaYTTUF+QYO/Nan13S0tf+Jwi+p45Bf0= github.com/terminalstatic/go-xsd-validate v0.1.8/go.mod h1:1kb47fi2c6onlf+B7UrrQ9VYraOhcYwFm3iG+J6F4Zo= -github.com/tetafro/godot v1.5.6 h1:IEkrFCwXaYHlOn4mGzGS3F3dkP6m9t0jpwqBFPIkKiA= -github.com/tetafro/godot v1.5.6/go.mod h1:eOkMrVQurDui411nBY2FA05EYH01r14LuWY/NrVDVcU= -github.com/timakin/bodyclose v0.0.0-20260129054331-73d1f95b84b4 h1:SiHe5XLTn9sFWJ5pBwJ5FN/4j34q9ZlOAD//kMoMYp0= -github.com/timakin/bodyclose v0.0.0-20260129054331-73d1f95b84b4/go.mod h1:sDHLK7rb/59v/ZxZ7KtymgcoxuUMxjXq8gtu9VMOK8M= -github.com/timonwong/loggercheck v0.11.0 h1:jdaMpYBl+Uq9mWPXv1r8jc5fC3gyXx4/WGwTnnNKn4M= -github.com/timonwong/loggercheck v0.11.0/go.mod h1:HEAWU8djynujaAVX7QI65Myb8qgfcZ1uKbdpg3ZzKl8= -github.com/tomarrell/wrapcheck/v2 v2.12.0 h1:H/qQ1aNWz/eeIhxKAFvkfIA+N7YDvq6TWVFL27Of9is= -github.com/tomarrell/wrapcheck/v2 v2.12.0/go.mod h1:AQhQuZd0p7b6rfW+vUwHm5OMCGgp63moQ9Qr/0BpIWo= -github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw= -github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0= github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= -github.com/ultraware/funlen v0.2.0 h1:gCHmCn+d2/1SemTdYMiKLAHFYxTYz7z9VIDRaTGyLkI= -github.com/ultraware/funlen v0.2.0/go.mod h1:ZE0q4TsJ8T1SQcjmkhN/w+MceuatI6pBFSxxyteHIJA= -github.com/ultraware/whitespace v0.2.0 h1:TYowo2m9Nfj1baEQBjuHzvMRbp19i+RCcRYrSWoFa+g= -github.com/ultraware/whitespace v0.2.0/go.mod h1:XcP1RLD81eV4BW8UhQlpaR+SDc2givTvyI8a586WjW8= -github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= -github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/uudashr/gocognit v1.2.1 h1:CSJynt5txTnORn/DkhiB4mZjwPuifyASC8/6Q0I/QS4= -github.com/uudashr/gocognit v1.2.1/go.mod h1:acaubQc6xYlXFEMb9nWX2dYBzJ/bIjEkc1zzvyIZg5Q= -github.com/uudashr/iface v1.5.0 h1:PgdMt4uAettGG8K/Kbamc4B9FABgUgnS3TLbl6fnjEk= -github.com/uudashr/iface v1.5.0/go.mod h1:pbeBPlbuU2qkNDn0mmfrxP2X+wjPMIQAy+r1MBXSXtg= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xen0n/gosmopolitan v1.3.0 h1:zAZI1zefvo7gcpbCOrPSHJZJYA9ZgLfJqtKzZ5pHqQM= -github.com/xen0n/gosmopolitan v1.3.0/go.mod h1:rckfr5T6o4lBtM1ga7mLGKZmLxswUoH1zxHgNXOsEt4= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= -github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM= -github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= -github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs= -github.com/yeya24/promlinter v0.3.0/go.mod h1:cDfJQQYv9uYciW60QT0eeHlFodotkYZlL+YcPQN+mW4= -github.com/ykadowak/zerologlint v0.1.5 h1:Gy/fMz1dFQN9JZTPjv1hxEk+sRWm05row04Yoolgdiw= -github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -gitlab.com/bosi/decorder v0.4.2 h1:qbQaV3zgwnBZ4zPMhGLW4KZe7A7NwxEhJx39R3shffo= -gitlab.com/bosi/decorder v0.4.2/go.mod h1:muuhHoaJkA9QLcYHq4Mj8FJUwDZ+EirSHRiaTcTf6T8= -go-simpler.org/assert v0.9.0 h1:PfpmcSvL7yAnWyChSjOz6Sp6m9j5lyK8Ok9pEL31YkQ= -go-simpler.org/assert v0.9.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28= -go-simpler.org/musttag v0.14.0 h1:XGySZATqQYSEV3/YTy+iX+aofbZZllJaqwFWs+RTtSo= -go-simpler.org/musttag v0.14.0/go.mod h1:uP8EymctQjJ4Z1kUnjX0u2l60WfUdQxCwSNKzE1JEOE= -go-simpler.org/sloglint v0.12.0 h1:UzWDlLWNE5FLqsvyq3tWYHuQMbqrervOhT8qPl4Mmw4= -go-simpler.org/sloglint v0.12.0/go.mod h1:jBjjC2bm8rYrs88oTRlFX497kWjJsyZWYoNaXkGRI6I= -go.augendre.info/arangolint v0.4.0 h1:xSCZjRoS93nXazBSg5d0OGCi9APPLNMmmLrC995tR50= -go.augendre.info/arangolint v0.4.0/go.mod h1:l+f/b4plABuFISuKnTGD4RioXiCCgghv2xqst/xOvAA= -go.augendre.info/fatcontext v0.10.0 h1:HhFopmivh8U1+AU7f0kuwUeg2eiIns7YsGQOMHwSJ90= -go.augendre.info/fatcontext v0.10.0/go.mod h1:pqpGvA9GlrXy+aXkp8L2dKz12Zp4g2FhzcAtwToU+2w= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= @@ -732,123 +301,53 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.6 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= -go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= -go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= -go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= -go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= -go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= -go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= -go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= -go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= -go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= -go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/metric/x v0.67.0 h1:PcicCNZFkZ4bXfSooXdo3WN7RBOVOtjVdo1wD358Uns= +go.opentelemetry.io/otel/metric/x v0.67.0/go.mod h1:FBjCWZe6wgcqxcMtjdGiClDKXb2YxxXii0CXftE4QtI= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= -go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= gocloud.dev v0.46.0 h1:niIuZwSjMtBx8K+ITB2s5kZullB13PGOS2ZoQPZxQ4Q= gocloud.dev v0.46.0/go.mod h1:ACQe+2qO+hEO+pdcvvsM+RB63r8TyGD1W3ESCLFyzvM= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= -golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= -golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= -golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= -golang.org/x/exp/typeparams v0.0.0-20260811152304-ee035b5b010f h1:+lI8cDJ4uceLipg2f1ODay7fEuLkk0BIHXd6PB8icxo= -golang.org/x/exp/typeparams v0.0.0-20260811152304-ee035b5b010f/go.mod h1:PqrXSW65cXDZH0k4IeUbhmg/bcAZDbzNz3byBpKCsXo= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= -golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= -golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= -golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= -golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= -golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= -golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= -golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= @@ -861,8 +360,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1: google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 h1:qEHAMpSaUhtD0p3NbEEI83HwNGFxEwaSJ1G9PLnCBZE= google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= -google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -870,49 +369,38 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.8.0 h1:UacpzPr7D6i5BAjTkA7sNVcx4kIbhAZcQ4zYtKiXx68= -honnef.co/go/tools v0.8.0/go.mod h1:XA+OnlRA9EDh/ukGvXMNSZNKGwFQJ+5dER0ioUkOxks= -modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= -modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= -modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= -modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= -modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= +modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg= -modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= +modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -mvdan.cc/gofumpt v0.11.0 h1:0H01XB95PnN2QgCSR9ELdZyTlJqNZ7181B0BTMh5VZc= -mvdan.cc/gofumpt v0.11.0/go.mod h1:BeT5wCsOJt6J9zT2MZIOGszjUHzFkn1/l9g6xAzqsXo= -mvdan.cc/unparam v0.0.0-20260818115549-3f964bcb5673 h1:dEE6li4OPIE54oojY2qaayFS1fSp17G14si0gXRxl0U= -mvdan.cc/unparam v0.0.0-20260818115549-3f964bcb5673/go.mod h1:62roFV3D3nYOWIXv3PfGO4UYEKAotz2WgLywT87ONd8= -sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= -sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/accesslog/accesslog.go b/internal/accesslog/accesslog.go index 6a3cb011..9ff4a0af 100644 --- a/internal/accesslog/accesslog.go +++ b/internal/accesslog/accesslog.go @@ -33,7 +33,18 @@ type Entry struct { StatusCode int `json:"status_code,omitempty"` DurationMS int64 `json:"duration_ms"` RemoteAddr string `json:"remote_addr,omitempty"` - Error string `json:"error,omitempty"` + // RemoteIP is the address the request is attributed to: the TCP peer, or + // the leftmost X-Forwarded-For entry when that header is trusted. + RemoteIP string `json:"remote_ip,omitempty"` + // UserAgent is recorded verbatim; Client is it reduced to a known tool + // name, so log queries can group without parsing. + UserAgent string `json:"user_agent,omitempty"` + Client string `json:"client,omitempty"` + Ecosystem string `json:"ecosystem,omitempty"` + // Bytes is always emitted, including as 0, so a log pipeline summing the + // field does not have to treat a bodyless response as null. + Bytes int64 `json:"bytes"` + Error string `json:"error,omitempty"` } // Logger appends complete JSON objects to a file, one per line. diff --git a/internal/config/config.go b/internal/config/config.go index 137d7f73..54f826f5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -71,6 +71,7 @@ import ( "strings" "time" + "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/purl" "gopkg.in/yaml.v3" ) @@ -97,6 +98,23 @@ type Config struct { // Example: "https://proxy.example.com/ui" UIBaseURL string `json:"ui_base_url" yaml:"ui_base_url"` + // TrustForwardedFor attributes requests to the leftmost X-Forwarded-For + // entry instead of the TCP peer address, in the structured log, the access + // log and the request-source table on /ui/analytics. + // + // Enable this only when the proxy sits behind a load balancer or ingress + // that sets the header, because any client can send it: behind one it is + // the only way to see past the hop, in front of one it lets a caller choose + // what address it is attributed to. + TrustForwardedFor bool `json:"trust_forwarded_for" yaml:"trust_forwarded_for"` + + // UIRequestSources shows the request-source table on /ui/analytics, which + // reports caller addresses, the tool each ran and how much each pulled. + // + // Off by default. The proxy has no authentication of its own, so until this + // is enabled /ui exposes what is cached rather than who called. + UIRequestSources bool `json:"ui_request_sources" yaml:"ui_request_sources"` + // Storage configures artifact storage. Storage StorageConfig `json:"storage" yaml:"storage"` @@ -115,6 +133,9 @@ type Config struct { // Cooldown configures version age filtering to mitigate supply chain attacks. Cooldown CooldownConfig `json:"cooldown" yaml:"cooldown"` + // Denylist blocks explicitly configured package versions. + Denylist DenylistConfig `json:"denylist" yaml:"denylist"` + // Scanning configures pre-cache artifact scanning (trivy, ClamAV, Wiz, // or a custom service) to mitigate supply chain attacks. Scanning ScanningConfig `json:"scanning" yaml:"scanning"` @@ -134,10 +155,18 @@ type Config struct { // size return ErrMetadataTooLarge. Default: "100MB". MetadataMaxSize string `json:"metadata_max_size" yaml:"metadata_max_size"` + // MetadataRewriteCacheSize is how much rewritten npm and Composer + // metadata to keep in memory, so a document is rewritten once rather than + // on every request (e.g. "256MB", "1GB"). Default: "256MB". Set to "0" to + // rewrite on every request. + MetadataRewriteCacheSize string `json:"metadata_rewrite_cache_size" yaml:"metadata_rewrite_cache_size"` + // HTTPTimeout is the timeout for individual upstream HTTP requests made // by protocol handlers (metadata fetches, pass-through file requests). // Uses Go duration syntax (e.g. "30s", "2m"). Default: "30s". - // Set to "0" to disable the timeout entirely. + // Set to "0" to disable the timeout entirely. Independently of this + // setting, the shared transport gives up on an upstream that has not sent + // response headers within 60 seconds. HTTPTimeout string `json:"http_timeout" yaml:"http_timeout"` // MirrorAPI enables the /api/mirror endpoints for starting mirror jobs via HTTP. @@ -163,6 +192,10 @@ type CooldownConfig struct { // Packages overrides the cooldown for specific packages (keyed by PURL). // Valid PURL keys are normalized to canonical form before use. Packages map[string]string `json:"packages" yaml:"packages"` + + // PackagePatterns overrides the cooldown for packages whose PURLs match a glob. + // Exact package overrides take precedence over matching patterns. + PackagePatterns map[string]string `json:"package_patterns" yaml:"package_patterns"` } // NormalizedPackages returns a copy of the package overrides with valid PURL @@ -364,6 +397,14 @@ type StorageConfig struct { // storage at an internal address (e.g. 127.0.0.1 or a Docker hostname) // but clients must use a public one. DirectServeBaseURL string `json:"direct_serve_base_url" yaml:"direct_serve_base_url"` + + // CacheArtifacts stores fetched artifacts so later downloads are served + // from storage. When false, every download streams from upstream and + // nothing is stored; metadata is still cached, and cooldown and the + // denylist still apply. Useful when another caching layer sits in front + // of the proxy. False is incompatible with scanning, direct_serve and + // mirror_api, which all depend on stored artifacts. Default: true. + CacheArtifacts bool `json:"cache_artifacts" yaml:"cache_artifacts"` } // GradleConfig configures Gradle-specific features. @@ -412,6 +453,12 @@ type DatabaseConfig struct { // URL is the PostgreSQL connection string. URL string `json:"url" yaml:"url"` + + // HitFlushInterval is how often cache hit counts and last-access times + // are written, batched in one transaction. Uses Go duration syntax + // (e.g. "1s", "10s"). Default: "1s". Set to "0" to write each hit as it + // happens. + HitFlushInterval string `json:"hit_flush_interval" yaml:"hit_flush_interval"` } // String returns a human-readable description of the configured database @@ -549,11 +596,16 @@ type UpstreamConfig struct { // Default: https://tuist.dev/api/registry/swift Swift string `json:"swift" yaml:"swift"` - // Debian is the upstream APT repository base URL. + // Debian is the upstream APT repository base URL, served at /debian/. // Example: http://archive.ubuntu.com/ubuntu would get Ubuntu. // Default: http://deb.debian.org/debian Debian string `json:"debian" yaml:"debian"` + // DebianRepositories maps repository names to additional APT repository + // base URLs, served at /debian/{name}/. + // Example: {"security": "https://security.debian.org/debian-security"}. + DebianRepositories map[string]string `json:"debian_repositories" yaml:"debian_repositories"` + // RPM is the upstream RPM repository base URL. // Default: https://dl.fedoraproject.org/pub/fedora/linux RPM string `json:"rpm" yaml:"rpm"` @@ -642,9 +694,24 @@ func (u *UpstreamConfig) Validate() error { if err := validateNamedUpstreams("upstream.generic", u.Generic); err != nil { return err } + if err := validateNamedUpstreams("upstream.debian_repositories", u.DebianRepositories); err != nil { + return err + } + for _, name := range debianReservedRepositoryNames { + if _, found := u.DebianRepositories[name]; found { + return fmt.Errorf( + "invalid upstream.debian_repositories name %q: reserved for the upstream.debian archive", + name, + ) + } + } return nil } +// debianReservedRepositoryNames are the upstream.debian archive's own root +// paths, which a repository of the same name would shadow. +var debianReservedRepositoryNames = []string{"pool", "dists"} + func validateNamedUpstreams(field string, upstreams map[string]string) error { for name, upstreamURL := range upstreams { if name == "" || name == "." || name == ".." || strings.ContainsAny(name, `/\\`) { @@ -731,8 +798,9 @@ func Default() *Config { Listen: ":8080", BaseURL: "http://localhost:8080", Storage: StorageConfig{ - Path: "./cache/artifacts", - MaxSize: "", + Path: "./cache/artifacts", + MaxSize: "", + CacheArtifacts: true, }, Database: DatabaseConfig{ Driver: "sqlite", @@ -855,21 +923,27 @@ func setEnvStringSlice(dst *[]string, key string) { // - PROXY_LOG_LEVEL // - PROXY_LOG_FORMAT // - PROXY_ACCESS_LOG_PATH +// - PROXY_TRUST_FORWARDED_FOR +// - PROXY_UI_REQUEST_SOURCES // - PROXY_UPSTREAM_SWIFT // - PROXY_HEALTH_STORAGE_PROBE_INTERVAL func (c *Config) LoadFromEnv() { setEnvString(&c.Listen, "PROXY_LISTEN") setEnvString(&c.BaseURL, "PROXY_BASE_URL") setEnvString(&c.UIBaseURL, "PROXY_UI_URL") + setEnvBool(&c.TrustForwardedFor, "PROXY_TRUST_FORWARDED_FOR") + setEnvBool(&c.UIRequestSources, "PROXY_UI_REQUEST_SOURCES") setEnvString(&c.Storage.URL, "PROXY_STORAGE_URL") setEnvString(&c.Storage.Path, "PROXY_STORAGE_PATH") setEnvString(&c.Storage.MaxSize, "PROXY_STORAGE_MAX_SIZE") setEnvBool(&c.Storage.DirectServe, "PROXY_STORAGE_DIRECT_SERVE") setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL") setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL") + setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS") setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER") setEnvString(&c.Database.Path, "PROXY_DATABASE_PATH") setEnvString(&c.Database.URL, "PROXY_DATABASE_URL") + setEnvString(&c.Database.HitFlushInterval, "PROXY_DATABASE_HIT_FLUSH_INTERVAL") setEnvString(&c.Log.Level, "PROXY_LOG_LEVEL") setEnvString(&c.Log.Format, "PROXY_LOG_FORMAT") setEnvString(&c.AccessLog.Path, "PROXY_ACCESS_LOG_PATH") @@ -912,6 +986,7 @@ func (c *Config) LoadFromEnv() { setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") setEnvString(&c.MetadataTTL, "PROXY_METADATA_TTL") setEnvString(&c.MetadataMaxSize, "PROXY_METADATA_MAX_SIZE") + setEnvString(&c.MetadataRewriteCacheSize, "PROXY_METADATA_REWRITE_CACHE_SIZE") setEnvString(&c.HTTPTimeout, "PROXY_HTTP_TIMEOUT") setEnvBool(&c.Gradle.BuildCache.ReadOnly, "PROXY_GRADLE_BUILD_CACHE_READ_ONLY") setEnvString(&c.Gradle.BuildCache.MaxUploadSize, "PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE") @@ -997,6 +1072,10 @@ func (c *Config) Validate() error { } } + if err := c.validateCacheArtifacts(); err != nil { + return err + } + // Validate metadata TTL if specified if c.MetadataTTL != "" && c.MetadataTTL != "0" { if _, err := time.ParseDuration(c.MetadataTTL); err != nil { @@ -1012,10 +1091,37 @@ func (c *Config) Validate() error { return err } + if err := validateHitFlushInterval(c.Database.HitFlushInterval); err != nil { + return err + } + return c.validateComponents() } +func (c *Config) validateCacheArtifacts() error { + if c.Storage.CacheArtifacts { + return nil + } + switch { + case c.Scanning.Enabled: + return fmt.Errorf("storage.cache_artifacts: false cannot be combined with scanning.enabled: scanning needs stored artifacts") + case c.Storage.DirectServe: + return fmt.Errorf("storage.cache_artifacts: false cannot be combined with storage.direct_serve: no artifacts are stored to redirect to") + case c.MirrorAPI: + return fmt.Errorf("storage.cache_artifacts: false cannot be combined with mirror_api: mirrored artifacts would never be served") + } + return nil +} + func (c *Config) validateComponents() error { + if err := validateMetadataRewriteCacheSize(c.MetadataRewriteCacheSize); err != nil { + return err + } + + if _, err := denylist.New(c.Denylist.Packages); err != nil { + return err + } + if err := c.Upstream.Validate(); err != nil { return err } @@ -1090,7 +1196,9 @@ const ( defaultMetadataTTL = 5 * time.Minute //nolint:mnd // sensible default defaultDirectServeTTL = 15 * time.Minute //nolint:mnd // sensible default defaultHTTPTimeout = 30 * time.Second //nolint:mnd // sensible default + defaultHitFlushInterval = time.Second defaultMetadataMaxSize = 100 << 20 + defaultMetadataRewriteCacheSize = 256 << 20 defaultGradleBuildCacheMaxUploadSize = 100 << 20 defaultGradleBuildCacheSweepInterval = 10 * time.Minute defaultGradleMaxUploadSizeStr = "100MB" @@ -1139,6 +1247,36 @@ func validateMetadataMaxSize(s string) error { return nil } +func validateMetadataRewriteCacheSize(s string) error { + if s == "" || s == "0" { + return nil + } + size, err := ParseSize(s) + if err != nil { + return fmt.Errorf("invalid metadata_rewrite_cache_size: %w", err) + } + if size < 0 { + return fmt.Errorf("invalid metadata_rewrite_cache_size %q: must not be negative", s) + } + return nil +} + +// ParseMetadataRewriteCacheSize returns how many bytes of rewritten metadata +// to keep in memory. Returns 256MB if unset or invalid, 0 if disabled. +func (c *Config) ParseMetadataRewriteCacheSize() int64 { + if c.MetadataRewriteCacheSize == "" { + return defaultMetadataRewriteCacheSize + } + if c.MetadataRewriteCacheSize == "0" { + return 0 + } + size, err := ParseSize(c.MetadataRewriteCacheSize) + if err != nil || size < 0 { + return defaultMetadataRewriteCacheSize + } + return size +} + // ParseMetadataMaxSize returns the maximum metadata response size in bytes. // Returns 100MB if unset or invalid. func (c *Config) ParseMetadataMaxSize() int64 { @@ -1168,6 +1306,36 @@ func (c *Config) ParseHTTPTimeout() time.Duration { return d } +func validateHitFlushInterval(s string) error { + if s == "" || s == "0" { + return nil + } + d, err := time.ParseDuration(s) + if err != nil { + return fmt.Errorf("invalid database.hit_flush_interval %q: %w", s, err) + } + if d < 0 { + return fmt.Errorf("invalid database.hit_flush_interval %q: must be non-negative", s) + } + return nil +} + +// ParseHitFlushInterval returns how often batched cache hits are written. +// Returns 1 second if unset or invalid, 0 if explicitly disabled. +func (c *Config) ParseHitFlushInterval() time.Duration { + if c.Database.HitFlushInterval == "" { + return defaultHitFlushInterval + } + if c.Database.HitFlushInterval == "0" { + return 0 + } + d, err := time.ParseDuration(c.Database.HitFlushInterval) + if err != nil || d < 0 { + return defaultHitFlushInterval + } + return d +} + // ParseMetadataTTL returns the metadata TTL duration. // Returns 5 minutes if unset, 0 if explicitly disabled. func (c *Config) ParseMetadataTTL() time.Duration { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 80fcc68c..b1818cea 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -461,6 +461,8 @@ func TestLoadFromEnv(t *testing.T) { t.Setenv("PROXY_STORAGE_PATH", "/env/cache") t.Setenv("PROXY_LOG_LEVEL", testLevelDebug) t.Setenv("PROXY_ACCESS_LOG_PATH", "/tmp/proxy-access.jsonl") + t.Setenv("PROXY_TRUST_FORWARDED_FOR", "true") + t.Setenv("PROXY_UI_REQUEST_SOURCES", "true") t.Setenv("PROXY_UPSTREAM_ALLOW_PRIVATE_HOSTS", "registry.internal, 10.0.0.12") t.Setenv("PROXY_UPSTREAM_ALLOW_LOOPBACK", "true") t.Setenv("PROXY_GRADLE_BUILD_CACHE_READ_ONLY", "true") @@ -489,6 +491,15 @@ func TestLoadFromEnv(t *testing.T) { if cfg.AccessLog.Path != "/tmp/proxy-access.jsonl" { t.Errorf("AccessLog.Path = %q, want %q", cfg.AccessLog.Path, "/tmp/proxy-access.jsonl") } + // Containers configure the proxy through the environment, so a setting + // reachable only from YAML is unreachable in the deployment that most needs + // it -- which is the one sitting behind an ingress. + if !cfg.TrustForwardedFor { + t.Error("TrustForwardedFor = false, want true from the environment") + } + if !cfg.UIRequestSources { + t.Error("UIRequestSources = false, want true from the environment") + } if got := strings.Join(cfg.Upstream.AllowPrivateHosts, ","); got != "registry.internal,10.0.0.12" { t.Errorf("Upstream.AllowPrivateHosts = %q, want %q", got, "registry.internal,10.0.0.12") } @@ -551,6 +562,8 @@ cooldown: packages: "pkg:npm/lodash": "0" "pkg:npm/@babel/core": "14d" + package_patterns: + "pkg:npm/@example/*": "0" ` if err := os.WriteFile(path, []byte(content), 0644); err != nil { t.Fatalf("writing config file: %v", err) @@ -579,6 +592,9 @@ cooldown: if got := cfg.Cooldown.NormalizedPackages()["pkg:npm/%40babel/core"]; got != "14d" { t.Errorf("normalized Cooldown.Packages[@babel/core] = %q, want %q", got, "14d") } + if cfg.Cooldown.PackagePatterns["pkg:npm/@example/*"] != "0" { + t.Errorf("Cooldown.PackagePatterns[example] = %q, want %q", cfg.Cooldown.PackagePatterns["pkg:npm/@example/*"], "0") + } } func TestCooldownConfigNormalizedPackages(t *testing.T) { @@ -929,6 +945,54 @@ func TestValidateHTTPTimeout(t *testing.T) { } } +func TestParseMetadataRewriteCacheSize(t *testing.T) { + tests := []struct { + name string + size string + want int64 + }{ + {"empty defaults to 256MB", "", 256 << 20}, + {"explicit zero disables", "0", 0}, + {"1GB", "1GB", 1 << 30}, + {"invalid defaults to 256MB", "lots", 256 << 20}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.MetadataRewriteCacheSize = tt.size + if got := cfg.ParseMetadataRewriteCacheSize(); got != tt.want { + t.Errorf("ParseMetadataRewriteCacheSize() = %d, want %d", got, tt.want) + } + }) + } +} + +func TestValidateMetadataRewriteCacheSize(t *testing.T) { + cfg := Default() + cfg.MetadataRewriteCacheSize = "lots" + if err := cfg.Validate(); err == nil { + t.Error("expected validation error for invalid metadata_rewrite_cache_size") + } + + for _, ok := range []string{"512MB", "0", ""} { + cfg.MetadataRewriteCacheSize = ok + if err := cfg.Validate(); err != nil { + t.Errorf("unexpected error for metadata_rewrite_cache_size %q: %v", ok, err) + } + } +} + +func TestLoadMetadataRewriteCacheSizeFromEnv(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_METADATA_REWRITE_CACHE_SIZE", "1GB") + cfg.LoadFromEnv() + + if cfg.MetadataRewriteCacheSize != "1GB" { + t.Errorf("MetadataRewriteCacheSize = %q, want %q", cfg.MetadataRewriteCacheSize, "1GB") + } +} + func TestLoadHTTPTimeoutFromEnv(t *testing.T) { cfg := Default() t.Setenv("PROXY_HTTP_TIMEOUT", "90s") @@ -939,6 +1003,61 @@ func TestLoadHTTPTimeoutFromEnv(t *testing.T) { } } +func TestParseHitFlushInterval(t *testing.T) { + tests := []struct { + name string + interval string + want time.Duration + }{ + {"empty defaults to 1s", "", time.Second}, + {"explicit zero disables", "0", 0}, + {"10 seconds", "10s", 10 * time.Second}, + {"invalid defaults to 1s", "not-a-duration", time.Second}, + {"negative defaults to 1s", "-5s", time.Second}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.Database.HitFlushInterval = tt.interval + got := cfg.ParseHitFlushInterval() + if got != tt.want { + t.Errorf("ParseHitFlushInterval() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestValidateHitFlushInterval(t *testing.T) { + cfg := Default() + cfg.Database.HitFlushInterval = "not-a-duration" + if err := cfg.Validate(); err == nil { + t.Error("expected validation error for invalid hit_flush_interval") + } + + cfg.Database.HitFlushInterval = "-5s" + if err := cfg.Validate(); err == nil { + t.Error("expected validation error for negative hit_flush_interval") + } + + for _, ok := range []string{"10s", "0", ""} { + cfg.Database.HitFlushInterval = ok + if err := cfg.Validate(); err != nil { + t.Errorf("unexpected error for hit_flush_interval %q: %v", ok, err) + } + } +} + +func TestLoadHitFlushIntervalFromEnv(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_DATABASE_HIT_FLUSH_INTERVAL", "10s") + cfg.LoadFromEnv() + + if cfg.Database.HitFlushInterval != "10s" { + t.Errorf("HitFlushInterval = %q, want %q", cfg.Database.HitFlushInterval, "10s") + } +} + func TestLoadMetadataTTLFromEnv(t *testing.T) { cfg := Default() t.Setenv("PROXY_METADATA_TTL", "10m") @@ -1243,6 +1362,53 @@ func TestValidateNamedUpstreams(t *testing.T) { }, wantErr: true, }, + { + name: "valid Debian repository", + modify: func(cfg *Config) { + cfg.Upstream.DebianRepositories = map[string]string{ + "security": "https://security.debian.org/debian-security", + } + }, + }, + { + name: "Debian repository name contains path separator", + modify: func(cfg *Config) { + cfg.Upstream.DebianRepositories = map[string]string{ + "debian/security": "https://security.debian.org/debian-security", + } + }, + wantErr: true, + }, + { + name: "Debian repository name is a relative path element", + modify: func(cfg *Config) { + cfg.Upstream.DebianRepositories = map[string]string{ + ".": "https://security.debian.org/debian-security", + } + }, + wantErr: true, + }, + { + name: "Debian repository URL is not absolute", + modify: func(cfg *Config) { + cfg.Upstream.DebianRepositories = map[string]string{"security": "security.debian.org"} + }, + wantErr: true, + }, + { + name: "Debian repository shadows the pool prefix", + modify: func(cfg *Config) { + cfg.Upstream.DebianRepositories = map[string]string{"pool": "https://security.debian.org"} + }, + wantErr: true, + }, + { + name: "Debian repository shadows the dists prefix", + modify: func(cfg *Config) { + cfg.Upstream.DebianRepositories = map[string]string{"dists": "https://security.debian.org"} + }, + wantErr: true, + }, } for _, tt := range tests { @@ -1256,3 +1422,61 @@ func TestValidateNamedUpstreams(t *testing.T) { }) } } + +func TestValidateCacheArtifactsDisabled(t *testing.T) { + tests := []struct { + name string + modify func(*Config) + wantErr string + }{ + {"alone", func(*Config) {}, ""}, + {"with scanning", func(c *Config) { c.Scanning.Enabled = true }, "scanning.enabled"}, + {"with direct_serve", func(c *Config) { c.Storage.DirectServe = true }, "storage.direct_serve"}, + {"with mirror_api", func(c *Config) { c.MirrorAPI = true }, "mirror_api"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := Default() + cfg.Storage.CacheArtifacts = false + tt.modify(cfg) + err := cfg.Validate() + if tt.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("Validate() = %v, want an error mentioning %q", err, tt.wantErr) + } + }) + } +} + +func TestCacheArtifactsDefaultsToTrue(t *testing.T) { + if !Default().Storage.CacheArtifacts { + t.Fatal("Default().Storage.CacheArtifacts = false, want true") + } + + path := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(path, []byte("storage:\n url: \"file:///tmp/cache\"\n"), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if !cfg.Storage.CacheArtifacts { + t.Error("a config file without storage.cache_artifacts disabled artifact caching") + } +} + +func TestLoadCacheArtifactsFromEnv(t *testing.T) { + cfg := Default() + t.Setenv("PROXY_STORAGE_CACHE_ARTIFACTS", "false") + cfg.LoadFromEnv() + + if cfg.Storage.CacheArtifacts { + t.Error("Storage.CacheArtifacts should be false") + } +} diff --git a/internal/config/denylist.go b/internal/config/denylist.go new file mode 100644 index 00000000..20c1fcae --- /dev/null +++ b/internal/config/denylist.go @@ -0,0 +1,6 @@ +package config + +// DenylistConfig identifies exact package versions that must not be served. +type DenylistConfig struct { + Packages []string `json:"packages" yaml:"packages"` +} diff --git a/internal/config/denylist_test.go b/internal/config/denylist_test.go new file mode 100644 index 00000000..f17ad94d --- /dev/null +++ b/internal/config/denylist_test.go @@ -0,0 +1,35 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDenylistConfig(t *testing.T) { + for _, tc := range []struct{ name, body string }{ + {"yaml", "denylist:\n packages:\n - 'pkg:pypi/requests@2.31.0'\n"}, + {"json", `{"denylist":{"packages":["pkg:pypi/requests@2.31.0"]}}`}, + } { + t.Run(tc.name, func(t *testing.T) { + file := filepath.Join(t.TempDir(), "config."+tc.name) + if err := os.WriteFile(file, []byte(tc.body), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(file) + if err != nil { + t.Fatal(err) + } + if len(cfg.Denylist.Packages) != 1 || cfg.Denylist.Packages[0] != "pkg:pypi/requests@2.31.0" { + t.Fatalf("denylist = %+v", cfg.Denylist) + } + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + cfg.Denylist.Packages = []string{"pkg:pypi/requests"} + if err := cfg.Validate(); err == nil { + t.Fatal("unversioned denylist entry accepted") + } + }) + } +} diff --git a/internal/cooldownpolicy/policy.go b/internal/cooldownpolicy/policy.go new file mode 100644 index 00000000..096c4a4a --- /dev/null +++ b/internal/cooldownpolicy/policy.go @@ -0,0 +1,114 @@ +// Package cooldownpolicy applies package-pattern overrides to cooldown checks. +package cooldownpolicy + +import ( + "fmt" + "path" + "sort" + "strings" + "time" + + "github.com/git-pkgs/cooldown" +) + +// Policy applies exact PURL overrides before package-pattern overrides. +type Policy struct { + base *cooldown.Config + patterns []pattern + enabled bool +} + +type pattern struct { + glob string + duration time.Duration + config *cooldown.Config +} + +// New creates a Policy using the supplied exact and pattern overrides. +func New(base *cooldown.Config, packagePatterns map[string]string) (*Policy, error) { + if base == nil { + base = &cooldown.Config{} + } + + keys := make([]string, 0, len(packagePatterns)) + for glob := range packagePatterns { + keys = append(keys, glob) + } + sort.Strings(keys) + patterns := make([]pattern, 0, len(packagePatterns)) + seen := make(map[string]pattern) + enabled := base.Enabled() + for _, glob := range keys { + value := packagePatterns[glob] + if strings.ContainsAny(glob, "[\\") { + return nil, fmt.Errorf("invalid cooldown package pattern %q: character classes and escapes are not supported", glob) + } + canonicalGlob := strings.ReplaceAll(glob, "@", "%40") + if _, err := path.Match(canonicalGlob, ""); err != nil { + return nil, fmt.Errorf("invalid cooldown package pattern %q: %w", glob, err) + } + duration, err := cooldown.ParseDuration(value) + if err != nil { + return nil, fmt.Errorf("invalid cooldown duration for package pattern %q: %w", glob, err) + } + if previous, exists := seen[canonicalGlob]; exists { + if previous.duration != duration { + return nil, fmt.Errorf("conflicting cooldown package patterns %q and %q", previous.glob, glob) + } + continue + } + seen[canonicalGlob] = pattern{glob: glob, duration: duration} + config := &cooldown.Config{Default: value} + enabled = config.Enabled() || enabled + patterns = append(patterns, pattern{glob: canonicalGlob, duration: duration, config: config}) + } + sort.Slice(patterns, func(i, j int) bool { + left, right := literalLength(patterns[i].glob), literalLength(patterns[j].glob) + if left != right { + return left > right + } + return patterns[i].glob < patterns[j].glob + }) + + return &Policy{base: base, patterns: patterns, enabled: enabled}, nil +} + +func literalLength(glob string) int { + return len(glob) - strings.Count(glob, "*") - strings.Count(glob, "?") +} + +// For returns the duration, with exact overrides taking precedence over patterns. +func (p *Policy) For(ecosystem, packagePURL string) time.Duration { + return p.configFor(packagePURL).For(ecosystem, packagePURL) +} + +func (p *Policy) configFor(packagePURL string) *cooldown.Config { + if _, exact := p.base.Packages[packagePURL]; exact { + return p.base + } + + for _, candidate := range p.patterns { + matched, _ := path.Match(candidate.glob, packagePURL) + if !matched { + continue + } + return candidate.config + } + + return p.base +} + +// IsAllowed reports whether the package version has completed its cooldown. +func (p *Policy) IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool { + return p.Evaluate(ecosystem, packagePURL, publishedAt, time.Now()).Allowed +} + +// Evaluate returns the cooldown decision at the supplied evaluation time. +func (p *Policy) Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision { + return p.configFor(packagePURL).Evaluate(ecosystem, packagePURL, publishedAt, evaluatedAt) +} + +// Enabled reports whether any configured cooldown can filter a package version. +func (p *Policy) Enabled() bool { + return p.enabled +} diff --git a/internal/cooldownpolicy/policy_test.go b/internal/cooldownpolicy/policy_test.go new file mode 100644 index 00000000..9e956c14 --- /dev/null +++ b/internal/cooldownpolicy/policy_test.go @@ -0,0 +1,223 @@ +package cooldownpolicy + +import ( + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +func TestPatternOverride(t *testing.T) { + policy, err := New(&cooldown.Config{ + Default: "7d", + Ecosystems: map[string]string{"npm": "7d"}, + }, map[string]string{ + "pkg:npm/@example/*": "0", + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + if !policy.IsAllowed("npm", "pkg:npm/%40example/widget", time.Now()) { + t.Fatal("matching package pattern should disable cooldown") + } + if policy.IsAllowed("npm", "pkg:npm/public-package", time.Now()) { + t.Fatal("non-matching package should use ecosystem cooldown") + } +} + +func TestExactOverrideTakesPrecedenceOverPattern(t *testing.T) { + purl := "pkg:npm/%40example/widget" + policy, err := New(&cooldown.Config{ + Default: "7d", + Packages: map[string]string{purl: "2d"}, + }, map[string]string{ + "pkg:npm/@example/*": "0", + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + if policy.IsAllowed("npm", purl, time.Now()) { + t.Fatal("exact package override should take precedence over pattern") + } +} + +func TestMoreSpecificPatternTakesPrecedence(t *testing.T) { + policy, err := New(&cooldown.Config{Default: "7d"}, map[string]string{ + "pkg:npm/@example/*": "0", + "pkg:npm/@example/critical": "2d", + }) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + if policy.IsAllowed("npm", "pkg:npm/%40example/critical", time.Now()) { + t.Fatal("more specific pattern should take precedence") + } +} + +func TestNewRejectsInvalidPattern(t *testing.T) { + for _, glob := range []string{"pkg:npm/[", "pkg:npm/[@a]*", "pkg:npm/[abcdef]*", `pkg:npm/\*`, `pkg:npm/\@example/*`} { + t.Run(glob, func(t *testing.T) { + if _, err := New(nil, map[string]string{glob: "0"}); err == nil || !strings.Contains(err.Error(), "character classes and escapes are not supported") { + t.Fatalf("unsupported pattern %q: error = %v", glob, err) + } + }) + } +} + +func TestQuestionMarkPattern(t *testing.T) { + policy, err := New(&cooldown.Config{Default: "7d"}, map[string]string{"pkg:npm/@example/widget-?": "0"}) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + purl string + want time.Duration + }{ + {"pkg:npm/%40example/widget-a", 0}, + {"pkg:npm/%40example/widget-ab", 7 * 24 * time.Hour}, + {"pkg:npm/%40example/widget-/", 7 * 24 * time.Hour}, + } { + if got := policy.For("npm", tc.purl); got != tc.want { + t.Errorf("For(%q) = %s, want %s", tc.purl, got, tc.want) + } + } +} + +func TestNormalizedPatternCollisions(t *testing.T) { + for _, duration := range []string{"0", "24h"} { + t.Run(duration, func(t *testing.T) { + for range 20 { + policy, err := New(nil, map[string]string{ + "pkg:npm/@example/*": "1d", + "pkg:npm/%40example/*": duration, + }) + if duration == "0" { + if err == nil || !strings.Contains(err.Error(), `conflicting cooldown package patterns "pkg:npm/%40example/*" and "pkg:npm/@example/*"`) { + t.Fatalf("conflict error = %v", err) + } + continue + } + if err != nil { + t.Fatal(err) + } + if got := policy.For("npm", "pkg:npm/%40example/widget"); got != 24*time.Hour { + t.Fatalf("duration = %s", got) + } + } + }) + } +} + +func TestForPrecedence(t *testing.T) { + policy, err := New(&cooldown.Config{ + Default: "1h", + Ecosystems: map[string]string{"npm": "2h"}, + Packages: map[string]string{"pkg:npm/%40example/exact": "0"}, + }, map[string]string{ + "pkg:npm/@example/*": "3h", + "pkg:npm/@example/specific*": "4h", + "pkg:npm/@example/ab*": "5h", + "pkg:npm/@example/a*c": "6h", + }) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + ecosystem, purl string + want time.Duration + }{ + {"npm", "pkg:npm/%40example/exact", 0}, + {"npm", "pkg:npm/%40example/widget", 3 * time.Hour}, + {"npm", "pkg:npm/%40example/specific-widget", 4 * time.Hour}, + {"npm", "pkg:npm/%40example/abc", 6 * time.Hour}, + {"npm", "pkg:npm/other", 2 * time.Hour}, + {"npm", "pkg:npm/%40example/nested/package", 2 * time.Hour}, + {"cargo", "pkg:cargo/serde", time.Hour}, + } { + t.Run(tc.purl, func(t *testing.T) { + if got := policy.For(tc.ecosystem, tc.purl); got != tc.want { + t.Errorf("For = %s, want %s", got, tc.want) + } + }) + } +} + +func TestPatternOnlyPolicy(t *testing.T) { + policy, err := New(nil, map[string]string{"pkg:nuget/example.*": "24h"}) + if err != nil { + t.Fatal(err) + } + if !policy.Enabled() { + t.Fatal("pattern-only policy should be enabled") + } + for _, tc := range []struct { + published time.Time + allowed bool + }{ + {time.Now().Add(-time.Hour), false}, + {time.Now().Add(-48 * time.Hour), true}, + {time.Time{}, true}, + } { + if got := policy.IsAllowed("nuget", "pkg:nuget/example.widget", tc.published); got != tc.allowed { + t.Errorf("published=%s: allowed=%t, want %t", tc.published, got, tc.allowed) + } + } + if !policy.IsAllowed("nuget", "pkg:nuget/other", time.Now()) { + t.Error("unmatched package should have no cooldown") + } +} + +func TestDisabledPolicy(t *testing.T) { + for _, patterns := range []map[string]string{nil, {"pkg:npm/@example/*": "0"}} { + policy, err := New(nil, patterns) + if err != nil { + t.Fatal(err) + } + if policy.Enabled() { + t.Error("zero cooldown should be disabled") + } + } +} + +func TestNewRejectsInvalidDuration(t *testing.T) { + if _, err := New(nil, map[string]string{"pkg:npm/*": "invalid"}); err == nil { + t.Fatal("invalid duration accepted") + } +} + +func TestEvaluate(t *testing.T) { + policy, err := New(&cooldown.Config{ + Default: "48h", + Ecosystems: map[string]string{"npm": "72h"}, + Packages: map[string]string{"pkg:npm/%40example/exact": "0"}, + }, map[string]string{"pkg:npm/@example/*": "7d"}) + if err != nil { + t.Fatal(err) + } + published := time.Date(2026, time.October, 1, 0, 0, 0, 0, time.UTC) + eligible := published.Add(7 * 24 * time.Hour) + for _, tc := range []struct { + name, ecosystem, purl string + published, at time.Time + want cooldown.Decision + }{ + {"waiting", "npm", "pkg:npm/%40example/widget", published, eligible.Add(-time.Nanosecond), cooldown.Decision{Cooldown: 7 * 24 * time.Hour, AvailableAt: eligible, Reason: cooldown.ReasonWaiting}}, + {"boundary", "npm", "pkg:npm/%40example/widget", published, eligible, cooldown.Decision{Allowed: true, Cooldown: 7 * 24 * time.Hour, AvailableAt: eligible, Reason: cooldown.ReasonElapsed}}, + {"elapsed", "npm", "pkg:npm/%40example/widget", published, eligible.Add(time.Hour), cooldown.Decision{Allowed: true, Cooldown: 7 * 24 * time.Hour, AvailableAt: eligible, Reason: cooldown.ReasonElapsed}}, + {"unknown publication", "npm", "pkg:npm/%40example/widget", time.Time{}, eligible, cooldown.Decision{Allowed: true, Cooldown: 7 * 24 * time.Hour, Reason: cooldown.ReasonUnknownPublicationTime}}, + {"exact exemption", "npm", "pkg:npm/%40example/exact", published, published, cooldown.Decision{Allowed: true, AvailableAt: published, Reason: cooldown.ReasonDisabled}}, + {"exemption without publication", "npm", "pkg:npm/%40example/exact", time.Time{}, eligible, cooldown.Decision{Allowed: true, Reason: cooldown.ReasonUnknownPublicationTime}}, + {"ecosystem fallback", "npm", "pkg:npm/other", published, published, cooldown.Decision{Cooldown: 72 * time.Hour, AvailableAt: published.Add(72 * time.Hour), Reason: cooldown.ReasonWaiting}}, + {"global fallback", "cargo", "pkg:cargo/serde", published, published, cooldown.Decision{Cooldown: 48 * time.Hour, AvailableAt: published.Add(48 * time.Hour), Reason: cooldown.ReasonWaiting}}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := policy.Evaluate(tc.ecosystem, tc.purl, tc.published, tc.at); got != tc.want { + t.Errorf("Evaluate = %+v, want %+v", got, tc.want) + } + }) + } +} diff --git a/internal/database/analytics_test.go b/internal/database/analytics_test.go new file mode 100644 index 00000000..269b9d2f --- /dev/null +++ b/internal/database/analytics_test.go @@ -0,0 +1,375 @@ +package database + +import ( + "database/sql" + "path/filepath" + "testing" + "time" + + "github.com/git-pkgs/purl" +) + +// seedArtifact inserts a package, a version and a cached artifact so that the +// ecosystem aggregation has a complete join path to walk. +func seedArtifact(t *testing.T, db *DB, ecosystem, name, version string, size, hits int64) { + t.Helper() + + pkgPURL := "pkg:" + ecosystem + "/" + name + versionPURL := pkgPURL + "@" + version + + if err := db.UpsertPackage(&Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: name}); err != nil { + t.Fatalf("UpsertPackage(%s): %v", pkgPURL, err) + } + if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion(%s): %v", versionPURL, err) + } + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: versionPURL, + Filename: name + "-" + version + ".tgz", + UpstreamURL: "https://example.test/" + name, + StoragePath: sql.NullString{String: "objects/" + name, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + HitCount: hits, + }); err != nil { + t.Fatalf("UpsertArtifact(%s): %v", versionPURL, err) + } +} + +func newTestDB(t *testing.T) *DB { + t.Helper() + db, err := Create(filepath.Join(t.TempDir(), "analytics.db")) + if err != nil { + t.Fatalf("Create: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return db +} + +func TestGetEcosystemStats(t *testing.T) { + db := newTestDB(t) + + // npm: 2 artifacts across 2 packages, 1000*3 + 500*1 = 3500 bytes served. + seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 3) + seedArtifact(t, db, "npm", "express", "4.18.2", 500, 1) + // cargo: a single heavily-hit artifact, 200*10 = 2000 bytes served. + seedArtifact(t, db, "cargo", "serde", "1.0.0", 200, 10) + + stats, err := db.GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + if len(stats) != 2 { + t.Fatalf("expected 2 ecosystems, got %d: %+v", len(stats), stats) + } + + // Ordered by accumulated download volume, so npm (3500) precedes cargo (2000). + npm, cargo := stats[0], stats[1] + if npm.Ecosystem != "npm" || cargo.Ecosystem != "cargo" { + t.Fatalf("expected npm then cargo, got %q then %q", npm.Ecosystem, cargo.Ecosystem) + } + + if npm.DownloadedBytes != 3500 { + t.Errorf("npm DownloadedBytes = %d, want 3500", npm.DownloadedBytes) + } + if npm.Downloads != 4 { + t.Errorf("npm Downloads = %d, want 4", npm.Downloads) + } + if npm.CacheSize != 1500 { + t.Errorf("npm CacheSize = %d, want 1500", npm.CacheSize) + } + if npm.Artifacts != 2 { + t.Errorf("npm Artifacts = %d, want 2", npm.Artifacts) + } + if npm.Packages != 2 { + t.Errorf("npm Packages = %d, want 2", npm.Packages) + } + if npm.Versions != 2 { + t.Errorf("npm Versions = %d, want 2", npm.Versions) + } + + if cargo.DownloadedBytes != 2000 { + t.Errorf("cargo DownloadedBytes = %d, want 2000", cargo.DownloadedBytes) + } + if cargo.Downloads != 10 { + t.Errorf("cargo Downloads = %d, want 10", cargo.Downloads) + } +} + +func TestGetEcosystemStatsEmptyDatabase(t *testing.T) { + stats, err := newTestDB(t).GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + if len(stats) != 0 { + t.Errorf("expected no ecosystems, got %+v", stats) + } +} + +// An ecosystem known from metadata but with nothing cached should still appear, +// so the analytics table can show it as idle rather than omitting it. +func TestGetEcosystemStatsIncludesEcosystemsWithoutArtifacts(t *testing.T) { + db := newTestDB(t) + seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 2) + + if err := db.UpsertPackage(&Package{PURL: "pkg:gem/rails", Ecosystem: "gem", Name: "rails"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + + stats, err := db.GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + if len(stats) != 2 { + t.Fatalf("expected 2 ecosystems, got %d: %+v", len(stats), stats) + } + + // gem has no download volume, so it sorts last. + gem := stats[1] + if gem.Ecosystem != "gem" { + t.Fatalf("expected gem last, got %q", gem.Ecosystem) + } + if gem.Packages != 1 { + t.Errorf("gem Packages = %d, want 1", gem.Packages) + } + if gem.Artifacts != 0 || gem.CacheSize != 0 || gem.DownloadedBytes != 0 { + t.Errorf("expected gem to report no cached artifacts, got %+v", gem) + } +} + +// Eviction clears storage_path and size, which drops the artifact's historical +// hits out of the accumulated total. Pinning that here so the documented +// behaviour of GetEcosystemStats does not drift. +func TestGetEcosystemStatsExcludesEvictedArtifacts(t *testing.T) { + db := newTestDB(t) + seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 3) + seedArtifact(t, db, "npm", "express", "4.18.2", 500, 2) + + cleared, err := db.ClearArtifactCache("pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz", "objects/lodash") + if err != nil { + t.Fatalf("ClearArtifactCache: %v", err) + } + if !cleared { + t.Fatal("ClearArtifactCache cleared nothing") + } + + stats, err := db.GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + if len(stats) != 1 { + t.Fatalf("expected 1 ecosystem, got %d: %+v", len(stats), stats) + } + + // Only express remains cached: 500 * 2 = 1000. + if got := stats[0].DownloadedBytes; got != 1000 { + t.Errorf("DownloadedBytes = %d, want 1000 (evicted artifact must not count)", got) + } + if got := stats[0].Artifacts; got != 1 { + t.Errorf("Artifacts = %d, want 1", got) + } + // The package row survives eviction, so both packages still count. + if got := stats[0].Packages; got != 2 { + t.Errorf("Packages = %d, want 2", got) + } +} + +func TestSortEcosystemStatsTieBreaks(t *testing.T) { + stats := []EcosystemStats{ + {Ecosystem: "zzz"}, + {Ecosystem: "aaa"}, + {Ecosystem: "mid", CacheSize: 10}, + {Ecosystem: "top", DownloadedBytes: 5}, + } + sortEcosystemStats(stats) + + want := []string{"top", "mid", "aaa", "zzz"} + for i, name := range want { + if stats[i].Ecosystem != name { + t.Errorf("position %d = %q, want %q (full order: %+v)", i, stats[i].Ecosystem, name, stats) + } + } +} + +// A cached artifact whose version row is missing must still be counted, so the +// per-ecosystem totals reconcile with GetTotalCacheSize rather than quietly +// coming up short. Nothing at the schema level enforces the link. +func TestGetEcosystemStatsCountsUnattributedArtifacts(t *testing.T) { + db := newTestDB(t) + seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 2) + + // An artifact pointing at a version that was never recorded. + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: "pkg:npm/orphan@9.9.9", + Filename: "orphan-9.9.9.tgz", + UpstreamURL: "https://example.test/orphan", + StoragePath: sql.NullString{String: "objects/orphan", Valid: true}, + Size: sql.NullInt64{Int64: 500, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + HitCount: 4, + }); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + + stats, err := db.GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + + var cacheSize, artifacts, downloaded int64 + var sawUnattributed bool + for _, e := range stats { + cacheSize += e.CacheSize + artifacts += e.Artifacts + downloaded += e.DownloadedBytes + if e.Ecosystem == unattributedEcosystem { + sawUnattributed = true + } + } + + if !sawUnattributed { + t.Errorf("the orphaned artifact was dropped; got %+v", stats) + } + + // These must match what the unjoined queries report, or the UI shows two + // totals that do not add up. + wantSize, err := db.GetTotalCacheSize() + if err != nil { + t.Fatalf("GetTotalCacheSize: %v", err) + } + if cacheSize != wantSize { + t.Errorf("per-ecosystem cache size sums to %d, but GetTotalCacheSize reports %d", cacheSize, wantSize) + } + + wantCount, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("GetCachedArtifactCount: %v", err) + } + if artifacts != wantCount { + t.Errorf("per-ecosystem artifacts sum to %d, but GetCachedArtifactCount reports %d", artifacts, wantCount) + } + + // 1000*2 from lodash plus 500*4 from the orphan. + if downloaded != 4000 { + t.Errorf("downloaded bytes = %d, want 4000", downloaded) + } +} + +// The proxy writes "gem" and git-pkgs writes "rubygems". A database that has +// seen both must report one ecosystem, not two rows splitting its share. +func TestGetEcosystemStatsMergesAliasedEcosystems(t *testing.T) { + db := newTestDB(t) + seedArtifact(t, db, "gem", "colorize", "1.1.0", 1000, 3) + seedArtifact(t, db, "rubygems", "rails", "7.1.0", 4000, 2) + seedArtifact(t, db, "npm", "lodash", "4.17.21", 500, 1) + + stats, err := db.GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + if len(stats) != 2 { + t.Fatalf("expected gem and rubygems merged into one row alongside npm, got %+v", stats) + } + + var ruby *EcosystemStats + for i := range stats { + if purl.NormalizeEcosystem(stats[i].Ecosystem) == "rubygems" { + ruby = &stats[i] + } + } + if ruby == nil { + t.Fatal("no row for the rubygems ecosystem") + } + + // 1000*3 + 4000*2 = 11000 + if ruby.DownloadedBytes != 11000 { + t.Errorf("DownloadedBytes = %d, want 11000", ruby.DownloadedBytes) + } + if ruby.CacheSize != 5000 { + t.Errorf("CacheSize = %d, want 5000", ruby.CacheSize) + } + if ruby.Artifacts != 2 || ruby.Packages != 2 { + t.Errorf("Artifacts=%d Packages=%d, want 2 and 2", ruby.Artifacts, ruby.Packages) + } + + // The surviving row keeps the spelling with the most cached bytes, because + // the UI filters and links by it and the packages table stores it raw. + if ruby.Ecosystem != "rubygems" { + t.Errorf("Ecosystem = %q, want the dominant raw spelling %q", ruby.Ecosystem, "rubygems") + } +} + +// A version whose package row is missing must be bucketed like an orphaned +// artifact, so the per-ecosystem figures reconcile with GetCacheStats. +func TestGetEcosystemStatsCountsUnattributedVersions(t *testing.T) { + db := newTestDB(t) + seedArtifact(t, db, "npm", "lodash", "4.17.21", 1000, 1) + if err := db.UpsertVersion(&Version{PURL: "pkg:npm/ghost@1.0.0", PackagePURL: "pkg:npm/ghost"}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + + stats, err := db.GetEcosystemStats() + if err != nil { + t.Fatalf("GetEcosystemStats: %v", err) + } + + var versions int64 + for _, e := range stats { + versions += e.Versions + } + + cacheStats, err := db.GetCacheStats() + if err != nil { + t.Fatalf("GetCacheStats: %v", err) + } + if versions != cacheStats.TotalVersions { + t.Errorf("per-ecosystem versions sum to %d, but GetCacheStats reports %d", + versions, cacheStats.TotalVersions) + } +} + +// The surviving spelling must be the one with the most cached bytes whatever +// order the rows arrive in. GetEcosystemStats builds its input by ranging a +// map, so an implementation that compared against the running total instead of +// each row's own size would pick a different name between refreshes — flipping +// the analytics row's badge and its /ui/packages filter link. +func TestMergeAliasedEcosystemsPicksLargestWhateverTheOrder(t *testing.T) { + rows := []EcosystemStats{ + {Ecosystem: "gem", CacheSize: 5, Artifacts: 1}, + {Ecosystem: "rubygems", CacheSize: 4, Artifacts: 1}, + {Ecosystem: "RubyGems", CacheSize: 6, Artifacts: 1}, + } + + for _, order := range [][]int{ + {0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}, + } { + in := make([]EcosystemStats, 0, len(order)) + for _, i := range order { + in = append(in, rows[i]) + } + + got := mergeAliasedEcosystems(in) + if len(got) != 1 { + t.Fatalf("order %v: got %d rows, want 1", order, len(got)) + } + if got[0].Ecosystem != "RubyGems" { + t.Errorf("order %v: Ecosystem = %q, want the largest spelling %q", + order, got[0].Ecosystem, "RubyGems") + } + if got[0].CacheSize != 15 { + t.Errorf("order %v: CacheSize = %d, want 15", order, got[0].CacheSize) + } + } +} + +// Equal sizes still have to resolve to one answer, for the same reason. +func TestMergeAliasedEcosystemsBreaksSizeTiesByName(t *testing.T) { + a := []EcosystemStats{{Ecosystem: "gem", CacheSize: 5}, {Ecosystem: "rubygems", CacheSize: 5}} + b := []EcosystemStats{{Ecosystem: "rubygems", CacheSize: 5}, {Ecosystem: "gem", CacheSize: 5}} + + first := mergeAliasedEcosystems(a)[0].Ecosystem + second := mergeAliasedEcosystems(b)[0].Ecosystem + if first != second { + t.Errorf("input order changed the surviving spelling: %q vs %q", first, second) + } +} diff --git a/internal/database/database.go b/internal/database/database.go index eded6d2a..f438f676 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "time" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" @@ -14,6 +15,16 @@ const SchemaVersion = 1 const dirPermissions = 0755 +// Postgres connection pool limits. database/sql keeps only two idle +// connections by default, which opens a new Postgres session for almost every +// request under load. +const ( + postgresMaxOpenConns = 32 + postgresMaxIdleConns = 32 + postgresConnMaxIdleTime = 5 * time.Minute + postgresConnMaxLifetime = 30 * time.Minute +) + type Dialect string const ( @@ -25,6 +36,7 @@ type DB struct { *sqlx.DB dialect Dialect path string + hits *hitBatch } func (db *DB) Dialect() Dialect { @@ -94,6 +106,11 @@ func OpenPostgres(url string) (*DB, error) { return nil, fmt.Errorf("opening postgres database: %w", err) } + sqlDB.SetMaxOpenConns(postgresMaxOpenConns) + sqlDB.SetMaxIdleConns(postgresMaxIdleConns) + sqlDB.SetConnMaxIdleTime(postgresConnMaxIdleTime) + sqlDB.SetConnMaxLifetime(postgresConnMaxLifetime) + if err := sqlDB.Ping(); err != nil { _ = sqlDB.Close() return nil, fmt.Errorf("connecting to postgres: %w", err) diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 0fd42fcc..13f6828c 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -610,8 +610,10 @@ func createTestPostgresDB(t *testing.T) *DB { t.Fatalf("OpenPostgres failed: %v", err) } - // Drop and recreate tables for clean test state - tables := []string{"artifacts", "versions", "packages", "schema_info"} + // Drop and recreate every table CreateSchema creates for clean test state; + // leftover migration records make the next CreateSchema fail on the + // migrations primary key. + tables := []string{"artifacts", "pending_deletes", "versions", "packages", "vulnerabilities", "metadata_cache", "migrations", "schema_info"} for _, table := range tables { _, _ = db.Exec("DROP TABLE IF EXISTS " + table + " CASCADE") } @@ -779,6 +781,10 @@ func TestMigrationFromOldSchema(t *testing.T) { t.Errorf("expected package name test-package, got %s", pkg.Name) } + if has, err := db.HasTable("pending_deletes"); err != nil || !has { + t.Errorf("pending_deletes table missing after migration (err %v)", err) + } + // Verify migrations were recorded applied, err := db.appliedMigrations() if err != nil { diff --git a/internal/database/hits.go b/internal/database/hits.go new file mode 100644 index 00000000..249c6633 --- /dev/null +++ b/internal/database/hits.go @@ -0,0 +1,146 @@ +package database + +import ( + "cmp" + "log/slog" + "slices" + "sync" + "time" +) + +// Recording each cache hit as its own UPDATE makes every cached download a +// write transaction, and SQLite runs on a single connection, so concurrent +// downloads queue behind those writes. BatchHits counts hits in memory and +// writes them in one transaction per interval instead. + +type hitKey struct{ versionPURL, filename string } + +type hitEntry struct { + count int64 + last time.Time +} + +type hitBatch struct { + mu sync.Mutex + pending map[hitKey]hitEntry + stop chan struct{} + stopOnce sync.Once + done chan struct{} +} + +func (b *hitBatch) add(k hitKey, e hitEntry) { + b.mu.Lock() + defer b.mu.Unlock() + cur := b.pending[k] + cur.count += e.count + if e.last.After(cur.last) { + cur.last = e.last + } + b.pending[k] = cur +} + +func (b *hitBatch) take() map[hitKey]hitEntry { + b.mu.Lock() + defer b.mu.Unlock() + pending := b.pending + b.pending = map[hitKey]hitEntry{} + return pending +} + +// BatchHits makes RecordArtifactHit buffer hits and write them every +// interval. An interval of zero or less leaves every hit written immediately. +// Close writes whatever is still pending. Failed writes are logged to logger. +func (db *DB) BatchHits(interval time.Duration, logger *slog.Logger) { + if interval <= 0 || db.hits != nil { + return + } + if logger == nil { + logger = slog.New(slog.DiscardHandler) + } + b := &hitBatch{pending: map[hitKey]hitEntry{}, stop: make(chan struct{}), done: make(chan struct{})} + db.hits = b + go func() { + defer close(b.done) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + if err := db.flushHits(); err != nil { + logger.Warn("failed to write cache hits, retrying next flush", "error", err) + } + case <-b.stop: + if err := db.flushHits(); err != nil { + logger.Error("failed to write cache hits on close, dropping them", "error", err) + } + return + } + } + }() +} + +// flushHits writes the pending hits in one transaction. If the transaction +// fails they are put back for the next flush, so a failed write delays +// counts rather than losing them. +func (db *DB) flushHits() error { + pending := db.hits.take() + if len(pending) == 0 { + return nil + } + + err := db.writeHits(pending) + if err != nil { + for k, e := range pending { + db.hits.add(k, e) + } + } + return err +} + +func (db *DB) writeHits(pending map[hitKey]hitEntry) error { + tx, err := db.Beginx() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + // Timestamps only move forward: with proxies sharing a database, an older + // batch can flush after a newer hit is already written. + stmt, err := tx.Preparex(db.Rebind(` + UPDATE artifacts + SET hit_count = hit_count + ?, + last_accessed_at = CASE WHEN last_accessed_at IS NULL OR last_accessed_at < ? THEN ? ELSE last_accessed_at END, + updated_at = CASE WHEN updated_at IS NULL OR updated_at < ? THEN ? ELSE updated_at END + WHERE version_purl = ? AND filename = ? + `)) + if err != nil { + return err + } + defer func() { _ = stmt.Close() }() + + // A fixed order keeps proxies sharing a Postgres database from locking + // the same rows in opposite orders and deadlocking. + keys := make([]hitKey, 0, len(pending)) + for k := range pending { + keys = append(keys, k) + } + slices.SortFunc(keys, func(a, b hitKey) int { + return cmp.Or(cmp.Compare(a.versionPURL, b.versionPURL), cmp.Compare(a.filename, b.filename)) + }) + for _, k := range keys { + e := pending[k] + if _, err := stmt.Exec(e.count, e.last, e.last, e.last, e.last, k.versionPURL, k.filename); err != nil { + return err + } + } + return tx.Commit() +} + +// Close writes any batched hits, then closes the database. +func (db *DB) Close() error { + if b := db.hits; b != nil { + b.stopOnce.Do(func() { close(b.stop) }) + <-b.done + } + return db.DB.Close() +} diff --git a/internal/database/hits_test.go b/internal/database/hits_test.go new file mode 100644 index 00000000..99126f6b --- /dev/null +++ b/internal/database/hits_test.go @@ -0,0 +1,246 @@ +package database + +import ( + "bytes" + "log/slog" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +var discardLogger = slog.New(slog.DiscardHandler) + +func seedHitTestArtifact(t *testing.T, db *DB) (string, string) { + t.Helper() + versionPURL, filename := "pkg:npm/lodash@4.17.21", "lodash-4.17.21.tgz" + if err := db.UpsertPackage(&Package{PURL: "pkg:npm/lodash", Ecosystem: "npm", Name: "lodash"}); err != nil { + t.Fatalf("UpsertPackage failed: %v", err) + } + if err := db.UpsertVersion(&Version{PURL: versionPURL, PackagePURL: "pkg:npm/lodash"}); err != nil { + t.Fatalf("UpsertVersion failed: %v", err) + } + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: versionPURL, + Filename: filename, + UpstreamURL: "https://registry.npmjs.org/lodash/-/" + filename, + }); err != nil { + t.Fatalf("UpsertArtifact failed: %v", err) + } + return versionPURL, filename +} + +func hitCount(t *testing.T, db *DB, versionPURL, filename string) int64 { + t.Helper() + a, err := db.GetArtifact(versionPURL, filename) + if err != nil || a == nil { + t.Fatalf("GetArtifact failed: %v", err) + } + return a.HitCount +} + +func TestBatchHitsWritesOnFlush(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + versionPURL, filename := seedHitTestArtifact(t, db) + db.BatchHits(time.Hour, discardLogger) + + for range 3 { + if err := db.RecordArtifactHit(versionPURL, filename); err != nil { + t.Fatalf("RecordArtifactHit failed: %v", err) + } + } + if got := hitCount(t, db, versionPURL, filename); got != 0 { + t.Fatalf("hit count before flush = %d, want 0", got) + } + + if err := db.flushHits(); err != nil { + t.Fatalf("flushHits failed: %v", err) + } + a, err := db.GetArtifact(versionPURL, filename) + if err != nil { + t.Fatalf("GetArtifact failed: %v", err) + } + if a.HitCount != 3 { + t.Errorf("hit count after flush = %d, want 3", a.HitCount) + } + if !a.LastAccessedAt.Valid { + t.Error("expected last_accessed_at to be set") + } + }) +} + +func TestBatchHitsWritesOnClose(t *testing.T) { + path := filepath.Join(t.TempDir(), "test.db") + db, err := Create(path) + if err != nil { + t.Fatalf("Create failed: %v", err) + } + versionPURL, filename := seedHitTestArtifact(t, db) + db.BatchHits(time.Hour, discardLogger) + + for range 2 { + _ = db.RecordArtifactHit(versionPURL, filename) + } + if err := db.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + + db, err = Open(path) + if err != nil { + t.Fatalf("Open failed: %v", err) + } + defer func() { _ = db.Close() }() + if got := hitCount(t, db, versionPURL, filename); got != 2 { + t.Errorf("hit count after Close = %d, want 2", got) + } +} + +// TestBatchHitsKeepNewerAccessTime flushes a batch older than a hit already +// written, as when proxies share a database. The count still adds up, but +// neither timestamp may move backwards. +func TestBatchHitsKeepNewerAccessTime(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + versionPURL, filename := seedHitTestArtifact(t, db) + k := hitKey{versionPURL, filename} + now := time.Now() + read := func() *Artifact { + t.Helper() + a, err := db.GetArtifact(versionPURL, filename) + if err != nil || a == nil { + t.Fatalf("GetArtifact failed: %v", err) + } + return a + } + write := func(last time.Time) { + t.Helper() + if err := db.writeHits(map[hitKey]hitEntry{k: {count: 1, last: last}}); err != nil { + t.Fatalf("writeHits failed: %v", err) + } + } + + write(now) + newer := read() + if !newer.LastAccessedAt.Valid { + t.Fatal("last_accessed_at not set from NULL") + } + + write(now.Add(-time.Minute)) + got := read() + if got.HitCount != 2 { + t.Errorf("hit count = %d, want 2", got.HitCount) + } + if !got.LastAccessedAt.Time.Equal(newer.LastAccessedAt.Time) { + t.Errorf("last_accessed_at moved from %v to %v", newer.LastAccessedAt.Time, got.LastAccessedAt.Time) + } + if !got.UpdatedAt.Equal(newer.UpdatedAt) { + t.Errorf("updated_at moved from %v to %v", newer.UpdatedAt, got.UpdatedAt) + } + + write(now.Add(time.Minute)) + got = read() + if !got.LastAccessedAt.Time.After(newer.LastAccessedAt.Time) || !got.UpdatedAt.After(newer.UpdatedAt) { + t.Error("a newer batch did not advance the timestamps") + } + }) +} + +func TestBatchHitsZeroIntervalWritesImmediately(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + versionPURL, filename := seedHitTestArtifact(t, db) + db.BatchHits(0, discardLogger) + + _ = db.RecordArtifactHit(versionPURL, filename) + if got := hitCount(t, db, versionPURL, filename); got != 1 { + t.Errorf("hit count = %d, want 1", got) + } + }) +} + +func TestBatchHitsKeepsHitsWhenWriteFails(t *testing.T) { + db := createTestDB(t) + versionPURL, filename := seedHitTestArtifact(t, db) + db.BatchHits(time.Hour, discardLogger) + + for range 2 { + _ = db.RecordArtifactHit(versionPURL, filename) + } + _ = db.DB.Close() + if err := db.flushHits(); err == nil { + t.Fatal("expected flushHits to fail on a closed database") + } + if got := db.hits.take()[hitKey{versionPURL, filename}].count; got != 2 { + t.Errorf("pending hits after failed flush = %d, want 2", got) + } + _ = db.Close() +} + +// TestBatchHitsCountsEveryHitWhileFlushing records hits from several goroutines +// while another flushes, so no hit may be lost between taking the pending +// hits and writing them. +func TestBatchHitsCountsEveryHitWhileFlushing(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + versionPURL, filename := seedHitTestArtifact(t, db) + db.BatchHits(time.Hour, discardLogger) + + const writers, hitsEach = 8, 100 + stop := make(chan struct{}) + flushed := make(chan struct{}) + go func() { + defer close(flushed) + for { + select { + case <-stop: + return + default: + if err := db.flushHits(); err != nil { + t.Errorf("flushHits failed: %v", err) + return + } + } + } + }() + var wg sync.WaitGroup + for range writers { + wg.Go(func() { + for range hitsEach { + if err := db.RecordArtifactHit(versionPURL, filename); err != nil { + t.Errorf("RecordArtifactHit failed: %v", err) + } + } + }) + } + wg.Wait() + close(stop) + <-flushed + + if err := db.flushHits(); err != nil { + t.Fatalf("final flushHits failed: %v", err) + } + if got := hitCount(t, db, versionPURL, filename); got != writers*hitsEach { + t.Errorf("hit count = %d, want %d", got, writers*hitsEach) + } + }) +} + +func TestBatchHitsLogsHitsDroppedOnClose(t *testing.T) { + db := createTestDB(t) + versionPURL, filename := seedHitTestArtifact(t, db) + var logs bytes.Buffer + db.BatchHits(time.Hour, slog.New(slog.NewTextHandler(&logs, nil))) + + _ = db.RecordArtifactHit(versionPURL, filename) + _ = db.DB.Close() + _ = db.Close() + + if !strings.Contains(logs.String(), "dropping them") { + t.Errorf("no error logged for hits dropped on close, logs: %q", logs.String()) + } +} + +func TestCloseTwiceWithBatchHits(t *testing.T) { + db := createTestDB(t) + db.BatchHits(time.Hour, discardLogger) + _ = db.Close() + _ = db.Close() +} diff --git a/internal/database/pending_deletes_test.go b/internal/database/pending_deletes_test.go new file mode 100644 index 00000000..9af6c48d --- /dev/null +++ b/internal/database/pending_deletes_test.go @@ -0,0 +1,199 @@ +package database + +import ( + "database/sql" + "fmt" + "slices" + "sync" + "testing" + "time" +) + +const ( + pendingVersionPURL = "pkg:npm/pending@1.0.0" + pendingFilename = "pending-1.0.0.tgz" +) + +func upsertPendingArtifact(t *testing.T, db *DB, storagePath string) { + t.Helper() + a := &Artifact{ + VersionPURL: pendingVersionPURL, + Filename: pendingFilename, + UpstreamURL: "https://example.com/" + pendingFilename, + StoragePath: sql.NullString{String: storagePath, Valid: true}, + Size: sql.NullInt64{Int64: 1, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + } + if err := db.UpsertArtifact(a); err != nil { + t.Fatalf("UpsertArtifact(%q): %v", storagePath, err) + } +} + +func recordedPath(t *testing.T, db *DB) sql.NullString { + t.Helper() + a, err := db.GetArtifact(pendingVersionPURL, pendingFilename) + if err != nil || a == nil { + t.Fatalf("GetArtifact: %v, %v", a, err) + } + return a.StoragePath +} + +// allPendingDeletes returns every queued path, due or not. +func allPendingDeletes(t *testing.T, db *DB) []string { + t.Helper() + paths, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 100) + if err != nil { + t.Fatalf("GetDuePendingDeletes: %v", err) + } + return paths +} + +func TestUpsertArtifactQueuesThePathItReplaces(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") + upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") + if got := allPendingDeletes(t, db); len(got) != 0 { + t.Fatalf("queued %v, want nothing while the record keeps its path", got) + } + + upsertPendingArtifact(t, db, "npm/pending/1.0.0/b/pending-1.0.0.tgz") + want := []string{"npm/pending/1.0.0/a/pending-1.0.0.tgz"} + if got := allPendingDeletes(t, db); !slices.Equal(got, want) { + t.Errorf("queued %v, want %v", got, want) + } + }) +} + +// TestUpsertArtifactSkipsRecordThatMoved is the race UpsertArtifact retries: +// another commit moved the record after this one read it, so the write must +// not apply, or the path that commit stored would never be queued. +func TestUpsertArtifactSkipsRecordThatMoved(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") + read := recordedPath(t, db) + upsertPendingArtifact(t, db, "npm/pending/1.0.0/b/pending-1.0.0.tgz") + + late := &Artifact{ + VersionPURL: pendingVersionPURL, + Filename: pendingFilename, + UpstreamURL: "https://example.com/" + pendingFilename, + StoragePath: sql.NullString{String: "npm/pending/1.0.0/c/pending-1.0.0.tgz", Valid: true}, + } + applied, err := db.upsertArtifactFrom(late, read) + if err != nil { + t.Fatalf("upsertArtifactFrom: %v", err) + } + if applied { + t.Error("write applied over a record that moved since it was read") + } + if got := recordedPath(t, db).String; got != "npm/pending/1.0.0/b/pending-1.0.0.tgz" { + t.Errorf("record points at %q, want the newer commit kept", got) + } + + missing := &Artifact{VersionPURL: "pkg:npm/pending@2.0.0", Filename: pendingFilename, UpstreamURL: "u"} + applied, err = db.upsertArtifactFrom(missing, sql.NullString{}) + if err != nil || !applied { + t.Errorf("insert of a new record: applied=%v err=%v", applied, err) + } + }) +} + +// TestConcurrentUpsertsQueueEveryReplacedPath commits one artifact from several +// fetches at once: whichever commit ends up recorded, every other path must be +// queued, or its object is never deleted. +func TestConcurrentUpsertsQueueEveryReplacedPath(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + // A commit only retries after another commit succeeded, so this many + // always fit within upsertArtifactAttempts. + const commits = upsertArtifactAttempts - 1 + paths := make([]string, commits) + errs := make(chan error, commits) + var wg sync.WaitGroup + for i := range paths { + paths[i] = fmt.Sprintf("npm/pending/1.0.0/%d/pending-1.0.0.tgz", i) + wg.Go(func() { + errs <- db.UpsertArtifact(&Artifact{ + VersionPURL: pendingVersionPURL, + Filename: pendingFilename, + UpstreamURL: "https://example.com/" + pendingFilename, + StoragePath: sql.NullString{String: paths[i], Valid: true}, + }) + }) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } + } + + recorded := recordedPath(t, db).String + want := slices.DeleteFunc(slices.Clone(paths), func(p string) bool { return p == recorded }) + got := allPendingDeletes(t, db) + slices.Sort(want) + slices.Sort(got) + if !slices.Equal(got, want) { + t.Errorf("recorded %q, queued %v, want %v", recorded, got, want) + } + }) +} + +func TestClearArtifactCacheLeavesRecordThatMoved(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") + upsertPendingArtifact(t, db, "npm/pending/1.0.0/b/pending-1.0.0.tgz") + + if cleared, err := db.ClearArtifactCache(pendingVersionPURL, pendingFilename, "npm/pending/1.0.0/a/pending-1.0.0.tgz"); err != nil || cleared { + t.Fatalf("ClearArtifactCache: cleared=%v err=%v, want nothing cleared", cleared, err) + } + if err := db.DiscardArtifact(pendingVersionPURL, pendingFilename, "npm/pending/1.0.0/a/pending-1.0.0.tgz"); err != nil { + t.Fatalf("DiscardArtifact: %v", err) + } + if got := recordedPath(t, db).String; got != "npm/pending/1.0.0/b/pending-1.0.0.tgz" { + t.Errorf("record points at %q, want the newer commit kept", got) + } + }) +} + +func TestDiscardArtifactClearsAndQueues(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + upsertPendingArtifact(t, db, "npm/pending/1.0.0/a/pending-1.0.0.tgz") + + if err := db.DiscardArtifact(pendingVersionPURL, pendingFilename, "npm/pending/1.0.0/a/pending-1.0.0.tgz"); err != nil { + t.Fatalf("DiscardArtifact: %v", err) + } + if got := recordedPath(t, db); got.Valid { + t.Errorf("record still points at %q", got.String) + } + want := []string{"npm/pending/1.0.0/a/pending-1.0.0.tgz"} + if got := allPendingDeletes(t, db); !slices.Equal(got, want) { + t.Errorf("queued %v, want %v", got, want) + } + }) +} + +func TestGetDuePendingDeletes(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + for _, path := range []string{"old", "referenced"} { + if err := db.QueuePendingDelete(path); err != nil { + t.Fatalf("QueuePendingDelete: %v", err) + } + } + upsertPendingArtifact(t, db, "referenced") + + if got, err := db.GetDuePendingDeletes(time.Now().Add(-time.Hour), 100); err != nil || len(got) != 0 { + t.Errorf("before the grace period: got %v (err %v), want nothing", got, err) + } + if got := allPendingDeletes(t, db); !slices.Equal(got, []string{"old"}) { + t.Errorf("after the grace period: got %v, want [old] and not the path a record points at", got) + } + + if err := db.RemovePendingDelete("old"); err != nil { + t.Fatalf("RemovePendingDelete: %v", err) + } + if got := allPendingDeletes(t, db); len(got) != 0 { + t.Errorf("after removal: got %v", got) + } + }) +} diff --git a/internal/database/postgres_pool_test.go b/internal/database/postgres_pool_test.go new file mode 100644 index 00000000..bd39bee4 --- /dev/null +++ b/internal/database/postgres_pool_test.go @@ -0,0 +1,57 @@ +package database + +import ( + "context" + "database/sql" + "os" + "testing" +) + +// TestOpenPostgresKeepsConnectionsIdle checks the connection-count limits +// OpenPostgres sets: the open cap admits a burst of postgresMaxIdleConns +// connections, and releasing them again leaves all of them idle in the pool. +// database/sql's default keeps only two, so the next burst would open a new +// Postgres session for almost every request. The idle-time and lifetime +// settings are not exercised here. +func TestOpenPostgresKeepsConnectionsIdle(t *testing.T) { + url := os.Getenv("PROXY_DATABASE_URL") + if url == "" { + t.Skip("PROXY_DATABASE_URL not set, skipping postgres pool test") + } + + db, err := OpenPostgres(url) + if err != nil { + t.Fatalf("OpenPostgres failed: %v", err) + } + defer func() { _ = db.Close() }() + + const burst = postgresMaxIdleConns + // database/sql keeps two idle connections by default; a burst that small + // could not tell the tuned pool from the default one. + if burst <= 2 { + t.Fatalf("postgresMaxIdleConns = %d, want more than database/sql's default of 2", burst) + } + if got := db.Stats().MaxOpenConnections; got <= 0 || got < burst { + t.Fatalf("MaxOpenConnections = %d, want a cap of at least %d", got, burst) + } + + conns := make([]*sql.Conn, 0, burst) + for range burst { + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatalf("taking connection %d: %v", len(conns)+1, err) + } + t.Cleanup(func() { _ = conn.Close() }) // release the session if an assertion below fails + conns = append(conns, conn) + } + if got := db.Stats().InUse; got != burst { + t.Fatalf("InUse = %d while holding %d connections", got, burst) + } + + for _, conn := range conns { + _ = conn.Close() + } + if got := db.Stats().Idle; got != burst { + t.Errorf("Idle = %d after releasing %d connections, want all of them kept", got, burst) + } +} diff --git a/internal/database/queries.go b/internal/database/queries.go index a7c4c7a4..95030224 100644 --- a/internal/database/queries.go +++ b/internal/database/queries.go @@ -2,10 +2,13 @@ package database import ( "database/sql" + "errors" "fmt" + "sort" "time" "github.com/git-pkgs/artifacts" + "github.com/git-pkgs/purl" "github.com/opencontainers/go-digest" ) @@ -310,7 +313,42 @@ func (db *DB) GetArtifactsByVersionPURL(versionPURL string) ([]Artifact, error) return artifacts, nil } +// upsertArtifactAttempts bounds how often UpsertArtifact retries a record that +// concurrent commits keep moving. +const upsertArtifactAttempts = 5 + +// UpsertArtifact records a. A storage path the record stops pointing at is +// queued for deletion, since each fetch stores its own object and nothing else +// would remove it. The write applies only if the record still holds the path +// read before it, so commits racing on one artifact each queue the path they +// replaced. func (db *DB) UpsertArtifact(a *Artifact) error { + for range upsertArtifactAttempts { + var previous sql.NullString + query := db.Rebind(`SELECT storage_path FROM artifacts WHERE version_purl = ? AND filename = ?`) + if err := db.Get(&previous, query, a.VersionPURL, a.Filename); err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("reading artifact: %w", err) + } + applied, err := db.upsertArtifactFrom(a, previous) + if err != nil { + return err + } + if !applied { + continue + } + if previous.Valid && previous.String != a.StoragePath.String { + if err := db.QueuePendingDelete(previous.String); err != nil { + return fmt.Errorf("queueing replaced artifact: %w", err) + } + } + return nil + } + return errors.New("upserting artifact: record kept changing") +} + +// upsertArtifactFrom writes a if the record is absent or still holds +// previous, and reports whether it did. +func (db *DB) upsertArtifactFrom(a *Artifact, previous sql.NullString) (bool, error) { now := time.Now() var query string @@ -327,6 +365,7 @@ func (db *DB) UpsertArtifact(a *Artifact) error { content_type = EXCLUDED.content_type, fetched_at = EXCLUDED.fetched_at, updated_at = EXCLUDED.updated_at + WHERE artifacts.storage_path IS NOT DISTINCT FROM $13 ` } else { query = ` @@ -341,21 +380,30 @@ func (db *DB) UpsertArtifact(a *Artifact) error { content_type = excluded.content_type, fetched_at = excluded.fetched_at, updated_at = excluded.updated_at + WHERE artifacts.storage_path IS ? ` } - _, err := db.Exec(query, + res, err := db.Exec(query, a.VersionPURL, a.Filename, a.UpstreamURL, a.StoragePath, a.ContentHash, - a.Size, a.ContentType, a.FetchedAt, a.HitCount, a.LastAccessedAt, now, now, + a.Size, a.ContentType, a.FetchedAt, a.HitCount, a.LastAccessedAt, now, now, previous, ) if err != nil { - return fmt.Errorf("upserting artifact: %w", err) + return false, fmt.Errorf("upserting artifact: %w", err) } - return nil + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("upserting artifact: %w", err) + } + return n > 0, nil } func (db *DB) RecordArtifactHit(versionPURL, filename string) error { now := time.Now() + if db.hits != nil { + db.hits.add(hitKey{versionPURL, filename}, hitEntry{count: 1, last: now}) + return nil + } query := db.Rebind(` UPDATE artifacts SET hit_count = hit_count + 1, last_accessed_at = ?, updated_at = ? @@ -415,14 +463,68 @@ func (db *DB) GetCachedArtifactCount() (int64, error) { return count, err } -func (db *DB) ClearArtifactCache(versionPURL, filename string) error { +// ClearArtifactCache marks an artifact uncached if its record still points at +// storagePath, and reports whether it did. A record a newer fetch has moved +// elsewhere is left alone, so a clear never orphans the object that fetch +// committed. +func (db *DB) ClearArtifactCache(versionPURL, filename, storagePath string) (bool, error) { query := db.Rebind(` UPDATE artifacts SET storage_path = NULL, content_hash = NULL, size = NULL, content_type = NULL, fetched_at = NULL, updated_at = ? - WHERE version_purl = ? AND filename = ? + WHERE version_purl = ? AND filename = ? AND storage_path = ? + `) + res, err := db.Exec(query, time.Now(), versionPURL, filename, storagePath) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + return n > 0, err +} + +// DiscardArtifact clears the record as ClearArtifactCache does and queues +// storagePath for deletion, for callers that must not delete an object another +// request may be reading. +func (db *DB) DiscardArtifact(versionPURL, filename, storagePath string) error { + cleared, err := db.ClearArtifactCache(versionPURL, filename, storagePath) + if err != nil || !cleared { + return err + } + return db.QueuePendingDelete(storagePath) +} + +// QueuePendingDelete queues a storage path no record points at any more. +// Queueing a path again, as reclaim does after a failed delete, restarts its +// grace period and moves it behind the rest. +func (db *DB) QueuePendingDelete(path string) error { + query := db.Rebind(` + INSERT INTO pending_deletes (path, queued_at) VALUES (?, ?) + ON CONFLICT(path) DO UPDATE SET queued_at = excluded.queued_at + `) + _, err := db.Exec(query, path, time.Now().UTC()) + return err +} + +// GetDuePendingDeletes returns up to limit paths queued before cutoff, oldest +// first. A path a record points at again is skipped. +func (db *DB) GetDuePendingDeletes(cutoff time.Time, limit int) ([]string, error) { + var paths []string + query := db.Rebind(` + SELECT path FROM pending_deletes + WHERE queued_at < ? + AND NOT EXISTS (SELECT 1 FROM artifacts WHERE artifacts.storage_path = pending_deletes.path) + ORDER BY queued_at + LIMIT ? `) - _, err := db.Exec(query, time.Now(), versionPURL, filename) + if err := db.Select(&paths, query, cutoff.UTC(), limit); err != nil { + return nil, err + } + return paths, nil +} + +// RemovePendingDelete drops a deleted path from the queue. +func (db *DB) RemovePendingDelete(path string) error { + _, err := db.Exec(db.Rebind(`DELETE FROM pending_deletes WHERE path = ?`), path) return err } @@ -933,6 +1035,41 @@ func (db *DB) ListCachedPackages(ecosystem string, sortBy string, limit int, off return packages, nil } +// CountCachedPackagesByEcosystem counts packages with at least one stored artifact. +func (db *DB) CountCachedPackagesByEcosystem() (map[string]int64, error) { + hasArtifacts, err := db.HasTable("artifacts") + if err != nil { + return nil, err + } + counts := make(map[string]int64) + if !hasArtifacts { + return counts, nil + } + + rows, err := db.Query(` + SELECT p.ecosystem, COUNT(DISTINCT p.purl) + FROM packages p + JOIN versions v ON v.package_purl = p.purl + JOIN artifacts a ON a.version_purl = v.purl + WHERE a.storage_path IS NOT NULL + GROUP BY p.ecosystem + `) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + for rows.Next() { + var ecosystem string + var count int64 + if err := rows.Scan(&ecosystem, &count); err != nil { + return nil, err + } + counts[ecosystem] = count + } + return counts, rows.Err() +} + func (db *DB) CountCachedPackages(ecosystem string) (int64, error) { hasArtifacts, err := db.HasTable("artifacts") if err != nil { @@ -1032,3 +1169,202 @@ func (db *DB) UpsertMetadataCache(entry *MetadataCacheEntry) error { } return nil } + +// Analytics queries + +// EcosystemStats aggregates cache and download activity for one ecosystem. +// +// DownloadedBytes is the accumulated download volume: every cache hit on an +// artifact served its full size, so the sum of hit_count * size is the number +// of bytes the proxy has handed to clients from cache for this ecosystem. +type EcosystemStats struct { + Ecosystem string `db:"ecosystem"` + Packages int64 `db:"packages"` + Versions int64 `db:"versions"` + Artifacts int64 `db:"artifacts"` + CacheSize int64 `db:"cache_size"` + Downloads int64 `db:"downloads"` + DownloadedBytes int64 `db:"downloaded_bytes"` +} + +// GetEcosystemStats returns per-ecosystem cache and download totals, ordered by +// accumulated download volume descending. Ecosystems with rows in packages but +// nothing cached are included with zeroed artifact counters. +// +// Artifacts evicted from the cache no longer contribute: eviction clears the +// size column, so their historical hits drop out of the accumulated total. +func (db *DB) GetEcosystemStats() ([]EcosystemStats, error) { + byEcosystem := make(map[string]*EcosystemStats) + + get := func(ecosystem string) *EcosystemStats { + if s, ok := byEcosystem[ecosystem]; ok { + return s + } + s := &EcosystemStats{Ecosystem: ecosystem} + byEcosystem[ecosystem] = s + return s + } + + if err := db.eachCount(`SELECT ecosystem, COUNT(*) FROM packages GROUP BY ecosystem`, + func(ecosystem string, n int64) { get(ecosystem).Packages = n }); err != nil { + return nil, err + } + + // Left joined and bucketed for the same reason as artifacts below: a version + // whose package row is missing would otherwise vanish here while still + // counting in GetCacheStats' COUNT(*), leaving the two unable to reconcile. + if err := db.eachCount(` + SELECT COALESCE(p.ecosystem, '`+unattributedEcosystem+`'), COUNT(*) + FROM versions v + LEFT JOIN packages p ON p.purl = v.package_purl + GROUP BY COALESCE(p.ecosystem, '`+unattributedEcosystem+`') + `, func(ecosystem string, n int64) { get(ecosystem).Versions = n }); err != nil { + return nil, err + } + + // The artifacts table is proxy-specific: a database inherited from + // git-pkgs carries packages and versions without it. + hasArtifacts, err := db.HasTable("artifacts") + if err != nil { + return nil, err + } + if hasArtifacts { + if err := db.eachArtifactStat(get); err != nil { + return nil, err + } + } + + stats := make([]EcosystemStats, 0, len(byEcosystem)) + for _, s := range byEcosystem { + stats = append(stats, *s) + } + stats = mergeAliasedEcosystems(stats) + sortEcosystemStats(stats) + return stats, nil +} + +// mergeAliasedEcosystems combines rows whose ecosystem names normalize to the +// same canonical name. The proxy writes "gem" and git-pkgs writes "rubygems", +// so a database that has seen both carries two rows for one ecosystem; left +// split they would render as two table rows and two chart slices, each with +// half the real share. +// +// The surviving row keeps the raw spelling of whichever input held the most +// cached bytes, because that string is what the UI filters and links by: the +// packages table stores the raw value, so substituting the canonical name would +// produce links that match nothing. That comparison is against each input's own +// size, not the running total, which would otherwise let the first spelling win +// simply by being merged into first. +func mergeAliasedEcosystems(stats []EcosystemStats) []EcosystemStats { + merged := make(map[string]*EcosystemStats, len(stats)) + largest := make(map[string]int64, len(stats)) + order := make([]string, 0, len(stats)) + + for i := range stats { + key := purl.NormalizeEcosystem(stats[i].Ecosystem) + into, ok := merged[key] + if !ok { + row := stats[i] + merged[key] = &row + largest[key] = stats[i].CacheSize + order = append(order, key) + continue + } + + // The name tiebreak matters: GetEcosystemStats builds its input by + // ranging a map, so without it two spellings of equal size would swap + // between refreshes and flip the row's badge and filter link. + if stats[i].CacheSize > largest[key] || + (stats[i].CacheSize == largest[key] && stats[i].Ecosystem < into.Ecosystem) { + largest[key] = stats[i].CacheSize + into.Ecosystem = stats[i].Ecosystem + } + into.Packages += stats[i].Packages + into.Versions += stats[i].Versions + into.Artifacts += stats[i].Artifacts + into.CacheSize += stats[i].CacheSize + into.Downloads += stats[i].Downloads + into.DownloadedBytes += stats[i].DownloadedBytes + } + + out := make([]EcosystemStats, 0, len(order)) + for _, key := range order { + out = append(out, *merged[key]) + } + return out +} + +// unattributedEcosystem collects cached artifacts whose version or package row +// is missing. Nothing enforces that link at the schema level, so an inner join +// would silently drop such rows and leave the per-ecosystem totals short of +// GetTotalCacheSize — two numbers that sit side by side in the UI and in +// Grafana. Bucketing them keeps the two reconcilable. +const unattributedEcosystem = "unattributed" + +func (db *DB) eachArtifactStat(get func(string) *EcosystemStats) error { + rows, err := db.Query(` + SELECT COALESCE(p.ecosystem, '` + unattributedEcosystem + `'), + COUNT(*), + COALESCE(SUM(a.size), 0), + COALESCE(SUM(a.hit_count), 0), + COALESCE(SUM(a.hit_count * a.size), 0) + FROM artifacts a + LEFT JOIN versions v ON v.purl = a.version_purl + LEFT JOIN packages p ON p.purl = v.package_purl + WHERE a.storage_path IS NOT NULL + GROUP BY COALESCE(p.ecosystem, '` + unattributedEcosystem + `') + `) + if err != nil { + return err + } + defer func() { _ = rows.Close() }() + + for rows.Next() { + var ecosystem string + var artifacts, cacheSize, downloads, downloadedBytes int64 + if err := rows.Scan(&ecosystem, &artifacts, &cacheSize, &downloads, &downloadedBytes); err != nil { + return err + } + s := get(ecosystem) + s.Artifacts = artifacts + s.CacheSize = cacheSize + s.Downloads = downloads + s.DownloadedBytes = downloadedBytes + } + return rows.Err() +} + +// eachCount runs a two-column "group by" query and hands each (key, count) pair to fn. +func (db *DB) eachCount(query string, fn func(key string, n int64)) error { + rows, err := db.Query(query) + if err != nil { + return err + } + defer func() { _ = rows.Close() }() + + for rows.Next() { + var key string + var n int64 + if err := rows.Scan(&key, &n); err != nil { + return err + } + fn(key, n) + } + return rows.Err() +} + +// sortEcosystemStats orders by accumulated download volume, then by cache size, +// then by name, so that ecosystems with no traffic yet still sort predictably. +func sortEcosystemStats(stats []EcosystemStats) { + sort.Slice(stats, func(i, j int) bool { + a, b := stats[i], stats[j] + switch { + case a.DownloadedBytes != b.DownloadedBytes: + return a.DownloadedBytes > b.DownloadedBytes + case a.CacheSize != b.CacheSize: + return a.CacheSize > b.CacheSize + default: + return a.Ecosystem < b.Ecosystem + } + }) +} diff --git a/internal/database/queries_packages_list_test.go b/internal/database/queries_packages_list_test.go index fc9356c0..e30d3640 100644 --- a/internal/database/queries_packages_list_test.go +++ b/internal/database/queries_packages_list_test.go @@ -2,12 +2,57 @@ package database import ( "database/sql" + "maps" "testing" "time" ) const testEcosystemNPM = "npm" +func TestCountCachedPackagesByEcosystem(t *testing.T) { + runWithBothDatabases(t, func(t *testing.T, db *DB) { + counts, err := db.CountCachedPackagesByEcosystem() + if err != nil || len(counts) != 0 { + t.Fatalf("empty cache counts = %v, error = %v", counts, err) + } + + seedArtifact(t, db, "npm", "express", "1.0.0", 100, 1) + seedArtifact(t, db, "npm", "express", "2.0.0", 200, 1) + seedArtifact(t, db, "npm", "lodash", "1.0.0", 100, 1) + seedArtifact(t, db, "cargo", "serde", "1.0.0", 100, 1) + seedArtifact(t, db, "pypi", "evicted", "1.0.0", 100, 1) + if _, err := db.Exec(`UPDATE artifacts SET storage_path = NULL WHERE version_purl = 'pkg:pypi/evicted@1.0.0'`); err != nil { + t.Fatal(err) + } + if err := db.UpsertPackage(&Package{PURL: "pkg:gem/metadata-only", Ecosystem: "gem", Name: "metadata-only"}); err != nil { + t.Fatal(err) + } + if err := db.UpsertArtifact(&Artifact{ + VersionPURL: "pkg:npm/express@1.0.0", + Filename: "extra.tgz", + UpstreamURL: "https://example.test/extra.tgz", + StoragePath: sql.NullString{String: "extra.tgz", Valid: true}, + }); err != nil { + t.Fatal(err) + } + + counts, err = db.CountCachedPackagesByEcosystem() + if err != nil { + t.Fatal(err) + } + want := map[string]int64{"npm": 2, "cargo": 1} + if !maps.Equal(counts, want) { + t.Fatalf("counts = %v, want %v", counts, want) + } + for ecosystem, count := range counts { + listed, err := db.CountCachedPackages(ecosystem) + if err != nil || count != listed { + t.Errorf("%s count = %d, list count = %d, error = %v", ecosystem, count, listed, err) + } + } + }) +} + func setupListCachedPackagesDB(t *testing.T) *DB { t.Helper() diff --git a/internal/database/schema.go b/internal/database/schema.go index 564745fb..eb457899 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -78,6 +78,11 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(ve CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); +CREATE TABLE IF NOT EXISTS pending_deletes ( + path TEXT NOT NULL PRIMARY KEY, + queued_at DATETIME NOT NULL +); + CREATE TABLE IF NOT EXISTS vulnerabilities ( id INTEGER PRIMARY KEY, vuln_id TEXT NOT NULL, @@ -181,6 +186,11 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_artifacts_version_filename ON artifacts(ve CREATE INDEX IF NOT EXISTS idx_artifacts_storage_path ON artifacts(storage_path); CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at); +CREATE TABLE IF NOT EXISTS pending_deletes ( + path TEXT NOT NULL PRIMARY KEY, + queued_at TIMESTAMP NOT NULL +); + CREATE TABLE IF NOT EXISTS vulnerabilities ( id SERIAL PRIMARY KEY, vuln_id TEXT NOT NULL, @@ -368,6 +378,7 @@ var migrations = []migration{ {"006_add_metadata_content_digest", migrateAddMetadataContentDigest}, {"007_add_metadata_link", migrateAddMetadataLink}, {"008_add_metadata_content_encoding", migrateAddMetadataContentEncoding}, + {"009_add_pending_deletes", migrateAddPendingDeletes}, } // isTableNotFound returns true if the error indicates a missing table. @@ -640,6 +651,23 @@ func migrateAddMetadataContentEncoding(db *DB) error { return nil } +// migrateAddPendingDeletes creates the queue of storage paths that no record +// points at any more, which the server deletes after a grace period. +func migrateAddPendingDeletes(db *DB) error { + ts := sqliteDatetime + if db.dialect == DialectPostgres { + ts = postgresTimestamp + } + query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS pending_deletes ( + path TEXT NOT NULL PRIMARY KEY, + queued_at %s NOT NULL + )`, ts) + if _, err := db.Exec(query); err != nil { + return fmt.Errorf("creating pending_deletes table: %w", err) + } + return nil +} + // EnsureMetadataCacheTable creates the metadata_cache table if it doesn't exist. func (db *DB) EnsureMetadataCacheTable() error { has, err := db.HasTable("metadata_cache") diff --git a/internal/denylist/denylist.go b/internal/denylist/denylist.go new file mode 100644 index 00000000..c4384ad4 --- /dev/null +++ b/internal/denylist/denylist.go @@ -0,0 +1,67 @@ +// Package denylist implements an immutable, exact-version package policy. +package denylist + +import ( + "fmt" + "strings" + + "github.com/git-pkgs/purl" +) + +// Policy is safe for concurrent reads. A nil policy allows every version. +type Policy struct { + packages map[string]map[string]bool +} + +// New validates and canonicalizes versioned PURLs. Qualifiers and subpaths are +// rejected because the proxy cannot reliably distinguish them at every endpoint. +func New(packages []string) (*Policy, error) { + if len(packages) == 0 { + return nil, nil + } + p := &Policy{packages: make(map[string]map[string]bool)} + for _, value := range packages { + pkg, err := purl.Parse(value) + if err != nil { + return nil, fmt.Errorf("invalid denylist package %q: %w", value, err) + } + if pkg.Version == "" || len(pkg.Qualifiers) != 0 || pkg.Subpath != "" || + strings.ContainsAny(pkg.Namespace+pkg.Name+pkg.Version, "*?[]<>=| \t\r\n") { + return nil, fmt.Errorf("invalid denylist package %q: expected an exact versioned PURL without qualifiers or subpath", value) + } + key := pkg.WithoutVersion().String() + if p.packages[key] == nil { + p.packages[key] = make(map[string]bool) + } + p.packages[key][pkg.Version] = true + } + return p, nil +} + +// Denied reports whether a versioned PURL is denied. +func (p *Policy) Denied(value string) bool { + if p == nil || len(p.packages) == 0 { + return false + } + pkg, err := purl.Parse(value) + if err != nil { + return false + } + return p.packages[pkg.WithoutVersion().String()][pkg.Version] +} + +// Versions returns a caller-owned set of denied versions for a package PURL. +func (p *Policy) Versions(packagePURL string) map[string]bool { + if p == nil { + return nil + } + versions := p.packages[packagePURL] + if len(versions) == 0 { + return nil + } + result := make(map[string]bool, len(versions)) + for version := range versions { + result[version] = true + } + return result +} diff --git a/internal/denylist/denylist_test.go b/internal/denylist/denylist_test.go new file mode 100644 index 00000000..380f1ee6 --- /dev/null +++ b/internal/denylist/denylist_test.go @@ -0,0 +1,39 @@ +package denylist + +import "testing" + +func TestPolicy(t *testing.T) { + p, err := New([]string{"pkg:pypi/My_Package@1.0", "pkg:npm/@scope/name@2.0.0", "pkg:npm/%40scope/name@2.0.0"}) + if err != nil { + t.Fatal(err) + } + for _, value := range []string{"pkg:pypi/my-package@1.0", "pkg:npm/%40scope/name@2.0.0"} { + if !p.Denied(value) { + t.Errorf("not denied: %s", value) + } + } + for _, value := range []string{"pkg:pypi/my-package@1.1", "pkg:pypi/my-package", "pkg:npm/other@2.0.0", "invalid"} { + if p.Denied(value) { + t.Errorf("unexpected denial: %s", value) + } + } + versions := p.Versions("pkg:pypi/my-package") + delete(versions, "1.0") + if !p.Denied("pkg:pypi/my-package@1.0") { + t.Fatal("caller mutated policy") + } + var empty *Policy + if empty.Denied("pkg:npm/example@1") || empty.Versions("pkg:npm/example") != nil { + t.Fatal("nil policy must allow everything") + } +} + +func TestInvalidPolicy(t *testing.T) { + for _, value := range []string{"", "not-a-purl", "pkg:npm/foo", "pkg:npm/foo@*", "pkg:npm/foo@>=1", "pkg:npm/foo@1?arch=x86", "pkg:npm/foo@1#src"} { + t.Run(value, func(t *testing.T) { + if _, err := New([]string{value}); err == nil { + t.Fatal("invalid denylist accepted") + } + }) + } +} diff --git a/internal/handler/apk.go b/internal/handler/apk.go index 9acc3cce..e0b4b31b 100644 --- a/internal/handler/apk.go +++ b/internal/handler/apk.go @@ -132,7 +132,7 @@ func (h *APKHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques if result.Artifact.MediaType == "" { result.Artifact.MediaType = "application/octet-stream" } - serveArtifact(w, r.Method, result) + ServeArtifactRequest(w, r, result) } // handleMetadata serves repository indexes and signatures through the diff --git a/internal/handler/cargo.go b/internal/handler/cargo.go index 4cf8591e..f87819c1 100644 --- a/internal/handler/cargo.go +++ b/internal/handler/cargo.go @@ -124,12 +124,16 @@ type crateIndexEntry struct { } func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWriter, body []byte) { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { + cooldownEnabled := h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() + if !cooldownEnabled && h.proxy.Denylist == nil { _, _ = downstreamResponse.Write(body) return } scanner := bufio.NewScanner(strings.NewReader(string(body))) + // The response is already size-bounded by FetchOrCacheMetadata. Crate + // entries with many dependencies can exceed Scanner's default token limit. + scanner.Buffer(nil, len(body)+1) for scanner.Scan() { line := scanner.Text() @@ -142,9 +146,12 @@ func (h *CargoHandler) applyCooldownFiltering(downstreamResponse http.ResponseWr continue } + if h.proxy.versionDenied("cargo", crate.Name, crate.Version) { + continue + } publishedAt, err := time.Parse(time.RFC3339, crate.PublishTime) - if crate.PublishTime == "" || err != nil { + if !cooldownEnabled || crate.PublishTime == "" || err != nil { _, _ = downstreamResponse.Write([]byte(line + "\n")) continue } @@ -211,5 +218,5 @@ func (h *CargoHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } diff --git a/internal/handler/coalesce_semantics_test.go b/internal/handler/coalesce_semantics_test.go new file mode 100644 index 00000000..2ca52022 --- /dev/null +++ b/internal/handler/coalesce_semantics_test.go @@ -0,0 +1,615 @@ +package handler + +import ( + "context" + "errors" + "io" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/artifacts" + "github.com/git-pkgs/registries/fetch" +) + +// runConcurrent runs fn in n goroutines released together and returns their errors. +func runConcurrent(n int, fn func(i int) error) []error { + errs := make([]error, n) + start := make(chan struct{}) + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + errs[i] = fn(i) + }(i) + } + close(start) + wg.Wait() + return errs +} + +// artifactBody builds a one-shot upstream artifact carrying the given bytes. +func artifactBody(content string) *fetch.Artifact { + return &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader(content)), + ContentType: "application/gzip", + } +} + +// drain consumes and closes a CacheResult reader, if there is one. +func drain(res *CacheResult) { + if res != nil && res.Reader != nil { + _, _ = io.Copy(io.Discard, res.Reader) + _ = res.Reader.Close() + } +} + +// TestCoalesceKey_DifferentUpstreamHashDoesNotShare is the safety property that +// makes coalescing sound: callers expecting different bytes must never share a +// fetch, so a re-published version cannot serve stale bytes to a caller that +// asked for the new digest. +func TestCoalesceKey_DifferentUpstreamHashDoesNotShare(t *testing.T) { + const content = "artifact bytes" + proxy, _, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: content, delay: fetchHoldTime} + proxy.Fetcher = fetcher + + // The digest must carry the "sha256:" prefix; without it the API treats the + // value as unverifiable and clears the hash, which would legitimately let + // the two callers share one fetch. + hashes := []string{ + "sha256:" + sha256Hex(content), + "sha256:" + sha256Hex("something else entirely"), + } + + _ = runConcurrent(2, func(i int) error { + res, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), + "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", + "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz", hashes[i]) + drain(res) + return err + }) + + if got := fetcher.calls.Load(); got != 2 { + t.Errorf("upstream fetches = %d, want 2: callers expecting different digests must not share a fetch", got) + } +} + +// TestCoalesceKey_HashCasingSharesOneFetch is the other half of that property. +// artifactHashMatches compares digests case-insensitively, so one digest in two +// casings describes one artifact and must not split into two fetches. +func TestCoalesceKey_HashCasingSharesOneFetch(t *testing.T) { + const content = "artifact bytes" + proxy, _, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: content, delay: fetchHoldTime} + proxy.Fetcher = fetcher + + hex := sha256Hex(content) + digests := []string{"sha256:" + hex, "sha256:" + strings.ToUpper(hex)} + + for i, err := range runConcurrent(2, func(i int) error { + res, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), + "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", + "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz", digests[i]) + drain(res) + return err + }) { + if err != nil { + t.Fatalf("caller %d failed: %v", i, err) + } + } + + if got := fetcher.calls.Load(); got != 1 { + t.Errorf("upstream fetches = %d, want 1: one digest in two casings is one artifact", got) + } +} + +// TestCoalesceKey_DifferentDownloadURLDoesNotShare covers the other half of the +// key: same package, different upstream URL, must not collapse into one fetch. +func TestCoalesceKey_DifferentDownloadURLDoesNotShare(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: "artifact bytes", delay: fetchHoldTime} + proxy.Fetcher = fetcher + + urls := []string{ + "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz", + "https://mirror.example.com/pkg/-/pkg-1.0.0.tgz", + } + + _ = runConcurrent(2, func(i int) error { + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", urls[i]) + drain(res) + return err + }) + + if got := fetcher.calls.Load(); got != 2 { + t.Errorf("upstream fetches = %d, want 2: different upstream URLs must not share a fetch", got) + } +} + +// TestCoalesceKey_DistinctArtifactsDoNotSerialize guards against an over-broad +// key: four packages fetched at once must still produce four fetches. +func TestCoalesceKey_DistinctArtifactsDoNotSerialize(t *testing.T) { + const n = 4 + proxy, _, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: "artifact bytes", delay: fetchHoldTime} + proxy.Fetcher = fetcher + + names := []string{"alpha", "beta", "gamma", "delta"} + errs := runConcurrent(n, func(i int) error { + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "npm", names[i], "1.0.0", names[i]+"-1.0.0.tgz", + "https://registry.npmjs.org/"+names[i]+"/-/"+names[i]+"-1.0.0.tgz") + drain(res) + return err + }) + for i, err := range errs { + if err != nil { + t.Errorf("caller %d (%s): %v", i, names[i], err) + } + } + if got := fetcher.calls.Load(); got != n { + t.Errorf("upstream fetches = %d, want %d: distinct artifacts must not share a fetch", got, n) + } +} + +// TestCoalesce_FailedFetchReachesEveryCallerAndIsRetriable verifies both claims +// in coalesceFetch's doc comment: a failed fetch reaches every caller sharing +// it, and the key is released so a later request retries. +func TestCoalesce_FailedFetchReachesEveryCallerAndIsRetriable(t *testing.T) { + const callers = 8 + proxy, _, _, fetcher := setupTestProxy(t) + boom := errors.New("upstream unavailable") + fetcher.fetchErr = boom + + errs := runConcurrent(callers, func(int) error { + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", + "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz") + drain(res) + return err + }) + for i, err := range errs { + if err == nil { + t.Errorf("caller %d: got nil error, want the shared fetch's failure", i) + } else if !errors.Is(err, boom) { + t.Errorf("caller %d: got %v, want it to wrap %v", i, err, boom) + } + } + + // The key must be released: a later request retries rather than inheriting + // the failure. + fetcher.fetchErr = nil + fetcher.artifact = artifactBody("recovered bytes") + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", + "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz") + if err != nil { + t.Fatalf("retry after failed coalesced fetch: %v", err) + } + body, _ := io.ReadAll(res.Reader) + _ = res.Reader.Close() + if string(body) != "recovered bytes" { + t.Errorf("retry body = %q, want %q", body, "recovered bytes") + } +} + +// TestCoalesce_ResolverPath covers the other entry point: GetOrFetchArtifact +// resolves the URL itself, so it is keyed without one. +func TestCoalesce_ResolverPath(t *testing.T) { + const callers = 8 + proxy, _, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: "resolved artifact bytes", delay: fetchHoldTime} + proxy.Fetcher = fetcher + + errs := runConcurrent(callers, func(int) error { + res, err := proxy.GetOrFetchArtifact(context.Background(), + "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz") + drain(res) + return err + }) + for i, err := range errs { + if err != nil { + t.Errorf("caller %d: %v", i, err) + } + } + if got := fetcher.calls.Load(); got != 1 { + t.Errorf("upstream fetches = %d, want 1", got) + } +} + +// TestCoalesce_ResolverPathEmptyFilename exercises that path when the filename +// is left to be resolved, which the key cannot know up front. +func TestCoalesce_ResolverPathEmptyFilename(t *testing.T) { + const callers = 8 + proxy, _, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: "resolved artifact bytes", delay: fetchHoldTime} + proxy.Fetcher = fetcher + + errs := runConcurrent(callers, func(int) error { + res, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "left-pad", "1.3.0", "") + drain(res) + return err + }) + for i, err := range errs { + if err != nil { + t.Errorf("caller %d: %v", i, err) + } + } + if got := fetcher.calls.Load(); got != 1 { + t.Errorf("upstream fetches = %d, want 1", got) + } +} + +// TestCoalesce_SubsequentRequestIsACacheHit confirms the coalesced fetch was +// committed and is visible later, not just streamed to the waiting callers. +func TestCoalesce_SubsequentRequestIsACacheHit(t *testing.T) { + const callers = 8 + const url = "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz" + proxy, _, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: "artifact bytes", delay: fetchHoldTime} + proxy.Fetcher = fetcher + + _ = runConcurrent(callers, func(int) error { + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) + drain(res) + return err + }) + + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) + if err != nil { + t.Fatalf("follow-up request: %v", err) + } + defer func() { _ = res.Reader.Close() }() + if !res.Cached { + t.Error("follow-up request should be served from cache") + } + if got := fetcher.calls.Load(); got != 1 { + t.Errorf("upstream fetches = %d, want 1 after a follow-up cache hit", got) + } +} + +// TestCoalesce_ReadersAreIndependent guards openStoredArtifact: callers sharing +// a fetch each need their own reader, or one closing early breaks the rest. +func TestCoalesce_ReadersAreIndependent(t *testing.T) { + const callers = 8 + const content = "artifact bytes that every caller must receive intact" + proxy, _, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: content, delay: fetchHoldTime} + proxy.Fetcher = fetcher + + results := make([]*CacheResult, callers) + errs := runConcurrent(callers, func(i int) error { + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", + "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz") + results[i] = res + return err + }) + for i, err := range errs { + if err != nil { + t.Fatalf("caller %d: %v", i, err) + } + } + + // Close the first caller's reader before anyone else has read a byte. + _ = results[0].Reader.Close() + + for i := 1; i < callers; i++ { + body, err := io.ReadAll(results[i].Reader) + _ = results[i].Reader.Close() + if err != nil { + t.Errorf("caller %d read after another caller closed: %v", i, err) + continue + } + if string(body) != content { + t.Errorf("caller %d got %q, want %q", i, body, content) + } + } +} + +// TestCoalesce_CanceledWaiterDoesNotWaitForTheSharedFetch checks that joining a +// coalesced fetch does not cost a caller its own cancellation. Without the +// leader/waiter split a waiter is pinned until the shared fetch resolves, +// bounded only by the artifact client timeout, so clients that have already +// gone away keep handler goroutines alive for minutes. +func TestCoalesce_CanceledWaiterDoesNotWaitForTheSharedFetch(t *testing.T) { + const leaderFetch = 2 * time.Second + const url = "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz" + + proxy, _, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: "artifact bytes", delay: leaderFetch, entered: make(chan struct{})} + proxy.Fetcher = fetcher + + leaderDone := make(chan error, 1) + go func() { + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) + drain(res) + leaderDone <- err + }() + + select { + case <-fetcher.entered: // the leader holds the key and is inside its fetch + case <-time.After(5 * time.Second): + t.Fatal("leader never started its fetch") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + start := time.Now() + _, err := proxy.GetOrFetchArtifactFromURL(ctx, "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) + blocked := time.Since(start) + + if !errors.Is(err, context.Canceled) { + t.Errorf("waiter error = %v, want context.Canceled", err) + } + if blocked > leaderFetch/4 { + t.Errorf("canceled waiter blocked %v, want well under %v: it is pinned to the shared fetch", + blocked, leaderFetch/4) + } + + // A waiter leaving must not disturb the fetch the others share. + if err := <-leaderDone; err != nil { + t.Fatalf("leader failed after a waiter canceled: %v", err) + } + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) + if err != nil { + t.Fatalf("follow-up after leader completed: %v", err) + } + defer func() { _ = res.Reader.Close() }() + if !res.Cached { + t.Error("leader's fetch should have been committed to the cache") + } + if got := fetcher.calls.Load(); got != 1 { + t.Errorf("upstream fetches = %d, want 1", got) + } +} + +// inFlightLen reports how many coalesced fetches are currently registered. +func inFlightLen(p *Proxy) int { + p.fetchMu.Lock() + defer p.fetchMu.Unlock() + return len(p.inFlight) +} + +// TestCoalesce_KeyIsReleasedAfterFetch guards the bug this hand-rolled map can +// have that singleflight could not: a key left behind means later callers join +// a finished entry, see its closed done channel, and are served that stale +// result forever, while the map grows without bound. +func TestCoalesce_KeyIsReleasedAfterFetch(t *testing.T) { + const url = "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz" + proxy, _, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: "artifact bytes", delay: fetchHoldTime} + proxy.Fetcher = fetcher + + _ = runConcurrent(8, func(int) error { + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) + drain(res) + return err + }) + if n := inFlightLen(proxy); n != 0 { + t.Errorf("in-flight entries after a successful fetch = %d, want 0", n) + } + + // A fresh miss for the same key must start a new fetch, not rejoin the old + // entry. Clearing the cache record forces the miss path again. + if err := proxy.ClearCachedArtifact("npm", "pkg", "1.0.0", "pkg-1.0.0.tgz"); err != nil { + t.Fatalf("clear cached artifact: %v", err) + } + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "npm", "pkg", "1.0.0", "pkg-1.0.0.tgz", url) + if err != nil { + t.Fatalf("second miss for the same key: %v", err) + } + drain(res) + if got := fetcher.calls.Load(); got != 2 { + t.Errorf("upstream fetches = %d, want 2: the second miss must not reuse the finished entry", got) + } + if n := inFlightLen(proxy); n != 0 { + t.Errorf("in-flight entries at end = %d, want 0", n) + } +} + +// missingFromCache is a recheck that always reports a miss, so the shared fetch +// runs. +func missingFromCache() (artifacts.Artifact, string, bool) { + return artifacts.Artifact{}, "", false +} + +// TestCoalesce_LeaderRechecksCacheBeforeFetching covers the window between a +// caller's own cache lookup and it becoming the leader: a concurrent fetch can +// commit the artifact in that gap, and the leader must serve that rather than +// fetch it a second time. +func TestCoalesce_LeaderRechecksCacheBeforeFetching(t *testing.T) { + const content = "artifact bytes" + proxy, _, store, _ := setupTestProxy(t) + + const storagePath = "npm/pkg/1.0.0/pkg-1.0.0.tgz" + if _, _, err := store.Store(context.Background(), storagePath, strings.NewReader(content)); err != nil { + t.Fatalf("seeding storage: %v", err) + } + committed := artifacts.Artifact{ + PURL: "pkg:npm/pkg@1.0.0", + Filename: "pkg-1.0.0.tgz", + Size: int64(len(content)), + } + + res, err := proxy.coalesceFetch(context.Background(), "any-key", + func() (artifacts.Artifact, string, bool) { return committed, storagePath, true }, + func(context.Context) (artifacts.Artifact, string, error) { + t.Error("fetched an artifact that was already in the cache") + return artifacts.Artifact{}, "", errors.New("commit must not run") + }) + if err != nil { + t.Fatalf("coalesceFetch failed: %v", err) + } + defer drain(res) + got, err := io.ReadAll(res.Reader) + if err != nil { + t.Fatalf("reading result: %v", err) + } + if string(got) != content { + t.Errorf("got %q, want %q", got, content) + } + if n := inFlightLen(proxy); n != 0 { + t.Errorf("in-flight entries = %d, want 0", n) + } +} + +// TestCachedArtifactRecord covers the recheck itself: it must report the row a +// concurrent fetch committed, match its digest the way artifactHashMatches +// does, and report a miss for anything else. +func TestCachedArtifactRecord(t *testing.T) { + const ( + content = "artifact bytes" + pkgPURL = "pkg:npm/pkg" + versionPURL = "pkg:npm/pkg@1.0.0" + filename = "pkg-1.0.0.tgz" + storagePath = "npm/pkg/1.0.0/pkg-1.0.0.tgz" + ) + proxy, _, _, _ := setupTestProxy(t) + + hex := sha256Hex(content) + committed := testArtifact(content, versionPURL, filename, "application/gzip") + if err := proxy.updateCacheDB("npm", "pkg", pkgPURL, + "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz", storagePath, committed); err != nil { + t.Fatalf("seeding cache record: %v", err) + } + + for _, tc := range []struct { + name, filename, hash string + want bool + }{ + {"no upstream hash", filename, "", true}, + {"matching hash", filename, hex, true}, + {"matching hash in upper case", filename, strings.ToUpper(hex), true}, + {"different hash", filename, sha256Hex("something else entirely"), false}, + {"unknown filename", "pkg-1.0.0.zip", hex, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, path, ok := proxy.cachedArtifactRecord(pkgPURL, versionPURL, tc.filename, tc.hash) + if ok != tc.want { + t.Fatalf("ok = %v, want %v", ok, tc.want) + } + if !ok { + return + } + if path != storagePath { + t.Errorf("storage path = %q, want %q", path, storagePath) + } + if got.Digest.Encoded() != hex { + t.Errorf("digest = %q, want %q", got.Digest.Encoded(), hex) + } + }) + } +} + +// TestCoalesce_LeaderFetchesWhenRecheckedBytesAreGone covers the other branch +// of the recheck: a record whose bytes no longer open is not served, and the +// shared fetch runs instead, the same recovery the cache lookup makes. +func TestCoalesce_LeaderFetchesWhenRecheckedBytesAreGone(t *testing.T) { + const content = "fetched bytes" + const storagePath = "npm/pkg/1.0.0/pkg-1.0.0.tgz" + proxy, _, store, _ := setupTestProxy(t) + + stale := artifacts.Artifact{PURL: "pkg:npm/pkg@1.0.0", Filename: "pkg-1.0.0.tgz"} + if _, err := store.Open(context.Background(), storagePath); err == nil { + t.Fatal("stale bytes were present, so the test proves nothing") + } + + // The leader runs commit on its own goroutine, so a plain counter is safe. + fetches := 0 + res, err := proxy.coalesceFetch(context.Background(), "any-key", + func() (artifacts.Artifact, string, bool) { return stale, storagePath, true }, + func(ctx context.Context) (artifacts.Artifact, string, error) { + fetches++ + if _, _, err := store.Store(ctx, storagePath, strings.NewReader(content)); err != nil { + return artifacts.Artifact{}, "", err + } + return testArtifact(content, stale.PURL, stale.Filename, "application/gzip"), storagePath, nil + }) + if err != nil { + t.Fatalf("coalesceFetch failed: %v", err) + } + defer drain(res) + if fetches != 1 { + t.Errorf("shared fetches = %d, want 1: a record without bytes must be refetched", fetches) + } + got, err := io.ReadAll(res.Reader) + if err != nil { + t.Fatalf("reading result: %v", err) + } + if string(got) != content { + t.Errorf("got %q, want %q", got, content) + } + if n := inFlightLen(proxy); n != 0 { + t.Errorf("in-flight entries = %d, want 0", n) + } +} + +// TestCoalesce_PanicInSharedFetchDoesNotStrandWaiters checks the failure mode +// that matters most: a caller parked on a shared fetch must never be left +// blocked forever when that fetch dies. +// +// This drives coalesceFetch directly and holds the shared entry itself, because +// whether a second caller has reached the wait is not observable from outside: +// it runs a cache lookup against the database first, so releasing the leader on +// a timer races that query. Losing the race made a second caller the leader +// instead of a waiter, and its panic was unrecovered, killing the test binary +// rather than failing the test. +func TestCoalesce_PanicInSharedFetchDoesNotStrandWaiters(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + const key = "pkg:npm/pkg@1.0.0\x00pkg-1.0.0.tgz" + + inCommit := make(chan struct{}) + release := make(chan struct{}) + leaderPanicked := make(chan struct{}) + + go func() { + defer func() { + _ = recover() // the panic surfaces in the leader, as it would in a handler + close(leaderPanicked) + }() + _, _ = proxy.coalesceFetch(context.Background(), key, missingFromCache, + func(context.Context) (artifacts.Artifact, string, error) { + close(inCommit) + <-release + panic("upstream fetch exploded") + }) + }() + + <-inCommit // the leader holds the key and is inside the fetch + + // Take the entry a waiter would park on, while the leader is still held. + proxy.fetchMu.Lock() + shared := proxy.inFlight[key] + proxy.fetchMu.Unlock() + if shared == nil { + t.Fatal("no in-flight entry registered for a running fetch") + } + + close(release) + + select { + case <-shared.done: + case <-time.After(5 * time.Second): + t.Fatal("waiter stranded: a panicking shared fetch never released its waiters") + } + if !errors.Is(shared.err, errSharedFetchAbandoned) { + t.Errorf("waiter error = %v, want errSharedFetchAbandoned", shared.err) + } + + <-leaderPanicked + if n := inFlightLen(proxy); n != 0 { + t.Errorf("in-flight entries after a panic = %d, want 0", n) + } +} diff --git a/internal/handler/coalesce_test.go b/internal/handler/coalesce_test.go new file mode 100644 index 00000000..baf052d1 --- /dev/null +++ b/internal/handler/coalesce_test.go @@ -0,0 +1,185 @@ +package handler + +import ( + "bytes" + "context" + "io" + "log/slog" + "net/http" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" + "github.com/git-pkgs/registries/fetch" +) + +// fetchHoldTime holds each stub fetch open long enough that concurrent callers +// reliably overlap inside it. The exact value is not significant. +const fetchHoldTime = 50 * time.Millisecond + +// countingFetcher counts upstream fetches and holds each one open. +type countingFetcher struct { + calls atomic.Int64 + content string + delay time.Duration + + // entered, if set, is closed when the first fetch begins. A test can wait + // on it to know the leader holds the key, rather than guessing with a + // sleep. + entered chan struct{} + enterOnce sync.Once +} + +func (f *countingFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { + return f.FetchWithHeaders(ctx, url, nil) +} + +func (f *countingFetcher) FetchWithHeaders(_ context.Context, _ string, _ http.Header) (*fetch.Artifact, error) { + f.calls.Add(1) + if f.entered != nil { + f.enterOnce.Do(func() { close(f.entered) }) + } + time.Sleep(f.delay) + return &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader(f.content)), + ContentType: "application/gzip", + }, nil +} + +func (f *countingFetcher) Head(context.Context, string) (int64, string, error) { + return 0, "", nil +} + +// TestGetOrFetchArtifactFromURL_ConcurrentMissesCoalesce asserts that N +// simultaneous misses for one artifact produce a single upstream fetch. That is +// the CI shape: parallel jobs installing overlapping dependencies cold. +func TestGetOrFetchArtifactFromURL_ConcurrentMissesCoalesce(t *testing.T) { + const goroutines = 8 + const content = "left-pad tarball bytes" + + proxy, _, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: content, delay: fetchHoldTime} + proxy.Fetcher = fetcher + + start := make(chan struct{}) + var wg sync.WaitGroup + errs := make([]error, goroutines) + bodies := make([]string, goroutines) + + for i := 0; i < goroutines; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + res, err := proxy.GetOrFetchArtifactFromURL(context.Background(), + "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", + "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz") + if err != nil { + errs[i] = err + return + } + defer func() { _ = res.Reader.Close() }() + b, err := io.ReadAll(res.Reader) + errs[i] = err + bodies[i] = string(b) + }(i) + } + + close(start) + wg.Wait() + + for i, err := range errs { + if err != nil { + t.Errorf("goroutine %d: unexpected error: %v", i, err) + } + } + // Every caller must get its own intact copy of the bytes. + for i, b := range bodies { + if b != content { + t.Errorf("goroutine %d: body = %q, want %q", i, b, content) + } + } + if got := fetcher.calls.Load(); got != 1 { + t.Errorf("upstream fetches = %d, want 1 (%d concurrent callers stampeded the upstream)", got, goroutines) + } +} + +// TestGetOrFetchArtifactFromURL_ConcurrentMissesFileStorage runs the same +// scenario against the real file:// backend, the default in production. +// +// Uncoalesced this fails outright, not merely wastefully. Every caller stores +// to one key, and fileblob rewrites a ".attrs" sidecar per key with os.Create, +// truncating in place outside the rename that protects the blob. Decoding that +// sidecar mid-truncate gives "opening reader: EOF", served as a 502. +// +// Only the fetcher is stubbed, because the real one refuses loopback so an +// httptest upstream is unreachable. The storage, where this fails, is real. +func TestGetOrFetchArtifactFromURL_ConcurrentMissesFileStorage(t *testing.T) { + const goroutines = 16 + content := bytes.Repeat([]byte("tarball-bytes-"), 512) + + ctx := context.Background() + dir := t.TempDir() + + db, err := database.Create(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatalf("create database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + store, err := storage.OpenBucket(ctx, "file://"+filepath.Join(dir, "cache")) + if err != nil { + t.Fatalf("open storage: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + fetcher := &countingFetcher{content: string(content), delay: fetchHoldTime} + proxy := NewProxy(db, store, fetcher, fetch.NewResolver(), + slog.New(slog.NewTextHandler(io.Discard, nil))) + + start := make(chan struct{}) + var wg sync.WaitGroup + errs := make([]error, goroutines) + bodies := make([][]byte, goroutines) + + for i := 0; i < goroutines; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + res, err := proxy.GetOrFetchArtifactFromURL(ctx, + "npm", "left-pad", "1.3.0", "left-pad-1.3.0.tgz", + "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz") + if err != nil { + errs[i] = err + return + } + defer func() { _ = res.Reader.Close() }() + body, readErr := io.ReadAll(res.Reader) + errs[i] = readErr + bodies[i] = body + }(i) + } + + close(start) + wg.Wait() + + for i, err := range errs { + if err != nil { + t.Errorf("caller %d failed: %v", i, err) + } + } + for i, body := range bodies { + if !bytes.Equal(body, content) { + t.Errorf("caller %d got %d bytes, want %d", i, len(body), len(content)) + } + } + if got := fetcher.calls.Load(); got != 1 { + t.Errorf("upstream fetches = %d, want 1", got) + } +} diff --git a/internal/handler/composer.go b/internal/handler/composer.go index fbbd7a4e..2f475ac8 100644 --- a/internal/handler/composer.go +++ b/internal/handler/composer.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "net/http" "path" "strings" @@ -110,8 +109,11 @@ func (h *ComposerHandler) handlePackageMetadata(w http.ResponseWriter, r *http.R return } - rewritten, err := h.rewriteMetadata(body) + rewritten, err := h.proxy.cachedRewrite(r.Context(), "composer", h.proxyURL, packageName, body, h.rewriteMetadata) if err != nil { + if r.Context().Err() != nil { + return // the client left while waiting on a shared rewrite + } h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) w.Header().Set(headerContentType, "application/json") _, _ = w.Write(body) @@ -178,10 +180,12 @@ func expandMinifiedVersions(versionList []any) []any { } // Merge inherited fields into a new map, then overlay current fields. - // Deep copy values to avoid shared references between versions. + // Inherited values are shared between versions, not copied: the + // only one rewritten afterwards is dist, and rewriteDistURL copies + // it before changing it. merged := make(map[string]any, len(inherited)+len(vmap)) for k, val := range inherited { - merged[k] = deepCopyValue(val) + merged[k] = val } for k, val := range vmap { if val == composerUnset { @@ -200,26 +204,6 @@ func expandMinifiedVersions(versionList []any) []any { return expanded } -// deepCopyValue returns a deep copy of JSON-like values (maps, slices, scalars). -func deepCopyValue(v any) any { - switch val := v.(type) { - case map[string]any: - m := make(map[string]any, len(val)) - for k, v := range val { - m[k] = deepCopyValue(v) - } - return m - case []any: - s := make([]any, len(val)) - for i, v := range val { - s[i] = deepCopyValue(v) - } - return s - default: - return v - } -} - // filterAndRewriteVersions applies cooldown filtering and rewrites dist URLs // for a single package's version list. func (h *ComposerHandler) filterAndRewriteVersions(packageName string, versionList []any) []any { @@ -299,7 +283,14 @@ func (h *ComposerHandler) rewriteDistURL(vmap map[string]any, packageName, versi if len(parts) == vendorPackageParts { newURL := fmt.Sprintf("%s/composer/files/%s/%s/%s/%s", h.proxyURL, parts[0], parts[1], version, filename) - dist["url"] = newURL + // Expanded versions can share one inherited dist map, so give this + // version its own before changing its URL. + own := make(map[string]any, len(dist)) + for k, v := range dist { + own[k] = v + } + own["url"] = newURL + vmap["dist"] = own } } @@ -359,7 +350,7 @@ func (h *ComposerHandler) handleDownload(w http.ResponseWriter, r *http.Request) return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // isDevVersion reports whether a Composer version string refers to a @@ -491,12 +482,5 @@ func (h *ComposerHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) } defer func() { _ = resp.Body.Close() }() - for k, vv := range resp.Header { - for _, v := range vv { - w.Header().Add(k, v) - } - } - - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) + h.proxy.relayResponse(w, r, resp, nil) } diff --git a/internal/handler/conan.go b/internal/handler/conan.go index 7142f0dd..aa41668b 100644 --- a/internal/handler/conan.go +++ b/internal/handler/conan.go @@ -2,7 +2,6 @@ package handler import ( "fmt" - "io" "net/http" "strings" ) @@ -95,7 +94,7 @@ func (h *ConanHandler) handleRecipeFile(w http.ResponseWriter, r *http.Request) return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handlePackageFile serves a package file, fetching and caching from upstream if needed. @@ -132,7 +131,7 @@ func (h *ConanHandler) handlePackageFile(w http.ResponseWriter, r *http.Request) return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // shouldCacheFile returns true if the file should be cached. @@ -194,13 +193,5 @@ func (h *ConanHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { } defer func() { _ = resp.Body.Close() }() - // Copy response headers - for k, vv := range resp.Header { - for _, v := range vv { - w.Header().Add(k, v) - } - } - - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) + h.proxy.relayResponse(w, r, resp, nil) } diff --git a/internal/handler/conda.go b/internal/handler/conda.go index cef814b5..07c2aea1 100644 --- a/internal/handler/conda.go +++ b/internal/handler/conda.go @@ -1,6 +1,7 @@ package handler import ( + "compress/gzip" "encoding/json" "io" "net/http" @@ -83,7 +84,7 @@ func (h *CondaHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // isPackageFile returns true if the filename is a Conda package. @@ -155,19 +156,25 @@ func (h *CondaHandler) handleRepodata(w http.ResponseWriter, r *http.Request) { defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - for k, vv := range resp.Header { - for _, v := range vv { - w.Header().Add(k, v) - } - } - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) + h.proxy.relayResponse(w, r, resp, nil) return } - body, err := h.proxy.ReadMetadata(resp.Body) + // Explicitly requesting gzip disables the transport's automatic decoding. + // Cooldown needs JSON, so decode here and bound the decompressed size too. + var reader io.Reader = resp.Body + if strings.EqualFold(strings.TrimSpace(resp.Header.Get(headerContentEncoding)), "gzip") { + decoded, err := gzip.NewReader(resp.Body) + if err != nil { + http.Error(w, "failed to decode response", http.StatusBadGateway) + return + } + defer func() { _ = decoded.Close() }() + reader = decoded + } + body, err := h.proxy.ReadMetadata(reader) if err != nil { - http.Error(w, "failed to read response", http.StatusInternalServerError) + http.Error(w, "failed to read response", http.StatusBadGateway) return } @@ -240,7 +247,14 @@ func (h *CondaHandler) applyCooldownFiltering(body []byte) ([]byte, error) { func (h *CondaHandler) proxyCached(w http.ResponseWriter, r *http.Request) { cacheKey := strings.TrimPrefix(r.URL.Path, "/") cacheKey = strings.ReplaceAll(cacheKey, "/", "_") - h.proxy.ProxyCached(w, r, h.upstreamURL+r.URL.Path, "conda", cacheKey, "*/*") + // Large JSON indexes can exceed the metadata limit uncompressed. Conda + // clients decode Content-Encoding, so cache and relay gzip verbatim. The + // already-compressed repodata.json.bz2 route must stay on identity. + acceptEncoding := "identity" + if strings.HasSuffix(r.URL.Path, ".json") { + acceptEncoding = "gzip" + } + h.proxy.proxyCachedWithEncoding(w, r, h.upstreamURL+r.URL.Path, "conda", cacheKey, acceptEncoding, "*/*") } // proxyUpstream forwards a request to Anaconda without caching. diff --git a/internal/handler/conda_encoding_test.go b/internal/handler/conda_encoding_test.go new file mode 100644 index 00000000..b453a003 --- /dev/null +++ b/internal/handler/conda_encoding_test.go @@ -0,0 +1,219 @@ +package handler + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" +) + +func TestCondaRepodataGzip(t *testing.T) { + // Model a large index without allocating hundreds of megabytes: only its + // compressed representation fits the configured metadata limit. + plain := []byte(`{"packages":{},"padding":"` + strings.Repeat("x", 8192) + `"}`) + compressed := gzipPayload(t, plain) + for _, tt := range []struct { + filename string + mode string + }{ + {"repodata.json", "stream"}, + {"repodata.json", "cached"}, + {"repodata.json", "stale"}, + {"current_repodata.json", "stream"}, + {"current_repodata.json", "cached"}, + {"current_repodata.json", "stale"}, + } { + t.Run(tt.filename+"/"+tt.mode, func(t *testing.T) { + upstream := newGzipWhenAskedUpstream(plain, compressed) + defer upstream.Close() + proxy, db, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = tt.mode != "stream" + proxy.MetadataMaxSize = int64(len(compressed)) + if tt.mode == "cached" { + proxy.MetadataTTL = time.Hour + } + handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() + serve := func() *httptest.ResponseRecorder { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+tt.filename, nil)) + return w + } + assertGzipResponse(t, "first", serve(), compressed) + if got := upstream.sawAcceptEncoding(); got != "gzip" { + t.Fatalf("upstream Accept-Encoding = %q, want gzip", got) + } + if tt.mode == "stream" { + return + } + entry, err := db.GetMetadataCache("conda", "conda-forge_linux-64_"+tt.filename) + if err != nil { + t.Fatal(err) + } + if entry == nil || entry.ContentEncoding.String != "gzip" || entry.Size.Int64 != int64(len(compressed)) { + t.Fatalf("cache entry does not describe compressed bytes: %+v", entry) + } + upstream.available.Store(false) + assertGzipResponse(t, "offline replay", serve(), compressed) + wantRequests := int32(1) + if tt.mode == "stale" { + wantRequests = 2 + } + if got := upstream.requests.Load(); got != wantRequests { + t.Errorf("upstream requests = %d, want %d", got, wantRequests) + } + }) + } +} + +func TestCondaRepodataBzip2StaysIdentity(t *testing.T) { + plain := []byte("BZh already compressed repodata") + for _, cache := range []bool{false, true} { + t.Run(map[bool]string{false: "stream", true: "cached"}[cache], func(t *testing.T) { + upstream := newGzipWhenAskedUpstream(plain, gzipPayload(t, plain)) + defer upstream.Close() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = cache + handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json.bz2", nil)) + if got := upstream.sawAcceptEncoding(); got != "identity" { + t.Errorf("upstream Accept-Encoding = %q, want identity", got) + } + if w.Code != http.StatusOK || !bytes.Equal(w.Body.Bytes(), plain) || w.Header().Get(headerContentEncoding) != "" { + t.Fatalf("unexpected response: %d, %v, %q", w.Code, w.Header(), w.Body.String()) + } + }) + } +} + +func TestCondaRepodataCacheSizeLimit(t *testing.T) { + plain := []byte(`{"packages":{},"padding":"` + strings.Repeat("x", 8192) + `"}`) + compressed := gzipPayload(t, plain) + for _, tt := range []struct { + name string + body []byte + encoding string + limit int64 + status int + }{ + {"identity fallback", plain, "", int64(len(plain)), http.StatusOK}, + {"identity too large", plain, "", int64(len(plain) - 1), http.StatusBadGateway}, + {"gzip too large", compressed, "gzip", int64(len(compressed) - 1), http.StatusBadGateway}, + } { + t.Run(tt.name, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get(headerAcceptEncoding); got != "gzip" { + t.Errorf("upstream Accept-Encoding = %q, want gzip", got) + } + w.Header().Set(headerContentEncoding, tt.encoding) + _, _ = w.Write(tt.body) + })) + defer upstream.Close() + proxy, db, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.CacheMetadata = true + proxy.MetadataMaxSize = tt.limit + handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) + if w.Code != tt.status { + t.Fatalf("status = %d, want %d: %s", w.Code, tt.status, w.Body.String()) + } + if tt.status == http.StatusOK { + if !bytes.Equal(w.Body.Bytes(), plain) || w.Header().Get(headerContentEncoding) != "" { + t.Fatal("identity response was changed or mislabeled as gzip") + } + } else if entry, _ := db.GetMetadataCache("conda", "conda-forge_linux-64_repodata.json"); entry != nil { + t.Fatal("oversized response was cached") + } + }) + } +} + +func TestCondaRepodataCooldownEncoding(t *testing.T) { + plain, err := json.Marshal(map[string]any{ + "packages": map[string]any{ + "old.tar.bz2": map[string]any{"name": "demo", "timestamp": time.Now().Add(-7 * 24 * time.Hour).UnixMilli()}, + "new.tar.bz2": map[string]any{"name": "demo", "timestamp": time.Now().UnixMilli()}, + }, + "packages.conda": map[string]any{ + "old.conda": map[string]any{"name": "demo", "timestamp": time.Now().Add(-7 * 24 * time.Hour).UnixMilli()}, + "new.conda": map[string]any{"name": "demo", "timestamp": time.Now().UnixMilli()}, + }, + }) + if err != nil { + t.Fatal(err) + } + for _, encoding := range []string{"identity", "gzip"} { + t.Run(encoding, func(t *testing.T) { + body := plain + if encoding == "gzip" { + body = gzipPayload(t, plain) + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentEncoding, encoding) + _, _ = w.Write(body) + })) + defer upstream.Close() + proxy := testProxy() + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() + for _, filename := range []string{"repodata.json", "current_repodata.json"} { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/"+filename, nil)) + if w.Code != http.StatusOK || w.Header().Get(headerContentEncoding) != "" { + t.Fatalf("unexpected response: %d, %v", w.Code, w.Header()) + } + var result map[string]map[string]any + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + for key, old := range map[string]string{"packages": "old.tar.bz2", "packages.conda": "old.conda"} { + if len(result[key]) != 1 || result[key][old] == nil { + t.Errorf("%s: cooldown did not retain only the old package: %s", key, w.Body.String()) + } + } + } + }) + } +} + +func TestCondaRepodataCooldownRejectsInvalidGzip(t *testing.T) { + compressed := gzipPayload(t, []byte(`{"padding":"`+strings.Repeat("x", 8192)+`"}`)) + for _, tt := range []struct { + name string + body []byte + }{ + {"invalid header", []byte("not gzip")}, + {"truncated body", compressed[:len(compressed)-4]}, + {"decoded size exceeds limit", compressed}, + } { + t.Run(tt.name, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerContentEncoding, "gzip") + _, _ = w.Write(tt.body) + })) + defer upstream.Close() + proxy := testProxy() + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + if tt.name == "decoded size exceeds limit" { + proxy.MetadataMaxSize = int64(len(compressed)) + } + handler := NewCondaHandlerWithUpstream(proxy, "http://proxy.local", upstream.URL).Routes() + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/conda-forge/linux-64/repodata.json", nil)) + if w.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want 502", w.Code) + } + }) + } +} diff --git a/internal/handler/container.go b/internal/handler/container.go index 62d839eb..f73f0f54 100644 --- a/internal/handler/container.go +++ b/internal/handler/container.go @@ -164,7 +164,7 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req if cached.Artifact.MediaType == "" { cached.Artifact.MediaType = "application/octet-stream" } - serveArtifact(w, r.Method, cached) + ServeArtifactRequest(w, r, cached) return } @@ -190,7 +190,7 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req h.containerError(w, http.StatusNotFound, "BLOB_UNKNOWN", "blob unknown to registry") return } - if errors.Is(err, ErrArtifactBlocked) { + if errors.Is(err, ErrArtifactBlocked) || errors.Is(err, ErrVersionDenied) { h.containerError(w, http.StatusForbidden, "DENIED", err.Error()) return } @@ -208,7 +208,7 @@ func (h *ContainerHandler) handleBlobDownload(w http.ResponseWriter, r *http.Req if result.Artifact.MediaType == "" { result.Artifact.MediaType = "application/octet-stream" } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handleManifest serves immutable manifests from cache and revalidates mutable tags. @@ -274,16 +274,16 @@ func (h *ContainerHandler) proxyBlobHead(w http.ResponseWriter, r *http.Request, } defer func() { _ = resp.Body.Close() }() - for _, header := range []string{headerContentType, headerContentLength, "Docker-Content-Digest", headerETag, headerLastModified} { - if v := resp.Header.Get(header); v != "" { - w.Header().Set(header, v) + h.proxy.relayResponse(w, r, resp, func(dst, src http.Header) { + for _, header := range []string{headerContentType, headerContentLength, "Docker-Content-Digest", headerETag, headerLastModified} { + if v := src.Get(header); v != "" { + dst.Set(header, v) + } } - } - if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices && w.Header().Get("Docker-Content-Digest") == "" { - w.Header().Set("Docker-Content-Digest", digest) - } - - w.WriteHeader(resp.StatusCode) + if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices && dst.Get("Docker-Content-Digest") == "" { + dst.Set("Docker-Content-Digest", digest) + } + }) } // registryForName resolves a client-visible OCI repository name to an upstream diff --git a/internal/handler/container_manifest.go b/internal/handler/container_manifest.go index 7b030868..01df3afb 100644 --- a/internal/handler/container_manifest.go +++ b/internal/handler/container_manifest.go @@ -5,7 +5,6 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - "io" "mime" "net/http" "regexp" @@ -77,15 +76,12 @@ func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, writeContainerManifest(w, r, cached, true) return } - copyContainerManifestHeaders(w.Header(), resp.Header) - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) + h.proxy.relayResponse(w, r, resp, copyContainerManifestHeaders) return } if r.Method == http.MethodHead { - copyContainerManifestHeaders(w.Header(), resp.Header) - w.WriteHeader(http.StatusOK) + h.proxy.relayResponse(w, r, resp, copyContainerManifestHeaders) return } diff --git a/internal/handler/container_tags.go b/internal/handler/container_tags.go index dcc08f81..e9803ed1 100644 --- a/internal/handler/container_tags.go +++ b/internal/handler/container_tags.go @@ -5,7 +5,6 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - "io" "net/http" "net/url" "regexp" @@ -73,9 +72,7 @@ func (h *ContainerHandler) serveTagsList(w http.ResponseWriter, r *http.Request, writeContainerTags(w, cached, true) return } - copyContainerTagsHeaders(w.Header(), resp.Header) - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) + h.proxy.relayResponse(w, r, resp, copyContainerTagsHeaders) return } diff --git a/internal/handler/container_test.go b/internal/handler/container_test.go index 5b7aa032..bd237cf7 100644 --- a/internal/handler/container_test.go +++ b/internal/handler/container_test.go @@ -665,6 +665,7 @@ func TestContainerHandler_BlobDownload_CacheHitSkipsAuth(t *testing.T) { proxy, db, store, fetcher := setupTestProxy(t) digest := "sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd" seedPackage(t, db, store, "oci", "library/nginx", digest, digest, "cached blob") + store.seekable = true upstreamRequests := 0 upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -680,14 +681,18 @@ func TestContainerHandler_BlobDownload_CacheHitSkipsAuth(t *testing.T) { } req := httptest.NewRequest(http.MethodGet, "/library/nginx/blobs/"+digest, nil) + req.Header.Set("Range", "bytes=0-5") w := httptest.NewRecorder() h.Routes().ServeHTTP(w, req) - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + if w.Code != http.StatusPartialContent { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusPartialContent, w.Body.String()) + } + if got := w.Body.String(); got != "cached" { + t.Errorf("body = %q, want %q", got, "cached") } - if got := w.Body.String(); got != "cached blob" { - t.Errorf("body = %q, want %q", got, "cached blob") + if got := w.Header().Get("Content-Range"); got != "bytes 0-5/11" { + t.Errorf("Content-Range = %q, want %q", got, "bytes 0-5/11") } if upstreamRequests != 0 { t.Errorf("upstream requests = %d, want 0", upstreamRequests) diff --git a/internal/handler/cooldown_patterns_test.go b/internal/handler/cooldown_patterns_test.go new file mode 100644 index 00000000..83034cf8 --- /dev/null +++ b/internal/handler/cooldown_patterns_test.go @@ -0,0 +1,105 @@ +package handler + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/cooldownpolicy" +) + +func TestNuGetCooldownPackagePatterns(t *testing.T) { + for _, tc := range []struct { + name, defaultDuration, pattern string + exact map[string]string + blocked bool + }{ + {"pattern enables cooldown", "", "14d", nil, true}, + {"pattern exempts package", "14d", "0", nil, false}, + {"exact overrides exemption", "", "0", map[string]string{"pkg:nuget/testpkg": "14d"}, true}, + {"exact exempts package", "", "14d", map[string]string{"pkg:nuget/testpkg": "0"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + p, db, store, fetcher := setupTestProxy(t) + policy, err := cooldownpolicy.New(&cooldown.Config{Default: tc.defaultDuration, Packages: tc.exact}, map[string]string{"pkg:nuget/test*": tc.pattern}) + if err != nil { + t.Fatal(err) + } + p.Cooldown = policy + seedPackage(t, db, store, "nuget", "testpkg", "2.0.0", "testpkg.2.0.0.nupkg", "cached package") + requests := 0 + upstream := newNuGetCooldownUpstream(t, &requests) + defer upstream.Close() + p.HTTPClient = upstream.Client() + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + list := nugetGet(t, h.Routes(), "/v3-flatcontainer/TestPkg/index.json", http.StatusOK) + if strings.Contains(list.Body.String(), "2.0.0") == tc.blocked { + t.Fatalf("incorrect version list: %s", list.Body.String()) + } + status := http.StatusOK + if tc.blocked { + status = http.StatusNotFound + } + nugetGet(t, h.Routes(), "/v3-flatcontainer/TestPkg/2.0.0/testpkg.2.0.0.nupkg", status) + if fetcher.fetchCalled { + t.Fatal("cached or withheld download fetched upstream artifact") + } + }) + } +} + +func TestNPMCooldownPatternExemptionRespectsDenylist(t *testing.T) { + p, db, store, fetcher := setupTestProxy(t) + var err error + p.Cooldown, err = cooldownpolicy.New(&cooldown.Config{Default: "7d"}, map[string]string{"pkg:npm/@example/*": "0"}) + if err != nil { + t.Fatal(err) + } + setTestDenylist(t, p, "pkg:npm/@example/widget@1.0.0") + published := time.Now().Add(-time.Hour) + for _, version := range []string{"1.0.0", "2.0.0"} { + seedPackage(t, db, store, "npm", "@example/widget", version, "widget-"+version+".tgz", "cached package") + if err := db.SetVersionPublishedAt("pkg:npm/%40example/widget@"+version, "pkg:npm/%40example/widget", published); err != nil { + t.Fatal(err) + } + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/@example/widget" { + t.Errorf("unexpected upstream request: %s", r.URL) + } + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"name":"@example/widget","dist-tags":{"latest":"1.0.0"},"versions":{"1.0.0":{},"2.0.0":{}},"time":{"1.0.0":%q,"2.0.0":%q}}`, published.Format(time.RFC3339), published.Format(time.RFC3339)) + })) + defer upstream.Close() + p.HTTPClient = upstream.Client() + h := NewNPMHandler(p, "http://proxy.test", upstream.URL) + server := httptest.NewServer(h.Routes()) + defer server.Close() + for _, tc := range []struct { + path string + status int + contains, absent string + }{ + {"/@example%2Fwidget", http.StatusOK, `"latest":"2.0.0"`, "1.0.0"}, + {"/@example%2Fwidget/-/widget-1.0.0.tgz", http.StatusForbidden, "denylist", "cached package"}, + {"/@example%2Fwidget/-/widget-2.0.0.tgz", http.StatusOK, "cached package", "denylist"}, + } { + response, err := server.Client().Get(server.URL + tc.path) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(response.Body) + _ = response.Body.Close() + if err != nil || response.StatusCode != tc.status || !strings.Contains(string(body), tc.contains) || strings.Contains(string(body), tc.absent) { + t.Errorf("GET %s: status=%d body=%s error=%v", tc.path, response.StatusCode, body, err) + } + } + if fetcher.fetchCalled { + t.Error("cached or denied downloads fetched an upstream artifact") + } +} diff --git a/internal/handler/cran.go b/internal/handler/cran.go index 4a6ded88..6a363a45 100644 --- a/internal/handler/cran.go +++ b/internal/handler/cran.go @@ -83,7 +83,7 @@ func (h *CRANHandler) handleSourceDownload(w http.ResponseWriter, r *http.Reques return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handleBinaryDownload serves a binary package, fetching and caching from upstream. @@ -117,7 +117,7 @@ func (h *CRANHandler) handleBinaryDownload(w http.ResponseWriter, r *http.Reques return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // parseSourceFilename extracts name and version from a CRAN source filename. diff --git a/internal/handler/debian.go b/internal/handler/debian.go index 42d26e7a..7b6635ef 100644 --- a/internal/handler/debian.go +++ b/internal/handler/debian.go @@ -1,9 +1,12 @@ package handler import ( + "crypto/sha256" + "encoding/hex" "fmt" "net/http" "regexp" + "sort" "strings" ) @@ -14,22 +17,37 @@ const ( // DebianHandler handles APT/Debian repository protocol requests. // It proxies requests to upstream Debian/Ubuntu repositories and caches .deb packages. +// +// The main archive is served at /debian/. Additional archives are mounted at +// /debian/{repository}/ and the remaining path mirrors the upstream layout. type DebianHandler struct { - proxy *Proxy - upstreamURL string - proxyURL string + proxy *Proxy + upstreamURL string + proxyURL string + repositories map[string]string } // NewDebianHandler creates a new Debian/APT protocol handler. -func NewDebianHandler(proxy *Proxy, proxyURL string, upstreamURL string) *DebianHandler { +// When repositories is empty, only the main archive is reachable. +func NewDebianHandler( + proxy *Proxy, + proxyURL string, + upstreamURL string, + repositories map[string]string, +) *DebianHandler { if upstreamURL == "" { upstreamURL = debianUpstream } - return &DebianHandler{ - proxy: proxy, - upstreamURL: strings.TrimSuffix(upstreamURL, "/"), - proxyURL: strings.TrimSuffix(proxyURL, "/"), + h := &DebianHandler{ + proxy: proxy, + upstreamURL: strings.TrimSuffix(upstreamURL, "/"), + proxyURL: strings.TrimSuffix(proxyURL, "/"), + repositories: make(map[string]string, len(repositories)), } + for name, repositoryURL := range repositories { + h.repositories[name] = strings.TrimSuffix(repositoryURL, "/") + } + return h } // Routes returns the HTTP handler for Debian requests. @@ -48,60 +66,135 @@ func (h *DebianHandler) Routes() http.Handler { return } + // Unlike the APK handler, an unconfigured first segment is not a 404: + // the main archive is unnamed and serves paths of its own at the root + // (README, indices/, project/), so an unknown name stays a + // main-archive path. + upstreamURL, repository, rest := h.upstreamURL, "", path + if !strings.HasPrefix(path, "pool/") && !strings.HasPrefix(path, "dists/") { + if name, tail, ok := strings.Cut(path, "/"); ok && tail != "" { + if named, found := h.repositories[name]; found { + upstreamURL, repository, rest = named, name, tail + } else if strings.HasPrefix(tail, "dists/") || strings.HasPrefix(tail, "pool/") { + // The main archive has no {name}/dists/ or {name}/pool/ of + // its own, so this is a misspelled repository rather than a + // main-archive path. Answering here keeps a typo from + // surfacing as the upstream's opaque HTML 404. + h.repositoryNotFound(w, name) + return + } + } + } + // Route based on path type switch { - case strings.HasPrefix(path, "pool/"): + case strings.HasPrefix(rest, "pool/"): // Package downloads - cache these - h.handlePackageDownload(w, r, path) - case strings.HasPrefix(path, "dists/"): - // Repository metadata - proxy without caching (changes frequently) - h.handleMetadata(w, r, path) + h.handlePackageDownload(w, r, repository, upstreamURL, rest) + case strings.HasPrefix(rest, "dists/"): + // Repository metadata - served through the metadata cache + h.handleMetadata(w, r, repository, upstreamURL, rest) default: // Other files (like README, etc.) - proxy directly - h.proxyFile(w, r, path) + h.proxyFile(w, r, upstreamURL, rest) } }) } +// repositoryNotFound answers a request addressed to a repository that is not +// configured, naming the ones that are so a misspelling is self-evident. +func (h *DebianHandler) repositoryNotFound(w http.ResponseWriter, name string) { + if len(h.repositories) == 0 { + http.Error(w, + fmt.Sprintf("unknown debian repository %q: none are configured", name), + http.StatusNotFound) + return + } + + configured := make([]string, 0, len(h.repositories)) + for repository := range h.repositories { + configured = append(configured, repository) + } + sort.Strings(configured) + + http.Error(w, + fmt.Sprintf("unknown debian repository %q: configured repositories are %s", + name, strings.Join(configured, ", ")), + http.StatusNotFound) +} + // handlePackageDownload fetches and caches .deb packages from the pool. // Pool path format: pool/{component}/{prefix}/{name}/{filename} // Example: pool/main/n/nginx/nginx_1.18.0-6_amd64.deb -func (h *DebianHandler) handlePackageDownload(w http.ResponseWriter, r *http.Request, path string) { +func (h *DebianHandler) handlePackageDownload( + w http.ResponseWriter, + r *http.Request, + repository, upstreamURL, path string, +) { // Parse the path to extract package info name, version, arch := h.parsePoolPath(path) if name == "" { // Can't parse, just proxy directly - h.proxyFile(w, r, path) + h.proxyFile(w, r, upstreamURL, path) return } filename := path[strings.LastIndex(path, "/")+1:] - downloadURL := fmt.Sprintf("%s/%s", h.upstreamURL, path) + downloadURL := fmt.Sprintf("%s/%s", upstreamURL, path) + + cacheFilename := h.artifactCacheFilename(repository, filename) h.proxy.Logger.Info("debian package download", + "repository", repository, "name", name, "version", version, "arch", arch, "filename", filename) result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), "deb", name, version, filename, downloadURL) + r.Context(), "deb", name, version, cacheFilename, downloadURL) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch package") return } w.Header().Set(headerContentType, "application/vnd.debian.binary-package") - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) +} + +// handleMetadata serves repository metadata files through the metadata cache, +// so they keep resolving while the upstream archive is unreachable. +func (h *DebianHandler) handleMetadata( + w http.ResponseWriter, + r *http.Request, + repository, upstreamURL, path string, +) { + cacheKey := h.metadataCacheKeyFor(repository, upstreamURL, path) + h.proxy.ProxyCached(w, r, fmt.Sprintf("%s/%s", upstreamURL, path), "debian", cacheKey, "*/*") } -// handleMetadata proxies repository metadata files. -// These change frequently so we don't cache them. -func (h *DebianHandler) handleMetadata(w http.ResponseWriter, r *http.Request, path string) { - cacheKey := strings.ReplaceAll(path, "/", "_") - h.proxy.ProxyCached(w, r, fmt.Sprintf("%s/%s", h.upstreamURL, path), "debian", cacheKey, "*/*") +// artifactCacheFilename scopes a package by repository, since the same +// filename can hold different bytes in different archives. The main archive +// keeps the unscoped filename its existing cache entries were stored under. +func (h *DebianHandler) artifactCacheFilename(repository, filename string) string { + if repository == "" { + return filename + } + return repository + "/" + filename +} + +// metadataCacheKeyFor returns the metadata cache key for a request. The main +// archive keeps its separator-based key so existing cache entries stay valid; +// a named repository hashes its identity as APKHandler.metadataCacheKey does. +func (h *DebianHandler) metadataCacheKeyFor(repository, upstreamURL, path string) string { + if repository == "" { + return strings.ReplaceAll(path, "/", "_") + } + identity := repository + "\x00" + upstreamURL + "\x00" + path + digest := sha256.Sum256([]byte(identity)) + return hex.EncodeToString(digest[:]) } // proxyFile proxies any file directly without caching. -func (h *DebianHandler) proxyFile(w http.ResponseWriter, r *http.Request, path string) { - h.proxy.ProxyFile(w, r, fmt.Sprintf("%s/%s", h.upstreamURL, path)) +func (h *DebianHandler) proxyFile(w http.ResponseWriter, r *http.Request, upstreamURL, path string) { + h.proxy.ProxyFile(w, r, fmt.Sprintf("%s/%s", upstreamURL, path)) } // debPackagePattern matches .deb filenames to extract name, version, and arch. diff --git a/internal/handler/debian_test.go b/internal/handler/debian_test.go index b086fdf4..acfd165b 100644 --- a/internal/handler/debian_test.go +++ b/internal/handler/debian_test.go @@ -1,7 +1,14 @@ package handler import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" "testing" + "time" + + "github.com/git-pkgs/registries/fetch" ) func TestDebianHandler_parsePoolPath(t *testing.T) { @@ -23,6 +30,231 @@ func TestDebianHandler_parsePoolPath(t *testing.T) { } func TestDebianHandler_Routes(t *testing.T) { - h := NewDebianHandler(nil, "http://localhost:8080", "") + h := NewDebianHandler(nil, "http://localhost:8080", "", nil) assertRoutesBasics(t, h.Routes(), "/dists/stable/Release", "/pool/../../../etc/passwd") } + +// TestDebianHandler_LegacyCacheKeysUnchanged pins the cache identities the +// main archive used before named repositories existed. Deployments carry warm +// caches across upgrades, so a changed key here silently discards them. +func TestDebianHandler_LegacyCacheKeysUnchanged(t *testing.T) { + const ( + poolPath = "pool/main/h/hello/hello_2.10-3_amd64.deb" + distPath = "dists/trixie/InRelease" + ) + + h := NewDebianHandler(nil, "http://proxy.example", "https://archive.test", map[string]string{ + "security": "https://security.test", + }) + + // Artifact cache: the bare filename, with no repository prefix. + filename := poolPath[strings.LastIndex(poolPath, "/")+1:] + if got := h.artifactCacheFilename("", filename); got != "hello_2.10-3_amd64.deb" { + t.Errorf("legacy artifact cache filename = %q, want %q", got, "hello_2.10-3_amd64.deb") + } + + // Metadata cache: path separators replaced with underscores. + if got := h.metadataCacheKeyFor("", h.upstreamURL, distPath); got != "dists_trixie_InRelease" { + t.Errorf("legacy metadata cache key = %q, want %q", got, "dists_trixie_InRelease") + } + + // A named repository must not reuse either identity. + if got := h.artifactCacheFilename("security", filename); got == "hello_2.10-3_amd64.deb" { + t.Error("named repository reused the legacy artifact cache filename") + } + if got := h.metadataCacheKeyFor("security", "https://security.test", distPath); got == "dists_trixie_InRelease" { + t.Error("named repository reused the legacy metadata cache key") + } +} + +// TestDebianHandler_NamedRepositoryRouting checks that a named repository +// reaches its own archive while the main archive keeps serving /dists/ and +// /pool/ unchanged. +func TestDebianHandler_NamedRepositoryRouting(t *testing.T) { + mainRelease := "main archive InRelease" + mainArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/dists/trixie/InRelease" { + http.NotFound(w, r) + return + } + _, _ = fmt.Fprint(w, mainRelease) + })) + defer mainArchive.Close() + + securityRelease := "security archive InRelease" + securityArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/dists/trixie-security/InRelease" { + http.NotFound(w, r) + return + } + _, _ = fmt.Fprint(w, securityRelease) + })) + defer securityArchive.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = http.DefaultClient + + h := NewDebianHandler(proxy, "http://proxy.example", mainArchive.URL, map[string]string{ + "security": securityArchive.URL, + }) + + got := serveDebianRequest(h, "/security/dists/trixie-security/InRelease") + if got.Code != http.StatusOK || got.Body.String() != securityRelease { + t.Errorf("named repository: status = %d, body = %q, want 200 %q", + got.Code, got.Body.String(), securityRelease) + } + + got = serveDebianRequest(h, "/dists/trixie/InRelease") + if got.Code != http.StatusOK || got.Body.String() != mainRelease { + t.Errorf("main archive: status = %d, body = %q, want 200 %q", + got.Code, got.Body.String(), mainRelease) + } + + // An unconfigured name is not a repository, so it addresses the main + // archive as a plain path. + got = serveDebianRequest(h, "/unknown/dists/trixie/InRelease") + if got.Code == http.StatusOK { + t.Errorf("unknown repository: status = 200, want the main archive's 404") + } +} + +// TestDebianHandler_UnknownRepositoryReportsConfiguredNames covers a misspelled +// repository name. {name}/dists/ is never a main-archive path, so the handler +// answers directly rather than forwarding upstream, where the reply would be an +// opaque HTML 404 that does not mention the repository at all. +func TestDebianHandler_UnknownRepositoryReportsConfiguredNames(t *testing.T) { + var upstreamHits int + mainArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + upstreamHits++ + http.Error(w, "404 Not Found", http.StatusNotFound) + })) + defer mainArchive.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = http.DefaultClient + + h := NewDebianHandler(proxy, "http://proxy.example", mainArchive.URL, map[string]string{ + "security": "http://security.example", + "ghcli": "http://ghcli.example", + }) + + got := serveDebianRequest(h, "/secuirty/dists/trixie-security/InRelease") + if got.Code != http.StatusNotFound { + t.Errorf("status = %d, want %d", got.Code, http.StatusNotFound) + } + body := got.Body.String() + if !strings.Contains(body, `"secuirty"`) { + t.Errorf("body = %q, want it to name the misspelled repository", body) + } + // Sorted, so the message is stable across map iteration order. + if !strings.Contains(body, "ghcli, security") { + t.Errorf("body = %q, want it to list the configured repositories", body) + } + if upstreamHits != 0 { + t.Errorf("upstream hits = %d, want 0: the handler should answer without forwarding", upstreamHits) + } + + // A root-level main-archive path still falls through, since the main + // archive really does serve files of its own there. + got = serveDebianRequest(h, "/project/trace/README") + if got.Code != http.StatusNotFound || upstreamHits != 1 { + t.Errorf("main-archive path: status = %d, upstream hits = %d, want 404 and 1", + got.Code, upstreamHits) + } +} + +// TestDebianHandler_MetadataCacheKeysDoNotCollideAcrossRepositories guards the +// hashed metadata cache key. The main archive's key replaces '/' with '_', so +// a repository named "security" serving dists/trixie/InRelease would otherwise +// collide with the main archive's own security_dists_trixie_InRelease entry, +// serving one archive's signed metadata to clients of the other. +func TestDebianHandler_MetadataCacheKeysDoNotCollideAcrossRepositories(t *testing.T) { + mainRelease := "main archive InRelease" + mainArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, mainRelease) + })) + defer mainArchive.Close() + + securityRelease := "security archive InRelease" + securityArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, securityRelease) + })) + defer securityArchive.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = http.DefaultClient + + h := NewDebianHandler(proxy, "http://proxy.example", mainArchive.URL, map[string]string{ + "security": securityArchive.URL, + }) + + // Main archive path whose separator-based key is "security_dists_...". + first := serveDebianRequest(h, "/dists/security/dists/trixie/InRelease") + if first.Code != http.StatusOK || first.Body.String() != mainRelease { + t.Fatalf("main archive: status = %d, body = %q, want 200 %q", + first.Code, first.Body.String(), mainRelease) + } + + // Served within the metadata TTL: a colliding key would return + // mainRelease here. + second := serveDebianRequest(h, "/security/dists/trixie/InRelease") + if second.Code != http.StatusOK { + t.Fatalf("named repository: status = %d, want 200: %s", second.Code, second.Body.String()) + } + if second.Body.String() != securityRelease { + t.Errorf("named repository served %q, want %q (cache key collision)", + second.Body.String(), securityRelease) + } +} + +// TestDebianHandler_PackageCacheScopedPerRepository checks that the same +// package filename in two archives does not resolve to one cached artifact. +func TestDebianHandler_PackageCacheScopedPerRepository(t *testing.T) { + const pkgPath = "/pool/main/h/hello/hello_2.10-3_amd64.deb" + + mainPkg := "main archive package bytes" + mainArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, mainPkg) + })) + defer mainArchive.Close() + + securityPkg := "security archive package bytes" + securityArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, securityPkg) + })) + defer securityArchive.Close() + + proxy, _, _, _ := setupTestProxy(t) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(mainArchive.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + + h := NewDebianHandler(proxy, "http://proxy.example", mainArchive.URL, map[string]string{ + "security": securityArchive.URL, + }) + + first := serveDebianRequest(h, pkgPath) + if first.Code != http.StatusOK || first.Body.String() != mainPkg { + t.Fatalf("main archive: status = %d, body = %q, want 200 %q", + first.Code, first.Body.String(), mainPkg) + } + + second := serveDebianRequest(h, "/security"+pkgPath) + if second.Code != http.StatusOK { + t.Fatalf("named repository: status = %d, want 200: %s", second.Code, second.Body.String()) + } + if second.Body.String() != securityPkg { + t.Errorf("named repository served %q, want %q (artifact cache collision)", + second.Body.String(), securityPkg) + } +} + +func serveDebianRequest(h *DebianHandler, target string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil)) + return w +} diff --git a/internal/handler/denylist.go b/internal/handler/denylist.go new file mode 100644 index 00000000..a03f03e1 --- /dev/null +++ b/internal/handler/denylist.go @@ -0,0 +1,8 @@ +package handler + +func (p *Proxy) versionDenied(ecosystem, name, version string) bool { + if p.Denylist == nil { + return false + } + return p.Denylist.Denied(canonicalVersionPURL(ecosystem, name, version)) +} diff --git a/internal/handler/denylist_test.go b/internal/handler/denylist_test.go new file mode 100644 index 00000000..383f95c7 --- /dev/null +++ b/internal/handler/denylist_test.go @@ -0,0 +1,248 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/denylist" +) + +func setTestDenylist(t testing.TB, p *Proxy, packages ...string) { + t.Helper() + var err error + p.Denylist, err = denylist.New(packages) + if err != nil { + t.Fatal(err) + } +} + +func TestDenylistBlocksArtifactAccess(t *testing.T) { + for _, tc := range []struct{ cached, direct bool }{{false, false}, {false, true}, {true, false}, {true, true}} { + cached, direct := tc.cached, tc.direct + t.Run(fmt.Sprintf("cached=%t/direct=%t", cached, direct), func(t *testing.T) { + p, db, store, _ := setupTestProxy(t) + if cached { + seedPackage(t, db, store, "npm", "demo", "1.0.0", "demo.tgz", "content") + } + p.DirectServe = direct + store.signedURL = "https://storage.example/demo" + setTestDenylist(t, p, "pkg:npm/demo@1.0.0") + ctx := context.Background() + for _, get := range []func() (*CacheResult, error){ + func() (*CacheResult, error) { return p.GetOrFetchArtifact(ctx, "npm", "demo", "1.0.0", "demo.tgz") }, + func() (*CacheResult, error) { return p.GetCachedArtifact(ctx, "npm", "demo", "1.0.0", "demo.tgz") }, + func() (*CacheResult, error) { + return p.GetOrFetchArtifactFromURL(ctx, "npm", "demo", "1.0.0", "demo.tgz", "https://upstream.invalid/demo") + }, + func() (*CacheResult, error) { + return p.GetOrFetchArtifactFromURLWithHeaders(ctx, "npm", "demo", "1.0.0", "demo.tgz", "https://upstream.invalid/demo", nil) + }, + func() (*CacheResult, error) { + return p.GetOrFetchArtifactFromURLWithDigest(ctx, "npm", "demo", "1.0.0", "demo.tgz", "https://upstream.invalid/demo", "sha256:abc") + }, + } { + result, err := get() + if result != nil || !errors.Is(err, ErrVersionDenied) { + t.Fatalf("result=%+v err=%v; want denial", result, err) + } + w := httptest.NewRecorder() + p.serveArtifactError(w, err, "fetch failed") + if w.Code != http.StatusForbidden { + t.Fatalf("status = %d", w.Code) + } + } + // Denying does not purge the cache. Removing the policy restores it. + p.Denylist = nil + result, err := p.GetCachedArtifact(ctx, "npm", "demo", "1.0.0", "demo.tgz") + if err != nil || (result != nil) != cached { + t.Fatalf("cache changed: result=%+v err=%v", result, err) + } + if result != nil && result.Reader != nil { + _ = result.Reader.Close() + } + }) + } +} + +func TestNPMDenylistMetadata(t *testing.T) { + p := testProxy() + setTestDenylist(t, p, "pkg:npm/@scope/demo@2.0.0") + h := NewNPMHandler(p, "https://proxy.example", "") + for _, times := range []string{"", `,"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z"}`} { + body := `{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0-beta.1":{}},"dist-tags":{"latest":"2.0.0","beta":"2.0.0","next":"3.0.0-beta.1"}` + times + `}` + out, err := h.rewriteMetadata("@scope/demo", []byte(body)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(out), "2.0.0") || !strings.Contains(string(out), `"latest":"1.0.0"`) || !strings.Contains(string(out), `"next":"3.0.0-beta.1"`) { + t.Fatalf("incorrect metadata: %s", out) + } + } + out, err := h.rewriteMetadata("@scope/demo", []byte(`{"versions":{"2.0.0":{}},"dist-tags":{"latest":"2.0.0"}}`)) + if err != nil || strings.Contains(string(out), "2.0.0") { + t.Fatalf("all denied: %s, %v", out, err) + } +} + +func TestCargoDenylistWithoutTimestamps(t *testing.T) { + p := testProxy() + setTestDenylist(t, p, "pkg:cargo/demo@1.0.0") + h := &CargoHandler{proxy: p} + input := "{\"name\":\"demo\",\"vers\":\"1.0.0\"}\n{\"name\":\"demo\",\"vers\":\"2.0.0\"}\n" + for _, cd := range []CooldownPolicy{nil, &cooldown.Config{Default: "3d"}} { + p.Cooldown = cd + w := httptest.NewRecorder() + h.applyCooldownFiltering(w, []byte(input)) + if w.Body.String() != "{\"name\":\"demo\",\"vers\":\"2.0.0\"}\n" { + t.Fatalf("index: %s", w.Body.String()) + } + } +} + +func TestPyPIDenylistSimpleRepresentations(t *testing.T) { + for _, tc := range []struct{ name, contentType, body string }{ + {"html", "text/html", `downloaddemo-2.0.0.tar.gz`}, + {"json", pypiSimpleJSON, `{"meta":{"api-version":"1.0"},"files":[{"filename":"demo-1.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/a/b/demo-1.0.0.tar.gz"},{"filename":"demo-2.0.0.tar.gz","url":"https://files.pythonhosted.org/packages/a/b/demo-2.0.0.tar.gz"}]}`}, + } { + t.Run(tc.name, func(t *testing.T) { + h, p := setupPyPIHandler(t, func(r *http.Request) (*http.Response, error) { + if r.URL.Path != "/simple/demo/" { + t.Fatalf("denylist must not need timestamp metadata: %s", r.URL.Path) + } + return pypiHTTPResponse(r, tc.contentType, tc.body), nil + }) + p.CacheMetadata = true + p.MetadataTTL = time.Hour + // Seed raw metadata before configuring the denylist. + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/simple/demo/", nil)) + setTestDenylist(t, p, "pkg:pypi/Demo@1.0.0") + p.HTTPClient = &http.Client{Transport: pypiRoundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errors.New("offline") + })} + for _, ttl := range []time.Duration{time.Hour, 0} { + p.MetadataTTL = ttl + w = httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/simple/demo/", nil)) + if w.Code != http.StatusOK || strings.Contains(w.Body.String(), "1.0.0") || !strings.Contains(w.Body.String(), "2.0.0") { + t.Fatalf("ttl=%s status=%d body=%s", ttl, w.Code, w.Body.String()) + } + } + }) + } +} + +func TestPyPIDenylistJSONMetadata(t *testing.T) { + p := testProxy() + setTestDenylist(t, p, "pkg:pypi/demo@1.0.0") + h := NewPyPIHandler(p, "https://proxy.example") + body := `{"info":{"name":"Demo","version":"1.0.0"},"releases":{"1.0.0":[{"url":"https://files.pythonhosted.org/packages/demo-1.0.0.tar.gz"}],"2.0.0":[]},"urls":[{"url":"https://files.pythonhosted.org/packages/demo-1.0.0.tar.gz"}]}` + out, err := h.rewriteJSONMetadata([]byte(body)) + if err != nil { + t.Fatal(err) + } + var metadata map[string]any + if err := json.Unmarshal(out, &metadata); err != nil { + t.Fatal(err) + } + if _, exists := metadata["releases"].(map[string]any)["1.0.0"]; exists || len(metadata["urls"].([]any)) != 0 { + t.Fatalf("denied release survived: %s", out) + } + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/pypi/demo/1.0.0/json", nil)) + if w.Code != http.StatusNotFound { + t.Fatalf("denied version metadata status=%d", w.Code) + } +} + +func TestDenylistDownloadRoutes(t *testing.T) { + for _, tc := range []struct { + ecosystem, path, filename string + handler func(*Proxy) http.Handler + }{ + {"npm", "/demo/-/demo-1.0.0.tgz", "demo-1.0.0.tgz", func(p *Proxy) http.Handler { return NewNPMHandler(p, "http://proxy.test", "").Routes() }}, + {"pypi", "/packages/a/b/demo-1.0.0.tar.gz", "demo-1.0.0.tar.gz", func(p *Proxy) http.Handler { return NewPyPIHandler(p, "http://proxy.test").Routes() }}, + {"cargo", "/crates/demo/1.0.0/download", "demo-1.0.0.crate", func(p *Proxy) http.Handler { return NewCargoHandler(p, "http://proxy.test", "", "").Routes() }}, + {"deb", "/pool/main/d/demo/demo_1.0.0_amd64.deb", "demo_1.0.0_amd64.deb", func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://proxy.test", "", nil).Routes() }}, + } { + t.Run(tc.ecosystem, func(t *testing.T) { + p, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, tc.ecosystem, "demo", "1.0.0", tc.filename, "cached content") + setTestDenylist(t, p, "pkg:"+tc.ecosystem+"/demo@1.0.0") + w := httptest.NewRecorder() + tc.handler(p).ServeHTTP(w, httptest.NewRequest(http.MethodGet, tc.path, nil)) + if w.Code != http.StatusForbidden || !strings.Contains(w.Body.String(), "denylist") { + t.Fatalf("denied download status=%d body=%s", w.Code, w.Body.String()) + } + // An allowed version remains readable while the policy is enabled. + filename := strings.ReplaceAll(tc.filename, "1.0.0", "2.0.0") + seedPackage(t, db, store, tc.ecosystem, "demo", "2.0.0", filename, "allowed content") + result, err := p.GetCachedArtifact(context.Background(), tc.ecosystem, "demo", "2.0.0", filename) + if err != nil || result == nil { + t.Fatalf("allowed download: result=%v err=%v", result, err) + } + defer func() { _ = result.Reader.Close() }() + body, err := io.ReadAll(result.Reader) + if err != nil || string(body) != "allowed content" { + t.Fatalf("allowed body=%s err=%v", body, err) + } + }) + } +} + +func TestDenylistAndCooldownCompose(t *testing.T) { + p := testProxy() + p.Cooldown = &cooldown.Config{Default: "3d"} + setTestDenylist(t, p, "pkg:npm/demo@1.0.0") + h := NewNPMHandler(p, "https://proxy.test", "") + body := fmt.Sprintf(`{"versions":{"1.0.0":{},"2.0.0":{},"3.0.0":{}},"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":"2021-01-01T00:00:00Z","3.0.0":%q},"dist-tags":{"latest":"3.0.0"}}`, time.Now().UTC().Format(time.RFC3339)) + out, err := h.rewriteMetadata("demo", []byte(body)) + if err != nil || strings.Contains(string(out), "1.0.0") || strings.Contains(string(out), "3.0.0") || !strings.Contains(string(out), `"latest":"2.0.0"`) { + t.Fatalf("combined policy: %s, %v", out, err) + } +} + +func TestCargoDenylistLargeEntry(t *testing.T) { + p := testProxy() + setTestDenylist(t, p, "pkg:cargo/demo@1.0.0") + h := &CargoHandler{proxy: p} + line := fmt.Sprintf("{\"name\":\"demo\",\"vers\":\"2.0.0\",\"extra\":%q}\n", strings.Repeat("x", 70<<10)) + w := httptest.NewRecorder() + h.applyCooldownFiltering(w, []byte(line)) + if w.Body.String() != line { + t.Fatal("large allowed index entry was truncated") + } +} + +func TestDenylistMalformedMetadataFailsClosed(t *testing.T) { + for _, tc := range []struct { + name, path, body, contentType string + handler func(*Proxy) http.Handler + }{ + {"npm", "/demo", `{"versions":`, contentTypeJSON, func(p *Proxy) http.Handler { return NewNPMHandler(p, "https://proxy.test", "").Routes() }}, + {"pypi-json", "/pypi/demo/json", `{"releases":`, contentTypeJSON, func(p *Proxy) http.Handler { return NewPyPIHandler(p, "https://proxy.test").Routes() }}, + {"pypi-simple", "/simple/demo/", `{"files":{}}`, pypiSimpleJSON, func(p *Proxy) http.Handler { return NewPyPIHandler(p, "https://proxy.test").Routes() }}, + } { + t.Run(tc.name, func(t *testing.T) { + p := testProxy() + setTestDenylist(t, p, "pkg:npm/demo@1.0.0", "pkg:pypi/demo@1.0.0") + p.HTTPClient = &http.Client{Transport: pypiRoundTripFunc(func(r *http.Request) (*http.Response, error) { + return pypiHTTPResponse(r, tc.contentType, tc.body), nil + })} + w := httptest.NewRecorder() + tc.handler(p).ServeHTTP(w, httptest.NewRequest(http.MethodGet, tc.path, nil)) + if w.Code != http.StatusBadGateway { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + }) + } +} diff --git a/internal/handler/download_test.go b/internal/handler/download_test.go index e0cf9cca..d91b68ad 100644 --- a/internal/handler/download_test.go +++ b/internal/handler/download_test.go @@ -1213,7 +1213,7 @@ func TestDebianHandler_DownloadCacheMiss(t *testing.T) { ContentType: "application/vnd.debian.binary-package", } - h := NewDebianHandler(proxy, "http://localhost", "") + h := NewDebianHandler(proxy, "http://localhost", "", nil) srv := httptest.NewServer(h.Routes()) defer srv.Close() diff --git a/internal/handler/fetch_path_test.go b/internal/handler/fetch_path_test.go new file mode 100644 index 00000000..4f7602ac --- /dev/null +++ b/internal/handler/fetch_path_test.go @@ -0,0 +1,163 @@ +package handler + +import ( + "context" + "errors" + "io" + "net/http" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/registries/fetch" +) + +// These tests cover fetches of one artifact that do not share a coalescing +// key because their URLs differ. Each must keep its own stored object. + +const ( + fetchPathURLA = "https://mirror-a.example/pkg-1.0.0.tgz" + fetchPathURLB = "https://mirror-b.example/pkg-1.0.0.tgz" +) + +// barrierFetcher serves a body per URL and holds each fetch until both have +// started, so both requests are past the cache check before either commits. +type barrierFetcher struct { + bodies map[string]string + arrived sync.WaitGroup +} + +func newBarrierFetcher(bodies map[string]string) *barrierFetcher { + f := &barrierFetcher{bodies: bodies} + f.arrived.Add(len(bodies)) + return f +} + +func (f *barrierFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, error) { + return f.FetchWithHeaders(ctx, url, nil) +} + +func (f *barrierFetcher) FetchWithHeaders(_ context.Context, url string, _ http.Header) (*fetch.Artifact, error) { + f.arrived.Done() + f.arrived.Wait() + return artifactBody(f.bodies[url]), nil +} + +func (f *barrierFetcher) Head(_ context.Context, _ string) (int64, string, error) { + return 0, "", nil +} + +// gatedStorage holds the first Open until ready reports true, to order it +// after the other request's store or delete. +type gatedStorage struct { + *mockStorage + ready func(stores, deletes int32) bool + stores atomic.Int32 + deletes atomic.Int32 + opened atomic.Bool + release chan struct{} + releaseMu sync.Once +} + +func newGatedStorage(store *mockStorage, ready func(stores, deletes int32) bool) *gatedStorage { + return &gatedStorage{mockStorage: store, ready: ready, release: make(chan struct{})} +} + +func (g *gatedStorage) check() { + if g.ready(g.stores.Load(), g.deletes.Load()) { + g.releaseMu.Do(func() { close(g.release) }) + } +} + +func (g *gatedStorage) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { + size, hash, err := g.mockStorage.Store(ctx, path, r) + g.stores.Add(1) + g.check() + return size, hash, err +} + +func (g *gatedStorage) Delete(ctx context.Context, path string) error { + err := g.mockStorage.Delete(ctx, path) + g.deletes.Add(1) + g.check() + return err +} + +func (g *gatedStorage) Open(ctx context.Context, path string) (io.ReadCloser, error) { + if g.opened.CompareAndSwap(false, true) { + select { + case <-g.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return g.mockStorage.Open(ctx, path) +} + +func readResult(res *CacheResult) string { + b, _ := io.ReadAll(res.Reader) + _ = res.Reader.Close() + return string(b) +} + +// TestFetchesWithDifferentURLsServeTheirOwnBytes has both fetches store before +// either opens. Sharing one object, one of them would serve the other's bytes. +func TestFetchesWithDifferentURLsServeTheirOwnBytes(t *testing.T) { + proxy, _, store, _ := setupTestProxy(t) + bodies := map[string]string{fetchPathURLA: "bytes from a", fetchPathURLB: "bytes from b"} + proxy.Fetcher = newBarrierFetcher(bodies) + proxy.Storage = newGatedStorage(store, func(stores, _ int32) bool { return stores == 2 }) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + var wg sync.WaitGroup + for url, want := range bodies { + wg.Go(func() { + res, err := proxy.GetOrFetchArtifactFromURL(ctx, "npm", "pkg", "1.0.0", staleFilename, url) + if err != nil { + t.Errorf("fetch of %s: %v", url, err) + return + } + if got := readResult(res); got != want { + t.Errorf("fetch of %s served %q, want %q", url, got, want) + } + }) + } + wg.Wait() +} + +// TestDigestMismatchLeavesAnotherFetchsObject has one fetch discard bytes that +// fail the declared digest before another fetch, which stored good bytes, +// opens them. Sharing one object, the discard would delete the good bytes. +func TestDigestMismatchLeavesAnotherFetchsObject(t *testing.T) { + proxy, _, store, _ := setupTestProxy(t) + proxy.Fetcher = newBarrierFetcher(map[string]string{fetchPathURLA: "good bytes", fetchPathURLB: "tampered bytes"}) + proxy.Storage = newGatedStorage(store, func(_, deletes int32) bool { return deletes == 1 }) + declared := "sha256:" + sha256Hex("good bytes") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + var wg sync.WaitGroup + wg.Go(func() { + res, err := proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "npm", "pkg", "1.0.0", staleFilename, fetchPathURLA, declared) + if err != nil { + t.Errorf("good fetch: %v", err) + return + } + if got := readResult(res); got != "good bytes" { + t.Errorf("good fetch served %q", got) + } + }) + wg.Go(func() { + _, err := proxy.GetOrFetchArtifactFromURLWithDigest(ctx, "npm", "pkg", "1.0.0", staleFilename, fetchPathURLB, declared) + if !errors.Is(err, ErrArtifactDigestMismatch) { + t.Errorf("tampered fetch: got %v, want a digest mismatch", err) + } + }) + wg.Wait() + + if _, ok := store.files[recordedStoragePath(t, proxy.DB, staleVersionPURL, staleFilename)]; !ok { + t.Error("the recorded object was deleted") + } +} diff --git a/internal/handler/filename_download.go b/internal/handler/filename_download.go index e3c1162e..fc96586a 100644 --- a/internal/handler/filename_download.go +++ b/internal/handler/filename_download.go @@ -39,5 +39,5 @@ func (p *Proxy) handleFilenameDownload(w http.ResponseWriter, r *http.Request, d return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } diff --git a/internal/handler/gem.go b/internal/handler/gem.go index 8fc50399..efe964b6 100644 --- a/internal/handler/gem.go +++ b/internal/handler/gem.go @@ -4,7 +4,6 @@ import ( "bufio" "encoding/json" "fmt" - "io" "net/http" "strings" "time" @@ -117,17 +116,13 @@ func (h *GemHandler) handleCompactIndex(w http.ResponseWriter, r *http.Request) defer func() { _ = indexResp.Body.Close() }() if indexResp.StatusCode != http.StatusOK { - copyResponseHeaders(w, indexResp.Header) - w.WriteHeader(indexResp.StatusCode) - _, _ = io.Copy(w, indexResp.Body) + h.proxy.relayResponse(w, r, indexResp, nil) return } if filteredVersions == nil { h.proxy.Logger.Warn("failed to fetch version timestamps, proxying unfiltered", "name", name) - copyResponseHeaders(w, indexResp.Header) - w.WriteHeader(http.StatusOK) - _, _ = io.Copy(w, indexResp.Body) + h.proxy.relayResponse(w, r, indexResp, nil) return } @@ -215,15 +210,6 @@ func (h *GemHandler) writeFilteredIndex(w http.ResponseWriter, resp *http.Respon } } -// copyResponseHeaders copies HTTP headers from a response to a writer. -func copyResponseHeaders(w http.ResponseWriter, headers http.Header) { - for k, vv := range headers { - for _, v := range vv { - w.Header().Add(k, v) - } - } -} - // gemVersion represents a version entry from the RubyGems versions API. type gemVersion struct { Number string `json:"number"` @@ -314,15 +300,7 @@ func (h *GemHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { } defer func() { _ = resp.Body.Close() }() - // Copy response headers - for k, vv := range resp.Header { - for _, v := range vv { - w.Header().Add(k, v) - } - } - - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) + h.proxy.relayResponse(w, r, resp, nil) } func init() { diff --git a/internal/handler/generic.go b/internal/handler/generic.go index 31f5dc39..a2458b28 100644 --- a/internal/handler/generic.go +++ b/internal/handler/generic.go @@ -133,7 +133,7 @@ func (h *GenericHandler) handleReleaseAsset(w http.ResponseWriter, r *http.Reque if result.Artifact.MediaType == "" { result.Artifact.MediaType = "application/octet-stream" } - serveArtifact(w, r.Method, result) + ServeArtifactRequest(w, r, result) } // handleMetadata serves any other path through the metadata cache. The query diff --git a/internal/handler/generic_test.go b/internal/handler/generic_test.go index 7d47cb94..e67a6b10 100644 --- a/internal/handler/generic_test.go +++ b/internal/handler/generic_test.go @@ -100,7 +100,8 @@ func TestGenericHandler_ReleaseAssetIsCachedAndServedWhenUpstreamDown(t *testing })) defer upstream.Close() - proxy, _, _, _ := setupTestProxy(t) + proxy, _, store, _ := setupTestProxy(t) + store.seekable = true fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) proxy.Fetcher = fetcher t.Cleanup(func() { _ = fetcher.Close() }) @@ -120,6 +121,23 @@ func TestGenericHandler_ReleaseAssetIsCachedAndServedWhenUpstreamDown(t *testing // Second request must be served from cache, even with the upstream down. available.Store(false) + rangeRequest := httptest.NewRequest(http.MethodGet, "/github"+testReleaseAssetPath, nil) + rangeRequest.Header.Set("Range", "bytes=0-2") + rangeResponse := httptest.NewRecorder() + h.Routes().ServeHTTP(rangeResponse, rangeRequest) + if rangeResponse.Code != http.StatusPartialContent { + t.Fatalf("range: status = %d, want 206: %s", rangeResponse.Code, rangeResponse.Body.String()) + } + if got := rangeResponse.Body.String(); got != string(asset[:3]) { + t.Errorf("range: body = %q, want %q", got, asset[:3]) + } + if got := rangeResponse.Header().Get("Content-Range"); got != "bytes 0-2/15" { + t.Errorf("range: Content-Range = %q, want %q", got, "bytes 0-2/15") + } + if got := upstreamRequests.Load(); got != 1 { + t.Errorf("upstream requests after range cache hit = %d, want 1", got) + } + w = serveGenericRequest(h, "/github"+testReleaseAssetPath) if w.Code != http.StatusOK { t.Fatalf("cached: status = %d, want 200: %s", w.Code, w.Body.String()) diff --git a/internal/handler/go.go b/internal/handler/go.go index b562aca2..fb85587d 100644 --- a/internal/handler/go.go +++ b/internal/handler/go.go @@ -117,7 +117,7 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul http.Error(w, "not found", http.StatusNotFound) return } - if errors.Is(err, ErrArtifactBlocked) { + if errors.Is(err, ErrArtifactBlocked) || errors.Is(err, ErrVersionDenied) { http.Error(w, err.Error(), http.StatusForbidden) return } @@ -126,7 +126,7 @@ func (h *GoHandler) handleDownload(w http.ResponseWriter, r *http.Request, modul return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // proxyUpstream forwards a request to proxy.golang.org without caching. diff --git a/internal/handler/handler.go b/internal/handler/handler.go index a78393d9..20dce9ab 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -19,6 +19,7 @@ import ( "github.com/git-pkgs/artifacts" "github.com/git-pkgs/cooldown" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/metrics" "github.com/git-pkgs/proxy/internal/packageurl" "github.com/git-pkgs/proxy/internal/scanner" @@ -95,6 +96,7 @@ func packagePURLStrings(ecosystem, name, version string) (string, string, error) const contentTypeJSON = "application/json" const ( + headerAccept = "Accept" headerAcceptEncoding = "Accept-Encoding" headerContentType = "Content-Type" headerContentLength = "Content-Length" @@ -154,7 +156,8 @@ type Proxy struct { Fetcher fetch.FetcherInterface Resolver *fetch.Resolver Logger *slog.Logger - Cooldown *cooldown.Config + Cooldown CooldownPolicy + Denylist *denylist.Policy CacheMetadata bool MetadataTTL time.Duration MetadataMaxSize int64 @@ -172,6 +175,11 @@ type Proxy struct { HTTPClient *http.Client AuthForURL func(string) (headerName, headerValue string) + // StreamArtifacts streams artifacts from upstream without storing them. + // Each request fetches its own copy: there is no cache to check and + // nothing for concurrent misses to share. + StreamArtifacts bool + // Scanners runs pre-cache artifact scanning (e.g. trivy, ClamAV, Wiz). // Nil or disabled means artifacts are cached without scanning. Scanners *scanner.Group @@ -184,6 +192,28 @@ type Proxy struct { // ScanFetchBaseURL is the address scanners use to reach this proxy to // pull staged artifacts. ScanFetchBaseURL string + + // inFlight coalesces concurrent cache misses for one artifact, so a single + // upstream fetch serves every waiting caller. Keyed by artifactCoalesceKey. + fetchMu sync.Mutex + inFlight map[string]*inflightFetch + + // inFlightMeta does the same for metadata misses. Keyed by + // metadataCoalesceKey. + metaMu sync.Mutex + inFlightMeta map[string]*inflightMetadata + + // rewrites caches metadata documents after their handler rewrites them. + // Nil leaves every request to rewrite its own copy. + rewrites *rewriteCache +} + +// CooldownPolicy supplies version-age filtering and package-specific durations. +type CooldownPolicy interface { + IsAllowed(ecosystem, packagePURL string, publishedAt time.Time) bool + Evaluate(ecosystem, packagePURL string, publishedAt, evaluatedAt time.Time) cooldown.Decision + For(ecosystem, packagePURL string) time.Duration + Enabled() bool } // NewProxy creates a new Proxy with the given dependencies. @@ -218,6 +248,33 @@ func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version if err != nil { return nil, err } + // Deny before resolution as well as before reading the cache. Mirror callers + // do not supply a filename, and resolving it can contact an upstream registry. + if p.Denylist.Denied(versionPURL) { + return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) + } + var info *fetch.ArtifactInfo + if filename == "" { + info, err = p.resolveArtifact(ctx, ecosystem, name, version) + if err != nil { + return nil, err + } + filename = info.Filename + if filename == "" { + return nil, errors.New("resolved artifact has no filename") + } + } + if p.StreamArtifacts { + if info == nil { + if info, err = p.resolveArtifact(ctx, ecosystem, name, version); err != nil { + return nil, err + } + } + return p.streamFromUpstream(ctx, ecosystem, name, version, filename, versionPURL, info.URL, "", + func(fetchCtx context.Context) (*fetch.Artifact, error) { + return p.Fetcher.Fetch(fetchCtx, info.URL) + }) + } if cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename); err != nil { return nil, err } else if cached != nil { @@ -225,7 +282,13 @@ func (p *Proxy) GetOrFetchArtifact(ctx context.Context, ecosystem, name, version } metrics.RecordCacheMiss(ecosystem) - return p.fetchAndCache(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL) + key := artifactCoalesceKey(versionPURL, filename, "", "") + recheck := func() (artifacts.Artifact, string, bool) { + return p.cachedArtifactRecord(pkgPURL, versionPURL, filename, "") + } + return p.coalesceFetch(ctx, key, recheck, func(fetchCtx context.Context) (artifacts.Artifact, string, error) { + return p.fetchAndCache(fetchCtx, ecosystem, name, version, filename, pkgPURL, versionPURL, info) + }) } // GetCachedArtifact retrieves an artifact from cache without contacting an upstream. @@ -238,9 +301,9 @@ func (p *Proxy) GetCachedArtifact(ctx context.Context, ecosystem, name, version, return p.checkCache(ctx, pkgPURL, versionPURL, filename) } -// ClearCachedArtifact removes both an artifact cache record and its stored -// bytes after an external integrity check fails. -func (p *Proxy) ClearCachedArtifact(ctx context.Context, ecosystem, name, version, filename string) error { +// ClearCachedArtifact clears an artifact cache record after an external +// integrity check fails, and queues its stored bytes for deletion. +func (p *Proxy) ClearCachedArtifact(ecosystem, name, version, filename string) error { if p.DB == nil || p.Storage == nil { return nil } @@ -255,14 +318,19 @@ func (p *Proxy) ClearCachedArtifact(ctx context.Context, ecosystem, name, versio if cached == nil { return nil } - if err := p.Storage.Delete(ctx, cached.StoragePath); err != nil { - return fmt.Errorf("deleting cached artifact: %w", err) - } - return p.DB.ClearArtifactCache(versionPURL, filename) + return p.DB.DiscardArtifact(versionPURL, filename, cached.StoragePath) } // checkCache looks up an artifact in the cache. Returns nil if not cached. +// With StreamArtifacts set it always reports a miss, so entries stored before the +// mode was enabled are never served. func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename string) (*CacheResult, error) { + if p.Denylist.Denied(versionPURL) { + return nil, fmt.Errorf("%w: %s", ErrVersionDenied, versionPURL) + } + if p.StreamArtifacts { + return nil, nil + } artifact, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) if err != nil { return nil, fmt.Errorf("checking artifact cache: %w", err) @@ -311,7 +379,7 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s "purl", versionPURL, "filename", filename, "path", artifact.StoragePath, "reason", reason) metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem)) - if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil { + if err := p.DB.DiscardArtifact(versionPURL, filename, artifact.StoragePath); err != nil { p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err) } }) @@ -354,13 +422,12 @@ func (p *Proxy) rejectUnusableCacheRecord(artifact *database.CachedArtifact, ver "purl", versionPURL, "filename", filename, "path", artifact.StoragePath, "error", cause) metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem)) - if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil { + if err := p.DB.DiscardArtifact(versionPURL, filename, artifact.StoragePath); err != nil { p.Logger.Warn("failed to clear unusable artifact from cache", "error", err) } } -func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string) (*CacheResult, error) { - // Resolve download URL +func (p *Proxy) resolveArtifact(ctx context.Context, ecosystem, name, version string) (*fetch.ArtifactInfo, error) { info, err := p.Resolver.Resolve(ctx, ecosystem, name, version) if err != nil { if errors.Is(err, fetch.ErrNotFound) { @@ -368,10 +435,18 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil } return nil, fmt.Errorf("resolving download URL: %w", err) } + return info, nil +} - // Use resolved filename if provided filename is empty - if filename == "" { - filename = info.Filename +func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string, info *fetch.ArtifactInfo) (artifacts.Artifact, string, error) { + // Reuse the resolution used for an unnamed cache lookup. Named requests + // still resolve only on a cache miss, inside the coalesced fetch. + if info == nil { + var err error + info, err = p.resolveArtifact(ctx, ecosystem, name, version) + if err != nil { + return artifacts.Artifact{}, "", err + } } p.Logger.Info("fetching from upstream", @@ -386,9 +461,9 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil metrics.RecordUpstreamFetch(ecosystem, fetchDuration) metrics.RecordUpstreamError(ecosystem, "fetch_failed") if errors.Is(err, fetch.ErrNotFound) { - return nil, ErrUpstreamNotFound + return artifacts.Artifact{}, "", ErrUpstreamNotFound } - return nil, fmt.Errorf("fetching from upstream: %w", err) + return artifacts.Artifact{}, "", fmt.Errorf("fetching from upstream: %w", err) } metrics.RecordUpstreamFetch(ecosystem, fetchDuration) @@ -405,8 +480,11 @@ func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, fil // verdict means a blocked artifact was never reachable by any client. On // block, the just-stored bytes are deleted and ErrArtifactBlocked is // returned; updateCacheDB is never called. -func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash string, artifact *fetch.Artifact) (*CacheResult, error) { - storagePath := storage.ArtifactPath(ecosystem, "", name, version, filename) +// +// It returns the artifact and its storage path, not a reader; callers get one +// from openStoredArtifact. +func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, upstreamURL, upstreamHash string, artifact *fetch.Artifact) (artifacts.Artifact, string, error) { + storagePath := storage.FetchPath(ecosystem, name, version, storage.NewFetchID(), filename) storeStart := time.Now() size, hash, err := p.Storage.Store(ctx, storagePath, artifact.Body) @@ -414,14 +492,14 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil metrics.RecordStorageOperation("write", time.Since(storeStart)) if err != nil { metrics.RecordStorageError("write") - return nil, fmt.Errorf("storing artifact: %w", err) + return artifacts.Artifact{}, "", fmt.Errorf("storing artifact: %w", err) } if !artifactHashMatches(hash, upstreamHash) { if delErr := p.Storage.Delete(ctx, storagePath); delErr != nil { p.Logger.Warn("failed to discard artifact with mismatched checksum", "path", storagePath, "error", delErr) } - return nil, fmt.Errorf("%w: upstream declared %s, got %s", ErrArtifactDigestMismatch, upstreamHash, hash) + return artifacts.Artifact{}, "", fmt.Errorf("%w: upstream declared %s, got %s", ErrArtifactDigestMismatch, upstreamHash, hash) } if p.Scanners != nil && p.Scanners.Enabled() { @@ -433,7 +511,7 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil p.Logger.Warn("failed to delete blocked artifact from storage", "path", storagePath, "error", delErr) } - return nil, err + return artifacts.Artifact{}, "", err } } @@ -448,10 +526,21 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil // Update database if err := p.updateCacheDB(ecosystem, name, pkgPURL, upstreamURL, storagePath, sharedArtifact); err != nil { p.Logger.Warn("failed to update cache database", "error", err) - // Continue anyway - we have the file + // Continue anyway - we have the file. Queue it for deletion in case + // no record points at it; the queue skips it while one does. + if qErr := p.DB.QueuePendingDelete(storagePath); qErr != nil { + p.Logger.Warn("failed to queue unrecorded artifact for deletion", "path", storagePath, "error", qErr) + } } - // Open the stored file to return + return sharedArtifact, storagePath, nil +} + +// openStoredArtifact gives one caller its own reader over just-committed +// bytes. Callers sharing a fetch cannot share a handle: it has one read +// position, so they would consume each other's bytes and the first Close would +// break the rest. +func (p *Proxy) openStoredArtifact(ctx context.Context, artifact artifacts.Artifact, storagePath string) (*CacheResult, error) { readStart := time.Now() reader, err := p.Storage.Open(ctx, storagePath) metrics.RecordStorageOperation("read", time.Since(readStart)) @@ -463,11 +552,138 @@ func (p *Proxy) storeArtifact(ctx context.Context, ecosystem, name, version, fil return &CacheResult{ Reader: reader, - Artifact: sharedArtifact, + Artifact: artifact, Cached: false, }, nil } +// artifactCoalesceKey identifies one artifact fetch. downloadURL and +// upstreamHash are included so callers expecting different bytes (multiple +// upstreams, or a re-published version) never share a fetch. The hash is +// lowercased because artifactHashMatches compares case-insensitively, so one +// digest in two casings describes one artifact and must not split the fetch. +func artifactCoalesceKey(versionPURL, filename, downloadURL, upstreamHash string) string { + return strings.Join([]string{versionPURL, filename, downloadURL, strings.ToLower(upstreamHash)}, "\x00") +} + +// cachedArtifactRecord reports an artifact already committed to the cache, +// without opening it. A caller checks the cache before it gets here, so a +// concurrent fetch can commit the same artifact in between; rechecking the +// record keeps that caller from fetching it a second time. A lookup error is +// reported as a miss, which costs a redundant fetch rather than a failure. +func (p *Proxy) cachedArtifactRecord(pkgPURL, versionPURL, filename, upstreamHash string) (artifacts.Artifact, string, bool) { + record, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) + if err != nil || record == nil { + return artifacts.Artifact{}, "", false + } + if !artifactHashMatches(record.Artifact.Digest.Encoded(), upstreamHash) { + return artifacts.Artifact{}, "", false + } + return record.Artifact, record.StoragePath, true +} + +// errSharedFetchAbandoned is what waiters see if the caller running a shared +// fetch panicked out of it. +var errSharedFetchAbandoned = errors.New("shared upstream fetch did not complete") + +// inflightFetch is one upstream fetch that concurrent callers share. val and +// err are written before done closes and read only after, so the close is the +// handoff. +type inflightFetch struct { + done chan struct{} + val fetchedArtifact + err error +} + +// coalesceFetch runs commit at most once for concurrent callers sharing key, +// then gives each its own reader over the stored bytes. +// +// The first caller in runs the fetch and the rest wait on it. Roles are +// decided under fetchMu rather than inferred afterwards, because the two need +// different cancellation behaviour: a waiter may leave when its own client goes +// away, while the caller running the fetch must see it through so +// storeArtifact's scan-on-disconnect handling still decides the outcome. +// +// Before fetching, that caller rechecks the cache through recheck: its own +// lookup happened before it took the key, so a fetch that committed in +// between would otherwise be repeated. A hit fills the shared value as a +// fetch would. +// +// commit runs on that caller's context, so cancellation behaves as it did +// uncoalesced and mirroring still relies on it aborting the fetch. If that +// caller goes away, everyone sharing the fetch gets its error and the key is +// released for a later retry. +// +// Every sharing caller still records a cache miss, so the gap between +// proxy_cache_misses_total and upstream fetch observations is what coalescing +// saved. +func (p *Proxy) coalesceFetch(ctx context.Context, key string, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { + p.fetchMu.Lock() + if p.inFlight == nil { + p.inFlight = make(map[string]*inflightFetch) + } + f, joined := p.inFlight[key] + if !joined { + f = &inflightFetch{done: make(chan struct{})} + p.inFlight[key] = f + } + p.fetchMu.Unlock() + + if !joined { + return p.runSharedFetch(ctx, key, f, recheck, commit) + } + + select { + case <-ctx.Done(): + // This caller gave up; the fetch continues for everyone else. + return nil, ctx.Err() + case <-f.done: + } + if f.err != nil { + return nil, f.err + } + return p.openStoredArtifact(ctx, f.val.artifact, f.val.storagePath) +} + +// runSharedFetch performs the fetch that joined callers are waiting on. It is +// never abandoned early, and always releases the key and wakes the waiters. +func (p *Proxy) runSharedFetch(ctx context.Context, key string, f *inflightFetch, recheck func() (artifacts.Artifact, string, bool), commit func(context.Context) (artifacts.Artifact, string, error)) (*CacheResult, error) { + // Set before running so a panicking commit leaves waiters with an error + // rather than a zero-valued artifact. + f.err = errSharedFetchAbandoned + defer func() { + p.fetchMu.Lock() + delete(p.inFlight, key) + p.fetchMu.Unlock() + close(f.done) + }() + + // A caller checks the cache before reaching here, so a fetch that finished + // in between would otherwise be repeated. Serve that record only if its + // bytes are still present: a record can outlive them, and refetching is + // the same recovery the cache lookup makes. + if stored, path, ok := recheck(); ok { + if res, err := p.openStoredArtifact(ctx, stored, path); err == nil { + f.val, f.err = fetchedArtifact{artifact: stored, storagePath: path}, nil + return res, nil + } + } + + stored, path, err := commit(ctx) + f.val, f.err = fetchedArtifact{artifact: stored, storagePath: path}, err + if err != nil { + return nil, err + } + return p.openStoredArtifact(ctx, stored, path) +} + +// fetchedArtifact is what a shared fetch hands its callers: metadata and a +// storage path, neither holding reader state. +type fetchedArtifact struct { + artifact artifacts.Artifact + storagePath string +} + // runScan generates a signed fetch URL for the just-staged artifact and // asks the configured scanners for a verdict. Returns a wrapped // ErrArtifactBlocked if any scanner blocks, or a scan-infrastructure error. @@ -553,7 +769,21 @@ func ServeArtifact(w http.ResponseWriter, result *CacheResult) { serveArtifact(w, http.MethodGet, result) } +// ServeArtifactRequest writes a CacheResult to an HTTP response using request +// headers such as Range and If-Range. +func ServeArtifactRequest(w http.ResponseWriter, r *http.Request, result *CacheResult) { + if r == nil { + ServeArtifact(w, result) + return + } + serveArtifactResponse(w, r.Method, r, result) +} + func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { + serveArtifactResponse(w, method, nil, result) +} + +func serveArtifactResponse(w http.ResponseWriter, method string, request *http.Request, result *CacheResult) { contentHash := "" if result.Artifact.Digest != "" { contentHash = result.Artifact.Digest.Encoded() @@ -574,25 +804,169 @@ func serveArtifact(w http.ResponseWriter, method string, result *CacheResult) { if result.Artifact.MediaType != "" { w.Header().Set(headerContentType, result.Artifact.MediaType) } - if result.Artifact.Size > 0 || (method == http.MethodHead && result.Artifact.Size == 0) { - w.Header().Set(headerContentLength, strconv.FormatInt(result.Artifact.Size, 10)) - } if contentHash != "" { w.Header().Set(headerETag, `"`+contentHash+`"`) } + var seeker io.Seeker + if result.Reader != nil { + seeker, _ = result.Reader.(io.Seeker) + } + if seeker != nil && result.Artifact.Size >= 0 { + w.Header().Set("Accept-Ranges", "bytes") + if request != nil && serveArtifactRange(w, request, result, seeker) { + return + } + } + + if result.Artifact.Size > 0 || (method == http.MethodHead && result.Artifact.Size == 0) { + w.Header().Set(headerContentLength, strconv.FormatInt(result.Artifact.Size, 10)) + } + w.WriteHeader(http.StatusOK) if method != http.MethodHead && result.Reader != nil { - buffer := artifactCopyBufferPool.Get().(*[]byte) - defer artifactCopyBufferPool.Put(buffer) - // Hide optional ReaderFrom methods so io.CopyBuffer uses the pooled buffer. - _, _ = io.CopyBuffer(struct{ io.Writer }{w}, result.Reader, *buffer) + copyArtifactBody(w, result.Reader, result.Artifact.Size, false) + } +} + +type parsedByteRange struct { + start int64 + end int64 +} + +func serveArtifactRange(w http.ResponseWriter, request *http.Request, result *CacheResult, seeker io.Seeker) bool { + if request.Method != http.MethodGet { + return false + } + ranges := request.Header.Values("Range") + if len(ranges) != 1 || !ifRangeMatches(request, w.Header().Get(headerETag)) { + return false + } + + byteRange, valid, satisfiable := parseByteRange(ranges[0], result.Artifact.Size) + if !valid { + return false + } + if !satisfiable { + w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", result.Artifact.Size)) + w.Header().Set(headerContentLength, "0") + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return true + } + + if _, err := seeker.Seek(byteRange.start, io.SeekStart); err != nil { + w.Header().Del(headerContentType) + w.Header().Del(headerContentLength) + w.Header().Del(headerETag) + w.Header().Del("Accept-Ranges") + http.Error(w, "failed to seek cached artifact", http.StatusInternalServerError) + return true + } + + length := byteRange.end - byteRange.start + 1 + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", byteRange.start, byteRange.end, result.Artifact.Size)) + w.Header().Set(headerContentLength, strconv.FormatInt(length, 10)) + w.WriteHeader(http.StatusPartialContent) + copyArtifactBody(w, result.Reader, length, true) + return true +} + +func ifRangeMatches(request *http.Request, etag string) bool { + values := request.Header.Values("If-Range") + if len(values) == 0 { + return true + } + return len(values) == 1 && etag != "" && strings.TrimSpace(values[0]) == etag +} + +func parseByteRange(value string, size int64) (parsedByteRange, bool, bool) { + unit, spec, ok := strings.Cut(strings.TrimSpace(value), "=") + if !ok || !strings.EqualFold(strings.TrimSpace(unit), "bytes") { + return parsedByteRange{}, false, false + } + spec = strings.TrimSpace(spec) + if spec == "" || strings.Contains(spec, ",") { + return parsedByteRange{}, false, false + } + first, last, ok := strings.Cut(spec, "-") + if !ok { + return parsedByteRange{}, false, false + } + + if first == "" { + suffixLength, ok := parseRangeNumber(last) + if !ok { + return parsedByteRange{}, false, false + } + if size == 0 || suffixLength == 0 { + return parsedByteRange{}, true, false + } + if suffixLength >= size { + return parsedByteRange{start: 0, end: size - 1}, true, true + } + return parsedByteRange{start: size - suffixLength, end: size - 1}, true, true + } + + start, ok := parseRangeNumber(first) + if !ok { + return parsedByteRange{}, false, false + } + if last == "" { + if size == 0 || start >= size { + return parsedByteRange{}, true, false + } + return parsedByteRange{start: start, end: size - 1}, true, true + } + end, ok := parseRangeNumber(last) + if !ok || start > end { + return parsedByteRange{}, false, false + } + if size == 0 || start >= size { + return parsedByteRange{}, true, false + } + if end >= size { + end = size - 1 + } + return parsedByteRange{start: start, end: end}, true, true +} + +func parseRangeNumber(value string) (int64, bool) { + if value == "" { + return 0, false + } + for _, digit := range value { + if digit < '0' || digit > '9' { + return 0, false + } + } + number, err := strconv.ParseInt(value, 10, 64) + return number, err == nil +} + +func copyArtifactBody(w http.ResponseWriter, reader io.Reader, expectedSize int64, bounded bool) { + buffer := artifactCopyBufferPool.Get().(*[]byte) + defer artifactCopyBufferPool.Put(buffer) + + source := reader + if bounded { + source = io.LimitReader(reader, expectedSize) + } + // Hide optional ReaderFrom methods so io.CopyBuffer uses the pooled buffer. + written, err := io.CopyBuffer(struct{ io.Writer }{w}, source, *buffer) + if err != nil || (bounded && written != expectedSize) || (!bounded && expectedSize > 0 && written != expectedSize) { + // Headers are already committed, so an error status is no longer + // possible. Aborting leaves the response unterminated so clients discard + // a truncated or unverified artifact. + // net/http recovers ErrAbortHandler for this request and keeps the server + // running; on HTTP/1.x it may close this connection as well. + panic(http.ErrAbortHandler) } } // ProxyUpstream forwards a request to an upstream URL without caching. // It copies the request, forwards specified headers, and streams the response back. -// If forwardHeaders is nil, all response headers are copied. +// forwardHeaders controls the request headers sent upstream. End-to-end response +// headers and trailers are relayed independently of that list. func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamURL string, forwardHeaders []string) { p.Logger.Debug("proxying to upstream", "url", upstreamURL) @@ -618,17 +992,10 @@ func (p *Proxy) ProxyUpstream(w http.ResponseWriter, r *http.Request, upstreamUR } defer func() { _ = resp.Body.Close() }() - for k, vv := range resp.Header { - for _, v := range vv { - w.Header().Add(k, v) - } - } - - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) + p.relayResponse(w, r, resp, nil) } -// ProxyFile forwards a file request to upstream, copying all response headers. +// ProxyFile forwards a file request, relaying end-to-end headers and trailers. func (p *Proxy) ProxyFile(w http.ResponseWriter, r *http.Request, upstreamURL string) { req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) if err != nil { @@ -644,14 +1011,7 @@ func (p *Proxy) ProxyFile(w http.ResponseWriter, r *http.Request, upstreamURL st } defer func() { _ = resp.Body.Close() }() - for key, values := range resp.Header { - for _, v := range values { - w.Header().Add(key, v) - } - } - - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) + p.relayResponse(w, r, resp, nil) } // JSONError writes a JSON error response. @@ -667,15 +1027,18 @@ var ErrUpstreamNotFound = fmt.Errorf("upstream: %w", fetch.ErrNotFound) // ErrArtifactBlocked indicates a pre-cache security scan blocked the artifact. var ErrArtifactBlocked = errors.New("artifact blocked by security scan") +// ErrVersionDenied indicates an operator explicitly denied this package version. +var ErrVersionDenied = errors.New("package version blocked by denylist") + // serveArtifactError writes response for a failed fetch: -// 404 when upstream reports artifact missing, 403 when a security scan -// blocked the artifact, 502 otherwise. +// 404 when upstream reports artifact missing, 403 when a security scan or +// denylist blocked the artifact, 502 otherwise. func (p *Proxy) serveArtifactError(w http.ResponseWriter, err error, clientMsg string) { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) return } - if errors.Is(err, ErrArtifactBlocked) { + if errors.Is(err, ErrArtifactBlocked) || errors.Is(err, ErrVersionDenied) { JSONError(w, http.StatusForbidden, err.Error()) return } @@ -697,45 +1060,32 @@ func metadataStoragePath(ecosystem, cacheKey string) string { // cacheKey is typically the package name but can include subpath components. // Optional acceptHeaders specify the Accept header(s) to send; defaults to application/json. func (p *Proxy) FetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL string, acceptHeaders ...string) ([]byte, string, error) { - return p.fetchOrCacheMetadata(ctx, ecosystem, cacheKey, upstreamURL, false, acceptHeaders...) + body, contentType, _, err := p.fetchOrCacheMetadata(ctx, ecosystem, cacheKey, upstreamURL, "", nil, acceptHeaders...) + return body, contentType, err } -// fetchOrCacheMetadata implements FetchOrCacheMetadata. When verbatim is true -// (the ProxyCached path, which serves upstream bytes through unchanged) the -// upstream is fetched with Accept-Encoding: identity so signed and hash-pinned -// index files are cached exactly as sent. Direct callers that parse or rewrite -// the body pass verbatim=false and keep transparent transfer compression. -func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL string, verbatim bool, acceptHeaders ...string) ([]byte, string, error) { +// fetchOrCacheMetadata implements FetchOrCacheMetadata. acceptEncoding controls +// the upstream Accept-Encoding: an empty string leaves it unset so Go +// transparently decompresses (for direct callers that parse or rewrite the +// body); any non-empty value is sent verbatim, which disables Go's +// decompression so the wire bytes and their Content-Encoding are stored and +// replayed as sent. The ProxyCached path uses "identity" for signed indexes and +// "gzip" where both hops should stay compressed. +// validate, when supplied, runs before caching or serving a document. Validation +// failures follow the same stale-cache fallback path as upstream failures. It +// runs for every caller, including one that shares another caller's fetch, so +// it can also decode the document into request-local state; a caller that +// joined a fetch gets its own validation error directly. It must not modify the +// body, which joined callers share. +func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, upstreamURL, acceptEncoding string, validate func([]byte) error, acceptHeaders ...string) ([]byte, string, string, error) { if containsPathTraversal(cacheKey) { - return nil, "", fmt.Errorf("invalid cache key: %q", cacheKey) - } - - storagePath := metadataStoragePath(ecosystem, cacheKey) - - // Check for existing cache entry (for ETag revalidation and TTL) - var entry *database.MetadataCacheEntry - if p.CacheMetadata && p.DB != nil { - entry, _ = p.DB.GetMetadataCache(ecosystem, cacheKey) + return nil, "", "", fmt.Errorf("invalid cache key: %q", cacheKey) } // Serve from cache if within TTL (skip upstream entirely) - if entry != nil && p.MetadataTTL > 0 && entry.FetchedAt.Valid { - if time.Since(entry.FetchedAt.Time) < p.MetadataTTL { - cached, readErr := p.Storage.Open(ctx, entry.StoragePath) - if readErr == nil { - defer func() { _ = cached.Close() }() - data, readErr := p.ReadMetadata(cached) - if readErr == nil { - ct := contentTypeJSON - if entry.ContentType.Valid { - ct = entry.ContentType.String - } - metrics.RecordCacheHit(ecosystem) - return data, ct, nil - } - } - // Cache file missing/unreadable, fall through to upstream - } + if _, hit := p.cachedMetadataState(ctx, ecosystem, cacheKey, validate); hit != nil { + metrics.RecordCacheHit(ecosystem) + return hit.body, hit.contentType, hit.contentEncoding, nil } p.recordMetadataCacheMiss(ecosystem) @@ -744,44 +1094,191 @@ func (p *Proxy) fetchOrCacheMetadata(ctx context.Context, ecosystem, cacheKey, u accept = acceptHeaders[0] } - // Try upstream - meta, err := p.fetchUpstreamMetadata(ctx, upstreamURL, entry, accept, verbatim) + res := p.coalescedMetadataMiss(ctx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate) + return res.body, res.contentType, res.contentEncoding, res.err +} + +// coalescedMetadataMiss handles a metadata cache miss, sharing one upstream +// fetch among concurrent callers with the same key. +func (p *Proxy) coalescedMetadataMiss(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error) metadataResult { + key := metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate != nil) + res, shared := p.coalesceMetadata(ctx, key, func(fetchCtx context.Context) metadataResult { + // The caller's lookup ran before it took the key, so a fetch that + // finished in between has already refreshed the row. Recheck it + // rather than fetching again, and revalidate against the row as it + // is now. + entry, hit := p.cachedMetadataState(fetchCtx, ecosystem, cacheKey, validate) + if hit != nil { + return *hit + } + return p.fetchMetadataFromUpstream(fetchCtx, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, validate, entry) + }) + // The fetch ran the first caller's validate. A caller that joined it runs + // its own on the shared bytes, since validate may also decode them for it. + if shared && res.err == nil && validate != nil { + if err := validate(res.body); err != nil { + return metadataResult{err: err} + } + } + return res +} + +// cachedMetadataState reads the cache row for a metadata lookup. hit is set +// when the row is within TTL and its bytes are usable. Otherwise entry is the +// row to revalidate against with its ETag: nil when there is none, or when +// validate rejected the cached body, since revalidating an unusable body would +// keep it. +func (p *Proxy) cachedMetadataState(ctx context.Context, ecosystem, cacheKey string, validate func([]byte) error) (*database.MetadataCacheEntry, *metadataResult) { + if !p.CacheMetadata || p.DB == nil { + return nil, nil + } + entry, _ := p.DB.GetMetadataCache(ecosystem, cacheKey) + if entry != nil && p.MetadataTTL > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) < p.MetadataTTL { + data, ct, err := p.readCachedMetadata(ctx, entry, validate) + if err == nil { + return entry, &metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String} + } + if validate != nil { + return nil, nil + } + // Cache file missing/unreadable, fall through to upstream + } + return entry, nil +} + +// fetchMetadataFromUpstream fetches metadata after a cache miss, caches it, +// and falls back to the cached copy if upstream fails. entry is the row to +// revalidate against, or nil. +func (p *Proxy) fetchMetadataFromUpstream(ctx context.Context, ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validate func([]byte) error, entry *database.MetadataCacheEntry) metadataResult { + meta, err := p.fetchUpstreamMetadata(ctx, upstreamURL, entry, accept, acceptEncoding) if errors.Is(err, errStale304) { // 304 but cached file is gone; retry without ETag - meta, err = p.fetchUpstreamMetadata(ctx, upstreamURL, nil, accept, verbatim) + meta, err = p.fetchUpstreamMetadata(ctx, upstreamURL, nil, accept, acceptEncoding) + } + if err == nil && validate != nil { + err = validate(meta.body) } if err == nil { if p.CacheMetadata { - p.cacheMetadataBlob(ctx, ecosystem, cacheKey, storagePath, meta) + p.cacheMetadataBlob(ctx, ecosystem, cacheKey, metadataStoragePath(ecosystem, cacheKey), meta) } - return meta.body, meta.contentType, nil + return metadataResult{body: meta.body, contentType: meta.contentType, contentEncoding: meta.contentEncoding} } // Upstream failed -- fall back to cache if available if !p.CacheMetadata || entry == nil { - return nil, "", fmt.Errorf("upstream failed and no cached metadata: %w", err) + return metadataResult{err: fmt.Errorf("upstream failed and no cached metadata: %w", err)} } p.Logger.Warn("upstream metadata fetch failed, checking cache", "ecosystem", ecosystem, "key", cacheKey, "error", err) - cached, readErr := p.Storage.Open(ctx, entry.StoragePath) + // Re-read the row so the encoding describes the blob as it is now: a + // concurrent refetch may have replaced both since entry was read above + // (an identity blob swapped for a gzip one during rollout). + entry = p.currentMetadataEntry(ecosystem, cacheKey, entry) + + data, ct, readErr := p.readCachedMetadata(ctx, entry, validate) if readErr != nil { - return nil, "", fmt.Errorf("upstream failed and cached file missing: %w", err) + return metadataResult{err: fmt.Errorf("upstream failed and cached metadata unusable (%v): %w", readErr, err)} } - defer func() { _ = cached.Close() }() - data, readErr := p.ReadMetadata(cached) - if readErr != nil { - return nil, "", fmt.Errorf("upstream failed and cached read error: %w", err) + p.Logger.Info("serving metadata from cache", + "ecosystem", ecosystem, "key", cacheKey) + return metadataResult{body: data, contentType: ct, contentEncoding: entry.ContentEncoding.String} +} + +// metadataResult is what a metadata lookup hands back. A shared fetch gives +// every waiter the same body, so callers must treat it as read-only. +type metadataResult struct { + body []byte + contentType string + contentEncoding string + err error +} + +// inflightMetadata is one metadata fetch that concurrent callers share. res is +// written before done closes and read only after, so the close is the handoff. +type inflightMetadata struct { + done chan struct{} + res metadataResult + + // waiters counts callers that joined the fetch, guarded by metaMu. Tests + // read it to know every caller has joined before the fetch finishes. + waiters int +} + +// metadataCoalesceKey identifies metadata requests that can share one fetch. +// Accept and Accept-Encoding are part of it because they change the bytes +// upstream returns: npm serves an abbreviated or a full document for the same +// package, and the cached-proxy paths ask for identity or gzip. Whether the +// caller validates is part of it so an unvalidated caller is never handed a +// result that skipped validation, or the reverse. +func metadataCoalesceKey(ecosystem, cacheKey, upstreamURL, accept, acceptEncoding string, validated bool) string { + return strings.Join([]string{ecosystem, cacheKey, upstreamURL, accept, acceptEncoding, strconv.FormatBool(validated)}, "\x00") +} + +// coalesceMetadata runs fetch at most once for concurrent callers sharing key +// and gives each the result, reporting whether this caller joined another's +// fetch rather than running it. It follows coalesceFetch with one difference: +// fetch runs on a context detached from the first caller's cancellation. A +// metadata fetch has no scan or mirror that depends on the caller aborting it, +// and CI jobs asking for the same Composer or npm metadata would otherwise all +// fail when the first of them disconnects. It stays bounded by the HTTP +// client's timeout. Waiters still leave when their own context ends. +func (p *Proxy) coalesceMetadata(ctx context.Context, key string, fetch func(context.Context) metadataResult) (metadataResult, bool) { + p.metaMu.Lock() + if p.inFlightMeta == nil { + p.inFlightMeta = make(map[string]*inflightMetadata) + } + f, joined := p.inFlightMeta[key] + if joined { + f.waiters++ + } else { + f = &inflightMetadata{done: make(chan struct{})} + p.inFlightMeta[key] = f + } + p.metaMu.Unlock() + + if joined { + select { + case <-ctx.Done(): + return metadataResult{err: ctx.Err()}, true + case <-f.done: + return f.res, true + } } + // Set before running so a panicking fetch leaves waiters with an error + // rather than an empty body. + f.res = metadataResult{err: errSharedFetchAbandoned} + defer func() { + p.metaMu.Lock() + delete(p.inFlightMeta, key) + p.metaMu.Unlock() + close(f.done) + }() + f.res = fetch(context.WithoutCancel(ctx)) + return f.res, false +} + +func (p *Proxy) readCachedMetadata(ctx context.Context, entry *database.MetadataCacheEntry, validate func([]byte) error) ([]byte, string, error) { + cached, err := p.Storage.Open(ctx, entry.StoragePath) + if err != nil { + return nil, "", err + } + defer func() { _ = cached.Close() }() + data, err := p.ReadMetadata(cached) + if err == nil && validate != nil { + err = validate(data) + } + if err != nil { + return nil, "", err + } ct := contentTypeJSON if entry.ContentType.Valid { ct = entry.ContentType.String } - p.Logger.Info("serving metadata from cache", - "ecosystem", ecosystem, "key", cacheKey) return data, ct, nil } @@ -801,20 +1298,19 @@ type upstreamMetadata struct { } // fetchUpstreamMetadata fetches metadata from upstream, using ETag for conditional revalidation. -// It requests the identity encoding and never transparently decompresses, so the returned -// bytes are exactly what the upstream sent; any Content-Encoding the upstream applied -// anyway is reported alongside so callers can store and replay it. -func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept string, verbatim bool) (*upstreamMetadata, error) { +// When acceptEncoding is non-empty it is sent as the Accept-Encoding header, which disables Go's +// transparent decompression (it only applies when the transport adds the header itself), so the +// returned bytes are exactly what the upstream sent and any Content-Encoding it applied is reported +// alongside for the caller to store and replay. An empty acceptEncoding leaves Go to negotiate and +// decompress transparently. +func (p *Proxy) fetchUpstreamMetadata(ctx context.Context, upstreamURL string, entry *database.MetadataCacheEntry, accept, acceptEncoding string) (*upstreamMetadata, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, upstreamURL, nil) if err != nil { return nil, fmt.Errorf("creating request: %w", err) } req.Header.Set("Accept", accept) - if verbatim { - // Setting Accept-Encoding explicitly disables Go's transparent gzip - // decompression (it only applies when the transport adds the header - // itself), so signed index files are cached byte-for-byte as sent. - req.Header.Set(headerAcceptEncoding, "identity") + if acceptEncoding != "" { + req.Header.Set(headerAcceptEncoding, acceptEncoding) } p.applyUpstreamAuth(req) @@ -893,7 +1389,7 @@ func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor return } - _ = p.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ + err = p.DB.UpsertMetadataCache(&database.MetadataCacheEntry{ Ecosystem: ecosystem, Name: cacheKey, StoragePath: storagePath, @@ -904,14 +1400,34 @@ func (p *Proxy) cacheMetadataBlob(ctx context.Context, ecosystem, cacheKey, stor LastModified: sql.NullTime{Time: meta.lastModified, Valid: !meta.lastModified.IsZero()}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, }) + if err != nil { + // The blob is written but the row describing it is not, so a later + // TTL hit or stale fallback would serve these bytes with the previous + // row's encoding. Drop the blob so row and bytes can never disagree; + // the next request refetches instead. + p.Logger.Warn("failed to record cached metadata, discarding blob", "ecosystem", ecosystem, "key", cacheKey, "error", err) + if delErr := p.Storage.Delete(ctx, storagePath); delErr != nil { + p.Logger.Warn("failed to discard metadata blob", "ecosystem", ecosystem, "key", cacheKey, "error", delErr) + } + } +} + +// currentMetadataEntry re-reads the metadata cache row and returns it, or +// fallback when the row cannot be read. Used before serving a stored blob so +// its encoding comes from the row as it is now rather than from a snapshot +// taken before the upstream fetch. +func (p *Proxy) currentMetadataEntry(ecosystem, cacheKey string, fallback *database.MetadataCacheEntry) *database.MetadataCacheEntry { + if fresh, err := p.DB.GetMetadataCache(ecosystem, cacheKey); err == nil && fresh != nil { + return fresh + } + return fallback } // cachedMeta holds cache validators and freshness state from a metadata cache entry. type cachedMeta struct { - etag string - lastModified time.Time - contentEncoding string - stale bool + etag string + lastModified time.Time + stale bool } // lookupCachedMeta retrieves cache validators for a metadata entry. @@ -930,9 +1446,6 @@ func (p *Proxy) lookupCachedMeta(ecosystem, cacheKey string) cachedMeta { if entry.LastModified.Valid { cm.lastModified = entry.LastModified.Time } - if entry.ContentEncoding.Valid { - cm.contentEncoding = entry.ContentEncoding.String - } // If FetchedAt is older than TTL, upstream must have failed and // we served from stale cache (successful fetches update FetchedAt). if p.MetadataTTL > 0 && entry.FetchedAt.Valid && time.Since(entry.FetchedAt.Time) > p.MetadataTTL { @@ -946,13 +1459,22 @@ func (p *Proxy) lookupCachedMeta(ecosystem, cacheKey string) cachedMeta { // When metadata caching is disabled, the response is streamed directly to avoid buffering // large metadata responses (e.g. npm packages with many versions) in memory. func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey string, acceptHeaders ...string) { + p.proxyCachedWithEncoding(w, r, upstreamURL, ecosystem, cacheKey, "identity", acceptHeaders...) +} + +// proxyCachedWithEncoding is ProxyCached with an explicit upstream Accept-Encoding. +// "identity" preserves signed index bytes (the default); "gzip" keeps both hops +// compressed for large, non-hash-pinned metadata whose clients decode gzip +// (Homebrew API and Conda repodata). The stored bytes and Content-Encoding are +// replayed verbatim either way. +func (p *Proxy) proxyCachedWithEncoding(w http.ResponseWriter, r *http.Request, upstreamURL, ecosystem, cacheKey, acceptEncoding string, acceptHeaders ...string) { if !p.CacheMetadata { // Stream directly without buffering when caching is off. - p.proxyMetadataStream(w, r, upstreamURL, acceptHeaders...) + p.proxyMetadataStream(w, r, upstreamURL, acceptEncoding, acceptHeaders...) return } - body, contentType, err := p.fetchOrCacheMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, true, acceptHeaders...) + body, contentType, contentEncoding, err := p.fetchOrCacheMetadata(r.Context(), ecosystem, cacheKey, upstreamURL, acceptEncoding, nil, acceptHeaders...) if err != nil { if errors.Is(err, ErrUpstreamNotFound) { http.Error(w, "not found", http.StatusNotFound) @@ -963,12 +1485,21 @@ func (p *Proxy) ProxyCached(w http.ResponseWriter, r *http.Request, upstreamURL, return } - p.writeMetadataCachedResponse(w, r, ecosystem, cacheKey, body, contentType) + p.writeMetadataCachedResponseWithEncoding(w, r, ecosystem, cacheKey, body, contentType, contentEncoding) } // writeMetadataCachedResponse writes a cached metadata response and handles // conditional request headers using metadata cache validators. func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType string) { + p.writeMetadataCachedResponseWithEncoding(w, r, ecosystem, cacheKey, body, contentType, "") +} + +// writeMetadataCachedResponseWithEncoding is writeMetadataCachedResponse with +// an explicit Content-Encoding. contentEncoding must describe the body being +// written; it is passed in rather than re-read from the cache row, which is +// missing or stale when the metadata cache write failed and would otherwise +// mislabel the bytes. +func (p *Proxy) writeMetadataCachedResponseWithEncoding(w http.ResponseWriter, r *http.Request, ecosystem, cacheKey string, body []byte, contentType, contentEncoding string) { cm := p.lookupCachedMeta(ecosystem, cacheKey) if cm.etag != "" { @@ -992,8 +1523,8 @@ func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Reque w.Header().Set(headerContentType, contentType) w.Header().Set(headerContentLength, strconv.Itoa(len(body))) - if cm.contentEncoding != "" { - w.Header().Set(headerContentEncoding, cm.contentEncoding) + if contentEncoding != "" { + w.Header().Set(headerContentEncoding, contentEncoding) } if cm.stale { w.Header().Set("Warning", `110 - "Response is Stale"`) @@ -1006,7 +1537,7 @@ func (p *Proxy) writeMetadataCachedResponse(w http.ResponseWriter, r *http.Reque // proxyMetadataStream forwards an upstream metadata response by streaming it to the client // without buffering the full body in memory. -func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upstreamURL string, acceptHeaders ...string) { +func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upstreamURL, acceptEncoding string, acceptHeaders ...string) { req, err := http.NewRequestWithContext(r.Context(), r.Method, upstreamURL, nil) if err != nil { http.Error(w, "failed to create request", http.StatusInternalServerError) @@ -1018,10 +1549,14 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst accept = acceptHeaders[0] } req.Header.Set("Accept", accept) - // ProxyCached serves bytes through verbatim, so request identity to keep - // Go from transparently decompressing (and stripping the Content-Encoding - // of) signed index files, regardless of what the client negotiated. - req.Header.Set(headerAcceptEncoding, "identity") + // Set Accept-Encoding explicitly (identity, or gzip for compressible + // verbatim metadata) so Go does not transparently decompress and strip the + // Content-Encoding of the bytes we forward, regardless of what the client + // negotiated. An empty value leaves the header unset, as in + // fetchUpstreamMetadata. + if acceptEncoding != "" { + req.Header.Set(headerAcceptEncoding, acceptEncoding) + } p.applyUpstreamAuth(req) for _, header := range []string{"If-Modified-Since", "If-None-Match"} { @@ -1037,16 +1572,13 @@ func (p *Proxy) proxyMetadataStream(w http.ResponseWriter, r *http.Request, upst } defer func() { _ = resp.Body.Close() }() - for _, header := range []string{headerContentType, headerContentLength, headerContentEncoding, headerLastModified, headerETag} { - if v := resp.Header.Get(header); v != "" { - w.Header().Set(header, v) + p.relayResponse(w, r, resp, func(dst, src http.Header) { + for _, header := range []string{headerContentType, headerContentLength, headerContentEncoding, headerLastModified, headerETag} { + if v := src.Get(header); v != "" { + dst.Set(header, v) + } } - } - - w.WriteHeader(resp.StatusCode) - if r.Method != http.MethodHead { - _, _ = io.Copy(w, resp.Body) - } + }) } func (p *Proxy) applyUpstreamAuth(req *http.Request) { @@ -1094,22 +1626,46 @@ func (p *Proxy) getOrFetchArtifactFromURL(ctx context.Context, ecosystem, name, } func (p *Proxy) getOrFetchArtifactFromURLWithCachePURLs(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash string) (*CacheResult, error) { + // Some protocols use qualified cache keys. Check their package identity too. + if p.versionDenied(ecosystem, name, version) { + return nil, fmt.Errorf("%w: %s", ErrVersionDenied, canonicalVersionPURL(ecosystem, name, version)) + } + if p.StreamArtifacts { + return p.streamFromUpstream(ctx, ecosystem, name, version, filename, versionPURL, downloadURL, upstreamHash, + func(fetchCtx context.Context) (*fetch.Artifact, error) { + return p.Fetcher.FetchWithHeaders(fetchCtx, downloadURL, headers) + }) + } if cached, err := p.getCachedArtifactWithUpstreamHash(ctx, pkgPURL, versionPURL, filename, upstreamHash); err != nil { return nil, err } else if cached != nil { return cached, nil } metrics.RecordCacheMiss(ecosystem) + return p.coalescedFetchFromURL(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers, upstreamHash) +} - return p.fetchAndCacheFromURL(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers, upstreamHash) +// coalescedFetchFromURL fetches an artifact the cache could not serve, sharing +// the fetch with concurrent callers. The caller running it discards a stale +// entry under the key, where it cannot delete a fetch that just replaced it. +func (p *Proxy) coalescedFetchFromURL(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash string) (*CacheResult, error) { + key := artifactCoalesceKey(versionPURL, filename, downloadURL, upstreamHash) + recheck := func() (artifacts.Artifact, string, bool) { + return p.cachedArtifactRecord(pkgPURL, versionPURL, filename, upstreamHash) + } + return p.coalesceFetch(ctx, key, recheck, func(fetchCtx context.Context) (artifacts.Artifact, string, error) { + p.discardStaleArtifact(pkgPURL, versionPURL, filename, upstreamHash) + return p.fetchAndCacheFromURL(fetchCtx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, headers, upstreamHash) + }) } // getCachedArtifactWithUpstreamHash returns a cached artifact whose recorded // content hash matches the checksum the upstream currently declares for it. // This detects an upstream re-publishing under the same version, which the // stream integrity check in checkCache cannot: that check only verifies the -// stored blob against the hash recorded when it was cached. On mismatch the -// stale entry is discarded and nil is returned so the caller re-fetches. +// stored blob against the hash recorded when it was cached. A stale entry is +// a miss and is left in place: the fetch that replaces it discards it under +// the coalescing key. func (p *Proxy) getCachedArtifactWithUpstreamHash(ctx context.Context, pkgPURL, versionPURL, filename, upstreamHash string) (*CacheResult, error) { cached, err := p.checkCache(ctx, pkgPURL, versionPURL, filename) if err != nil || cached == nil { @@ -1118,23 +1674,64 @@ func (p *Proxy) getCachedArtifactWithUpstreamHash(ctx context.Context, pkgPURL, if artifactHashMatches(cached.Artifact.Digest.Encoded(), upstreamHash) { return cached, nil } - if cached.Reader != nil { _ = cached.Reader.Close() } - p.Logger.Warn("cached artifact hash disagrees with upstream metadata, discarding", - "purl", versionPURL, "filename", filename, "cached", cached.Artifact.Digest.Encoded(), "upstream", upstreamHash) - p.discardCachedArtifact(ctx, versionPURL, filename, cached.storagePath) return nil, nil } -func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash string) (*CacheResult, error) { +// discardStaleArtifact clears the cached entry when its digest disagrees with +// upstreamHash, queueing its object for deletion. It runs under the coalescing +// key, after the recheck, so an entry a previous fetch refreshed is kept. +func (p *Proxy) discardStaleArtifact(pkgPURL, versionPURL, filename, upstreamHash string) { + record, err := p.DB.GetCachedArtifact(pkgPURL, versionPURL, filename) + if err != nil { + p.Logger.Warn("failed to read cache record before refetch", + "purl", versionPURL, "filename", filename, "error", err) + return + } + if record == nil || artifactHashMatches(record.Artifact.Digest.Encoded(), upstreamHash) { + return + } + p.Logger.Warn("cached artifact hash disagrees with upstream metadata, discarding", + "purl", versionPURL, "filename", filename, "cached", record.Artifact.Digest.Encoded(), "upstream", upstreamHash) + if err := p.DB.DiscardArtifact(versionPURL, filename, record.StoragePath); err != nil { + p.Logger.Warn("failed to clear artifact cache record", "purl", versionPURL, "filename", filename, "error", err) + } +} + +func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL string, headers http.Header, upstreamHash string) (artifacts.Artifact, string, error) { p.Logger.Info("fetching from upstream", "ecosystem", ecosystem, "name", name, "version", version, "url", downloadURL) fetchStart := time.Now() artifact, err := p.Fetcher.FetchWithHeaders(ctx, downloadURL, headers) metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) + if err != nil { + metrics.RecordUpstreamError(ecosystem, "fetch_failed") + if errors.Is(err, fetch.ErrNotFound) { + return artifacts.Artifact{}, "", ErrUpstreamNotFound + } + return artifacts.Artifact{}, "", fmt.Errorf("fetching from upstream: %w", err) + } + + return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, upstreamHash, artifact) +} + +// streamFromUpstream fetches an artifact when StreamArtifacts is set and hands its +// body to the caller without storing it. +// +// With upstreamHash set the body is verified as it streams. The size is left +// unknown so the response goes out chunked: a mismatch only shows at EOF, +// after the bytes have been sent, and an unterminated chunked response is the +// only way left to make the client reject them (see serveArtifact). +func (p *Proxy) streamFromUpstream(ctx context.Context, ecosystem, name, version, filename, versionPURL, upstreamURL, upstreamHash string, fetchArtifact func(context.Context) (*fetch.Artifact, error)) (*CacheResult, error) { + p.Logger.Info("streaming from upstream", + "ecosystem", ecosystem, "name", name, "version", version, "url", upstreamURL) + + fetchStart := time.Now() + artifact, err := fetchArtifact(ctx) + metrics.RecordUpstreamFetch(ecosystem, time.Since(fetchStart)) if err != nil { metrics.RecordUpstreamError(ecosystem, "fetch_failed") if errors.Is(err, fetch.ErrNotFound) { @@ -1143,7 +1740,64 @@ func (p *Proxy) fetchAndCacheFromURL(ctx context.Context, ecosystem, name, versi return nil, fmt.Errorf("fetching from upstream: %w", err) } - return p.storeArtifact(ctx, ecosystem, name, version, filename, pkgPURL, versionPURL, downloadURL, upstreamHash, artifact) + body := &streamErrorLogger{ + ReadCloser: artifact.Body, + onError: func(read int64, err error) { + p.Logger.Warn("streaming artifact from upstream failed", + "purl", versionPURL, "filename", filename, "url", upstreamURL, "bytes", read, "error", err) + metrics.RecordUpstreamError(ecosystem, "stream_failed") + }, + } + result := &CacheResult{ + Reader: body, + Artifact: artifacts.Artifact{ + PURL: versionPURL, + Size: artifact.Size, + Filename: filename, + MediaType: artifact.ContentType, + }, + } + if upstreamHash == "" { + return result, nil + } + + hash := strings.ToLower(upstreamHash) + checks, err := newIntegrityChecks(hash, "") + if err != nil { + _ = artifact.Body.Close() + return nil, fmt.Errorf("parsing upstream digest: %w", err) + } + result.Reader, err = checks.wrapFailOnMismatch(body, func(reason string) { + p.Logger.Error("streamed artifact failed integrity check", + "purl", versionPURL, "filename", filename, "url", upstreamURL, "reason", reason) + metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(ecosystem)) + }) + if err != nil { + _ = artifact.Body.Close() + return nil, err + } + result.Artifact.Digest = digest.Digest("sha256:" + hash) + result.Artifact.Size = -1 + return result, nil +} + +// streamErrorLogger reports the first read error of a streamed upstream body. +// The error itself still reaches serveArtifact, which aborts the response. +type streamErrorLogger struct { + io.ReadCloser + onError func(read int64, err error) + read int64 + logged bool +} + +func (r *streamErrorLogger) Read(p []byte) (int, error) { + n, err := r.ReadCloser.Read(p) + r.read += int64(n) + if err != nil && err != io.EOF && !r.logged { + r.logged = true + r.onError(r.read, err) + } + return n, err } // ErrArtifactDigestMismatch indicates that fetched bytes did not match the @@ -1153,14 +1807,3 @@ var ErrArtifactDigestMismatch = errors.New("artifact digest mismatch") func artifactHashMatches(got, expected string) bool { return expected == "" || strings.EqualFold(got, expected) } - -func (p *Proxy) discardCachedArtifact(ctx context.Context, versionPURL, filename, storagePath string) { - if storagePath != "" { - if err := p.Storage.Delete(ctx, storagePath); err != nil { - p.Logger.Warn("failed to discard cached artifact", "path", storagePath, "error", err) - } - } - if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil { - p.Logger.Warn("failed to clear artifact cache record", "purl", versionPURL, "filename", filename, "error", err) - } -} diff --git a/internal/handler/handler_test.go b/internal/handler/handler_test.go index 076b74bc..4c05cbd9 100644 --- a/internal/handler/handler_test.go +++ b/internal/handler/handler_test.go @@ -10,7 +10,9 @@ import ( "log/slog" "net/http" "net/http/httptest" + "slices" "strings" + "sync" "testing" "time" @@ -29,11 +31,13 @@ import ( // mockStorage implements storage.Storage for testing. type mockStorage struct { + mu sync.Mutex files map[string][]byte storeErr error openErr error signedURL string signErr error + seekable bool } func newMockStorage() *mockStorage { @@ -41,6 +45,8 @@ func newMockStorage() *mockStorage { } func (s *mockStorage) Store(_ context.Context, path string, r io.Reader) (int64, string, error) { + s.mu.Lock() + defer s.mu.Unlock() if s.storeErr != nil { return 0, "", s.storeErr } @@ -53,6 +59,8 @@ func (s *mockStorage) Store(_ context.Context, path string, r io.Reader) (int64, } func (s *mockStorage) Open(_ context.Context, path string) (io.ReadCloser, error) { + s.mu.Lock() + defer s.mu.Unlock() if s.openErr != nil { return nil, s.openErr } @@ -60,10 +68,21 @@ func (s *mockStorage) Open(_ context.Context, path string) (io.ReadCloser, error if !ok { return nil, storage.ErrNotFound } + if s.seekable { + return &mockSeekableReadCloser{Reader: bytes.NewReader(data)}, nil + } return io.NopCloser(bytes.NewReader(data)), nil } +type mockSeekableReadCloser struct { + *bytes.Reader +} + +func (r *mockSeekableReadCloser) Close() error { return nil } + func (s *mockStorage) Exists(_ context.Context, path string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() _, ok := s.files[path] return ok, nil } @@ -75,11 +94,15 @@ func (s *mockStorage) Delete(ctx context.Context, path string) error { if err := ctx.Err(); err != nil { return err } + s.mu.Lock() + defer s.mu.Unlock() delete(s.files, path) return nil } func (s *mockStorage) Size(_ context.Context, path string) (int64, error) { + s.mu.Lock() + defer s.mu.Unlock() data, ok := s.files[path] if !ok { return 0, storage.ErrNotFound @@ -88,6 +111,8 @@ func (s *mockStorage) Size(_ context.Context, path string) (int64, error) { } func (s *mockStorage) UsedSpace(_ context.Context) (int64, error) { + s.mu.Lock() + defer s.mu.Unlock() var total int64 for _, data := range s.files { total += int64(len(data)) @@ -109,11 +134,15 @@ func (s *mockStorage) URL() string { return "mem://" } func (s *mockStorage) Close() error { return nil } -// mockFetcher implements fetch.FetcherInterface for testing. +// mockFetcher implements fetch.FetcherInterface for testing. Recording is +// locked because coalescing tests call the handler from many goroutines; tests +// read the recorded fields only after those calls have returned. type mockFetcher struct { artifact *fetch.Artifact fetchErr error fetchErrByURL map[string]error + + mu sync.Mutex fetchCalled bool fetchedURL string fetchedHeader http.Header @@ -124,9 +153,11 @@ func (f *mockFetcher) Fetch(ctx context.Context, url string) (*fetch.Artifact, e } func (f *mockFetcher) FetchWithHeaders(_ context.Context, url string, headers http.Header) (*fetch.Artifact, error) { + f.mu.Lock() f.fetchCalled = true f.fetchedURL = url f.fetchedHeader = headers.Clone() + f.mu.Unlock() if f.fetchErrByURL != nil { if err, ok := f.fetchErrByURL[url]; ok { return nil, err @@ -226,6 +257,16 @@ func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, n } } +// recordedStoragePath returns the storage path the artifact's record points at. +func recordedStoragePath(t testing.TB, db *database.DB, versionPURL, filename string) string { + t.Helper() + art, err := db.GetArtifact(versionPURL, filename) + if err != nil || art == nil || !art.StoragePath.Valid { + t.Fatalf("no storage path recorded for %s %s (err %v)", versionPURL, filename, err) + } + return art.StoragePath.String +} + // pathParseCase holds a single test case for path parsing functions that return // (name, version, arch). type pathParseCase struct { @@ -374,6 +415,38 @@ func assertMalformedCacheRejected(t *testing.T, malformedHash, malformedIntegrit if artifact.StoragePath.Valid { t.Error("unusable cache record retained its storage path") } + assertQueuedForDeletion(t, db, storage.ArtifactPath("npm", "", packageName, version, filename)) +} + +// TestCorruptCachedArtifactIsQueuedForDeletion streams cached bytes that no +// longer match their recorded digest, which clears the record once the stream +// ends. With each fetch writing its own path, no later fetch overwrites them, +// so they must be queued for deletion. +func TestCorruptCachedArtifactIsQueuedForDeletion(t *testing.T) { + proxy, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "npm", "corrupt", "1.0.0", "corrupt-1.0.0.tgz", "cached content") + storagePath := storage.ArtifactPath("npm", "", "corrupt", "1.0.0", "corrupt-1.0.0.tgz") + store.files[storagePath] = []byte("tampered bytes") + + result, err := proxy.GetCachedArtifact(context.Background(), "npm", "corrupt", "1.0.0", "corrupt-1.0.0.tgz") + if err != nil || result == nil { + t.Fatalf("GetCachedArtifact = %v, %v", result, err) + } + drain(result) + artifact, err := db.GetArtifact("pkg:npm/corrupt@1.0.0", "corrupt-1.0.0.tgz") + if err != nil || artifact == nil || artifact.StoragePath.Valid { + t.Errorf("record = %+v (err %v), want it cleared", artifact, err) + } + assertQueuedForDeletion(t, db, storagePath) +} + +// assertQueuedForDeletion fails unless path waits in the pending delete queue. +func assertQueuedForDeletion(t *testing.T, db *database.DB, path string) { + t.Helper() + queued, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 100) + if err != nil || !slices.Contains(queued, path) { + t.Errorf("queued %v (err %v), want %q queued for deletion", queued, err, path) + } } func TestGetOrFetchArtifact_CacheMiss_NoPackage(t *testing.T) { @@ -434,7 +507,7 @@ func TestGetOrFetchArtifactFromURL_CacheMiss_StorageMissing(t *testing.T) { } // Verify the new content was stored - storagePath := storage.ArtifactPath("npm", "", "missing", "1.0.0", "missing-1.0.0.tgz") + storagePath := recordedStoragePath(t, db, "pkg:npm/missing@1.0.0", "missing-1.0.0.tgz") if _, ok := store.files[storagePath]; !ok { t.Error("refetched artifact should be stored") } @@ -674,6 +747,161 @@ func TestServeArtifact_Stream(t *testing.T) { } } +func TestServeArtifactRequestRanges(t *testing.T) { + const payload = "hello world" + tests := []struct { + name string + rangeHeader string + secondRange string + wantStatus int + wantRange string + wantLength string + wantBody string + }{ + {name: "bounded", rangeHeader: "bytes=1-3", wantStatus: http.StatusPartialContent, wantRange: "bytes 1-3/11", wantLength: "3", wantBody: "ell"}, + {name: "open ended", rangeHeader: "bytes=6-", wantStatus: http.StatusPartialContent, wantRange: "bytes 6-10/11", wantLength: "5", wantBody: "world"}, + {name: "suffix", rangeHeader: "bytes=-4", wantStatus: http.StatusPartialContent, wantRange: "bytes 7-10/11", wantLength: "4", wantBody: "orld"}, + {name: "suffix larger than artifact", rangeHeader: "bytes=-99", wantStatus: http.StatusPartialContent, wantRange: "bytes 0-10/11", wantLength: "11", wantBody: payload}, + {name: "unsatisfiable", rangeHeader: "bytes=11-", wantStatus: http.StatusRequestedRangeNotSatisfiable, wantRange: "bytes */11", wantLength: "0"}, + {name: "malformed", rangeHeader: "bytes=invalid", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, + {name: "reversed", rangeHeader: "bytes=4-2", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, + {name: "multiple ranges", rangeHeader: "bytes=0-1,4-5", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, + {name: "multiple range fields", rangeHeader: "bytes=0-1", secondRange: "bytes=4-5", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, + {name: "no range", wantStatus: http.StatusOK, wantLength: "11", wantBody: payload}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "/artifact", nil) + if test.rangeHeader != "" { + request.Header.Set("Range", test.rangeHeader) + } + if test.secondRange != "" { + request.Header.Add("Range", test.secondRange) + } + w := httptest.NewRecorder() + ServeArtifactRequest(w, request, newRangeTestResult(payload)) + + if w.Code != test.wantStatus { + t.Errorf("status = %d, want %d", w.Code, test.wantStatus) + } + if got := w.Header().Get("Content-Range"); got != test.wantRange { + t.Errorf("Content-Range = %q, want %q", got, test.wantRange) + } + if got := w.Header().Get(headerContentLength); got != test.wantLength { + t.Errorf("Content-Length = %q, want %q", got, test.wantLength) + } + if got := w.Body.String(); got != test.wantBody { + t.Errorf("body = %q, want %q", got, test.wantBody) + } + if got := w.Header().Get("Accept-Ranges"); got != "bytes" { + t.Errorf("Accept-Ranges = %q, want bytes", got) + } + }) + } +} + +func TestServeArtifactRequestIfRange(t *testing.T) { + const payload = "hello world" + tests := []struct { + name string + ifRange string + wantStatus int + wantRange string + wantBody string + }{ + {name: "matching etag", ifRange: `"sha256-matching"`, wantStatus: http.StatusPartialContent, wantRange: "bytes 0-4/11", wantBody: "hello"}, + {name: "mismatching etag", ifRange: `"sha256-stale"`, wantStatus: http.StatusOK, wantBody: payload}, + {name: "weak etag", ifRange: `W/"sha256-matching"`, wantStatus: http.StatusOK, wantBody: payload}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := newRangeTestResult(payload) + request := httptest.NewRequest(http.MethodGet, "/artifact", nil) + request.Header.Set("Range", "bytes=0-4") + if test.name == "matching etag" { + test.ifRange = `"` + result.Artifact.Digest.Encoded() + `"` + } + request.Header.Set("If-Range", test.ifRange) + w := httptest.NewRecorder() + ServeArtifactRequest(w, request, result) + + if w.Code != test.wantStatus { + t.Errorf("status = %d, want %d", w.Code, test.wantStatus) + } + if got := w.Header().Get("Content-Range"); got != test.wantRange { + t.Errorf("Content-Range = %q, want %q", got, test.wantRange) + } + if got := w.Body.String(); got != test.wantBody { + t.Errorf("body = %q, want %q", got, test.wantBody) + } + }) + } +} + +func TestServeArtifactRequestHeadIgnoresRange(t *testing.T) { + request := httptest.NewRequest(http.MethodHead, "/artifact", nil) + request.Header.Set("Range", "bytes=0-1") + w := httptest.NewRecorder() + ServeArtifactRequest(w, request, newRangeTestResult("hello world")) + + if w.Code != http.StatusOK { + t.Errorf("status = %d, want %d", w.Code, http.StatusOK) + } + if got := w.Header().Get(headerContentLength); got != "11" { + t.Errorf("Content-Length = %q, want 11", got) + } + if got := w.Header().Get("Content-Range"); got != "" { + t.Errorf("Content-Range = %q, want empty", got) + } + if w.Body.Len() != 0 { + t.Errorf("HEAD body = %q, want empty", w.Body.String()) + } +} + +func TestServeArtifactRequestWithoutSeekCapability(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "/artifact", nil) + request.Header.Set("Range", "bytes=1-2") + w := httptest.NewRecorder() + ServeArtifactRequest(w, request, &CacheResult{ + Reader: io.NopCloser(strings.NewReader("payload")), + Artifact: testArtifact("payload", "pkg:npm/example@1.0.0", "example.tgz", "application/gzip"), + }) + + if w.Code != http.StatusOK || w.Body.String() != "payload" { + t.Errorf("response = %d %q, want 200 with full payload", w.Code, w.Body.String()) + } + if got := w.Header().Get("Accept-Ranges"); got != "" { + t.Errorf("Accept-Ranges = %q, want empty", got) + } +} + +func TestServeArtifactRequestRedirectDoesNotAdvertiseRanges(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, "/artifact", nil) + request.Header.Set("Range", "bytes=1-2") + w := httptest.NewRecorder() + ServeArtifactRequest(w, request, &CacheResult{RedirectURL: "https://storage.example/artifact"}) + + if w.Code != http.StatusFound { + t.Errorf("status = %d, want %d", w.Code, http.StatusFound) + } + if got := w.Header().Get("Accept-Ranges"); got != "" { + t.Errorf("Accept-Ranges = %q, want empty", got) + } +} + +type rangeTestReadSeeker struct { + *bytes.Reader +} + +func (r *rangeTestReadSeeker) Close() error { return nil } + +func newRangeTestResult(payload string) *CacheResult { + return &CacheResult{ + Reader: &rangeTestReadSeeker{Reader: bytes.NewReader([]byte(payload))}, + Artifact: testArtifact(payload, "pkg:npm/example@1.0.0", "example.tgz", "application/gzip"), + } +} + func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) { proxy, db, store, fetcher := setupTestProxy(t) seedPackage(t, db, store, "pypi", "requests", "2.28.0", "requests-2.28.0.tar.gz", "pypi content") @@ -698,7 +926,7 @@ func TestGetOrFetchArtifactFromURL_CacheHit(t *testing.T) { } func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) { - proxy, _, store, fetcher := setupTestProxy(t) + proxy, db, store, fetcher := setupTestProxy(t) missesBefore := testutil.ToFloat64(metrics.CacheMisses.WithLabelValues("pypi")) fetchesBefore := histogramSampleCount(t, metrics.UpstreamFetchDuration.WithLabelValues("pypi")) writesBefore := histogramSampleCount(t, metrics.StorageOperationDuration.WithLabelValues("write")) @@ -743,7 +971,7 @@ func TestGetOrFetchArtifactFromURL_CacheMiss(t *testing.T) { } // Verify it was stored - storagePath := storage.ArtifactPath("pypi", "", "newpkg", "1.0.0", "newpkg-1.0.0.tar.gz") + storagePath := recordedStoragePath(t, db, "pkg:pypi/newpkg@1.0.0", "newpkg-1.0.0.tar.gz") if _, ok := store.files[storagePath]; !ok { t.Error("artifact was not stored in storage") } diff --git a/internal/handler/helm.go b/internal/handler/helm.go index 629d5c9f..31ef9015 100644 --- a/internal/handler/helm.go +++ b/internal/handler/helm.go @@ -23,9 +23,10 @@ const ( // HelmHandler serves read-only HTTP Helm chart repositories. Each configured // repository is mounted at /helm/{repository}/. type HelmHandler struct { - proxy *Proxy - proxyURL string - repositories map[string]string + proxy *Proxy + proxyURL string + repositories map[string]string + ociRegistries []helmOCIRegistry } // NewHelmHandler creates a Helm chart repository protocol handler. @@ -114,8 +115,14 @@ func (h *HelmHandler) handleChart(w http.ResponseWriter, r *http.Request) { return } - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), helmMetadataEcosystem, repository, digest, filename, downloadURL) + // A streamed fetch is never stored, so serveChart's digest check has + // nothing to compare against: verify the stream itself instead. + expectedDigest := "" + if h.proxy.StreamArtifacts { + expectedDigest = "sha256:" + digest + } + result, err := h.proxy.GetOrFetchArtifactFromURLWithDigest( + r.Context(), helmMetadataEcosystem, repository, digest, filename, downloadURL, expectedDigest) if err != nil { h.proxy.serveArtifactError(w, err, "failed to fetch chart") return @@ -128,7 +135,7 @@ func (h *HelmHandler) serveChart(w http.ResponseWriter, r *http.Request, reposit if result.Reader != nil { _ = result.Reader.Close() } - if clearErr := h.proxy.ClearCachedArtifact(r.Context(), helmMetadataEcosystem, repository, digest, filename); clearErr != nil { + if clearErr := h.proxy.ClearCachedArtifact(helmMetadataEcosystem, repository, digest, filename); clearErr != nil { h.proxy.Logger.Warn("failed to clear Helm chart with invalid digest", "error", clearErr) } http.Error(w, "chart digest verification failed", http.StatusBadGateway) @@ -138,7 +145,7 @@ func (h *HelmHandler) serveChart(w http.ResponseWriter, r *http.Request, reposit if result.Artifact.MediaType == "" { w.Header().Set(headerContentType, "application/gzip") } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } func (h *HelmHandler) repositoryForRequest(r *http.Request) (name, upstreamURL string, ok bool) { @@ -189,13 +196,19 @@ func (h *HelmHandler) rewriteIndex(repository, upstreamURL string, body []byte) for _, release := range releases.Content { chart, err := h.parseChartRelease(chartName, upstreamURL, release) if err != nil { - return nil, err + h.proxy.Logger.Warn("omitting malformed Helm chart release", + "repository", repository, "chart", chartName, "error", err) + continue } if h.chartOnCooldown(chartName, chart.created) { continue } for _, download := range chart.downloads { - download.node.Value = h.chartProxyURL(repository, chart.digest, download.filename) + if download.oci { + download.node.Value = h.ociChartProxyURL(download.url) + } else { + download.node.Value = h.chartProxyURL(repository, chart.digest, download.filename) + } } filtered = append(filtered, release) } @@ -220,13 +233,13 @@ func (h *HelmHandler) findChartDownload(upstreamURL string, body []byte, digest, for _, release := range releases.Content { chart, err := h.parseChartRelease(chartName, upstreamURL, release) if err != nil { - return "", err + continue } if chart.digest != digest || h.chartOnCooldown(chartName, chart.created) { continue } for _, download := range chart.downloads { - if download.filename == filename { + if !download.oci && download.filename == filename { return download.url, nil } } @@ -245,6 +258,7 @@ type helmChartDownload struct { node *yaml.Node url string filename string + oci bool } type helmChartRelease struct { @@ -280,30 +294,42 @@ func (h *HelmHandler) parseChartRelease(chartName, upstreamURL string, release * } for _, urlNode := range urlsNode.Content { - if urlNode.Kind != yaml.ScalarNode { - return helmChartRelease{}, fmt.Errorf("chart %q has invalid URL", chartName) - } - reference, err := url.Parse(urlNode.Value) + download, err := parseHelmChartDownload(chartName, baseURL, urlNode) if err != nil { - return helmChartRelease{}, fmt.Errorf("parsing chart %q URL: %w", chartName, err) - } - downloadURL := baseURL.ResolveReference(reference) - if (downloadURL.Scheme != "http" && downloadURL.Scheme != "https") || downloadURL.Host == "" { - return helmChartRelease{}, fmt.Errorf("chart %q URL must be HTTP(S)", chartName) + return helmChartRelease{}, err } - filename := path.Base(downloadURL.Path) - if filename == "." || filename == "/" || filename == "" || !strings.HasSuffix(filename, ".tgz") { - return helmChartRelease{}, fmt.Errorf("chart %q URL must point to a .tgz file", chartName) - } - chart.downloads = append(chart.downloads, helmChartDownload{ - node: urlNode, - url: downloadURL.String(), - filename: filename, - }) + chart.downloads = append(chart.downloads, download) } return chart, nil } +func parseHelmChartDownload(chartName string, baseURL *url.URL, node *yaml.Node) (helmChartDownload, error) { + if node.Kind != yaml.ScalarNode { + return helmChartDownload{}, fmt.Errorf("chart %q has invalid URL", chartName) + } + reference, err := url.Parse(node.Value) + if err != nil { + return helmChartDownload{}, fmt.Errorf("parsing chart %q URL: %w", chartName, err) + } + if reference.Scheme == "oci" { + if reference.Hostname() == "" || strings.Trim(reference.Path, "/") == "" || + reference.User != nil || reference.RawQuery != "" || reference.ForceQuery || reference.Fragment != "" || + containsPathTraversal(reference.Path) { + return helmChartDownload{}, fmt.Errorf("chart %q has invalid OCI reference", chartName) + } + return helmChartDownload{node: node, url: node.Value, oci: true}, nil + } + downloadURL := baseURL.ResolveReference(reference) + if (downloadURL.Scheme != "http" && downloadURL.Scheme != "https") || downloadURL.Host == "" { + return helmChartDownload{}, fmt.Errorf("chart %q URL must be HTTP(S) or OCI", chartName) + } + filename := path.Base(downloadURL.Path) + if filename == "." || filename == "/" || filename == "" || !strings.HasSuffix(filename, ".tgz") { + return helmChartDownload{}, fmt.Errorf("chart %q URL must point to a .tgz file", chartName) + } + return helmChartDownload{node: node, url: downloadURL.String(), filename: filename}, nil +} + func (h *HelmHandler) chartProxyURL(repository, digest, filename string) string { return fmt.Sprintf("%s/helm/%s/charts/%s/%s", h.proxyURL, url.PathEscape(repository), digest, url.PathEscape(filename)) diff --git a/internal/handler/helm_oci.go b/internal/handler/helm_oci.go new file mode 100644 index 00000000..00392e84 --- /dev/null +++ b/internal/handler/helm_oci.go @@ -0,0 +1,64 @@ +package handler + +import ( + "net/url" + "slices" + "strings" +) + +type helmOCIRegistry struct { + host string + prefix string +} + +// NewHelmHandlerWithOCIRegistries also rewrites OCI references for configured +// registries through the existing /v2 routes. Unmatched references stay intact. +func NewHelmHandlerWithOCIRegistries(proxy *Proxy, proxyURL string, repositories map[string]string, defaultRegistry string, registries map[string]string) *HelmHandler { + h := NewHelmHandler(proxy, proxyURL, repositories) + // Prefer named routes; use a stable order when aliases share a registry. + names := make([]string, 0, len(registries)) + for name := range registries { + names = append(names, name) + } + slices.Sort(names) + for _, name := range names { + h.addOCIRegistry(registries[name], "/upstream/"+url.PathEscape(name)) + } + h.addOCIRegistry(configuredUpstreamURL(defaultRegistry, dockerHubRegistry), "") + return h +} + +func (h *HelmHandler) addOCIRegistry(rawURL, prefix string) { + u, err := url.Parse(rawURL) + // A path-prefixed upstream is an API mount, not an OCI repository prefix; + // its mapping cannot be inferred from a chart's registry authority alone. + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || + strings.Trim(u.Path, "/") != "" || u.RawQuery != "" || u.Fragment != "" || u.User != nil { + return + } + h.ociRegistries = append(h.ociRegistries, helmOCIRegistry{host: u.Host, prefix: prefix}) +} + +func (h *HelmHandler) ociChartProxyURL(reference string) string { + u, err := url.Parse(reference) + if err != nil { + return reference + } + proxy, err := url.Parse(h.proxyURL) + // OCI clients place /v2 before the repository path, so a path-prefixed + // public HTTP base URL cannot be represented by simply prepending its path. + if err != nil || proxy.Host == "" || strings.Trim(proxy.Path, "/") != "" { + return reference + } + for _, registry := range h.ociRegistries { + if !strings.EqualFold(u.Host, registry.host) { + continue + } + if registry.prefix == "" && strings.HasPrefix(u.Path, "/upstream/") { + // This prefix is reserved by the named-registry router. + return reference + } + return "oci://" + proxy.Host + registry.prefix + u.EscapedPath() + } + return reference +} diff --git a/internal/handler/helm_oci_test.go b/internal/handler/helm_oci_test.go new file mode 100644 index 00000000..c5163bd8 --- /dev/null +++ b/internal/handler/helm_oci_test.go @@ -0,0 +1,216 @@ +package handler + +import ( + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +func TestHelmHandler_MixedOCIIndex(t *testing.T) { + chart := "http chart content" + digest := helmSHA256Hex([]byte(chart)) + const ociURL = "oci://ghcr.io/stakater/saap-catalog/charts/konfigurator-0.1.40.tgz" + index := fmt.Sprintf(`apiVersion: v1 +entries: + konfigurator: + - digest: %s + urls: [%s] + version: 0.1.40 + demo: + - digest: %s + urls: [oci://ghcr.io/owner/demo:1.0.0, demo-1.0.0.tgz] + version: 1.0.0 +`, digest, ociURL, digest) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/index.yaml": + _, _ = fmt.Fprint(w, index) + case "/demo-1.0.0.tgz": + _, _ = fmt.Fprint(w, chart) + default: + t.Errorf("unexpected HTTP upstream request: %s", r.URL.Path) + http.NotFound(w, r) + } + })) + defer upstream.Close() + p, _, _, _ := setupTestProxy(t) + p.CacheMetadata = true + p.MetadataTTL = time.Hour + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + p.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewHelmHandler(p, "https://proxy.example", map[string]string{"mixed": upstream.URL}) + response := serveHelmRequest(h, "/mixed/index.yaml") + if response.Code != http.StatusOK { + t.Fatalf("index status = %d: %s", response.Code, response.Body.String()) + } + if !strings.Contains(response.Body.String(), ociURL) || !strings.Contains(response.Body.String(), h.chartProxyURL("mixed", digest, "demo-1.0.0.tgz")) { + t.Fatalf("incorrectly rewritten index: %s", response.Body.String()) + } + download, err := h.findChartDownload(upstream.URL, []byte(index), digest, "demo-1.0.0.tgz") + if err != nil || download != upstream.URL+"/demo-1.0.0.tgz" { + t.Fatalf("HTTP chart lookup = %q, %v", download, err) + } + // Even a .tgz-shaped OCI reference must never reach the HTTP fetcher. + response = serveHelmRequest(h, "/mixed/charts/"+digest+"/konfigurator-0.1.40.tgz") + if response.Code != http.StatusNotFound { + t.Fatalf("OCI reference reached HTTP download route: status=%d", response.Code) + } + response = serveHelmRequest(h, "/mixed/charts/"+digest+"/demo-1.0.0.tgz") + if response.Code != http.StatusOK || response.Body.String() != chart { + t.Fatalf("cold HTTP chart status=%d body=%s", response.Code, response.Body.String()) + } + upstream.Close() + response = serveHelmRequest(h, "/mixed/charts/"+digest+"/demo-1.0.0.tgz") + if response.Code != http.StatusOK || response.Body.String() != chart { + t.Fatalf("cached HTTP chart status=%d body=%s", response.Code, response.Body.String()) + } +} + +func TestHelmOCIReferenceRewriting(t *testing.T) { + for _, tt := range []struct { + name, reference, defaultRegistry, proxyURL string + registries map[string]string + want string + }{ + {name: "unmatched", reference: "oci://ghcr.io/owner/chart:1.0.0", want: "oci://ghcr.io/owner/chart:1.0.0"}, + {name: "named", reference: "oci://ghcr.io/owner/chart:1.0.0", registries: map[string]string{"ghcr": "https://ghcr.io"}, want: "oci://proxy.example/upstream/ghcr/owner/chart:1.0.0"}, + {name: "digest", reference: "oci://ghcr.io/owner/chart@sha256:" + strings.Repeat("a", 64), registries: map[string]string{"ghcr": "https://ghcr.io/"}, want: "oci://proxy.example/upstream/ghcr/owner/chart@sha256:" + strings.Repeat("a", 64)}, + {name: "reported tgz reference", reference: "oci://ghcr.io/stakater/saap-catalog/charts/konfigurator-0.1.40.tgz", registries: map[string]string{"ghcr": "https://ghcr.io"}, want: "oci://proxy.example/upstream/ghcr/stakater/saap-catalog/charts/konfigurator-0.1.40.tgz"}, + {name: "default", reference: "oci://registry.example/owner/chart:1.0.0", defaultRegistry: "https://registry.example", want: "oci://proxy.example/owner/chart:1.0.0"}, + {name: "named preferred and stable", reference: "oci://ghcr.io/owner/chart", defaultRegistry: "https://ghcr.io", registries: map[string]string{"z": "https://ghcr.io", "a": "https://ghcr.io"}, want: "oci://proxy.example/upstream/a/owner/chart"}, + {name: "port", reference: "oci://registry.example:5000/owner/chart", registries: map[string]string{"local": "http://registry.example:5000"}, proxyURL: "http://localhost:8080", want: "oci://localhost:8080/upstream/local/owner/chart"}, + {name: "different port", reference: "oci://registry.example:5001/owner/chart", registries: map[string]string{"local": "http://registry.example:5000"}, want: "oci://registry.example:5001/owner/chart"}, + {name: "different host", reference: "oci://ghcr.io.evil.example/owner/chart", registries: map[string]string{"ghcr": "https://ghcr.io"}, want: "oci://ghcr.io.evil.example/owner/chart"}, + {name: "escaped path", reference: "oci://ghcr.io/owner/chart:1.0.0%2Bbuild", registries: map[string]string{"ghcr": "https://ghcr.io"}, want: "oci://proxy.example/upstream/ghcr/owner/chart:1.0.0%2Bbuild"}, + {name: "upstream API mount", reference: "oci://ghcr.io/owner/chart", registries: map[string]string{"ghcr": "https://ghcr.io/mount"}, want: "oci://ghcr.io/owner/chart"}, + {name: "proxy path prefix", reference: "oci://ghcr.io/owner/chart", proxyURL: "https://proxy.example/mount", registries: map[string]string{"ghcr": "https://ghcr.io"}, want: "oci://ghcr.io/owner/chart"}, + {name: "reserved default prefix", reference: "oci://ghcr.io/upstream/other/chart", defaultRegistry: "https://ghcr.io", want: "oci://ghcr.io/upstream/other/chart"}, + } { + t.Run(tt.name, func(t *testing.T) { + if tt.proxyURL == "" { + tt.proxyURL = "https://proxy.example" + } + h := NewHelmHandlerWithOCIRegistries(&Proxy{}, tt.proxyURL, nil, tt.defaultRegistry, tt.registries) + index := fmt.Sprintf("apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [%q]\n", strings.Repeat("a", 64), tt.reference) + rewritten, err := h.rewriteIndex("mixed", "https://charts.example", []byte(index)) + if err != nil { + t.Fatal(err) + } + _, entries, err := parseHelmIndex(rewritten) + if err != nil { + t.Fatal(err) + } + got := helmMappingValue(entries.Content[1].Content[0], "urls").Content[0].Value + if got != tt.want { + t.Fatalf("rewritten reference = %q, want %q", got, tt.want) + } + }) + } +} + +func TestHelmIndexOmitsMalformedChartReleases(t *testing.T) { + goodDigest := strings.Repeat("a", 64) + for _, reference := range []string{"oci:///chart", "oci://ghcr.io", "oci://user:secret@ghcr.io/chart", "oci://ghcr.io/../chart", "oci://ghcr.io/%2e%2e/chart", "oci://ghcr.io/chart?token=x", "file:///chart.tgz", "ftp://example.com/chart.tgz", "https://example.com/chart.zip"} { + t.Run(reference, func(t *testing.T) { + h := NewHelmHandler(&Proxy{Logger: slog.New(slog.NewTextHandler(io.Discard, nil))}, "https://proxy.example", nil) + index := fmt.Sprintf("apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [%q]\n - digest: %s\n urls: [demo-1.0.0.tgz]\n", strings.Repeat("b", 64), reference, goodDigest) + rewritten, err := h.rewriteIndex("mixed", "https://charts.example", []byte(index)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(rewritten), reference) || strings.Contains(string(rewritten), strings.Repeat("b", 64)) { + t.Fatalf("malformed release not omitted: %s", rewritten) + } + if !strings.Contains(string(rewritten), h.chartProxyURL("mixed", goodDigest, "demo-1.0.0.tgz")) { + t.Fatalf("valid release missing from rewritten index: %s", rewritten) + } + download, err := h.findChartDownload("https://charts.example", []byte(index), goodDigest, "demo-1.0.0.tgz") + if err != nil || download != "https://charts.example/demo-1.0.0.tgz" { + t.Fatalf("chart lookup = %q, %v", download, err) + } + if _, err := h.findChartDownload("https://charts.example", []byte(index), strings.Repeat("b", 64), "demo-1.0.0.tgz"); !errors.Is(err, errHelmChartNotFound) { + t.Fatalf("omitted release lookup error = %v, want errHelmChartNotFound", err) + } + }) + } +} + +func TestHelmOCICooldown(t *testing.T) { + p := &Proxy{Cooldown: &cooldown.Config{Default: "3d"}} + h := NewHelmHandlerWithOCIRegistries(p, "https://proxy.example", nil, "https://ghcr.io", nil) + index := fmt.Sprintf(`apiVersion: v1 +entries: + demo: + - digest: %s + created: %s + urls: [oci://ghcr.io/owner/demo:1.0.0] + - digest: %s + created: %s + urls: [oci://ghcr.io/owner/demo:2.0.0] +`, strings.Repeat("a", 64), time.Now().Add(-7*24*time.Hour).Format(time.RFC3339), + strings.Repeat("b", 64), time.Now().Add(-time.Hour).Format(time.RFC3339)) + rewritten, err := h.rewriteIndex("mixed", "https://charts.example", []byte(index)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(rewritten), "demo:2.0.0") || !strings.Contains(string(rewritten), "oci://proxy.example/owner/demo:1.0.0") { + t.Fatalf("incorrect cooldown filtering: %s", rewritten) + } +} + +func TestHelmRewrittenOCIRouteServesChart(t *testing.T) { + chart := "OCI Helm chart layer" + digest := "sha256:" + helmSHA256Hex([]byte(chart)) + manifest := `{"schemaVersion":2,"layers":[{"mediaType":"application/vnd.cncf.helm.chart.content.v1.tar+gzip","digest":"` + digest + `"}]}` + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v2/owner/demo/manifests/1.0.0": + w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") + _, _ = fmt.Fprint(w, manifest) + case "/v2/owner/demo/blobs/" + digest: + _, _ = fmt.Fprint(w, chart) + default: + t.Errorf("unexpected registry request: %s", r.URL.Path) + http.NotFound(w, r) + } + })) + defer upstream.Close() + p, _, _, _ := setupTestProxy(t) + p.HTTPClient = upstream.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + p.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + registries := map[string]string{"local": upstream.URL} + h := NewHelmHandlerWithOCIRegistries(p, "https://proxy.example", nil, "", registries) + reference := "oci://" + strings.TrimPrefix(upstream.URL, "http://") + "/owner/demo:1.0.0" + rewritten, err := url.Parse(h.ociChartProxyURL(reference)) + if err != nil { + t.Fatal(err) + } + if rewritten.Host != "proxy.example" { + t.Fatalf("OCI reference was not proxied: %s", rewritten) + } + repository := strings.TrimSuffix(rewritten.Path, ":1.0.0") + container := NewContainerHandlerWithRegistry(p, "https://proxy.example", "", registries) + for _, tt := range []struct{ path, want string }{ + {repository + "/manifests/1.0.0", manifest}, + {repository + "/blobs/" + digest, chart}, + } { + w := httptest.NewRecorder() + container.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, tt.path, nil)) + if w.Code != http.StatusOK || w.Body.String() != tt.want { + t.Fatalf("OCI GET %s: status=%d body=%s", tt.path, w.Code, w.Body.String()) + } + } +} diff --git a/internal/handler/helm_test.go b/internal/handler/helm_test.go index ec8d4a5a..c4a60fe4 100644 --- a/internal/handler/helm_test.go +++ b/internal/handler/helm_test.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "slices" "strings" "sync/atomic" "testing" @@ -164,11 +165,23 @@ func TestHelmHandler_RejectsChartDigestMismatch(t *testing.T) { if requests != 2 { t.Errorf("chart requests = %d, want 2 after invalid cache entry is cleared", requests) } - storagePath := storage.ArtifactPath(helmMetadataEcosystem, "", "test", digest, "demo.tgz") - if exists, err := store.Exists(t.Context(), storagePath); err != nil { - t.Fatalf("checking rejected chart storage: %v", err) - } else if exists { - t.Errorf("rejected chart remains in storage at %q", storagePath) + queued, err := proxy.DB.GetDuePendingDeletes(time.Now().Add(time.Hour), 10) + if err != nil { + t.Fatalf("listing queued deletes: %v", err) + } + chartDir := storage.ArtifactPath(helmMetadataEcosystem, "", "test", digest, "") + stored := 0 + for path := range store.files { + if !strings.HasPrefix(path, chartDir) { + continue + } + stored++ + if !slices.Contains(queued, path) { + t.Errorf("rejected chart at %q is not queued for deletion", path) + } + } + if stored != requests { + t.Errorf("found %d stored charts, want one per request (%d)", stored, requests) } } diff --git a/internal/handler/hex.go b/internal/handler/hex.go index 15d28915..c475e5a9 100644 --- a/internal/handler/hex.go +++ b/internal/handler/hex.go @@ -117,13 +117,7 @@ func (h *HexHandler) handlePackages(w http.ResponseWriter, r *http.Request) { defer func() { _ = protoResp.Body.Close() }() if protoResp.StatusCode != http.StatusOK { - for k, vv := range protoResp.Header { - for _, v := range vv { - w.Header().Add(k, v) - } - } - w.WriteHeader(protoResp.StatusCode) - _, _ = io.Copy(w, protoResp.Body) + h.proxy.relayResponse(w, r, protoResp, nil) return } diff --git a/internal/handler/homebrew.go b/internal/handler/homebrew.go index 0fd5b62f..0af67afc 100644 --- a/internal/handler/homebrew.go +++ b/internal/handler/homebrew.go @@ -55,7 +55,16 @@ func (h *HomebrewHandler) Routes() http.Handler { upstreamURL += "?" + r.URL.RawQuery } - h.proxy.ProxyCached(w, r, upstreamURL, homebrewMetadataEcosystem, homebrewMetadataCacheKey(requestPath, r.URL.RawQuery), "*/*") + // brew fetches every JSON API download with `curl --compressed` and + // decodes Content-Encoding itself, and formula.jws.json is ~33 MB plain + // versus ~5 MB gzip, so keep both hops compressed. The analytics + // endpoints are the one consumer brew fetches without --compressed; + // they stay identity. + acceptEncoding := "gzip" + if strings.HasPrefix(requestPath, "analytics/") { + acceptEncoding = "identity" + } + h.proxy.proxyCachedWithEncoding(w, r, upstreamURL, homebrewMetadataEcosystem, homebrewMetadataCacheKey(requestPath, r.URL.RawQuery), acceptEncoding, "*/*") }) } diff --git a/internal/handler/homebrew_test.go b/internal/handler/homebrew_test.go index e8931ad5..e5d7e5c5 100644 --- a/internal/handler/homebrew_test.go +++ b/internal/handler/homebrew_test.go @@ -1,11 +1,13 @@ package handler import ( + "bytes" "io" "net/http" "net/http/httptest" "strconv" "strings" + "sync/atomic" "testing" "time" @@ -361,3 +363,96 @@ func TestRegisterHomebrewArtifactsRejectsOtherHomebrewRoutes(t *testing.T) { t.Errorf("blocked Homebrew routes made %d upstream requests, want 0", upstreamRequests) } } + +// TestHomebrewHandler_RequestsGzipForAPIPaths covers #305's motivating case: +// the JSON API files are fetched, cached and served gzip-compressed with +// Content-Encoding: gzip (brew fetches them with --compressed), while the +// analytics endpoints, which brew fetches without --compressed, stay identity. +func TestHomebrewHandler_RequestsGzipForAPIPaths(t *testing.T) { + plain := []byte(`{"payload":"signed bytes","signatures":[]}`) + compressed := gzipPayload(t, plain) + + var available atomic.Bool + available.Store(true) + var requests atomic.Int32 + var sawAcceptEncoding atomic.Value // string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + sawAcceptEncoding.Store(r.Header.Get(headerAcceptEncoding)) + if !available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + w.Header().Set(headerContentType, "application/json") + if strings.Contains(r.Header.Get(headerAcceptEncoding), "gzip") { + w.Header().Set(headerContentEncoding, "gzip") + _, _ = w.Write(compressed) + return + } + _, _ = w.Write(plain) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + h := NewHomebrewHandler(proxy, upstream.URL+"/api").Routes() + + get := func(path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + return w + } + lastAE := func() string { + s, _ := sawAcceptEncoding.Load().(string) + return s + } + + first := get("/formula.jws.json") + if first.Code != http.StatusOK { + t.Fatalf("formula.jws.json: status = %d, want 200: %s", first.Code, first.Body.String()) + } + if got := lastAE(); got != "gzip" { + t.Errorf("formula.jws.json: upstream Accept-Encoding = %q, want %q", got, "gzip") + } + if !bytes.Equal(first.Body.Bytes(), compressed) { + t.Errorf("formula.jws.json: body is not the compressed bytes (got %d, want %d)", first.Body.Len(), len(compressed)) + } + if got := first.Header().Get(headerContentEncoding); got != "gzip" { + t.Errorf("formula.jws.json: Content-Encoding = %q, want %q", got, "gzip") + } + if got := first.Header().Get(headerContentLength); got != strconv.Itoa(len(compressed)) { + t.Errorf("formula.jws.json: Content-Length = %q, want %d", got, len(compressed)) + } + + // Replay from cache with the upstream down: same bytes and header, no refetch. + before := requests.Load() + available.Store(false) + cached := get("/formula.jws.json") + if cached.Code != http.StatusOK { + t.Fatalf("cached formula.jws.json: status = %d, want 200: %s", cached.Code, cached.Body.String()) + } + if !bytes.Equal(cached.Body.Bytes(), compressed) || cached.Header().Get(headerContentEncoding) != "gzip" { + t.Errorf("cached formula.jws.json: body/header not replayed verbatim") + } + if requests.Load() != before { + t.Errorf("cached formula.jws.json hit upstream: requests %d -> %d", before, requests.Load()) + } + available.Store(true) + + // Analytics is fetched by brew without --compressed: stays identity, no header. + analytics := get("/analytics/install/30d.json") + if analytics.Code != http.StatusOK { + t.Fatalf("analytics: status = %d, want 200: %s", analytics.Code, analytics.Body.String()) + } + if got := lastAE(); got != "identity" { + t.Errorf("analytics: upstream Accept-Encoding = %q, want %q", got, "identity") + } + if !bytes.Equal(analytics.Body.Bytes(), plain) { + t.Errorf("analytics: body = %q, want plain %q", analytics.Body.Bytes(), plain) + } + if got := analytics.Header().Get(headerContentEncoding); got != "" { + t.Errorf("analytics: Content-Encoding = %q, want empty", got) + } +} diff --git a/internal/handler/integrity.go b/internal/handler/integrity.go index 07963b9f..437252f7 100644 --- a/internal/handler/integrity.go +++ b/internal/handler/integrity.go @@ -40,6 +40,32 @@ func newIntegrityChecks(contentHash, native string) (integrityChecks, error) { } func (c integrityChecks) wrap(source io.ReadCloser, onMismatch func(string)) (io.ReadCloser, error) { + if len(c.algorithms) == 0 { + return source, nil + } + verified, err := c.newVerifyingReader(source, onMismatch, false) + if err != nil { + return nil, err + } + seeker, ok := source.(io.Seeker) + if !ok { + return verified, nil + } + return &seekableVerifyingReader{ + source: source, + verified: verified, + seeker: seeker, + }, nil +} + +// wrapFailOnMismatch is wrap for bytes that have not been checked anywhere +// else: a mismatch is also returned from Read as ErrArtifactDigestMismatch in +// place of io.EOF, so the caller can abort rather than complete the response. +func (c integrityChecks) wrapFailOnMismatch(source io.ReadCloser, onMismatch func(string)) (io.ReadCloser, error) { + return c.newVerifyingReader(source, onMismatch, true) +} + +func (c integrityChecks) newVerifyingReader(source io.ReadCloser, onMismatch func(string), failOnMismatch bool) (io.ReadCloser, error) { if len(c.algorithms) == 0 { return source, nil } @@ -48,10 +74,11 @@ func (c integrityChecks) wrap(source io.ReadCloser, onMismatch func(string)) (io return nil, fmt.Errorf("create integrity reader: %w", err) } return &verifyingReader{ - source: source, - reader: reader, - checks: c, - onMismatch: onMismatch, + source: source, + reader: reader, + checks: c, + onMismatch: onMismatch, + failOnMismatch: failOnMismatch, }, nil } @@ -63,12 +90,46 @@ type verifyingReader struct { checks integrityChecks onMismatch func(reason string) verified bool + + failOnMismatch bool + mismatched bool +} + +// seekableVerifyingReader verifies ordinary reads until a successful seek, +// after which reads bypass whole-object verification for range responses. +type seekableVerifyingReader struct { + source io.ReadCloser + verified io.ReadCloser + seeker io.Seeker + bypass bool +} + +func (r *seekableVerifyingReader) Read(p []byte) (int, error) { + if r.bypass { + return r.source.Read(p) + } + return r.verified.Read(p) +} + +func (r *seekableVerifyingReader) Seek(offset int64, whence int) (int64, error) { + position, err := r.seeker.Seek(offset, whence) + if err == nil { + r.bypass = true + } + return position, err +} + +func (r *seekableVerifyingReader) Close() error { + return r.verified.Close() } func (r *verifyingReader) Read(p []byte) (int, error) { n, err := r.reader.Read(p) if err == io.EOF { r.verify() + if r.failOnMismatch && r.mismatched { + return n, ErrArtifactDigestMismatch + } } return n, err } @@ -89,11 +150,13 @@ func (r *verifyingReader) verify() { if len(r.checks.contentHash) > 0 { if err := result.Verify(r.checks.contentHash); err != nil { + r.mismatched = true r.onMismatch("content_hash: " + err.Error()) } } if len(r.checks.native) > 0 { if err := result.Verify(r.checks.native); err != nil { + r.mismatched = true r.onMismatch("integrity: " + err.Error()) } } diff --git a/internal/handler/integrity_test.go b/internal/handler/integrity_test.go index 95992c03..9aa99589 100644 --- a/internal/handler/integrity_test.go +++ b/internal/handler/integrity_test.go @@ -1,6 +1,7 @@ package handler import ( + "bytes" "crypto/sha256" "crypto/sha512" "encoding/base64" @@ -126,6 +127,72 @@ func TestVerifyingReader(t *testing.T) { } } +func TestVerifyingReaderPreservesSeekCapability(t *testing.T) { + const data = "hello world" + t.Run("ordinary reads stay verified", func(t *testing.T) { + source := &seekableCloseTrackingReader{Reader: bytes.NewReader([]byte(data))} + var calls int + reader := wrapIntegrityReader(t, source, sha256Hex("different"), "", func(string) { calls++ }) + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if string(got) != data { + t.Errorf("data = %q, want %q", got, data) + } + if calls != 1 { + t.Errorf("onMismatch called %d times, want 1", calls) + } + if err := reader.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if source.closeCount != 1 { + t.Errorf("source closed %d times, want 1", source.closeCount) + } + }) + + t.Run("seek bypasses whole-object verification", func(t *testing.T) { + source := &seekableCloseTrackingReader{Reader: bytes.NewReader([]byte(data))} + var calls int + reader := wrapIntegrityReader(t, source, sha256Hex("different"), "", func(string) { calls++ }) + seeker, ok := reader.(io.Seeker) + if !ok { + t.Fatal("seekable source did not retain io.Seeker") + } + if _, err := seeker.Seek(6, io.SeekStart); err != nil { + t.Fatalf("Seek: %v", err) + } + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if string(got) != "world" { + t.Errorf("data after seek = %q, want %q", got, "world") + } + if calls != 0 { + t.Errorf("onMismatch called %d times for a partial read, want 0", calls) + } + if err := reader.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if source.closeCount != 1 { + t.Errorf("source closed %d times, want 1", source.closeCount) + } + }) +} + +type seekableCloseTrackingReader struct { + *bytes.Reader + closeCount int +} + +func (r *seekableCloseTrackingReader) Close() error { + r.closeCount++ + return nil +} + func TestVerifyingReaderUsesStrongestNativeAlgorithm(t *testing.T) { const data = "artifact" tests := []struct { diff --git a/internal/handler/julia.go b/internal/handler/julia.go index 211d0922..66a22459 100644 --- a/internal/handler/julia.go +++ b/internal/handler/julia.go @@ -103,7 +103,7 @@ func (h *JuliaHandler) handleRegistry(w http.ResponseWriter, r *http.Request) { go h.refreshNamesFromRegistry(uuid, hash) - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handlePackage serves an immutable package source tarball. @@ -129,7 +129,7 @@ func (h *JuliaHandler) handlePackage(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handleArtifact serves an immutable binary artifact tarball. Artifacts are @@ -150,7 +150,7 @@ func (h *JuliaHandler) handleArtifact(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // proxyUpstream forwards a request to the upstream Pkg server without caching. diff --git a/internal/handler/maven.go b/internal/handler/maven.go index 10e551e0..3f1cd9d1 100644 --- a/internal/handler/maven.go +++ b/internal/handler/maven.go @@ -134,7 +134,7 @@ func (h *MavenHandler) handleDownload(w http.ResponseWriter, r *http.Request, ur return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // parsePath extracts Maven coordinates from a URL path. diff --git a/internal/handler/metadata_coalesce_test.go b/internal/handler/metadata_coalesce_test.go new file mode 100644 index 00000000..af8e2eb9 --- /dev/null +++ b/internal/handler/metadata_coalesce_test.go @@ -0,0 +1,249 @@ +package handler + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +// metadataUpstream is a stub registry that counts metadata requests. It +// signals entered when the first request arrives and holds every request until +// release is closed, so a test can keep the first fetch in flight while others +// arrive. +type metadataUpstream struct { + *httptest.Server + calls atomic.Int64 + entered chan struct{} + release chan struct{} + once sync.Once +} + +func newMetadataUpstream(t *testing.T, status int) *metadataUpstream { + t.Helper() + u := &metadataUpstream{entered: make(chan struct{}), release: make(chan struct{})} + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.calls.Add(1) + u.once.Do(func() { close(u.entered) }) + <-u.release + if status != http.StatusOK { + w.WriteHeader(status) + return + } + w.Header().Set("Content-Type", "application/json") + // Echo the Accept header so a test can tell which variant it got. + _, _ = w.Write([]byte(`{"accept":"` + r.Header.Get("Accept") + `"}`)) + })) + t.Cleanup(u.Close) + return u +} + +func metadataTestProxy(t *testing.T, u *metadataUpstream) *Proxy { + t.Helper() + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = u.Client() + return proxy +} + +// waitForMetadataWaiters returns once n callers are waiting on the fetch for +// key, so a test can release upstream knowing every caller joined it. +func waitForMetadataWaiters(t *testing.T, p *Proxy, key string, n int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + p.metaMu.Lock() + f := p.inFlightMeta[key] + joined := 0 + if f != nil { + joined = f.waiters + } + p.metaMu.Unlock() + if joined >= n { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("timed out waiting for %d callers to join the metadata fetch", n) +} + +type metadataCall struct { + body []byte + err error +} + +// startMetadataCalls starts n FetchOrCacheMetadata calls for one package. +func startMetadataCalls(p *Proxy, ctx context.Context, n int, url, accept string) []chan metadataCall { + out := make([]chan metadataCall, n) + for i := range out { + out[i] = make(chan metadataCall, 1) + go func(ch chan metadataCall) { + body, _, err := p.FetchOrCacheMetadata(ctx, "npm", "left-pad", url, accept) + ch <- metadataCall{body, err} + }(out[i]) + } + return out +} + +// TestFetchOrCacheMetadata_ConcurrentMissesCoalesce asserts that concurrent +// misses for one package make a single upstream request: the CI shape, where +// parallel jobs resolve the same Composer or npm dependencies at once. +func TestFetchOrCacheMetadata_ConcurrentMissesCoalesce(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + p := metadataTestProxy(t, u) + const n = 10 + + calls := startMetadataCalls(p, context.Background(), 1, u.URL, contentTypeJSON) + <-u.entered + calls = append(calls, startMetadataCalls(p, context.Background(), n-1, u.URL, contentTypeJSON)...) + key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) + waitForMetadataWaiters(t, p, key, n-1) + close(u.release) + + for _, ch := range calls { + c := <-ch + if c.err != nil { + t.Fatalf("FetchOrCacheMetadata: %v", c.err) + } + if string(c.body) != `{"accept":"application/json"}` { + t.Errorf("body = %s", c.body) + } + } + if got := u.calls.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1", got) + } +} + +// TestFetchOrCacheMetadata_DifferentAcceptDoNotShare asserts that requests +// for different variants of one package, such as npm's abbreviated and full +// documents, each get their own fetch and their own bytes. +func TestFetchOrCacheMetadata_DifferentAcceptDoNotShare(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + p := metadataTestProxy(t, u) + const abbreviated = "application/vnd.npm.install-v1+json" + + full := startMetadataCalls(p, context.Background(), 1, u.URL, contentTypeJSON) + <-u.entered + abbr := startMetadataCalls(p, context.Background(), 1, u.URL, abbreviated) + deadline := time.Now().Add(5 * time.Second) + for u.calls.Load() < 2 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + close(u.release) + + if c := <-full[0]; c.err != nil || string(c.body) != `{"accept":"application/json"}` { + t.Errorf("full document: body %s, err %v", c.body, c.err) + } + if c := <-abbr[0]; c.err != nil || string(c.body) != `{"accept":"`+abbreviated+`"}` { + t.Errorf("abbreviated document: body %s, err %v", c.body, c.err) + } + if got := u.calls.Load(); got != 2 { + t.Errorf("upstream requests = %d, want 2", got) + } +} + +// TestFetchOrCacheMetadata_FirstCallerLeavingDoesNotFailOthers asserts that +// when the caller running the shared fetch disconnects, the fetch continues +// and everyone waiting on it still gets the metadata. +func TestFetchOrCacheMetadata_FirstCallerLeavingDoesNotFailOthers(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + p := metadataTestProxy(t, u) + const n = 5 + + leaderCtx, cancelLeader := context.WithCancel(context.Background()) + leader := startMetadataCalls(p, leaderCtx, 1, u.URL, contentTypeJSON) + <-u.entered + waiters := startMetadataCalls(p, context.Background(), n, u.URL, contentTypeJSON) + key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) + waitForMetadataWaiters(t, p, key, n) + cancelLeader() + close(u.release) + + <-leader[0] + for _, ch := range waiters { + if c := <-ch; c.err != nil { + t.Errorf("waiter failed after the first caller left: %v", c.err) + } + } + if got := u.calls.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1", got) + } +} + +// TestFetchOrCacheMetadata_WaiterLeavingKeepsFetch asserts that a waiter whose +// client disconnects returns its own context error without disturbing the +// fetch the others are waiting on. +func TestFetchOrCacheMetadata_WaiterLeavingKeepsFetch(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + p := metadataTestProxy(t, u) + + leader := startMetadataCalls(p, context.Background(), 1, u.URL, contentTypeJSON) + <-u.entered + waiterCtx, cancelWaiter := context.WithCancel(context.Background()) + leaving := startMetadataCalls(p, waiterCtx, 1, u.URL, contentTypeJSON) + key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) + waitForMetadataWaiters(t, p, key, 1) + cancelWaiter() + + if c := <-leaving[0]; !errors.Is(c.err, context.Canceled) { + t.Errorf("leaving waiter err = %v, want context.Canceled", c.err) + } + close(u.release) + if c := <-leader[0]; c.err != nil { + t.Errorf("first caller failed: %v", c.err) + } +} + +// TestFetchOrCacheMetadata_SharedNotFound asserts that an upstream 404 reaches +// every caller sharing the fetch as ErrUpstreamNotFound, from one request. +func TestFetchOrCacheMetadata_SharedNotFound(t *testing.T) { + u := newMetadataUpstream(t, http.StatusNotFound) + p := metadataTestProxy(t, u) + const n = 5 + + calls := startMetadataCalls(p, context.Background(), 1, u.URL, contentTypeJSON) + <-u.entered + calls = append(calls, startMetadataCalls(p, context.Background(), n-1, u.URL, contentTypeJSON)...) + key := metadataCoalesceKey("npm", "left-pad", u.URL, contentTypeJSON, "", false) + waitForMetadataWaiters(t, p, key, n-1) + close(u.release) + + for _, ch := range calls { + if c := <-ch; !errors.Is(c.err, ErrUpstreamNotFound) { + t.Errorf("err = %v, want ErrUpstreamNotFound", c.err) + } + } + if got := u.calls.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1", got) + } +} + +// TestCoalescedMetadataMiss_ServesRowCommittedSinceLookup asserts that a +// caller which missed the cache, but finds the row fresh once it takes the +// key, serves that row instead of fetching again. That is a fetch finishing +// between a caller's lookup and its turn at the key. +func TestCoalescedMetadataMiss_ServesRowCommittedSinceLookup(t *testing.T) { + u := newMetadataUpstream(t, http.StatusOK) + close(u.release) + p := metadataTestProxy(t, u) + p.CacheMetadata = true + p.MetadataTTL = time.Hour + + if _, _, err := p.FetchOrCacheMetadata(context.Background(), "npm", "left-pad", u.URL); err != nil { + t.Fatalf("priming fetch: %v", err) + } + + res := p.coalescedMetadataMiss(context.Background(), "npm", "left-pad", u.URL, contentTypeJSON, "", nil) + if res.err != nil { + t.Fatalf("coalescedMetadataMiss: %v", res.err) + } + if string(res.body) != `{"accept":"application/json"}` { + t.Errorf("body = %s", res.body) + } + if got := u.calls.Load(); got != 1 { + t.Errorf("upstream requests = %d, want 1 (the priming fetch only)", got) + } +} diff --git a/internal/handler/notfound_ecosystems_test.go b/internal/handler/notfound_ecosystems_test.go index bf3fe551..22cc3d29 100644 --- a/internal/handler/notfound_ecosystems_test.go +++ b/internal/handler/notfound_ecosystems_test.go @@ -16,7 +16,7 @@ func TestArtifactDownloadUpstreamNotFoundReturns404(t *testing.T) { handler func(p *Proxy) http.Handler }{ {"debian", "/pool/main/n/nginx/nginx_1.18.0-6_amd64.deb", - func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://localhost", "").Routes() }}, + func(p *Proxy) http.Handler { return NewDebianHandler(p, "http://localhost", "", nil).Routes() }}, {"rpm", "/releases/39/Everything/x86_64/os/Packages/n/nginx-1.24.0-1.fc39.x86_64.rpm", func(p *Proxy) http.Handler { return NewRPMHandler(p, "http://localhost").Routes() }}, {"apk", "/alpine/v3.22/main/x86_64/busybox-1.37.0-r12.apk", diff --git a/internal/handler/npm.go b/internal/handler/npm.go index 2cd88851..98ae10a3 100644 --- a/internal/handler/npm.go +++ b/internal/handler/npm.go @@ -1,13 +1,16 @@ package handler import ( + "bytes" "encoding/json" "errors" "fmt" + "io" "net/http" "net/url" "sort" "strings" + "sync" "time" ) @@ -15,6 +18,17 @@ const ( npmUpstream = "https://registry.npmjs.org" npmAcceptDefault = "application/vnd.npm.install-v1+json;q=1.0, application/json;q=0.8" scopedParts = 2 // scope + name in scoped packages + + // npmSecurityPrefix covers the audit endpoints: /-/npm/v1/security/audits, + // .../audits/quick and .../advisories/bulk. + npmSecurityPrefix = "/-/npm/v1/security/" + + // npmKeysPath serves the registry signing keys `npm audit signatures` reads. + npmKeysPath = "/-/npm/v1/keys" + + // npmSecurityMaxBody caps the audit payload. Buffering it lets an oversized + // body be refused before the upstream request starts. + npmSecurityMaxBody = 16 << 20 ) // NPMHandler handles npm registry protocol requests. @@ -41,11 +55,25 @@ func NewNPMHandler(proxy *Proxy, proxyURL, upstreamURL string) *NPMHandler { // Mount this at /npm on your router. func (h *NPMHandler) Routes() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // The /-/npm/v1 endpoints share the /-/ prefix with tarball paths, so + // they are routed before the tarball dispatch below. Audits are POSTs, + // so they also precede the GET-only gate. + if strings.HasPrefix(r.URL.Path, npmSecurityPrefix) { + h.handleSecurity(w, r) + return + } + if r.Method != http.MethodGet { http.Error(w, "method not allowed", http.StatusMethodNotAllowed) return } + if r.URL.Path == npmKeysPath { + h.proxy.ProxyUpstream(w, r, h.upstreamURL+npmKeysPath, + []string{headerAccept, headerAcceptEncoding}) + return + } + path := strings.TrimPrefix(r.URL.Path, "/") // Check if this is a tarball download (contains /-/) @@ -59,6 +87,73 @@ func (h *NPMHandler) Routes() http.Handler { }) } +// handleSecurity relays the npm audit endpoints to upstream. The request body +// is the dependency tree being audited, so no two requests share a cache key, +// and the response carries no tarball URLs to rewrite. +// +// Advisories come from upstream's database, not the proxy's own vulnerability +// data, and versions withheld by cooldown are not excluded from the report. +func (h *NPMHandler) handleSecurity(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + JSONError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + + if containsPathTraversal(r.URL.Path) { + JSONError(w, http.StatusBadRequest, "invalid path") + return + } + + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, npmSecurityMaxBody)) + if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + JSONError(w, http.StatusRequestEntityTooLarge, "audit request too large") + return + } + // A client aborting mid-upload must not be told its payload was too big. + h.proxy.Logger.Warn("npm audit request body unreadable", "path", r.URL.Path, "error", err) + JSONError(w, http.StatusBadRequest, "could not read audit request") + return + } + + // EscapedPath keeps an encoded "?" or "#" from turning the rest of the path + // into a query or fragment upstream. + upstreamURL := h.upstreamURL + r.URL.EscapedPath() + if r.URL.RawQuery != "" { + upstreamURL += "?" + r.URL.RawQuery + } + + h.proxy.Logger.Info("npm audit request", "path", r.URL.Path, "bytes", len(body)) + + req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, upstreamURL, bytes.NewReader(body)) + if err != nil { + JSONError(w, http.StatusInternalServerError, "failed to create request") + return + } + + // Clients may gzip the body, so the headers describing it travel with it. + for _, header := range []string{headerContentType, headerContentEncoding, headerAccept, headerAcceptEncoding} { + if v := r.Header.Get(header); v != "" { + req.Header.Set(header, v) + } + } + if req.Header.Get(headerContentType) == "" { + req.Header.Set(headerContentType, contentTypeJSON) + } + h.proxy.applyUpstreamAuth(req) + + resp, err := h.proxy.HTTPClient.Do(req) + if err != nil { + h.proxy.Logger.Error("npm audit request failed", "path", r.URL.Path, "error", err) + JSONError(w, http.StatusBadGateway, "failed to reach upstream registry") + return + } + defer func() { _ = resp.Body.Close() }() + + h.proxy.relayResponse(w, r, resp, nil) +} + // handlePackageMetadata proxies package metadata from upstream and rewrites tarball URLs. func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Request) { packageName := h.extractPackageName(r) @@ -94,8 +189,17 @@ func (h *NPMHandler) handlePackageMetadata(w http.ResponseWriter, r *http.Reques return } - rewritten, err := h.rewriteMetadata(packageName, body) + rewritten, err := h.proxy.cachedRewrite(r.Context(), "npm", h.proxyURL, packageName, body, func(b []byte) ([]byte, error) { + return h.rewriteMetadata(packageName, b) + }) if err != nil { + if r.Context().Err() != nil { + return // the client left while waiting on a shared rewrite + } + if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { + JSONError(w, http.StatusBadGateway, "failed to filter package metadata") + return + } // If rewriting fails, just proxy the original h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) w.Header().Set(headerContentType, contentTypeJSON) @@ -120,10 +224,14 @@ func (h *NPMHandler) rewriteMetadata(packageName string, body []byte) ([]byte, e // Rewrite tarball URLs in versions versions, ok := metadata["versions"].(map[string]any) if !ok { + if len(h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName))) != 0 { + return nil, errors.New("npm metadata has no versions object") + } return body, nil // No versions to rewrite } h.applyCooldownFiltering(metadata, versions, packageName) + h.applyDenylistFiltering(metadata, versions, packageName) h.rewriteTarballURLs(versions, packageName) return json.Marshal(metadata) @@ -208,6 +316,13 @@ func (h *NPMHandler) rewriteTarballURLs(versions map[string]any, packageName str if idx := strings.LastIndex(tarball, "/"); idx >= 0 { filename = tarball[idx+1:] } + if h.extractVersionFromFilename(packageName, filename) != version { + _, shortName, scoped := strings.Cut(packageName, "/") + if !scoped { + shortName = packageName + } + filename = shortName + "-" + version + ".tgz" + } escapedName := url.PathEscape(packageName) newTarball := fmt.Sprintf("%s/npm/%s/-/%s", h.proxyURL, escapedName, filename) @@ -270,27 +385,28 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { h.proxy.Logger.Info("npm download request", "package", packageName, "version", version, "filename", filename) - if h.versionInCooldown(r, packageName, version) { + if h.proxy.versionDenied("npm", packageName, version) { + JSONError(w, http.StatusForbidden, ErrVersionDenied.Error()+": "+canonicalVersionPURL("npm", packageName, version)) + return + } + metadata := sync.OnceValues(func() ([]byte, error) { + upstreamURL := h.upstreamURL + "/" + url.PathEscape(packageName) + body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, contentTypeJSON) + return body, err + }) + if h.versionInCooldown(packageName, version, metadata) { h.proxy.Logger.Info("cooldown: withholding npm tarball", "package", packageName, "version", version) JSONError(w, http.StatusNotFound, "version not found") return } - downloadURL := fmt.Sprintf( - "%s/%s/-/%s", - h.upstreamURL, - escapeNPMDownloadPackage(packageName), - url.PathEscape(filename), - ) - result, err := h.proxy.GetOrFetchArtifactFromURL( - r.Context(), "npm", packageName, version, filename, downloadURL, - ) + result, err := h.getTarball(r, packageName, version, filename, metadata) if err != nil { switch { case errors.Is(err, ErrUpstreamNotFound): JSONError(w, http.StatusNotFound, "package not found") - case errors.Is(err, ErrArtifactBlocked): + case errors.Is(err, ErrArtifactBlocked), errors.Is(err, ErrVersionDenied): JSONError(w, http.StatusForbidden, err.Error()) default: h.proxy.Logger.Error("failed to get artifact", "error", err) @@ -299,7 +415,61 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) +} + +func (h *NPMHandler) getTarball(r *http.Request, packageName, version, filename string, metadata func() ([]byte, error)) (*CacheResult, error) { + if cached, err := h.proxy.GetCachedArtifact(r.Context(), "npm", packageName, version, filename); err != nil || cached != nil { + return cached, err + } + downloadURL := fmt.Sprintf("%s/%s/-/%s", h.upstreamURL, escapeNPMDownloadPackage(packageName), url.PathEscape(filename)) + body, err := metadata() + if err == nil { + if tarball := npmVersionTarball(body, version); tarball != "" { + downloadURL, err = h.validateTarballURL(tarball) + if err != nil { + return nil, err + } + } + } + return h.proxy.GetOrFetchArtifactFromURL(r.Context(), "npm", packageName, version, filename, downloadURL) +} + +func npmVersionTarball(body []byte, version string) string { + var metadata struct { + Versions map[string]struct { + Dist struct { + Tarball string `json:"tarball"` + } `json:"dist"` + } `json:"versions"` + } + if err := json.Unmarshal(body, &metadata); err != nil { + return "" + } + return metadata.Versions[version].Dist.Tarball +} + +func (h *NPMHandler) validateTarballURL(raw string) (string, error) { + tarball, err := url.Parse(raw) + if err != nil { + return "", fmt.Errorf("parsing npm tarball URL: %w", err) + } + upstream, err := url.Parse(h.upstreamURL) + if err != nil { + return "", fmt.Errorf("parsing npm upstream URL: %w", err) + } + if tarball.User != nil || tarball.Fragment != "" || + tarball.Scheme != upstream.Scheme || !strings.EqualFold(tarball.Host, upstream.Host) { + return "", errors.New("npm tarball URL does not match upstream registry") + } + if containsPathTraversal(tarball.Path) || strings.Contains(tarball.Path, "\\") { + return "", errors.New("npm tarball URL contains path traversal") + } + basePath := strings.TrimRight(upstream.Path, "/") + if basePath != "" && !strings.HasPrefix(tarball.Path, basePath+"/") { + return "", errors.New("npm tarball URL is outside upstream base path") + } + return tarball.String(), nil } // versionInCooldown reports whether a version is still inside the cooldown @@ -313,7 +483,7 @@ func (h *NPMHandler) handleDownload(w http.ResponseWriter, r *http.Request) { // fetched and parsed at most once per version. A version with no usable // publish time is allowed through, matching how applyCooldownFiltering // treats it. -func (h *NPMHandler) versionInCooldown(r *http.Request, packageName, version string) bool { +func (h *NPMHandler) versionInCooldown(packageName, version string, metadata func() ([]byte, error)) bool { if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { return false } @@ -323,25 +493,23 @@ func (h *NPMHandler) versionInCooldown(r *http.Request, packageName, version str return !h.proxy.Cooldown.IsAllowed("npm", canonicalPackagePURL("npm", packageName), ver.PublishedAt.Time) } - upstreamURL := fmt.Sprintf("%s/%s", h.upstreamURL, url.PathEscape(packageName)) - - body, _, err := h.proxy.FetchOrCacheMetadata(r.Context(), "npm", packageName, upstreamURL, contentTypeJSON) + body, err := metadata() if err != nil { h.proxy.Logger.Warn("cooldown: could not fetch npm metadata for download check", "package", packageName, "version", version, "error", err) return false } - var metadata struct { + var document struct { Time map[string]string `json:"time"` } - if err := json.Unmarshal(body, &metadata); err != nil { + if err := json.Unmarshal(body, &document); err != nil { h.proxy.Logger.Warn("cooldown: could not parse npm metadata for download check", "package", packageName, "version", version, "error", err) return false } - published, ok := metadata.Time[version] + published, ok := document.Time[version] if !ok { return false } diff --git a/internal/handler/npm_denylist.go b/internal/handler/npm_denylist.go new file mode 100644 index 00000000..7f8b87db --- /dev/null +++ b/internal/handler/npm_denylist.go @@ -0,0 +1,49 @@ +package handler + +import "github.com/Masterminds/semver/v3" + +func (h *NPMHandler) applyDenylistFiltering(metadata, versions map[string]any, packageName string) { + if h.proxy.Denylist == nil { + return + } + denied := h.proxy.Denylist.Versions(canonicalPackagePURL("npm", packageName)) + if len(denied) == 0 { + return + } + times, _ := metadata["time"].(map[string]any) + for version := range denied { + delete(versions, version) + delete(times, version) + } + tags, _ := metadata["dist-tags"].(map[string]any) + for tag, value := range tags { + version, _ := value.(string) + if !denied[version] { + continue + } + delete(tags, tag) + // Custom tags describe publisher intent; do not retarget them. For + // latest, use the highest remaining stable version, even in abbreviated + // packuments without publication timestamps. + if tag == "latest" { + if latest := newestStableNPMVersion(versions); latest != "" { + tags[tag] = latest + } + } + } +} + +func newestStableNPMVersion(versions map[string]any) string { + var best *semver.Version + result := "" + for version := range versions { + parsed, err := semver.StrictNewVersion(version) + if err != nil || parsed.Prerelease() != "" { + continue + } + if best == nil || parsed.GreaterThan(best) || (parsed.Equal(best) && version > result) { + best, result = parsed, version + } + } + return result +} diff --git a/internal/handler/npm_tarball_test.go b/internal/handler/npm_tarball_test.go new file mode 100644 index 00000000..303f2184 --- /dev/null +++ b/internal/handler/npm_tarball_test.go @@ -0,0 +1,154 @@ +package handler + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +func TestNPMContentAddressedTarball(t *testing.T) { + for _, tc := range []struct { + cacheMetadata, stream bool + metadata, tarballs int64 + }{ + {false, false, 2, 1}, + {false, true, 3, 2}, + {true, false, 1, 1}, + {true, true, 1, 2}, + } { + t.Run(fmt.Sprintf("metadata=%t/stream=%t", tc.cacheMetadata, tc.stream), func(t *testing.T) { + var metadataCalls, tarballCalls atomic.Int64 + upstream := npmContentAddressedRegistry(t, &metadataCalls, &tarballCalls) + p, _, _, _ := setupTestProxy(t) + p.CacheMetadata = tc.cacheMetadata + p.MetadataTTL = time.Hour + p.StreamArtifacts = tc.stream + p.Cooldown = &cooldown.Config{Default: "7d"} + p.HTTPClient = upstream.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + p.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) + h := NewNPMHandler(p, "http://proxy.test", upstream.URL+"/registry") + routes := http.StripPrefix("/npm", h.Routes()) + metadata := httptest.NewRecorder() + routes.ServeHTTP(metadata, httptest.NewRequest(http.MethodGet, "/npm/@example/widget", nil)) + if metadata.Code != http.StatusOK { + t.Fatalf("metadata status = %d: %s", metadata.Code, metadata.Body.String()) + } + tarball := npmVersionTarball(metadata.Body.Bytes(), testVersion100) + if want := "http://proxy.test/npm/@example%2Fwidget/-/widget-1.0.0.tgz"; tarball != want { + t.Fatalf("rewritten tarball = %q, want %q", tarball, want) + } + for i := range 2 { + response := httptest.NewRecorder() + routes.ServeHTTP(response, httptest.NewRequest(http.MethodGet, tarball, nil)) + if response.Code != http.StatusOK || response.Body.String() != "tarball bytes" { + t.Fatalf("download %d = %d %q", i, response.Code, response.Body.String()) + } + if !tc.stream { + upstream.Close() + } + } + if metadataCalls.Load() != tc.metadata || tarballCalls.Load() != tc.tarballs { + t.Errorf("upstream requests: metadata=%d tarballs=%d, want %d %d", metadataCalls.Load(), tarballCalls.Load(), tc.metadata, tc.tarballs) + } + }) + } +} + +func npmContentAddressedRegistry(t *testing.T, metadataCalls, tarballCalls *atomic.Int64) *httptest.Server { + t.Helper() + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/registry/@example/widget": + metadataCalls.Add(1) + w.Header().Set("Content-Type", contentTypeJSON) + _, _ = fmt.Fprintf(w, `{"name":"@example/widget","time":{"1.0.0":%q},"versions":{"1.0.0":{"dist":{"tarball":%q}}}}`, + time.Now().Add(-30*24*time.Hour).Format(time.RFC3339), upstream.URL+"/registry/download/@example/widget/1.0.0/abc123?token=example") + case "/registry/download/@example/widget/1.0.0/abc123": + if r.URL.RawQuery != "token=example" { + t.Errorf("tarball query = %q", r.URL.RawQuery) + } + tarballCalls.Add(1) + _, _ = io.WriteString(w, "tarball bytes") + default: + t.Errorf("unexpected upstream request: %s", r.URL) + http.NotFound(w, r) + } + })) + t.Cleanup(upstream.Close) + return upstream +} + +func TestNPMTarballMetadataFallback(t *testing.T) { + for _, body := range []string{`{"versions":{}}`, `{"versions":{"1.0.0":{"dist":{}}}}`, `not json`, "unavailable"} { + t.Run(body, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if body == "unavailable" { + w.WriteHeader(http.StatusServiceUnavailable) + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(upstream.Close) + p, _, _, fetcher := setupTestProxy(t) + p.CacheMetadata = true + p.MetadataTTL = time.Hour + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package"))} + h := NewNPMHandler(p, "http://proxy.test", upstream.URL) + if body != "unavailable" { + if _, _, err := p.FetchOrCacheMetadata(t.Context(), "npm", "@example/widget", upstream.URL); err != nil { + t.Fatal(err) + } + upstream.Close() + } + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/@example/widget/-/widget-1.0.0.tgz", nil)) + if response.Code != http.StatusOK { + t.Fatalf("status = %d: %s", response.Code, response.Body.String()) + } + if want := upstream.URL + "/@example/widget/-/widget-1.0.0.tgz"; fetcher.fetchedURL != want { + t.Errorf("fetched URL = %q, want %q", fetcher.fetchedURL, want) + } + }) + } +} + +func TestNPMTarballRejectsUnsafeMetadataURL(t *testing.T) { + for _, target := range []string{ + "https://outside.invalid/package.tgz", "//outside.invalid/package.tgz", "/registry/package.tgz", + "UPSTREAM/outside/package.tgz", "UPSTREAM/registry-other/package.tgz", + "UPSTREAM/registry/../outside/package.tgz", "UPSTREAM/registry/%2e%2e/outside/package.tgz", + "UPSTREAM/registry/%252e%252e/outside/package.tgz", "UPSTREAM/registry/..%2foutside/package.tgz", + "UPSTREAM/registry/..%5coutside/package.tgz", "UPSTREAM/registry/package.tgz#fragment", + "USERINFO/registry/package.tgz", "SCHEME/registry/package.tgz", + } { + t.Run(target, func(t *testing.T) { + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + address, _ := url.Parse(upstream.URL) + address.User = url.UserPassword("user", "secret") + tarball := strings.NewReplacer("UPSTREAM", upstream.URL, "USERINFO", address.String(), "SCHEME", strings.Replace(upstream.URL, "http:", "https:", 1)).Replace(target) + _ = json.NewEncoder(w).Encode(map[string]any{"versions": map[string]any{testVersion100: map[string]any{"dist": map[string]string{"tarball": tarball}}}}) + })) + t.Cleanup(upstream.Close) + p, _, _, fetcher := setupTestProxy(t) + h := NewNPMHandler(p, "http://proxy.test", upstream.URL+"/registry") + response := httptest.NewRecorder() + h.Routes().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/widget/-/widget-1.0.0.tgz", nil)) + if response.Code != http.StatusBadGateway || fetcher.fetchCalled { + t.Errorf("status = %d, fetched = %t; body: %s", response.Code, fetcher.fetchCalled, response.Body.String()) + } + }) + } +} diff --git a/internal/handler/npm_test.go b/internal/handler/npm_test.go index c4a5f7ec..5d8378fb 100644 --- a/internal/handler/npm_test.go +++ b/internal/handler/npm_test.go @@ -1,8 +1,11 @@ package handler import ( + "bytes" + "compress/gzip" "encoding/json" "errors" + "fmt" "io" "log/slog" "net/http" @@ -10,6 +13,7 @@ import ( "strings" "sync/atomic" "testing" + "testing/iotest" "time" "github.com/git-pkgs/cooldown" @@ -534,22 +538,13 @@ func TestNPMDownloadCooldown(t *testing.T) { } func TestNPMDownloadCooldownDisabled(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - t.Error("metadata must not be fetched when cooldown is disabled") - w.WriteHeader(http.StatusInternalServerError) - })) - defer upstream.Close() - - proxy, _, _, fetcher := setupTestProxy(t) - proxy.HTTPClient = upstream.Client() - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("tarball data")), - ContentType: "application/octet-stream", - } - - h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + proxy, _, _, _ := setupTestProxy(t) + h := NewNPMHandler(proxy, "http://proxy.test", "") - if h.versionInCooldown(httptest.NewRequest(http.MethodGet, "/", nil), "leftpad", testVersion100) { + if h.versionInCooldown("leftpad", testVersion100, func() ([]byte, error) { + t.Fatal("cooldown must not request metadata when disabled") + return nil, nil + }) { t.Error("versionInCooldown = true, want false when cooldown is not configured") } } @@ -573,13 +568,10 @@ func TestNPMDownloadCooldownUsesStoredPublishTime(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - proxy, db, _, fetcher := setupTestProxy(t) + proxy, db, store, _ := setupTestProxy(t) proxy.HTTPClient = upstream.Client() proxy.Cooldown = &cooldown.Config{Default: "7d"} - fetcher.artifact = &fetch.Artifact{ - Body: io.NopCloser(strings.NewReader("tarball data")), - ContentType: "application/octet-stream", - } + seedPackage(t, db, store, "npm", "leftpad", tt.version, "leftpad-"+tt.version+".tgz", "tarball data") if err := db.SetVersionPublishedAt("pkg:npm/leftpad@"+tt.version, "pkg:npm/leftpad", tt.publishedAt); err != nil { t.Fatalf("seeding publish time failed: %v", err) @@ -709,3 +701,349 @@ func TestNPMDownloadErrorResponsesAreJSON(t *testing.T) { }) } } + +// newNPMAuditUpstream returns a handler pointed at a stub registry, plus the +// last request that registry saw. +func newNPMAuditUpstream(t *testing.T, respond http.HandlerFunc) (*NPMHandler, *npmAuditCapture) { + t.Helper() + + capture := &npmAuditCapture{} + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capture.method = r.Method + capture.path = r.URL.Path + capture.query = r.URL.RawQuery + capture.contentType = r.Header.Get("Content-Type") + capture.contentEncoding = r.Header.Get("Content-Encoding") + capture.authorization = r.Header.Get("Authorization") + capture.body, _ = io.ReadAll(r.Body) + respond(w, r) + })) + t.Cleanup(upstream.Close) + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + return NewNPMHandler(proxy, "http://proxy.test", upstream.URL), capture +} + +type npmAuditCapture struct { + method string + path string + query string + contentType string + contentEncoding string + authorization string + body []byte +} + +func npmAuditJSON(body string) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, body) + } +} + +func serveNPM(t *testing.T, h *NPMHandler, method, target string, body io.Reader, + headers map[string]string, +) *httptest.ResponseRecorder { + t.Helper() + + req := httptest.NewRequest(method, target, body) + for name, value := range headers { + req.Header.Set(name, value) + } + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + return w +} + +func TestNPMAuditRelaysRequestAndResponse(t *testing.T) { + const report = `{"actions":[],"advisories":{},"metadata":{"vulnerabilities":{"total":0}}}` + const payload = `{"name":"app","requires":{"lodash":"^4.17.21"},"dependencies":{}}` + + h, got := newNPMAuditUpstream(t, npmAuditJSON(report)) + w := serveNPM(t, h, http.MethodPost, "/-/npm/v1/security/audits?foo=bar", + strings.NewReader(payload), map[string]string{"Content-Type": "application/json"}) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if w.Body.String() != report { + t.Errorf("body = %q, want %q", w.Body.String(), report) + } + if ct := w.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("response Content-Type = %q, want application/json", ct) + } + if got.method != http.MethodPost { + t.Errorf("upstream method = %q, want POST", got.method) + } + if got.path != "/-/npm/v1/security/audits" { + t.Errorf("upstream path = %q, want /-/npm/v1/security/audits", got.path) + } + if got.query != "foo=bar" { + t.Errorf("upstream query = %q, want foo=bar", got.query) + } + if string(got.body) != payload { + t.Errorf("upstream body = %q, want %q", got.body, payload) + } + if got.contentType != "application/json" { + t.Errorf("upstream Content-Type = %q, want application/json", got.contentType) + } +} + +// npm gzips its audit payload, so the bytes and the header describing them +// must travel together. +func TestNPMAuditForwardsGzippedBody(t *testing.T) { + var gzipped bytes.Buffer + zw := gzip.NewWriter(&gzipped) + if _, err := io.WriteString(zw, `{"name":"app"}`); err != nil { + t.Fatalf("writing gzip body: %v", err) + } + if err := zw.Close(); err != nil { + t.Fatalf("closing gzip writer: %v", err) + } + want := gzipped.Bytes() + + h, got := newNPMAuditUpstream(t, npmAuditJSON(`{}`)) + w := serveNPM(t, h, http.MethodPost, "/-/npm/v1/security/audits", + bytes.NewReader(want), map[string]string{ + "Content-Type": "application/json", + "Content-Encoding": "gzip", + }) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got.contentEncoding != "gzip" { + t.Errorf("upstream Content-Encoding = %q, want gzip", got.contentEncoding) + } + if !bytes.Equal(got.body, want) { + t.Errorf("upstream body was altered: got %d bytes, want %d", len(got.body), len(want)) + } +} + +func TestNPMAuditCoversAllSecurityEndpoints(t *testing.T) { + paths := []string{ + "/-/npm/v1/security/audits", // pnpm audit, npm audit (full) + "/-/npm/v1/security/audits/quick", // yarn npm audit, npm audit fallback + "/-/npm/v1/security/advisories/bulk", // npm audit (npm 7+) + } + + for _, path := range paths { + t.Run(path, func(t *testing.T) { + h, got := newNPMAuditUpstream(t, npmAuditJSON(`{}`)) + w := serveNPM(t, h, http.MethodPost, path, strings.NewReader(`{}`), nil) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if got.path != path { + t.Errorf("upstream path = %q, want %q", got.path, path) + } + }) + } +} + +func TestNPMAuditAppliesUpstreamAuth(t *testing.T) { + h, got := newNPMAuditUpstream(t, npmAuditJSON(`{}`)) + h.proxy.AuthForURL = func(string) (string, string) { + return "Authorization", "Bearer npm-token" + } + + serveNPM(t, h, http.MethodPost, "/-/npm/v1/security/audits", strings.NewReader(`{}`), nil) + + if got.authorization != "Bearer npm-token" { + t.Errorf("Authorization = %q, want %q", got.authorization, "Bearer npm-token") + } +} + +func TestNPMAuditRelaysUpstreamError(t *testing.T) { + const upstreamBody = `{"error":"unauthorized"}` + + h, _ := newNPMAuditUpstream(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + _, _ = io.WriteString(w, upstreamBody) + }) + w := serveNPM(t, h, http.MethodPost, "/-/npm/v1/security/audits", strings.NewReader(`{}`), nil) + + if w.Code != http.StatusUnauthorized { + t.Errorf("status = %d, want %d", w.Code, http.StatusUnauthorized) + } + if w.Body.String() != upstreamBody { + t.Errorf("body = %q, want %q", w.Body.String(), upstreamBody) + } +} + +// A proxy-side failure must still be JSON, or the client reports it as a +// malformed audit response. +func TestNPMAuditUpstreamUnreachable(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + upstreamURL := upstream.URL + upstream.Close() // nothing is listening now + + proxy, _, _, _ := setupTestProxy(t) + h := NewNPMHandler(proxy, "http://proxy.test", upstreamURL) + + w := serveNPM(t, h, http.MethodPost, "/-/npm/v1/security/audits", strings.NewReader(`{}`), nil) + + if w.Code != http.StatusBadGateway { + t.Errorf("status = %d, want %d", w.Code, http.StatusBadGateway) + } + if ct := w.Header().Get("Content-Type"); ct != "application/json" { + t.Errorf("Content-Type = %q, want application/json", ct) + } + if !json.Valid(w.Body.Bytes()) { + t.Errorf("body is not valid JSON: %q", w.Body.String()) + } +} + +func TestNPMAuditRejectsBadRequests(t *testing.T) { + t.Run("non-POST method", func(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test") + + w := serveNPM(t, h, http.MethodGet, "/-/npm/v1/security/audits", nil, nil) + + if w.Code != http.StatusMethodNotAllowed { + t.Errorf("status = %d, want %d", w.Code, http.StatusMethodNotAllowed) + } + if !json.Valid(w.Body.Bytes()) { + t.Errorf("body is not valid JSON: %q", w.Body.String()) + } + }) + + t.Run("body over the size cap", func(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test") + + body := strings.NewReader(strings.Repeat("a", npmSecurityMaxBody+1)) + w := serveNPM(t, h, http.MethodPost, "/-/npm/v1/security/audits", body, nil) + + if w.Code != http.StatusRequestEntityTooLarge { + t.Errorf("status = %d, want %d", w.Code, http.StatusRequestEntityTooLarge) + } + }) + + t.Run("unreadable body is not reported as too large", func(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test") + + req := httptest.NewRequest(http.MethodPost, "/-/npm/v1/security/audits", + iotest.TimeoutReader(strings.NewReader("{}"))) + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, req) + + if w.Code != http.StatusBadRequest { + t.Errorf("status = %d, want %d", w.Code, http.StatusBadRequest) + } + if !json.Valid(w.Body.Bytes()) { + t.Errorf("body is not valid JSON: %q", w.Body.String()) + } + }) +} + +func TestNPMAuditDoesNotInjectUpstreamQuery(t *testing.T) { + h, got := newNPMAuditUpstream(t, npmAuditJSON(`{}`)) + + serveNPM(t, h, http.MethodPost, + "/-/npm/v1/security/audits%3Fevil=1", strings.NewReader(`{}`), nil) + + if got.query != "" { + t.Errorf("upstream query = %q, want empty: encoded ? leaked into the query", got.query) + } + if got.path != "/-/npm/v1/security/audits?evil=1" { + t.Errorf("upstream path = %q, want the encoded ? kept in the path", got.path) + } +} + +// The audit POST must go through relayResponse, not copy upstream headers +// wholesale: a relayed Connection header would be honoured downstream. +// TestRelayRoutes covers the GET paths; this covers the POST. +func TestNPMAuditStripsHopByHopHeaders(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + conn, rw, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + defer func() { _ = conn.Close() }() + _, _ = fmt.Fprint(rw, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n"+ + "Connection: X-Private\r\nX-Private: secret\r\nContent-Length: 2\r\n\r\n{}") + _ = rw.Flush() + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + downstream := httptest.NewServer(NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) + defer downstream.Close() + + resp, err := downstream.Client().Post( + downstream.URL+"/-/npm/v1/security/audits", contentTypeJSON, strings.NewReader(`{}`)) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.Header.Get("Connection") != "" || resp.Header.Get("X-Private") != "" { + t.Errorf("connection-scoped headers leaked: %v", resp.Header) + } +} + +// `npm audit signatures` reads the registry signing keys. The path used to fall +// through to the package dispatch and be escaped into a package name. +func TestNPMKeysProxiesUpstream(t *testing.T) { + const keys = `{"keys":[{"keyid":"SHA256:jl3bwswu","keytype":"ecdsa-sha2-nistp256"}]}` + + var gotPath, gotAuth string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // EscapedPath, not Path: the bug escaped the slashes into a package + // name, and the server decodes %2F back into Path either way. + gotPath, gotAuth = r.URL.EscapedPath(), r.Header.Get("Authorization") + if gotPath != npmKeysPath { + w.WriteHeader(http.StatusMethodNotAllowed) // what registry.npmjs.org answers + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, keys) + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.AuthForURL = func(string) (string, string) { + return "Authorization", "Bearer npm-token" + } + h := NewNPMHandler(proxy, "http://proxy.test", upstream.URL) + + w := serveNPM(t, h, http.MethodGet, npmKeysPath, nil, nil) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } + if w.Body.String() != keys { + t.Errorf("body = %q, want %q", w.Body.String(), keys) + } + if gotPath != npmKeysPath { + t.Errorf("upstream path = %q, want %q", gotPath, npmKeysPath) + } + if gotAuth != "Bearer npm-token" { + t.Errorf("Authorization = %q, want it applied", gotAuth) + } +} + +// Tarball paths share the /-/ prefix with the /-/npm/v1 endpoints. +func TestNPMTarballStillRoutesToDownload(t *testing.T) { + proxy, _, _, artifactFetcher := setupTestProxy(t) + artifactFetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader("package")), + ContentType: "application/gzip", + } + h := NewNPMHandler(proxy, "http://proxy.test", "https://npm.example.test") + + w := serveNPM(t, h, http.MethodGet, "/lodash/-/lodash-4.17.21.tgz", nil, nil) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body: %s", w.Code, http.StatusOK, w.Body.String()) + } +} diff --git a/internal/handler/nuget.go b/internal/handler/nuget.go index 7df62408..612f32ee 100644 --- a/internal/handler/nuget.go +++ b/internal/handler/nuget.go @@ -4,10 +4,8 @@ import ( "encoding/json" "errors" "fmt" - "io" "net/http" "strings" - "time" ) const ( @@ -51,10 +49,12 @@ func (h *NuGetHandler) Routes() http.Handler { // Package content (downloads) mux.HandleFunc("GET /v3-flatcontainer/{id}/{version}/{filename}", h.handleDownload) - mux.HandleFunc("GET /v3-flatcontainer/{id}/index.json", h.proxyUpstream) + mux.HandleFunc("GET /v3-flatcontainer/{id}/index.json", h.handleVersionList) // Registration (package metadata) - use prefix matching since {version}.json isn't allowed - mux.HandleFunc("GET /v3/registration5-gz-semver2/", h.handleRegistration) + for _, prefix := range nugetRegistrationPrefixes { + mux.HandleFunc("GET "+prefix, h.handleRegistration) + } // Search mux.HandleFunc("GET /query", h.proxyUpstream) @@ -84,6 +84,10 @@ func (h *NuGetHandler) handleServiceIndex(w http.ResponseWriter, r *http.Request rewritten, err := h.rewriteServiceIndex(body) if err != nil { + if h.cooldownEnabled() { + h.nugetMetadataError(w, err) + return + } h.proxy.Logger.Warn("failed to rewrite service index, proxying original", "error", err) w.Header().Set(headerContentType, "application/json") _, _ = w.Write(body) @@ -131,6 +135,10 @@ func (h *NuGetHandler) rewriteNuGetURL(origURL, serviceType string) string { switch serviceType { case "PackageBaseAddress/3.0.0": return h.proxyURL + "/nuget/v3-flatcontainer/" + case "RegistrationsBaseUrl", "RegistrationsBaseUrl/3.0.0-beta", "RegistrationsBaseUrl/3.0.0-rc": + return h.proxyURL + "/nuget/v3/registration5-semver1/" + case "RegistrationsBaseUrl/3.4.0": + return h.proxyURL + "/nuget/v3/registration5-gz-semver1/" case "RegistrationsBaseUrl/3.6.0", "RegistrationsBaseUrl/Versioned": return h.proxyURL + "/nuget/v3/registration5-gz-semver2/" case "SearchQueryService", "SearchQueryService/3.0.0-rc", "SearchQueryService/3.5.0": @@ -142,140 +150,6 @@ func (h *NuGetHandler) rewriteNuGetURL(origURL, serviceType string) string { } } -// handleRegistration proxies NuGet registration pages, applying cooldown filtering. -func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request) { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - h.proxyUpstream(w, r) - return - } - - upstreamURL := h.buildUpstreamURL(r) - - h.proxy.Logger.Debug("fetching registration for cooldown filtering", "url", upstreamURL) - - req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil) - if err != nil { - http.Error(w, "failed to create request", http.StatusInternalServerError) - return - } - req.Header.Set(headerAcceptEncoding, "gzip") - - resp, err := h.proxy.HTTPClient.Do(req) - if err != nil { - h.proxy.Logger.Error("upstream request failed", "error", err) - http.Error(w, "upstream request failed", http.StatusBadGateway) - return - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode != http.StatusOK { - for k, vv := range resp.Header { - for _, v := range vv { - w.Header().Add(k, v) - } - } - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) - return - } - - body, err := h.proxy.ReadMetadata(resp.Body) - if err != nil { - http.Error(w, "failed to read response", http.StatusInternalServerError) - return - } - - filtered, err := h.applyCooldownFiltering(body) - if err != nil { - h.proxy.Logger.Warn("failed to filter registration, proxying original", "error", err) - w.Header().Set(headerContentType, "application/json") - _, _ = w.Write(body) - return - } - - w.Header().Set(headerContentType, "application/json") - _, _ = w.Write(filtered) -} - -// applyCooldownFiltering filters versions from NuGet registration pages -// that are too recently published. -func (h *NuGetHandler) applyCooldownFiltering(body []byte) ([]byte, error) { - if h.proxy.Cooldown == nil || !h.proxy.Cooldown.Enabled() { - return body, nil - } - - var registration map[string]any - if err := json.Unmarshal(body, ®istration); err != nil { - return nil, err - } - - pages, ok := registration["items"].([]any) - if !ok { - return body, nil - } - - for _, page := range pages { - pageMap, ok := page.(map[string]any) - if !ok { - continue - } - - items, ok := pageMap["items"].([]any) - if !ok { - continue - } - - filtered := items[:0] - for _, item := range items { - itemMap, ok := item.(map[string]any) - if !ok { - continue - } - - catalogEntry, ok := itemMap["catalogEntry"].(map[string]any) - if !ok { - filtered = append(filtered, item) - continue - } - - version, _ := catalogEntry["version"].(string) - id, _ := catalogEntry["id"].(string) - publishedStr, _ := catalogEntry["published"].(string) - - if publishedStr == "" { - filtered = append(filtered, item) - continue - } - - publishedAt, err := time.Parse(time.RFC3339, publishedStr) - if err != nil { - // NuGet uses a slightly non-standard format, try parsing with fractional seconds - publishedAt, err = time.Parse("2006-01-02T15:04:05.999-07:00", publishedStr) - if err != nil { - filtered = append(filtered, item) - continue - } - } - - packagePURL := canonicalPackagePURL("nuget", strings.ToLower(id)) - - if !h.proxy.Cooldown.IsAllowed("nuget", packagePURL, publishedAt) { - h.proxy.Logger.Info("cooldown: filtering nuget version", - "package", id, "version", version, - "published", publishedStr) - continue - } - - filtered = append(filtered, item) - } - - pageMap["items"] = filtered - pageMap["count"] = len(filtered) - } - - return json.Marshal(registration) -} - // handleDownload serves a package file, fetching and caching from upstream if needed. func (h *NuGetHandler) handleDownload(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") @@ -287,6 +161,18 @@ func (h *NuGetHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } + if h.cooldownEnabled() { + allowed, err := h.nugetDownloadAllowed(r.Context(), id, version) + if err != nil { + h.nugetMetadataError(w, err) + return + } + if !allowed { + JSONError(w, http.StatusNotFound, "version not found") + return + } + } + // Only cache .nupkg files if !strings.HasSuffix(filename, ".nupkg") { h.proxyUpstream(w, r) @@ -306,7 +192,7 @@ func (h *NuGetHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // proxyUpstream forwards a request to NuGet without caching. @@ -335,14 +221,7 @@ func (h *NuGetHandler) proxyUpstream(w http.ResponseWriter, r *http.Request) { } defer func() { _ = resp.Body.Close() }() - for k, vv := range resp.Header { - for _, v := range vv { - w.Header().Add(k, v) - } - } - - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) + h.proxy.relayResponse(w, r, resp, nil) } // buildUpstreamURL constructs the upstream URL for a request. diff --git a/internal/handler/nuget_cooldown.go b/internal/handler/nuget_cooldown.go new file mode 100644 index 00000000..6b95ae9b --- /dev/null +++ b/internal/handler/nuget_cooldown.go @@ -0,0 +1,374 @@ +package handler + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "slices" + "strings" + "time" +) + +var nugetRegistrationPrefixes = []string{ + "/v3/registration5-semver1/", + "/v3/registration5-gz-semver1/", + "/v3/registration5-gz-semver2/", +} + +var nugetArtifactPrefixes = append([]string{"/v3-flatcontainer/"}, nugetRegistrationPrefixes...) + +const nugetRegistrationPath = "/v3/registration5-gz-semver2/" + +func (h *NuGetHandler) cooldownEnabled() bool { + return h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() +} + +func (h *NuGetHandler) nugetCooldownApplies(id string) bool { + return h.cooldownEnabled() && h.proxy.Cooldown.For("nuget", canonicalPackagePURL("nuget", strings.ToLower(id))) > 0 +} + +// Cache upstream documents, not filtered results, so policy changes and elapsed +// time take effect even while metadata is fresh. Include the upstream in the key. +func (h *NuGetHandler) nugetMetadata(ctx context.Context, path string) (map[string]any, error) { + target := h.upstreamURL + path + key := fmt.Sprintf("_cooldown/%x", sha256.Sum256([]byte(target))) + var document map[string]any + validate := func(body []byte) error { + var err error + document, err = h.decodeNuGetMetadata(body) + return err + } + _, _, _, err := h.proxy.fetchOrCacheMetadata(ctx, "nuget", key, target, "", validate) + if err != nil { + return nil, err + } + return document, nil +} + +func (h *NuGetHandler) decodeNuGetMetadata(body []byte) (map[string]any, error) { + // Normally the HTTP transport decodes gzip. Also support compressed cached + // bytes and clients with transparent decompression disabled, with the same + // metadata limit applied to the decompressed document. + if bytes.HasPrefix(body, []byte{0x1f, 0x8b}) { + reader, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer func() { _ = reader.Close() }() + body, err = h.proxy.ReadMetadata(reader) + if err != nil { + return nil, err + } + } + var document map[string]any + if err := json.Unmarshal(body, &document); err != nil { + return nil, fmt.Errorf("parsing NuGet metadata: %w", err) + } + if document == nil { + return nil, fmt.Errorf("empty NuGet metadata") + } + return document, nil +} + +// Prefer semver2, but a configured source may advertise only an older hive. +// Retry only advertised aliases on 404; transport/validation errors must not +// silently switch to a hive with less complete metadata. Keep requests on the +// configured upstream, consistent with the service-index route rewriting. +func (h *NuGetHandler) nugetRegistrationMetadata(ctx context.Context, suffix string) (map[string]any, string, error) { + path := nugetRegistrationPath + suffix + document, err := h.nugetMetadata(ctx, path) + if !errors.Is(err, ErrUpstreamNotFound) { + return document, path, err + } + index, indexErr := h.nugetMetadata(ctx, "/v3/index.json") + if indexErr != nil { + return nil, path, indexErr + } + resources, _ := index["resources"].([]any) + seen := map[string]bool{nugetRegistrationPath: true} + for _, resource := range resources { + entry, _ := resource.(map[string]any) + service, _ := entry["@type"].(string) + id, _ := entry["@id"].(string) + if id == "" || !strings.HasPrefix(service, "RegistrationsBaseUrl") { + continue + } + prefix := strings.TrimPrefix(h.rewriteNuGetURL(id, service), h.proxyURL+"/nuget") + if !slices.Contains(nugetRegistrationPrefixes, prefix) || seen[prefix] { + continue + } + seen[prefix] = true + path = prefix + suffix + document, err = h.nugetMetadata(ctx, path) + if !errors.Is(err, ErrUpstreamNotFound) { + return document, path, err + } + } + return nil, path, err +} + +func (h *NuGetHandler) nugetMetadataError(w http.ResponseWriter, err error) { + if errors.Is(err, ErrUpstreamNotFound) { + JSONError(w, http.StatusNotFound, "package metadata not found") + return + } + h.proxy.Logger.Warn("failed to process NuGet metadata", "error", err) + JSONError(w, http.StatusBadGateway, "failed to process package metadata") +} + +func (h *NuGetHandler) handleVersionList(w http.ResponseWriter, r *http.Request) { + if !h.cooldownEnabled() { + h.proxyUpstream(w, r) + return + } + id := strings.ToLower(r.PathValue("id")) + document, err := h.nugetMetadata(r.Context(), "/v3-flatcontainer/"+url.PathEscape(id)+"/index.json") + if err != nil { + h.nugetMetadataError(w, err) + return + } + blocked := make(map[string]bool) + // A globally enabled policy may still exempt this package or ecosystem. + // Keep metadata caching, but do not require publication data in that case. + if h.nugetCooldownApplies(id) { + registration, registrationPath, err := h.nugetRegistrationMetadata(r.Context(), url.PathEscape(id)+"/index.json") + if err == nil { + err = h.expandNuGetPages(r.Context(), registration, registrationPath) + } + if err != nil { + h.nugetMetadataError(w, err) + return + } + h.collectNuGetBlockedVersions(registration, id, blocked) + } + versions, ok := document["versions"].([]any) + if !ok { + h.nugetMetadataError(w, fmt.Errorf("missing NuGet versions")) + return + } + filtered := make([]any, 0, len(versions)) + for _, value := range versions { + version, ok := value.(string) + if !ok { + h.nugetMetadataError(w, fmt.Errorf("invalid NuGet version")) + return + } + if !blocked[nugetVersionKey(version)] { + filtered = append(filtered, value) + } + } + document["versions"] = filtered + w.Header().Set(headerContentType, contentTypeJSON) + _ = json.NewEncoder(w).Encode(document) +} + +func nugetVersionKey(version string) string { + version, _, _ = strings.Cut(version, "+") + return strings.ToLower(version) +} + +func (h *NuGetHandler) nugetDownloadAllowed(ctx context.Context, id, version string) (bool, error) { + if !h.nugetCooldownApplies(id) { + return true, nil + } + suffix := url.PathEscape(strings.ToLower(id)) + "/" + url.PathEscape(nugetVersionKey(version)) + ".json" + leaf, _, err := h.nugetRegistrationMetadata(ctx, suffix) + if err != nil { + return false, err + } + return h.nugetLeafAllowed(leaf, id), nil +} + +// A standalone leaf has published at its root; leaves embedded in pages carry +// it in catalogEntry. Missing/invalid timestamps retain the existing permissive +// behavior, but fetch and JSON errors must not bypass the policy. +func (h *NuGetHandler) nugetLeafAllowed(leaf map[string]any, id string) bool { + if !h.cooldownEnabled() { + return true + } + entry := nugetCatalogEntry(leaf) + if id == "" { + id, _ = entry["id"].(string) + } + published, _ := entry["published"].(string) + when, err := time.Parse(time.RFC3339, published) + if err != nil { + return true + } + return h.proxy.Cooldown.IsAllowed("nuget", canonicalPackagePURL("nuget", strings.ToLower(id)), when) +} + +func nugetCatalogEntry(leaf map[string]any) map[string]any { + if entry, ok := leaf["catalogEntry"].(map[string]any); ok { + return entry + } + return leaf +} + +func (h *NuGetHandler) collectNuGetBlockedVersions(document map[string]any, id string, blocked map[string]bool) { + entry := nugetCatalogEntry(document) + if version, ok := entry["version"].(string); ok && !h.nugetLeafAllowed(document, id) { + blocked[nugetVersionKey(version)] = true + } + items, _ := document["items"].([]any) + for _, item := range items { + if child, ok := item.(map[string]any); ok { + h.collectNuGetBlockedVersions(child, id, blocked) + } + } +} + +func (h *NuGetHandler) handleRegistration(w http.ResponseWriter, r *http.Request) { + if !h.cooldownEnabled() { + h.proxyUpstream(w, r) + return + } + id := nugetRegistrationID(r.URL.Path) + applyCooldown := h.nugetCooldownApplies(id) + document, err := h.nugetMetadata(r.Context(), r.URL.Path) + if err == nil && applyCooldown { + err = h.expandNuGetPages(r.Context(), document, r.URL.Path) + } + if err != nil { + h.nugetMetadataError(w, err) + return + } + _, hasItems := document["items"] + if applyCooldown && !h.filterNuGetRegistration(document, id) && !hasItems { + JSONError(w, http.StatusNotFound, "version not found") + return + } + h.rewriteNuGetRegistrationLinks(document) + w.Header().Set(headerContentType, contentTypeJSON) + _ = json.NewEncoder(w).Encode(document) +} + +func nugetRegistrationID(path string) string { + for _, prefix := range nugetRegistrationPrefixes { + if rest, ok := strings.CutPrefix(path, prefix); ok { + id, _, _ := strings.Cut(rest, "/") + return id + } + } + return "" +} + +// Only expand index pages, never recursively follow arbitrary upstream links. +// Pin requests to this configured upstream and the current package's page path. +func (h *NuGetHandler) expandNuGetPages(ctx context.Context, document map[string]any, path string) error { + if !strings.HasSuffix(path, "/index.json") { + return nil + } + items, ok := document["items"].([]any) + if !ok { + return fmt.Errorf("missing registration pages") + } + base, err := url.Parse(h.upstreamURL + path) + if err != nil { + return err + } + pagePrefix := strings.TrimSuffix(base.Path, "index.json") + "page/" + for _, item := range items { + page, ok := item.(map[string]any) + if !ok { + return fmt.Errorf("invalid registration page") + } + if _, ok := page["items"].([]any); ok { + continue + } + link, _ := page["@id"].(string) + target, err := base.Parse(link) + if err != nil || target.Scheme != base.Scheme || target.Host != base.Host || + !strings.HasPrefix(target.Path, pagePrefix) || containsPathTraversal(target.Path) || target.RawQuery != "" || target.Fragment != "" { + return fmt.Errorf("invalid registration page URL: %q", link) + } + upstream, _ := url.Parse(h.upstreamURL) + pageDocument, err := h.nugetMetadata(ctx, strings.TrimPrefix(target.Path, upstream.Path)) + if err != nil { + return err + } + leaves, ok := pageDocument["items"].([]any) + if !ok { + return fmt.Errorf("missing registration leaves") + } + page["items"] = leaves + } + return nil +} + +func (h *NuGetHandler) filterNuGetRegistration(document map[string]any, id string) bool { + items, ok := document["items"].([]any) + if !ok { + return h.nugetLeafAllowed(document, id) + } + filtered := make([]any, 0, len(items)) + for _, item := range items { + child, ok := item.(map[string]any) + if ok && h.filterNuGetRegistration(child, id) { + filtered = append(filtered, child) + } + } + document["items"] = filtered + document["count"] = len(filtered) + // Page bounds describe the retained leaves, not versions hidden by cooldown. + if _, isPage := document["lower"]; isPage && len(filtered) > 0 { + first, _ := filtered[0].(map[string]any) + last, _ := filtered[len(filtered)-1].(map[string]any) + document["lower"] = nugetCatalogEntry(first)["version"] + document["upper"] = nugetCatalogEntry(last)["version"] + } + return len(filtered) > 0 +} + +func (h *NuGetHandler) rewriteNuGetRegistrationLinks(value any) { + switch node := value.(type) { + case map[string]any: + for key, child := range node { + if link, ok := child.(string); ok { + switch key { + case "@id", "parent", "registration", "packageContent": + node[key] = h.nugetProxyLink(link) + } + } else { + h.rewriteNuGetRegistrationLinks(child) + } + } + case []any: + for _, child := range node { + h.rewriteNuGetRegistrationLinks(child) + } + } +} + +func (h *NuGetHandler) nugetProxyLink(link string) string { + u, err := url.Parse(link) + if err != nil { + return link + } + upstream, err := url.Parse(h.upstreamURL) + if err != nil { + return link + } + path := u.Path + if u.Host == upstream.Host { + path = strings.TrimPrefix(path, upstream.Path) + } + for _, prefix := range nugetArtifactPrefixes { + if strings.HasPrefix(path, prefix) { + proxy, err := url.Parse(h.proxyURL + "/nuget" + path) + if err != nil { + return link + } + proxy.RawQuery = u.RawQuery + proxy.Fragment = u.Fragment + return proxy.String() + } + } + return link +} diff --git a/internal/handler/nuget_cooldown_test.go b/internal/handler/nuget_cooldown_test.go new file mode 100644 index 00000000..3160bd57 --- /dev/null +++ b/internal/handler/nuget_cooldown_test.go @@ -0,0 +1,555 @@ +package handler + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/registries/fetch" +) + +func TestNuGetCooldownRoutes(t *testing.T) { + for _, disableCompression := range []bool{false, true} { + t.Run(map[bool]string{false: "transport gzip", true: "explicit gzip"}[disableCompression], func(t *testing.T) { + proxy, db, store, fetcher := setupTestProxy(t) + proxy.Cooldown = &cooldown.Config{Default: "14d"} + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + seedPackage(t, db, store, "nuget", "testpkg", "2.0.0", "testpkg.2.0.0.nupkg", "cached package") + seedPackage(t, db, store, "nuget", "testpkg", "1.0.0", "testpkg.1.0.0.nupkg", "old package") + metadataRequests := 0 + upstream := newNuGetCooldownUpstream(t, &metadataRequests) + defer upstream.Close() + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DisableCompression = disableCompression + defer transport.CloseIdleConnections() + proxy.HTTPClient = &http.Client{Transport: transport} + h := NewNuGetHandlerWithUpstreams(proxy, "http://proxy.test", upstream.URL, upstream.URL) + routes := http.StripPrefix("/nuget", h.Routes()) + get := func(path string, status int) *httptest.ResponseRecorder { + t.Helper() + return nugetGet(t, routes, path, status) + } + list := get("/nuget/v3-flatcontainer/testpkg/index.json", http.StatusOK) + if got := strings.TrimSpace(list.Body.String()); got != `{"versions":["1.0.0"]}` { + t.Fatalf("filtered list = %s", got) + } + index := get("/nuget"+nugetRegistrationPath+"testpkg/index.json", http.StatusOK) + if strings.Contains(index.Body.String(), `"version":"2.0.0"`) || strings.Contains(index.Body.String(), upstream.URL) { + t.Fatalf("registration leaks blocked leaf or upstream link: %s", index.Body.String()) + } + if index.Header().Get("Content-Encoding") != "" || !json.Valid(index.Body.Bytes()) { + t.Fatal("registration must be decoded JSON") + } + var doc struct { + Items []struct { + ID string `json:"@id"` + Count int + Lower, Upper string + Items []struct { + ID string `json:"@id"` + PackageContent string + } + } + } + if err := json.Unmarshal(index.Body.Bytes(), &doc); err != nil { + t.Fatal(err) + } + if len(doc.Items) != 1 || doc.Items[0].Count != 1 || doc.Items[0].Upper != "1.0.0" { + t.Fatalf("incorrect page: %+v", doc) + } + get(doc.Items[0].ID, http.StatusOK) + get(doc.Items[0].Items[0].ID, http.StatusOK) + get(doc.Items[0].Items[0].PackageContent, http.StatusOK) + get("/nuget"+nugetRegistrationPath+"testpkg/2.0.0.json", http.StatusNotFound) + get("/nuget/v3-flatcontainer/TestPkg/2.0.0/testpkg.2.0.0.nupkg", http.StatusNotFound) + get("/nuget/v3-flatcontainer/testpkg/2.0.0/testpkg.nuspec", http.StatusNotFound) + if fetcher.fetchCalled { + t.Fatal("blocked or cached downloads must not fetch artifacts") + } + + // Reevaluate fresh, unfiltered metadata under a changed package policy. + requestsBefore := metadataRequests + proxy.Cooldown = &cooldown.Config{Default: "14d", Packages: map[string]string{"pkg:nuget/testpkg": "1d"}} + list = get("/nuget/v3-flatcontainer/testpkg/index.json", http.StatusOK) + if !strings.Contains(list.Body.String(), "2.0.0") { + t.Fatal("fresh metadata retained the previous policy") + } + get("/nuget/v3-flatcontainer/testpkg/2.0.0/testpkg.2.0.0.nupkg", http.StatusOK) + if metadataRequests != requestsBefore { + t.Fatal("fresh metadata should be reused") + } + }) + } +} + +func nugetGet(t *testing.T, routes http.Handler, path string, status int) *httptest.ResponseRecorder { + t.Helper() + w := httptest.NewRecorder() + routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != status { + t.Fatalf("GET %s: status %d, want %d: %s", path, w.Code, status, w.Body.String()) + } + return w +} + +func TestNuGetMetadataWithoutEffectiveCooldown(t *testing.T) { + for _, tt := range []struct { + name string + policy *cooldown.Config + }{ + {"package exemption", &cooldown.Config{Default: "14d", Packages: map[string]string{"pkg:nuget/testpkg": "0"}}}, + {"ecosystem exemption", &cooldown.Config{Default: "14d", Ecosystems: map[string]string{"nuget": "0"}}}, + {"other ecosystem only", &cooldown.Config{Ecosystems: map[string]string{"npm": "14d"}}}, + {"other package only", &cooldown.Config{Packages: map[string]string{"pkg:nuget/other": "14d"}}}, + } { + t.Run(tt.name, func(t *testing.T) { + const body = `{"versions":["1.0.0","2.0.0"]}` + const pagePath = nugetRegistrationPath + "testpkg/page/1.0.0/2.0.0.json" + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v3-flatcontainer/testpkg/index.json": + _, _ = io.WriteString(w, body) + case nugetRegistrationPath + "testpkg/index.json": + _, _ = io.WriteString(w, `{"count":1,"items":[{"@id":"`+pagePath+`","count":2,"lower":"1.0.0","upper":"2.0.0"}]}`) + default: + t.Errorf("unnecessary registration request: %s", r.URL.Path) + http.Error(w, "registration unavailable", http.StatusServiceUnavailable) + } + })) + defer upstream.Close() + p := nugetTestProxy() + p.Cooldown = tt.policy + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + w := nugetGet(t, h.Routes(), "/v3-flatcontainer/TestPkg/index.json", http.StatusOK) + if got := strings.TrimSpace(w.Body.String()); got != body { + t.Fatalf("version list = %s, want %s", got, body) + } + w = nugetGet(t, h.Routes(), nugetRegistrationPath+"testpkg/index.json", http.StatusOK) + if !strings.Contains(w.Body.String(), `"@id":"http://proxy.test/nuget`+pagePath+`"`) { + t.Fatalf("registration page link was not rewritten: %s", w.Body.String()) + } + }) + } +} + +func TestNuGetCooldownColdDownload(t *testing.T) { + for _, tt := range []struct { + name, published string + policy *cooldown.Config + want int + }{ + {"recent", time.Now().Add(-time.Hour).Format(time.RFC3339), &cooldown.Config{Default: "14d"}, http.StatusNotFound}, + {"missing timestamp", "", &cooldown.Config{Default: "14d"}, http.StatusOK}, + {"package exemption", time.Now().Add(-time.Hour).Format(time.RFC3339), &cooldown.Config{Default: "14d", Packages: map[string]string{"pkg:nuget/testpkg": "0"}}, http.StatusOK}, + {"ecosystem override", time.Now().Add(-time.Hour).Format(time.RFC3339), &cooldown.Config{Ecosystems: map[string]string{"nuget": "14d"}}, http.StatusNotFound}, + } { + t.Run(tt.name, func(t *testing.T) { + p, _, _, fetcher := setupTestProxy(t) + p.Cooldown = tt.policy + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("package")), ContentType: "application/octet-stream"} + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"published": tt.published}) + })) + defer upstream.Close() + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + nugetGet(t, h.Routes(), "/v3-flatcontainer/testpkg/2.0.0/testpkg.2.0.0.nupkg", tt.want) + if fetcher.fetchCalled != (tt.want == http.StatusOK) { + t.Errorf("artifact fetch called = %v", fetcher.fetchCalled) + } + }) + } +} + +// TestNuGetCooldownConcurrentDownloads asserts that a download which joins +// another request's metadata fetch still applies the cooldown. Each caller +// decodes the leaf in its own validate callback, so a waiter on the shared +// fetch must run its own rather than finding no document and allowing it. +func TestNuGetCooldownConcurrentDownloads(t *testing.T) { + p, db, store, fetcher := setupTestProxy(t) + p.Cooldown = &cooldown.Config{Default: "14d"} + seedPackage(t, db, store, "nuget", "testpkg", "2.0.0", "testpkg.2.0.0.nupkg", "cached package") + entered, release := make(chan struct{}), make(chan struct{}) + var enteredOnce, releaseOnce sync.Once + releaseUpstream := func() { releaseOnce.Do(func() { close(release) }) } + var calls atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + enteredOnce.Do(func() { close(entered) }) + <-release + _ = json.NewEncoder(w).Encode(map[string]string{"published": time.Now().Add(-time.Hour).Format(time.RFC3339)}) + })) + // Cleanups run last first: release the held request before closing the + // server, so a failed wait can't hang the test. + t.Cleanup(upstream.Close) + t.Cleanup(releaseUpstream) + routes := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL).Routes() + + statuses := make(chan int, 2) + download := func() { + w := httptest.NewRecorder() + routes.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v3-flatcontainer/testpkg/2.0.0/testpkg.2.0.0.nupkg", nil)) + statuses <- w.Code + } + go download() + <-entered + go download() + target := upstream.URL + nugetRegistrationPath + "testpkg/2.0.0.json" + key := metadataCoalesceKey("nuget", fmt.Sprintf("_cooldown/%x", sha256.Sum256([]byte(target))), target, contentTypeJSON, "", true) + waitForMetadataWaiters(t, p, key, 1) + releaseUpstream() + + for range 2 { + if status := <-statuses; status != http.StatusNotFound { + t.Errorf("download status = %d, want %d", status, http.StatusNotFound) + } + } + if got := calls.Load(); got != 1 { + t.Errorf("upstream metadata requests = %d, want 1", got) + } + if fetcher.fetchCalled { + t.Error("blocked downloads must not fetch artifacts") + } +} + +func TestNuGetRegistrationServiceAliases(t *testing.T) { + h := NewNuGetHandler(nugetTestProxy(), "http://proxy.test") + for _, tt := range []struct{ service, path string }{ + {"RegistrationsBaseUrl", "/v3/registration5-semver1/"}, + {"RegistrationsBaseUrl/3.0.0-beta", "/v3/registration5-semver1/"}, + {"RegistrationsBaseUrl/3.0.0-rc", "/v3/registration5-semver1/"}, + {"RegistrationsBaseUrl/3.4.0", "/v3/registration5-gz-semver1/"}, + {"RegistrationsBaseUrl/3.6.0", nugetRegistrationPath}, + {"RegistrationsBaseUrl/Versioned", nugetRegistrationPath}, + } { + t.Run(tt.service, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != tt.path+"testpkg/index.json" { + t.Errorf("wrong hive: %s", r.URL.Path) + } + _, _ = io.WriteString(w, `{"count":0,"items":[]}`) + })) + defer upstream.Close() + h.upstreamURL = upstream.URL + h.proxy.Cooldown = &cooldown.Config{Default: "14d"} + body := []byte(`{"resources":[{"@id":"` + upstream.URL + tt.path + `","@type":"` + tt.service + `"}]}`) + out, err := h.rewriteServiceIndex(body) + if err != nil { + t.Fatal(err) + } + var doc struct { + Resources []struct { + ID string `json:"@id"` + } + } + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + http.StripPrefix("/nuget", h.Routes()).ServeHTTP(w, httptest.NewRequest(http.MethodGet, doc.Resources[0].ID+"testpkg/index.json", nil)) + if w.Code != http.StatusOK { + t.Fatalf("alias route status = %d: %s", w.Code, w.Body.String()) + } + }) + } +} + +func TestNuGetCooldownMetadataErrors(t *testing.T) { + for _, tt := range []struct { + name, body string + status int + }{ + {"upstream failure", "unavailable", http.StatusServiceUnavailable}, + {"invalid JSON", "broken JSON", http.StatusOK}, + {"null", "null", http.StatusOK}, + } { + t.Run(tt.name, func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(tt.status) + _, _ = io.WriteString(w, tt.body) + })) + defer upstream.Close() + p := nugetTestProxy() + p.Cooldown = &cooldown.Config{Default: "14d"} + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + for _, path := range []string{"/v3-flatcontainer/testpkg/index.json", "/v3-flatcontainer/testpkg/2.0.0/testpkg.2.0.0.nupkg", nugetRegistrationPath + "testpkg/index.json"} { + w := httptest.NewRecorder() + h.Routes().ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + if w.Code != http.StatusBadGateway { + t.Errorf("GET %s: %d, want 502", path, w.Code) + } + } + }) + } +} + +func TestNuGetCooldownRejectsUnsafePageLinks(t *testing.T) { + for _, link := range []string{"https://other.example/page.json", "/v3/registration5-gz-semver2/other/page/1/2.json", "page/../index.json", "index.json"} { + t.Run(link, func(t *testing.T) { + requests := 0 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + _ = json.NewEncoder(w).Encode(map[string]any{"items": []any{map[string]any{"@id": link}}}) + })) + defer upstream.Close() + p := nugetTestProxy() + p.Cooldown = &cooldown.Config{Default: "14d"} + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + nugetGet(t, h.Routes(), nugetRegistrationPath+"testpkg/index.json", http.StatusBadGateway) + if requests != 1 { + t.Fatalf("unsafe page link was followed (%d requests)", requests) + } + }) + } +} + +func TestNuGetCooldownDecompressedMetadataLimit(t *testing.T) { + var compressed bytes.Buffer + gz := gzip.NewWriter(&compressed) + _, _ = io.WriteString(gz, `{"padding":"`+strings.Repeat("x", 2048)+`"}`) + _ = gz.Close() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Encoding", "gzip") + _, _ = w.Write(compressed.Bytes()) + })) + defer upstream.Close() + p := nugetTestProxy() + p.Cooldown = &cooldown.Config{Default: "14d"} + p.MetadataMaxSize = 1024 + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DisableCompression = true + defer transport.CloseIdleConnections() + p.HTTPClient = &http.Client{Transport: transport} + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + nugetGet(t, h.Routes(), nugetRegistrationPath+"testpkg/index.json", http.StatusBadGateway) +} + +func newNuGetCooldownUpstream(t *testing.T, metadataRequests *int) *httptest.Server { + t.Helper() + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + (*metadataRequests)++ + base := upstream.URL + nugetRegistrationPath + "testpkg/" + leaf := func(version string, age time.Duration) map[string]any { + return map[string]any{ + "@id": base + version + ".json", + "packageContent": upstream.URL + "/v3-flatcontainer/testpkg/" + version + "/testpkg." + version + ".nupkg", + "catalogEntry": map[string]any{"id": "TestPkg", "version": version, "published": time.Now().Add(-age).Format(time.RFC3339)}, + } + } + page := map[string]any{"@id": base + "page/1.0.0/2.0.0.json", "lower": "1.0.0", "upper": "2.0.0", "count": 2, + "parent": base + "index.json", "items": []any{leaf("1.0.0", 30*24*time.Hour), leaf("2.0.0", 2*24*time.Hour)}} + var body any + switch r.URL.Path { + case "/v3-flatcontainer/testpkg/index.json": + body = map[string]any{"versions": []string{"1.0.0", "2.0.0"}} + case nugetRegistrationPath + "testpkg/index.json": + // This index deliberately does not inline its leaves. + body = map[string]any{"count": 1, "items": []any{map[string]any{ + "@id": page["@id"], "count": 2, "lower": "1.0.0", "upper": "2.0.0", + }}} + case nugetRegistrationPath + "testpkg/page/1.0.0/2.0.0.json": + body = page + case nugetRegistrationPath + "testpkg/1.0.0.json": + body = map[string]any{"published": time.Now().Add(-30 * 24 * time.Hour).Format(time.RFC3339)} + case nugetRegistrationPath + "testpkg/2.0.0.json": + body = map[string]any{"published": time.Now().Add(-2 * 24 * time.Hour).Format(time.RFC3339)} + default: + t.Errorf("unexpected metadata request: %s", r.URL.Path) + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + gz := gzip.NewWriter(w) + _ = json.NewEncoder(gz).Encode(body) + _ = gz.Close() + })) + + return upstream +} + +func TestNuGetCooldownLegacyRegistration(t *testing.T) { + for _, service := range []string{"RegistrationsBaseUrl", "RegistrationsBaseUrl/3.0.0-beta", "RegistrationsBaseUrl/3.0.0-rc", "RegistrationsBaseUrl/3.4.0"} { + t.Run(service, func(t *testing.T) { + prefix := "/v3/registration5-semver1/" + if service == "RegistrationsBaseUrl/3.4.0" { + prefix = "/v3/registration5-gz-semver1/" + } + upstream := newNuGetLegacyUpstream(t, service, prefix) + defer upstream.Close() + p, db, store, fetcher := setupTestProxy(t) + p.Cooldown = &cooldown.Config{Default: "14d"} + seedPackage(t, db, store, "nuget", "testpkg", "1.0.0", "testpkg.1.0.0.nupkg", "cached old package") + seedPackage(t, db, store, "nuget", "testpkg", "2.0.0", "testpkg.2.0.0.nupkg", "cached recent package") + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL+"/feed", upstream.URL) + list := nugetGet(t, h.Routes(), "/v3-flatcontainer/testpkg/index.json", http.StatusOK) + if strings.TrimSpace(list.Body.String()) != `{"versions":["1.0.0"]}` { + t.Fatalf("incorrect version list: %s", list.Body.String()) + } + nugetGet(t, h.Routes(), "/v3-flatcontainer/testpkg/1.0.0/testpkg.1.0.0.nupkg", http.StatusOK) + nugetGet(t, h.Routes(), "/v3-flatcontainer/testpkg/2.0.0/testpkg.2.0.0.nupkg", http.StatusNotFound) + if fetcher.fetchCalled { + t.Fatal("cached or blocked package must not be fetched") + } + }) + } +} + +func newNuGetLegacyUpstream(t *testing.T, service, prefix string) *httptest.Server { + t.Helper() + var upstream *httptest.Server + upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/feed") + base := upstream.URL + "/feed" + prefix + "testpkg/" + published := func(age time.Duration) string { return time.Now().Add(-age).Format(time.RFC3339) } + var body any + switch path { + case "/v3/index.json": + body = map[string]any{"resources": []any{map[string]string{"@id": upstream.URL + "/feed" + prefix, "@type": service}}} + case "/v3-flatcontainer/testpkg/index.json": + body = map[string]any{"versions": []string{"1.0.0", "2.0.0"}} + case prefix + "testpkg/index.json": + body = map[string]any{"items": []any{map[string]any{"@id": base + "page/1.0.0/2.0.0.json"}}} + case prefix + "testpkg/page/1.0.0/2.0.0.json": + body = map[string]any{"items": []any{ + map[string]any{"catalogEntry": map[string]string{"id": "testpkg", "version": "1.0.0", "published": published(30 * 24 * time.Hour)}}, + map[string]any{"catalogEntry": map[string]string{"id": "testpkg", "version": "2.0.0", "published": published(time.Hour)}}, + }} + case prefix + "testpkg/1.0.0.json": + body = map[string]string{"published": published(30 * 24 * time.Hour)} + case prefix + "testpkg/2.0.0.json": + body = map[string]string{"published": published(time.Hour)} + default: + if !strings.HasPrefix(path, nugetRegistrationPath) { + t.Errorf("unexpected request: %s", r.URL.Path) + } + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(body) + })) + return upstream +} + +func TestNuGetRegistrationDoesNotFallbackOnFailure(t *testing.T) { + for _, status := range []int{http.StatusServiceUnavailable, http.StatusUnauthorized, http.StatusOK} { + t.Run(http.StatusText(status), func(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != nugetRegistrationPath+"testpkg/index.json" { + t.Errorf("must not switch registration hive on failure: %s", r.URL.Path) + } + w.WriteHeader(status) + _, _ = io.WriteString(w, "invalid metadata") + })) + defer upstream.Close() + h := NewNuGetHandlerWithUpstreams(nugetTestProxy(), "http://proxy.test", upstream.URL, upstream.URL) + if _, _, err := h.nugetRegistrationMetadata(t.Context(), "testpkg/index.json"); err == nil { + t.Fatal("expected metadata error") + } + }) + } +} + +func TestNuGetMetadataPreservesValidCache(t *testing.T) { + for _, invalid := range []string{"broken JSON", "null", "[]", string([]byte{0x1f, 0x8b, 0x00})} { + t.Run(invalid, func(t *testing.T) { + const good = `{"published":"2020-01-01T00:00:00Z"}` + var response atomic.Value + response.Store(good) + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) > 1 && r.Header.Get("If-None-Match") != `"good"` { + t.Errorf("cached ETag was replaced: %s", r.Header.Get("If-None-Match")) + } + body := response.Load().(string) + etag := `"good"` + if body == invalid { + etag = `"bad"` + } + w.Header().Set("ETag", etag) + _, _ = io.WriteString(w, body) + })) + defer upstream.Close() + p, _, _, _ := setupTestProxy(t) + p.CacheMetadata = true + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + check := func(want string) { + t.Helper() + doc, err := h.nugetMetadata(t.Context(), nugetRegistrationPath+"testpkg/1.0.0.json") + if err != nil || doc["published"] != want { + t.Fatalf("metadata = %v, err = %v, want publication %s", doc, err, want) + } + } + check("2020-01-01T00:00:00Z") + response.Store(invalid) + check("2020-01-01T00:00:00Z") // Bad 200 must fall back without overwriting. + p.MetadataTTL = time.Hour + check("2020-01-01T00:00:00Z") // The on-disk cache must still be usable. + if requests.Load() != 2 { + t.Fatalf("requests = %d, want 2", requests.Load()) + } + p.MetadataTTL = 0 + response.Store(`{"published":"2021-01-01T00:00:00Z"}`) + check("2021-01-01T00:00:00Z") // A later valid response replaces the cache. + }) + } +} + +func TestNuGetMetadataInvalidResponseNotCached(t *testing.T) { + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + _, _ = io.WriteString(w, "invalid JSON") + return + } + _, _ = io.WriteString(w, `{"published":"2020-01-01T00:00:00Z"}`) + })) + defer upstream.Close() + p, _, _, _ := setupTestProxy(t) + p.CacheMetadata = true + p.MetadataTTL = time.Hour + h := NewNuGetHandlerWithUpstreams(p, "http://proxy.test", upstream.URL, upstream.URL) + path := nugetRegistrationPath + "testpkg/1.0.0.json" + if _, err := h.nugetMetadata(t.Context(), path); err == nil { + t.Fatal("invalid response without a usable cache must fail") + } + if _, err := h.nugetMetadata(t.Context(), path); err != nil { + t.Fatalf("invalid response was cached: %v", err) + } + if requests.Load() != 2 { + t.Fatalf("requests = %d, want 2", requests.Load()) + } +} + +func TestNuGetRegistrationDoesNotGuessUnadvertisedAliases(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case nugetRegistrationPath + "testpkg/index.json": + http.NotFound(w, r) + case "/v3/index.json": + _, _ = io.WriteString(w, `{"resources":[{"@type":"UnrelatedService","@id":"https://other.example/"}]}`) + default: + t.Errorf("unadvertised endpoint requested: %s", r.URL.Path) + http.NotFound(w, r) + } + })) + defer upstream.Close() + h := NewNuGetHandlerWithUpstreams(nugetTestProxy(), "http://proxy.test", upstream.URL, upstream.URL) + _, _, err := h.nugetRegistrationMetadata(t.Context(), "testpkg/index.json") + if !errors.Is(err, ErrUpstreamNotFound) { + t.Fatalf("error = %v, want metadata not found", err) + } +} diff --git a/internal/handler/nuget_test.go b/internal/handler/nuget_test.go index 09efe956..710b8eae 100644 --- a/internal/handler/nuget_test.go +++ b/internal/handler/nuget_test.go @@ -812,11 +812,6 @@ func TestNuGetCooldownFiltering(t *testing.T) { }, } - body, err := json.Marshal(registration) - if err != nil { - t.Fatal(err) - } - proxy := testProxy() proxy.Cooldown = &cooldown.Config{ Default: "3d", @@ -827,17 +822,11 @@ func TestNuGetCooldownFiltering(t *testing.T) { proxyURL: "http://localhost:8080", } - filtered, err := h.applyCooldownFiltering(body) - if err != nil { - t.Fatal(err) + if !h.filterNuGetRegistration(registration, "") { + t.Fatal("expected registration items to be retained") } - var result map[string]any - if err := json.Unmarshal(filtered, &result); err != nil { - t.Fatal(err) - } - - pages := result["items"].([]any) + pages := registration["items"].([]any) page := pages[0].(map[string]any) items := page["items"].([]any) @@ -851,7 +840,7 @@ func TestNuGetCooldownFiltering(t *testing.T) { } count := page["count"] - if count != float64(1) { + if count != 1 { t.Errorf("expected page count to be 1, got %v", count) } } @@ -877,11 +866,6 @@ func TestNuGetCooldownFilteringWithPackageOverride(t *testing.T) { }, } - body, err := json.Marshal(registration) - if err != nil { - t.Fatal(err) - } - proxy := testProxy() proxy.Cooldown = &cooldown.Config{ Default: "3d", @@ -893,17 +877,11 @@ func TestNuGetCooldownFilteringWithPackageOverride(t *testing.T) { proxyURL: "http://localhost:8080", } - filtered, err := h.applyCooldownFiltering(body) - if err != nil { - t.Fatal(err) + if !h.filterNuGetRegistration(registration, "") { + t.Fatal("expected registration items to be retained") } - var result map[string]any - if err := json.Unmarshal(filtered, &result); err != nil { - t.Fatal(err) - } - - pages := result["items"].([]any) + pages := registration["items"].([]any) page := pages[0].(map[string]any) items := page["items"].([]any) @@ -930,36 +908,20 @@ func TestNuGetCooldownNoCooldownConfig(t *testing.T) { }, } - body, err := json.Marshal(registration) - if err != nil { - t.Fatal(err) - } - - // No cooldown - applyCooldownFiltering still works, just doesn't filter + // No cooldown: all registration items are retained. h := &NuGetHandler{ proxy: testProxy(), proxyURL: "http://localhost:8080", } - filtered, err := h.applyCooldownFiltering(body) - if err != nil { - t.Fatal(err) + if !h.filterNuGetRegistration(registration, "") { + t.Fatal("expected registration items to be retained") } - var result map[string]any - if err := json.Unmarshal(filtered, &result); err != nil { - t.Fatal(err) - } - - pages := result["items"].([]any) + pages := registration["items"].([]any) page := pages[0].(map[string]any) items := page["items"].([]any) - // Without cooldown config on the handler, applyCooldownFiltering - // is called but proxy.Cooldown is nil, so IsAllowed is never called - // Actually, applyCooldownFiltering always runs the filter logic - - // but the caller (handleRegistration) short-circuits when cooldown is disabled. - // The function itself should still work fine with a nil Cooldown. if len(items) != 1 { t.Fatalf("expected 1 item, got %d", len(items)) } @@ -988,11 +950,6 @@ func TestNuGetCooldownFilteringNuGetTimestamp(t *testing.T) { }, } - body, err := json.Marshal(registration) - if err != nil { - t.Fatal(err) - } - proxy := testProxy() proxy.Cooldown = &cooldown.Config{ Default: "3d", @@ -1003,17 +960,11 @@ func TestNuGetCooldownFilteringNuGetTimestamp(t *testing.T) { proxyURL: "http://localhost:8080", } - filtered, err := h.applyCooldownFiltering(body) - if err != nil { - t.Fatal(err) + if !h.filterNuGetRegistration(registration, "") { + t.Fatal("expected registration items to be retained") } - var result map[string]any - if err := json.Unmarshal(filtered, &result); err != nil { - t.Fatal(err) - } - - pages := result["items"].([]any) + pages := registration["items"].([]any) page := pages[0].(map[string]any) items := page["items"].([]any) diff --git a/internal/handler/proxy_cached_encoding_test.go b/internal/handler/proxy_cached_encoding_test.go new file mode 100644 index 00000000..35c8793b --- /dev/null +++ b/internal/handler/proxy_cached_encoding_test.go @@ -0,0 +1,260 @@ +package handler + +import ( + "bytes" + "context" + "errors" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" +) + +// gzipWhenAskedUpstream serves compressed bytes with Content-Encoding: gzip +// when the request advertises gzip, plain bytes otherwise, like a CDN that +// compresses on the fly. It records the last Accept-Encoding it saw and counts +// every request before the availability gate so a cache-miss refetch during a +// simulated outage is observable. +type gzipWhenAskedUpstream struct { + *httptest.Server + available atomic.Bool + requests atomic.Int32 + acceptEncoding atomic.Value // string +} + +func newGzipWhenAskedUpstream(plain, compressed []byte) *gzipWhenAskedUpstream { + u := &gzipWhenAskedUpstream{} + u.available.Store(true) + u.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.requests.Add(1) + u.acceptEncoding.Store(r.Header.Get(headerAcceptEncoding)) + if !u.available.Load() { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + return + } + w.Header().Set(headerContentType, contentTypeJSON) + if strings.Contains(r.Header.Get(headerAcceptEncoding), "gzip") { + w.Header().Set(headerContentEncoding, "gzip") + _, _ = w.Write(compressed) + return + } + _, _ = w.Write(plain) + })) + return u +} + +func (u *gzipWhenAskedUpstream) sawAcceptEncoding() string { + s, _ := u.acceptEncoding.Load().(string) + return s +} + +// serveGzip issues one request through proxyCachedWithEncoding asking the +// upstream for gzip. +func serveGzip(proxy *Proxy, upstreamURL string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/index.json", nil) + proxy.proxyCachedWithEncoding(w, r, upstreamURL, "gzip-test", "index", "gzip", "*/*") + return w +} + +func assertGzipResponse(t *testing.T, label string, w *httptest.ResponseRecorder, compressed []byte) { + t.Helper() + if w.Code != http.StatusOK { + t.Fatalf("%s: status = %d, want 200: %s", label, w.Code, w.Body.String()) + } + if !bytes.Equal(w.Body.Bytes(), compressed) { + t.Errorf("%s: body is not the compressed bytes (got %d, want %d)", label, w.Body.Len(), len(compressed)) + } + if got := w.Header().Get(headerContentEncoding); got != "gzip" { + t.Errorf("%s: Content-Encoding = %q, want %q", label, got, "gzip") + } + if got := w.Header().Get(headerContentLength); got != strconv.Itoa(len(compressed)) { + t.Errorf("%s: Content-Length = %q, want %d", label, got, len(compressed)) + } +} + +// TestProxyCachedWithEncoding_GzipCachesAndReplays covers the cached path: +// requesting gzip upstream stores the compressed bytes plus Content-Encoding +// and replays both from cache without contacting the upstream again. +func TestProxyCachedWithEncoding_GzipCachesAndReplays(t *testing.T) { + plain := []byte(`{"packages":{}}`) + compressed := gzipPayload(t, plain) + upstream := newGzipWhenAskedUpstream(plain, compressed) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + + first := serveGzip(proxy, upstream.URL+"/index.json") + assertGzipResponse(t, "first", first, compressed) + if got := upstream.sawAcceptEncoding(); got != "gzip" { + t.Errorf("upstream Accept-Encoding = %q, want %q", got, "gzip") + } + + before := upstream.requests.Load() + upstream.available.Store(false) + cached := serveGzip(proxy, upstream.URL+"/index.json") + assertGzipResponse(t, "cached", cached, compressed) + if upstream.requests.Load() != before { + t.Errorf("cached replay hit upstream: requests %d -> %d", before, upstream.requests.Load()) + } +} + +// TestProxyCachedWithEncoding_GzipStreamPath covers the cache_metadata=false +// branch: the streaming path must request gzip and forward Content-Encoding. +func TestProxyCachedWithEncoding_GzipStreamPath(t *testing.T) { + plain := []byte(`{"packages":{}}`) + compressed := gzipPayload(t, plain) + upstream := newGzipWhenAskedUpstream(plain, compressed) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = false + proxy.HTTPClient = upstream.Client() + + w := serveGzip(proxy, upstream.URL+"/index.json") + assertGzipResponse(t, "stream", w, compressed) + if got := upstream.sawAcceptEncoding(); got != "gzip" { + t.Errorf("stream path upstream Accept-Encoding = %q, want %q", got, "gzip") + } +} + +// TestProxyCachedWithEncoding_GzipSurvivesCacheWriteFailure covers the failure +// the gzip mode makes reachable: when the metadata cache write fails the +// freshly fetched body is still served, so its Content-Encoding must come from +// the fetch and not from the (unwritten) cache row -- otherwise gzip bytes go +// out labelled application/json with no Content-Encoding. +func TestProxyCachedWithEncoding_GzipSurvivesCacheWriteFailure(t *testing.T) { + plain := []byte(`{"packages":{}}`) + compressed := gzipPayload(t, plain) + upstream := newGzipWhenAskedUpstream(plain, compressed) + defer upstream.Close() + + proxy, _, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = time.Hour + proxy.HTTPClient = upstream.Client() + store.storeErr = errors.New("disk full") + + w := serveGzip(proxy, upstream.URL+"/index.json") + assertGzipResponse(t, "store-failure", w, compressed) +} + +// TestProxyCachedWithEncoding_GzipStaleFallbackKeepsEncoding pins the +// stale-fallback return: when the upstream fails after the entry has expired, +// the stored gzip blob is served with its Content-Encoding taken from the +// cache row. +func TestProxyCachedWithEncoding_GzipStaleFallbackKeepsEncoding(t *testing.T) { + plain := []byte(`{"packages":{}}`) + compressed := gzipPayload(t, plain) + upstream := newGzipWhenAskedUpstream(plain, compressed) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = 0 // every request revalidates; an upstream failure falls back to the stale row + proxy.HTTPClient = upstream.Client() + + first := serveGzip(proxy, upstream.URL+"/index.json") + assertGzipResponse(t, "first", first, compressed) + + upstream.available.Store(false) + stale := serveGzip(proxy, upstream.URL+"/index.json") + assertGzipResponse(t, "stale", stale, compressed) +} + +// TestProxyCachedWithEncoding_UpsertFailureDiscardsBlob covers the row-write +// failure: when the gzip blob is stored but the cache row cannot be updated, +// the blob must be discarded so a later stale fallback cannot serve gzip +// bytes with the previous row's encoding. The fresh response is still +// correct because its encoding comes from the fetch. +func TestProxyCachedWithEncoding_UpsertFailureDiscardsBlob(t *testing.T) { + plain := []byte(`{"packages":{}}`) + compressed := gzipPayload(t, plain) + upstream := newGzipWhenAskedUpstream(plain, compressed) + defer upstream.Close() + + proxy, db, store, _ := setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = 0 // every request revalidates + proxy.HTTPClient = upstream.Client() + + // Seed an identity row + plain blob, as every key has before the gzip rollout. + w := httptest.NewRecorder() + proxy.proxyCachedWithEncoding(w, httptest.NewRequest(http.MethodGet, "/index.json", nil), + upstream.URL+"/index.json", "gzip-test", "index", "identity", "*/*") + if w.Code != http.StatusOK { + t.Fatalf("seed status = %d, want 200", w.Code) + } + + // Now DB writes fail while reads keep working. + db.SetMaxOpenConns(1) + if _, err := db.Exec("PRAGMA query_only=1"); err != nil { + t.Fatalf("PRAGMA query_only=1: %v", err) + } + fresh := serveGzip(proxy, upstream.URL+"/index.json") + assertGzipResponse(t, "fresh with failed row write", fresh, compressed) + + storagePath := metadataStoragePath("gzip-test", "index") + if exists, _ := store.Exists(context.Background(), storagePath); exists { + t.Fatalf("blob %s still present after the row write failed", storagePath) + } + + // Upstream down: the stale fallback must not serve the orphaned gzip + // blob under the old identity row. + if _, err := db.Exec("PRAGMA query_only=0"); err != nil { + t.Fatalf("PRAGMA query_only=0: %v", err) + } + upstream.available.Store(false) + stale := serveGzip(proxy, upstream.URL+"/index.json") + if stale.Code == http.StatusOK { + t.Fatalf("stale fallback served status 200 (Content-Encoding=%q, %d bytes) from an orphaned blob; want an error", + stale.Header().Get(headerContentEncoding), stale.Body.Len()) + } +} + +// TestProxyCachedWithEncoding_StaleFallbackRereadsRow covers the rollout race: +// a request that read the identity row, lost the upstream race to a request +// that stored the gzip blob, and then failed upstream must label the blob +// with the row as it is now, not with the row it read at the start. +func TestProxyCachedWithEncoding_StaleFallbackRereadsRow(t *testing.T) { + plain := []byte(`{"packages":{}}`) + compressed := gzipPayload(t, plain) + + var proxy *Proxy + var requests atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if requests.Add(1) == 1 { + w.Header().Set(headerContentType, contentTypeJSON) + _, _ = w.Write(plain) // seed request: identity + return + } + // Second request has already read the identity row. Simulate a + // concurrent request finishing first: store the gzip blob and row, + // then fail this request so it takes the stale fallback. + proxy.cacheMetadataBlob(r.Context(), "gzip-test", "index", metadataStoragePath("gzip-test", "index"), + &upstreamMetadata{body: compressed, contentType: contentTypeJSON, contentEncoding: "gzip"}) + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + defer upstream.Close() + + proxy, _, _, _ = setupTestProxy(t) + proxy.CacheMetadata = true + proxy.MetadataTTL = 0 + proxy.HTTPClient = upstream.Client() + + w := httptest.NewRecorder() + proxy.proxyCachedWithEncoding(w, httptest.NewRequest(http.MethodGet, "/index.json", nil), + upstream.URL+"/index.json", "gzip-test", "index", "identity", "*/*") + if w.Code != http.StatusOK { + t.Fatalf("seed status = %d, want 200", w.Code) + } + + raced := serveGzip(proxy, upstream.URL+"/index.json") + assertGzipResponse(t, "stale fallback after concurrent gzip store", raced, compressed) +} diff --git a/internal/handler/pub.go b/internal/handler/pub.go index 9d449bae..6e967e35 100644 --- a/internal/handler/pub.go +++ b/internal/handler/pub.go @@ -81,7 +81,7 @@ func (h *PubHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handlePackageMetadata proxies package metadata and rewrites archive URLs. diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index 03d178f1..e37fddb2 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "mime" "net/http" "regexp" @@ -118,15 +117,26 @@ func (h *PyPIHandler) handleSimplePackage(w http.ResponseWriter, r *http.Request if h.proxy.Cooldown != nil && h.proxy.Cooldown.Enabled() { filteredVersions = h.fetchFilteredVersions(r, name) } + for version := range h.proxy.Denylist.Versions(canonicalPackagePURL("pypi", name)) { + if filteredVersions == nil { + filteredVersions = make(map[string]bool) + } + filteredVersions[version] = true + } var rewritten []byte if isJSONMediaType(contentType) { rewritten, err = h.rewriteSimpleJSON(body, filteredVersions) if err != nil { + if len(h.proxy.Denylist.Versions(canonicalPackagePURL("pypi", name))) != 0 { + http.Error(w, "failed to filter package metadata", http.StatusBadGateway) + return + } h.proxy.Logger.Warn("failed to rewrite pypi simple json, proxying original", "error", err) rewritten = body } } else { + body = h.filterSimpleHTMLLinks(body, filteredVersions) rewritten = h.rewriteSimpleHTML(body, filteredVersions) } @@ -378,6 +388,10 @@ func (h *PyPIHandler) handleVersionJSON(w http.ResponseWriter, r *http.Request) http.Error(w, "invalid request", http.StatusBadRequest) return } + if h.proxy.versionDenied("pypi", name, version) { + http.Error(w, "not found", http.StatusNotFound) + return + } h.proxy.Logger.Info("pypi version json request", "package", name, "version", version) @@ -400,6 +414,10 @@ func (h *PyPIHandler) proxyAndRewriteJSON(w http.ResponseWriter, r *http.Request rewritten, err := h.rewriteJSONMetadata(body) if err != nil { + if len(h.proxy.Denylist.Versions(canonicalPackagePURL("pypi", r.PathValue("name")))) != 0 { + http.Error(w, "failed to filter package metadata", http.StatusBadGateway) + return + } h.proxy.Logger.Warn("failed to rewrite metadata, proxying original", "error", err) w.Header().Set(headerContentType, "application/json") _, _ = w.Write(body) @@ -439,8 +457,8 @@ func (h *PyPIHandler) filterAndRewriteReleases(metadata map[string]any, packageN } for version, files := range releases { - if h.shouldFilterRelease(packagePURL, files) { - h.proxy.Logger.Info("cooldown: filtering pypi version", + if h.proxy.versionDenied("pypi", packageName, version) || h.shouldFilterRelease(packagePURL, files) { + h.proxy.Logger.Info("policy: filtering pypi version", "package", packageName, "version", version) delete(releases, version) continue @@ -502,7 +520,10 @@ func (h *PyPIHandler) filterAndRewriteURLs(metadata map[string]any, packagePURL return } - if h.shouldFilterRelease(packagePURL, urls) { + info, _ := metadata["info"].(map[string]any) + name, _ := info["name"].(string) + version, _ := info["version"].(string) + if h.proxy.versionDenied("pypi", name, version) || h.shouldFilterRelease(packagePURL, urls) { metadata["urls"] = []any{} } @@ -605,7 +626,7 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { return } - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // archiveExtensions are sdist formats of the form {name}-{version}{ext}. They @@ -795,13 +816,8 @@ func (h *PyPIHandler) proxySimple(w http.ResponseWriter, r *http.Request, path s } defer func() { _ = resp.Body.Close() }() - for k, vv := range resp.Header { - for _, v := range vv { - w.Header().Add(k, v) - } - } - ensureVaryAccept(w.Header()) - - w.WriteHeader(resp.StatusCode) - _, _ = io.Copy(w, resp.Body) + h.proxy.relayResponse(w, r, resp, func(dst, src http.Header) { + copyRelayHeaders(dst, src) + ensureVaryAccept(dst) + }) } diff --git a/internal/handler/pypi_denylist.go b/internal/handler/pypi_denylist.go new file mode 100644 index 00000000..f39c0d8e --- /dev/null +++ b/internal/handler/pypi_denylist.go @@ -0,0 +1,59 @@ +package handler + +import ( + "bytes" + "net/url" + "path" + + "golang.org/x/net/html" +) + +// filterSimpleHTMLLinks reads hrefs rather than display text. Tokenization +// handles single quotes, entity escaping, and nested link text without +// reserializing the rest of the index. +func (h *PyPIHandler) filterSimpleHTMLLinks(body []byte, versions map[string]bool) []byte { + if len(versions) == 0 { + return body + } + var result bytes.Buffer + z := html.NewTokenizer(bytes.NewReader(body)) + skip := false + for { + tokenType := z.Next() + if tokenType == html.ErrorToken { + return result.Bytes() + } + raw := append([]byte(nil), z.Raw()...) + if tokenType == html.StartTagToken || tokenType == html.SelfClosingTagToken { + token := z.Token() + if token.Data == "a" { + skip = h.deniedSimpleLink(token, versions) + } + } else if tokenType == html.EndTagToken && z.Token().Data == "a" { + if skip { + skip = false + continue + } + } + if !skip { + result.Write(raw) + } + } +} + +func (h *PyPIHandler) deniedSimpleLink(token html.Token, versions map[string]bool) bool { + for _, attr := range token.Attr { + if attr.Key != "href" { + continue + } + u, err := url.Parse(attr.Val) + if err != nil { + continue + } + _, version := h.parseFilename(path.Base(u.Path)) + if versions[version] { + return true + } + } + return false +} diff --git a/internal/handler/relay.go b/internal/handler/relay.go new file mode 100644 index 00000000..75b859a6 --- /dev/null +++ b/internal/handler/relay.go @@ -0,0 +1,110 @@ +package handler + +import ( + "io" + "net/http" + "strings" + + "golang.org/x/net/http/httpguts" +) + +// relayResponse forwards an unmodified upstream body. Callers still own and +// close resp.Body, including when a failed transfer aborts the handler. A custom +// header copier may select or rewrite end-to-end headers; it only sees headers +// after hop-by-hop fields have been removed. +func (p *Proxy) relayResponse(w http.ResponseWriter, r *http.Request, resp *http.Response, copyHeaders func(http.Header, http.Header)) { + blocked := relayHopHeaders(resp.Header) + headers := resp.Header.Clone() + for name := range blocked { + headers.Del(name) + } + bodyAllowed := r.Method != http.MethodHead && resp.StatusCode >= http.StatusOK && + resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusResetContent && resp.StatusCode != http.StatusNotModified + if resp.StatusCode < http.StatusOK || resp.StatusCode == http.StatusNoContent { + headers.Del(headerContentLength) + } else if resp.StatusCode == http.StatusResetContent { + headers.Set(headerContentLength, "0") + } + if copyHeaders == nil { + copyHeaders = copyRelayHeaders + } + copyHeaders(w.Header(), headers) + + announced := make(map[string]bool) + if bodyAllowed { + for name := range resp.Trailer { + name = http.CanonicalHeaderKey(name) + if validRelayTrailer(name, blocked) { + announced[name] = true + w.Header().Add("Trailer", name) + } + } + if len(announced) > 0 { + // HTTP/1 trailers require chunking, not a fixed Content-Length. + w.Header().Del(headerContentLength) + } + } + w.WriteHeader(resp.StatusCode) + if !bodyAllowed || resp.Body == nil { + return + } + + written, err := io.Copy(w, resp.Body) + if err != nil { + upstreamURL := "" + if resp.Request != nil && resp.Request.URL != nil { + upstreamURL = resp.Request.URL.Redacted() + } + p.Logger.Warn("upstream response relay failed", "url", upstreamURL, + "status", resp.StatusCode, "bytes", written, "error", err) + // Headers are already committed: an error page would turn a truncated + // download into a seemingly successful response. Let net/http close the + // HTTP/1 connection or reset the HTTP/2 stream instead. + panic(http.ErrAbortHandler) + } + relayTrailers(w, resp.Trailer, announced, blocked) +} + +func copyRelayHeaders(dst, src http.Header) { + for name, values := range src { + for _, value := range values { + dst.Add(name, value) + } + } +} + +func relayHopHeaders(headers http.Header) map[string]bool { + blocked := map[string]bool{ + "Connection": true, "Proxy-Connection": true, "Keep-Alive": true, + "Proxy-Authenticate": true, "Proxy-Authorization": true, "Te": true, + "Trailer": true, "Transfer-Encoding": true, "Upgrade": true, + } + for _, value := range headers.Values("Connection") { + for _, name := range strings.Split(value, ",") { + if name = strings.TrimSpace(name); name != "" { + blocked[http.CanonicalHeaderKey(name)] = true + } + } + } + return blocked +} + +func validRelayTrailer(name string, blocked map[string]bool) bool { + return !blocked[name] && httpguts.ValidHeaderFieldName(name) && httpguts.ValidTrailerHeader(name) +} + +func relayTrailers(w http.ResponseWriter, trailers http.Header, announced, blocked map[string]bool) { + for name, values := range trailers { + name = http.CanonicalHeaderKey(name) + if !validRelayTrailer(name, blocked) { + continue + } + if !announced[name] { + // A trailer discovered only at EOF must not become a regular header + // or trigger automatic Content-Length on a short buffered response. + _ = http.NewResponseController(w).Flush() + name = http.TrailerPrefix + name + } + w.Header()[name] = append([]string(nil), values...) + } +} diff --git a/internal/handler/relay_test.go b/internal/handler/relay_test.go new file mode 100644 index 00000000..67af121c --- /dev/null +++ b/internal/handler/relay_test.go @@ -0,0 +1,277 @@ +package handler + +import ( + "bytes" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "testing/iotest" + + "github.com/git-pkgs/cooldown" + "github.com/go-chi/chi/v5/middleware" +) + +// Exercise the actual routes as well as the two shared entry points. Error +// responses also reach relay branches in handlers that normally parse metadata. +func relayTestRoutes(proxy *Proxy, upstream string) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/upstream", func(w http.ResponseWriter, r *http.Request) { + proxy.ProxyUpstream(w, r, upstream, nil) + }) + mux.HandleFunc("/file", func(w http.ResponseWriter, r *http.Request) { + proxy.ProxyFile(w, r, upstream) + }) + mux.HandleFunc("/metadata", func(w http.ResponseWriter, r *http.Request) { + proxy.ProxyCached(w, r, upstream, "test", "index") + }) + const proxyURL = "http://proxy.local" + mount := func(prefix string, h http.Handler) { + mux.Handle(prefix+"/", http.StripPrefix(prefix, h)) + } + mount("/nuget", NewNuGetHandlerWithUpstreams(proxy, proxyURL, upstream, upstream).Routes()) + mount("/conan", NewConanHandlerWithUpstream(proxy, proxyURL, upstream).Routes()) + mount("/composer", NewComposerHandlerWithUpstreams(proxy, proxyURL, upstream, upstream).Routes()) + mount("/pypi", NewPyPIHandlerWithUpstreams(proxy, proxyURL, upstream, upstream).Routes()) + mount("/gem", NewGemHandlerWithUpstream(proxy, proxyURL, upstream).Routes()) + mount("/conda", NewCondaHandlerWithUpstream(proxy, proxyURL, upstream).Routes()) + mount("/hex", NewHexHandlerWithUpstreams(proxy, proxyURL, upstream, upstream).Routes()) + mount("/swift", NewSwiftHandler(proxy, proxyURL, upstream).Routes()) + mount("/v2", NewContainerHandlerWithRegistry(proxy, proxyURL, upstream).Routes()) + mount("/npm", NewNPMHandler(proxy, proxyURL, upstream).Routes()) + // Match the production recovery middleware: it must not swallow the abort. + return middleware.Recoverer(mux) +} + +func TestRelayRoutes(t *testing.T) { + for _, route := range []string{ + "/upstream", "/file", "/metadata", "/nuget/query", + "/conan/v2/files/demo/1.0/user/stable/rev/recipe/other.txt", + "/composer/search.json", "/pypi/simple/", "/gem/api/v1/dependencies", + "/gem/info/demo", "/conda/conda-forge/noarch/repodata.json", + "/hex/packages/demo", "/swift/scope/demo/1.0.0/Package.swift", + "/v2/library/demo/manifests/latest", "/v2/library/demo/tags/list", + "/npm/-/npm/v1/keys", + } { + t.Run(route, func(t *testing.T) { + for _, truncated := range []bool{false, true} { + t.Run(fmt.Sprintf("truncated=%v", truncated), func(t *testing.T) { + testRelayRoute(t, route, truncated) + }) + } + }) + } +} + +func testRelayRoute(t *testing.T, route string, truncated bool) { + t.Helper() + // Large enough to commit downstream headers before a body-copy failure. + body := strings.Repeat("x", 64*1024) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + conn, rw, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + defer func() { _ = conn.Close() }() + _, _ = fmt.Fprintf(rw, "HTTP/1.1 502 Bad Gateway\r\nContent-Type: text/plain\r\nConnection: X-Private\r\nX-Private: secret\r\nTransfer-Encoding: chunked\r\nTrailer: X-Checksum, X-Private\r\n\r\n%x\r\n%s\r\n", len(body), body) + if !truncated { + _, _ = fmt.Fprint(rw, "0\r\nX-Checksum: verified\r\nX-Late: discovered-at-eof\r\nX-Private: still-secret\r\n\r\n") + } + // Two of these routes fan out a second upstream request for cooldown + // timestamps and abandon its body on a non-200, so this may be writing to + // a connection the proxy has already dropped. What the proxy made of the + // response under test is asserted downstream. + _ = rw.Flush() + })) + defer upstream.Close() + proxy := testProxy() + proxy.Logger = slog.New(slog.NewTextHandler(io.Discard, nil)) + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "3d"} + downstream := httptest.NewServer(relayTestRoutes(proxy, upstream.URL)) + defer downstream.Close() + + resp, err := downstream.Client().Get(downstream.URL + route) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + got, readErr := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusBadGateway { + t.Fatalf("status = %d, want 502", resp.StatusCode) + } + if resp.Header.Get("Connection") != "" || resp.Header.Get("X-Private") != "" || resp.Trailer.Get("X-Private") != "" { + t.Fatalf("connection-scoped fields leaked: headers=%v trailers=%v", resp.Header, resp.Trailer) + } + if truncated { + if readErr == nil { + t.Fatal("truncated upstream was delivered as a complete response") + } + return + } + if readErr != nil || string(got) != body { + t.Fatalf("body length = %d, want %d; error = %v", len(got), len(body), readErr) + } + if resp.Trailer.Get("X-Checksum") != "verified" || resp.Trailer.Get("X-Late") != "discovered-at-eof" { + t.Fatalf("trailers not relayed: %v", resp.Trailer) + } + if route == "/pypi/simple/" && !strings.Contains(resp.Header.Get("Vary"), "Accept") { + t.Error("PyPI lost Vary: Accept") + } +} + +func TestRelayResponseHeaders(t *testing.T) { + headers := http.Header{ + "Connection": {"keep-alive, x-private", " X-Second, Content-Length "}, + "X-Private": {"secret"}, "X-Second": {"secret"}, + "Keep-Alive": {"timeout=5"}, "Proxy-Connection": {"keep-alive"}, + "Proxy-Authenticate": {"challenge"}, "Proxy-Authorization": {"credentials"}, + "Te": {"trailers"}, "Trailer": {"X-Untrusted"}, + "Transfer-Encoding": {"chunked"}, "Upgrade": {"websocket"}, + "Content-Length": {"999"}, "Content-Type": {"application/octet-stream"}, + "Content-Encoding": {"gzip"}, "Etag": {`"v1"`}, + "Set-Cookie": {"a=1", "b=2"}, "Www-Authenticate": {"Bearer realm=test"}, + } + resp := &http.Response{StatusCode: http.StatusOK, Header: headers, Body: io.NopCloser(strings.NewReader("body"))} + w := httptest.NewRecorder() + w.Header().Set("X-Request-ID", "local") + testProxy().relayResponse(w, httptest.NewRequest(http.MethodGet, "/", nil), resp, nil) + for _, name := range []string{ + "Connection", "Keep-Alive", "Proxy-Connection", "Proxy-Authenticate", + "Proxy-Authorization", "Te", "Trailer", "Transfer-Encoding", "Upgrade", + "X-Private", "X-Second", "Content-Length", + } { + if got := w.Header().Get(name); got != "" { + t.Errorf("hop-by-hop header %s survived: %q", name, got) + } + } + for _, name := range []string{"Content-Type", "Content-Encoding", "Etag", "Set-Cookie", "Www-Authenticate"} { + if got := strings.Join(w.Header().Values(name), ","); got != strings.Join(headers.Values(name), ",") { + t.Errorf("end-to-end header %s changed: %q", name, got) + } + } + if w.Header().Get("X-Request-ID") != "local" || w.Body.String() != "body" { + t.Fatal("local header or response body changed") + } + if headers.Get("X-Private") != "secret" { + t.Fatal("relay mutated the upstream response headers") + } +} + +func TestRelayResponseBodiless(t *testing.T) { + for _, tt := range []struct { + method string + status int + length string + }{ + {http.MethodHead, http.StatusOK, "99"}, + {http.MethodGet, http.StatusNoContent, ""}, + {http.MethodGet, http.StatusResetContent, "0"}, + {http.MethodGet, http.StatusNotModified, "99"}, + {http.MethodGet, http.StatusEarlyHints, ""}, + } { + t.Run(fmt.Sprintf("%s/%d", tt.method, tt.status), func(t *testing.T) { + resp := &http.Response{ + StatusCode: tt.status, Header: http.Header{"Content-Length": {"99"}}, + Body: io.NopCloser(iotest.ErrReader(errors.New("body must not be read"))), + Trailer: http.Header{"X-Checksum": {"ignored"}}, + } + w := httptest.NewRecorder() + testProxy().relayResponse(w, httptest.NewRequest(tt.method, "/", nil), resp, nil) + if w.Code != tt.status || w.Body.Len() != 0 || w.Header().Get("Trailer") != "" { + t.Fatalf("unexpected bodiless response: %d %v %q", w.Code, w.Header(), w.Body.String()) + } + if got := w.Header().Get("Content-Length"); got != tt.length { + t.Errorf("Content-Length = %q, want %q", got, tt.length) + } + }) + } +} + +type relayFailWriter struct{ http.ResponseWriter } + +func (w relayFailWriter) Write([]byte) (int, error) { + return 0, errors.New("downstream disconnected") +} + +func TestRelayResponseCopyFailure(t *testing.T) { + for _, downstreamError := range []bool{false, true} { + t.Run(fmt.Sprintf("downstreamError=%v", downstreamError), func(t *testing.T) { + var logs bytes.Buffer + proxy := testProxy() + proxy.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + resp := &http.Response{ + StatusCode: http.StatusOK, Header: make(http.Header), + Request: httptest.NewRequest(http.MethodGet, "http://upstream.test/file", nil), + Body: io.NopCloser(io.MultiReader(strings.NewReader("part"), iotest.ErrReader(io.ErrUnexpectedEOF))), + } + var w http.ResponseWriter = httptest.NewRecorder() + wantBytes := "bytes=4" + if downstreamError { + w = relayFailWriter{httptest.NewRecorder()} + wantBytes = "bytes=0" + } + defer func() { + if got := recover(); got != http.ErrAbortHandler { + t.Errorf("panic = %v, want http.ErrAbortHandler", got) + } + for _, field := range []string{"url=http://upstream.test/file", "status=200", wantBytes, "error="} { + if !strings.Contains(logs.String(), field) { + t.Errorf("log missing %q: %s", field, logs.String()) + } + } + }() + proxy.relayResponse(w, httptest.NewRequest(http.MethodGet, "/", nil), resp, nil) + }) + } +} + +func TestRelayResponseTrailerValidation(t *testing.T) { + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Connection": {"X-Private"}, "Content-Length": {"4"}}, + Body: io.NopCloser(strings.NewReader("body")), + Trailer: http.Header{ + "X-Checksum": {"verified"}, "X-Private": {"secret"}, + "Content-Length": {"999"}, "Content-Type": {"bad/type"}, + "Authorization": {"secret"}, "Transfer-Encoding": {"chunked"}, + "If-Match": {"secret"}, "Invalid Name": {"invalid"}, + }, + } + w := httptest.NewRecorder() + testProxy().relayResponse(w, httptest.NewRequest(http.MethodGet, "/", nil), resp, nil) + result := w.Result() + defer func() { _ = result.Body.Close() }() + if result.Header.Get("Content-Length") != "" { + t.Error("declared trailers must remove Content-Length") + } + if len(result.Trailer) != 1 || result.Trailer.Get("X-Checksum") != "verified" { + t.Fatalf("invalid trailers leaked: %v", result.Trailer) + } +} + +func TestRelayClosesBodyOnAbort(t *testing.T) { + for _, mode := range []string{"upstream", "file", "metadata"} { + t.Run(mode, func(t *testing.T) { + body := &closeTrackingReader{Reader: iotest.ErrReader(io.ErrUnexpectedEOF)} + proxy := testProxy() + proxy.HTTPClient = &http.Client{Transport: pypiRoundTripFunc(func(r *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: body, Request: r}, nil + })} + defer func() { + if got := recover(); got != http.ErrAbortHandler { + t.Errorf("panic = %v, want http.ErrAbortHandler", got) + } + if !body.closed { + t.Error("upstream body was not closed on abort") + } + }() + relayTestRoutes(proxy, "http://upstream.test").ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/"+mode, nil)) + }) + } +} diff --git a/internal/handler/resolved_artifact_test.go b/internal/handler/resolved_artifact_test.go new file mode 100644 index 00000000..2a8f2864 --- /dev/null +++ b/internal/handler/resolved_artifact_test.go @@ -0,0 +1,201 @@ +package handler + +import ( + "context" + "errors" + "io" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/git-pkgs/registries" + "github.com/git-pkgs/registries/client" + "github.com/git-pkgs/registries/fetch" +) + +type artifactTestRegistry struct { + calls atomic.Int64 + err error + artifacts []registries.Artifact +} + +func (*artifactTestRegistry) Ecosystem() string { return "maven" } +func (*artifactTestRegistry) URLs() client.URLBuilder { //nolint:ireturn // required by fetch.Registry + return &client.BaseURLs{} +} +func (r *artifactTestRegistry) FetchVersions(ctx context.Context, _ string) ([]registries.Version, error) { + r.calls.Add(1) + if err := ctx.Err(); err != nil { + return nil, err + } + if r.err != nil { + return nil, r.err + } + return []registries.Version{{Number: "1.0.0", Artifacts: r.artifacts}}, nil +} + +func TestGetOrFetchArtifactResolvedCacheHit(t *testing.T) { + p, db, store, fetcher := setupTestProxy(t) + fetcher.fetchErr = errors.New("unexpected artifact fetch on a cache hit") + seedPackage(t, db, store, "maven", "org.example:demo", "1.0.0", "demo-1.0.0.pom", "pom") + seedPackage(t, db, store, "maven", "org.example:demo", "1.0.0", "demo-1.0.0.jar", "jar") + registry := &artifactTestRegistry{artifacts: []registries.Artifact{{URL: "https://registry.example/demo-1.0.0.jar", Filename: "demo-1.0.0.jar"}}} + p.Resolver.RegisterRegistry(registry) + result, err := p.GetOrFetchArtifact(context.Background(), "maven", "org.example:demo", "1.0.0", "") + if err != nil { + t.Fatal(err) + } + defer func() { _ = result.Reader.Close() }() + body, err := io.ReadAll(result.Reader) + if err != nil || string(body) != "jar" || !result.Cached || result.Artifact.Filename != "demo-1.0.0.jar" { + t.Fatalf("result=%+v body=%q err=%v", result, body, err) + } + if fetcher.fetchCalled || registry.calls.Load() != 1 { + t.Fatalf("fetchCalled=%t resolutions=%d", fetcher.fetchCalled, registry.calls.Load()) + } +} + +func TestGetOrFetchArtifactResolvedMiss(t *testing.T) { + p, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "maven", "org.example:demo", "1.0.0", "demo-1.0.0.pom", "pom") + registry := &artifactTestRegistry{artifacts: []registries.Artifact{{URL: "https://registry.example/demo-1.0.0.jar", Filename: "demo-1.0.0.jar"}}} + p.Resolver.RegisterRegistry(registry) + fetcher := &countingFetcher{content: "jar"} + p.Fetcher = fetcher + for i, cached := range []bool{false, true} { + result, err := p.GetOrFetchArtifact(context.Background(), "maven", "org.example:demo", "1.0.0", "") + if err != nil { + t.Fatal(err) + } + body, readErr := io.ReadAll(result.Reader) + _ = result.Reader.Close() + if readErr != nil || string(body) != "jar" || result.Cached != cached { + t.Fatalf("result=%+v body=%q err=%v; want cached=%t", result, body, readErr, cached) + } + if registry.calls.Load() != int64(i+1) { + t.Fatalf("resolutions=%d after request %d; want one per request", registry.calls.Load(), i+1) + } + } + if fetcher.calls.Load() != 1 || registry.calls.Load() != 2 { + t.Fatalf("downloads=%d resolutions=%d; want 1 and 2", fetcher.calls.Load(), registry.calls.Load()) + } +} + +func TestGetOrFetchArtifactNamedCacheHitDoesNotResolve(t *testing.T) { + p, db, store, _ := setupTestProxy(t) + seedPackage(t, db, store, "maven", "org.example:demo", "1.0.0", "demo-1.0.0.pom", "pom") + p.Resolver = nil // A named cache hit must not need a resolver. + result, err := p.GetOrFetchArtifact(context.Background(), "maven", "org.example:demo", "1.0.0", "demo-1.0.0.pom") + if err != nil { + t.Fatal(err) + } + defer func() { _ = result.Reader.Close() }() + if !result.Cached { + t.Fatal("expected cache hit") + } +} + +func TestGetOrFetchArtifactResolvedErrors(t *testing.T) { + for _, want := range []error{fetch.ErrNotFound, context.Canceled, errors.New("registry unavailable")} { + t.Run(want.Error(), func(t *testing.T) { + p, _, _, fetcher := setupTestProxy(t) + registry := &artifactTestRegistry{err: want} + p.Resolver.RegisterRegistry(registry) + _, err := p.GetOrFetchArtifact(context.Background(), "maven", "org.example:demo", "1.0.0", "") + if !errors.Is(err, want) || fetcher.fetchCalled { + t.Fatalf("err=%v fetched=%t", err, fetcher.fetchCalled) + } + if errors.Is(want, fetch.ErrNotFound) && !errors.Is(err, ErrUpstreamNotFound) { + t.Fatalf("missing upstream not-found classification: %v", err) + } + }) + } +} + +func TestGetOrFetchArtifactResolvedMissingStorage(t *testing.T) { + p, db, store, fetcher := setupTestProxy(t) + seedPackage(t, db, store, "npm", "demo", "1.0.0", "demo-1.0.0.tgz", "old") + clear(store.files) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("new"))} + result, err := p.GetOrFetchArtifact(context.Background(), "npm", "demo", "1.0.0", "") + if err != nil { + t.Fatal(err) + } + defer func() { _ = result.Reader.Close() }() + if result.Cached || !fetcher.fetchCalled { + t.Fatalf("expected refetch: result=%+v fetched=%t", result, fetcher.fetchCalled) + } +} + +func TestGetOrFetchArtifactResolvedDenyBeforeResolution(t *testing.T) { + p, _, _, fetcher := setupTestProxy(t) + registry := &artifactTestRegistry{err: errors.New("must not resolve")} + p.Resolver.RegisterRegistry(registry) + setTestDenylist(t, p, "pkg:maven/org.example/demo@1.0.0") + _, err := p.GetOrFetchArtifact(context.Background(), "maven", "org.example:demo", "1.0.0", "") + if !errors.Is(err, ErrVersionDenied) || registry.calls.Load() != 0 || fetcher.fetchCalled { + t.Fatalf("err=%v resolutions=%d fetched=%t", err, registry.calls.Load(), fetcher.fetchCalled) + } +} + +func TestGetOrFetchArtifactResolvedConcurrentMisses(t *testing.T) { + p, _, _, _ := setupTestProxy(t) + fetcher := &countingFetcher{content: "tarball", delay: fetchHoldTime} + p.Fetcher = fetcher + start := make(chan struct{}) + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + <-start + result, err := p.GetOrFetchArtifact(context.Background(), "npm", "demo", "1.0.0", "") + if err != nil { + t.Error(err) + return + } + defer func() { _ = result.Reader.Close() }() + body, err := io.ReadAll(result.Reader) + if err != nil || string(body) != "tarball" { + t.Errorf("body=%q err=%v", body, err) + } + }) + } + close(start) + wg.Wait() + if fetcher.calls.Load() != 1 { + t.Fatalf("downloads=%d, want 1", fetcher.calls.Load()) + } +} + +func TestGetOrFetchArtifactResolvedWithoutFilename(t *testing.T) { + p, _, _, fetcher := setupTestProxy(t) + registry := &artifactTestRegistry{artifacts: []registries.Artifact{{URL: "https://registry.example/"}}} + p.Resolver.RegisterRegistry(registry) + _, err := p.GetOrFetchArtifact(context.Background(), "maven", "org.example:demo", "1.0.0", "") + if err == nil || !strings.Contains(err.Error(), "no filename") || fetcher.fetchCalled { + t.Fatalf("err=%v fetched=%t; expected rejection before download", err, fetcher.fetchCalled) + } +} + +func TestGetOrFetchArtifactResolvedInvalidCacheHash(t *testing.T) { + p, db, store, fetcher := setupTestProxy(t) + seedPackage(t, db, store, "npm", "demo", "1.0.0", "demo-1.0.0.tgz", "old") + art, err := db.GetArtifact("pkg:npm/demo@1.0.0", "demo-1.0.0.tgz") + if err != nil { + t.Fatal(err) + } + art.ContentHash.String = "invalid-hash" + if err := db.UpsertArtifact(art); err != nil { + t.Fatal(err) + } + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("new"))} + result, err := p.GetOrFetchArtifact(context.Background(), "npm", "demo", "1.0.0", "") + if err != nil { + t.Fatal(err) + } + defer func() { _ = result.Reader.Close() }() + body, err := io.ReadAll(result.Reader) + if err != nil || string(body) != "new" || result.Cached || !fetcher.fetchCalled { + t.Fatalf("result=%+v body=%q err=%v fetched=%t", result, body, err, fetcher.fetchCalled) + } +} diff --git a/internal/handler/rewrite_cache.go b/internal/handler/rewrite_cache.go new file mode 100644 index 00000000..7626827b --- /dev/null +++ b/internal/handler/rewrite_cache.go @@ -0,0 +1,160 @@ +package handler + +import ( + "container/list" + "context" + "crypto/sha256" + "errors" + "strings" + "sync" +) + +// rewriteCache keeps metadata documents after a handler has rewritten them. +// Rewriting means decoding the whole document into generic maps, changing its +// download URLs and encoding it again, which for a large npm packument or an +// expanded Composer document costs milliseconds to hundreds of milliseconds +// and megabytes of allocations. It ran on every request, cached metadata +// included. The cache keeps one rewrite per distinct upstream document, and +// concurrent requests for a document that is not cached yet share a single +// rewrite. +// +// Entries are keyed by a hash of the raw document, so new bytes from upstream +// are rewritten again rather than served stale. Everything else a rewrite +// depends on is fixed for the life of the process: the proxy URL is part of +// the key, and the denylist is loaded at startup. +type rewriteCache struct { + maxBytes int64 + + mu sync.Mutex + size int64 + order *list.List // most recently used at the front + entries map[string]*list.Element + inFlight map[string]*inflightRewrite +} + +type rewriteEntry struct { + key string + out []byte +} + +// inflightRewrite is one rewrite that concurrent callers share. out and err +// are written before done closes and read only after, so the close is the +// handoff. +type inflightRewrite struct { + done chan struct{} + out []byte + err error +} + +// errSharedRewriteAbandoned is what waiters see if the caller running a +// shared rewrite panicked out of it. +var errSharedRewriteAbandoned = errors.New("shared metadata rewrite did not complete") + +// newRewriteCache returns a cache holding up to maxBytes of rewritten output. +// A size of zero or less disables it. +func newRewriteCache(maxBytes int64) *rewriteCache { + if maxBytes <= 0 { + return nil + } + return &rewriteCache{ + maxBytes: maxBytes, + order: list.New(), + entries: make(map[string]*list.Element), + inFlight: make(map[string]*inflightRewrite), + } +} + +// rewriteCacheKey identifies one rewrite: the ecosystem and proxy URL the +// handler rewrites for, the package, and the exact upstream bytes. +func rewriteCacheKey(ecosystem, proxyURL, name string, in []byte) string { + sum := sha256.Sum256(in) + return strings.Join([]string{ecosystem, proxyURL, name, string(sum[:])}, "\x00") +} + +// rewrite returns rewrite(in), from the cache when it can. Callers must treat +// the returned bytes as read-only: a cached result is shared. A caller waiting +// on another's rewrite leaves when ctx ends; the rewrite itself always runs +// to completion, so the result is cached for the next request. +func (c *rewriteCache) rewrite(ctx context.Context, key string, in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, error) { + if c == nil { + return rewrite(in) + } + + c.mu.Lock() + if el, ok := c.entries[key]; ok { + c.order.MoveToFront(el) + out := el.Value.(*rewriteEntry).out + c.mu.Unlock() + return out, nil + } + if f, ok := c.inFlight[key]; ok { + c.mu.Unlock() + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-f.done: + return f.out, f.err + } + } + f := &inflightRewrite{done: make(chan struct{}), err: errSharedRewriteAbandoned} + c.inFlight[key] = f + c.mu.Unlock() + + defer func() { + c.mu.Lock() + delete(c.inFlight, key) + if f.err == nil { + c.add(key, f.out) + } + c.mu.Unlock() + close(f.done) + }() + f.out, f.err = rewrite(in) + return f.out, f.err +} + +// add stores out under key and evicts least recently used entries until the +// cache is back under its limit. Output larger than the whole cache is not +// stored. Called with mu held. +func (c *rewriteCache) add(key string, out []byte) { + n := int64(len(out)) + if n > c.maxBytes { + return + } + c.entries[key] = c.order.PushFront(&rewriteEntry{key: key, out: out}) + c.size += n + for c.size > c.maxBytes { + oldest := c.order.Back() + e := oldest.Value.(*rewriteEntry) + c.order.Remove(oldest) + delete(c.entries, e.key) + c.size -= int64(len(e.out)) + } +} + +// len reports how many rewrites are cached. +func (c *rewriteCache) len() int { + if c == nil { + return 0 + } + c.mu.Lock() + defer c.mu.Unlock() + return len(c.entries) +} + +// cachedRewrite rewrites a metadata document through the proxy's rewrite +// cache. Cooldown filtering depends on the current time, so with cooldown on +// a cached rewrite could keep hiding a version past its cooldown; those +// rewrites always run. +func (p *Proxy) cachedRewrite(ctx context.Context, ecosystem, proxyURL, name string, in []byte, rewrite func([]byte) ([]byte, error)) ([]byte, error) { + if p.rewrites == nil || (p.Cooldown != nil && p.Cooldown.Enabled()) { + return rewrite(in) + } + return p.rewrites.rewrite(ctx, rewriteCacheKey(ecosystem, proxyURL, name, in), in, rewrite) +} + +// SetMetadataRewriteCacheSize enables the cache of rewritten metadata with +// room for maxBytes of output. Zero or less disables it. +func (p *Proxy) SetMetadataRewriteCacheSize(maxBytes int64) { + p.rewrites = newRewriteCache(maxBytes) +} diff --git a/internal/handler/rewrite_cache_test.go b/internal/handler/rewrite_cache_test.go new file mode 100644 index 00000000..0edb94fc --- /dev/null +++ b/internal/handler/rewrite_cache_test.go @@ -0,0 +1,258 @@ +package handler + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/git-pkgs/cooldown" +) + +// countingRewrite upper-cases its input and counts calls. If entered is set it +// is closed when the first call starts, and the call then waits on release. +type countingRewrite struct { + calls atomic.Int64 + entered chan struct{} + release chan struct{} + once sync.Once +} + +func (r *countingRewrite) fn(in []byte) ([]byte, error) { + r.calls.Add(1) + if r.entered != nil { + r.once.Do(func() { close(r.entered) }) + <-r.release + } + return []byte(strings.ToUpper(string(in))), nil +} + +func TestRewriteCache_RepeatServesCachedRewrite(t *testing.T) { + c := newRewriteCache(1 << 20) + var r countingRewrite + key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte("doc")) + + for range 3 { + out, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + if err != nil || string(out) != "DOC" { + t.Fatalf("rewrite = %q, %v", out, err) + } + } + if got := r.calls.Load(); got != 1 { + t.Errorf("rewrites = %d, want 1", got) + } +} + +func TestRewriteCache_NewUpstreamBytesRewriteAgain(t *testing.T) { + c := newRewriteCache(1 << 20) + var r countingRewrite + + for _, doc := range []string{"v1", "v2"} { + key := rewriteCacheKey("npm", "http://proxy", "left-pad", []byte(doc)) + out, err := c.rewrite(context.Background(), key, []byte(doc), r.fn) + if err != nil || string(out) != strings.ToUpper(doc) { + t.Fatalf("rewrite(%s) = %q, %v", doc, out, err) + } + } + if got := r.calls.Load(); got != 2 { + t.Errorf("rewrites = %d, want 2", got) + } +} + +// TestRewriteCache_ConcurrentRequestsShareRewrite asserts that requests +// arriving while a document is being rewritten wait for that rewrite instead +// of running their own. Callers that arrive after it finishes hit the cache, +// so the count holds however the goroutines are scheduled. +func TestRewriteCache_ConcurrentRequestsShareRewrite(t *testing.T) { + c := newRewriteCache(1 << 20) + r := countingRewrite{entered: make(chan struct{}), release: make(chan struct{})} + key := rewriteCacheKey("npm", "http://proxy", "typescript", []byte("doc")) + const n = 10 + + outs := make(chan string, n) + run := func() { + out, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + if err != nil { + t.Error(err) + } + outs <- string(out) + } + go run() + <-r.entered + for range n - 1 { + go run() + } + close(r.release) + + for range n { + if out := <-outs; out != "DOC" { + t.Errorf("out = %q", out) + } + } + if got := r.calls.Load(); got != 1 { + t.Errorf("rewrites = %d, want 1", got) + } +} + +// TestRewriteCache_WaiterLeavingKeepsRewrite asserts that a caller waiting on +// another's rewrite returns its own context error when its client leaves, +// while the rewrite completes and is cached for the next request. +func TestRewriteCache_WaiterLeavingKeepsRewrite(t *testing.T) { + c := newRewriteCache(1 << 20) + r := countingRewrite{entered: make(chan struct{}), release: make(chan struct{})} + key := rewriteCacheKey("npm", "http://proxy", "typescript", []byte("doc")) + + first := make(chan error, 1) + go func() { + _, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn) + first <- err + }() + <-r.entered + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := c.rewrite(ctx, key, []byte("doc"), r.fn); !errors.Is(err, context.Canceled) { + t.Errorf("waiter err = %v, want context.Canceled", err) + } + close(r.release) + if err := <-first; err != nil { + t.Fatal(err) + } + if out, err := c.rewrite(context.Background(), key, []byte("doc"), r.fn); err != nil || string(out) != "DOC" { + t.Errorf("cached rewrite = %q, %v", out, err) + } + if got := r.calls.Load(); got != 1 { + t.Errorf("rewrites = %d, want 1", got) + } +} + +func TestRewriteCache_EvictsLeastRecentlyUsed(t *testing.T) { + c := newRewriteCache(6) // room for two three-byte outputs + var r countingRewrite + keyFor := func(doc string) string { return rewriteCacheKey("npm", "http://proxy", doc, []byte(doc)) } + + for _, doc := range []string{"aaa", "bbb"} { + _, _ = c.rewrite(context.Background(), keyFor(doc), []byte(doc), r.fn) + } + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) // aaa is now the most recent + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) // evicts bbb + + before := r.calls.Load() + _, _ = c.rewrite(context.Background(), keyFor("aaa"), []byte("aaa"), r.fn) + _, _ = c.rewrite(context.Background(), keyFor("ccc"), []byte("ccc"), r.fn) + if got := r.calls.Load() - before; got != 0 { + t.Errorf("aaa and ccc rewritten %d times, want both still cached", got) + } + _, _ = c.rewrite(context.Background(), keyFor("bbb"), []byte("bbb"), r.fn) + if got := r.calls.Load() - before; got != 1 { + t.Errorf("bbb should have been evicted and rewritten") + } +} + +func TestRewriteCache_OutputLargerThanCacheIsNotStored(t *testing.T) { + c := newRewriteCache(2) + var r countingRewrite + key := rewriteCacheKey("npm", "http://proxy", "big", []byte("big")) + + _, _ = c.rewrite(context.Background(), key, []byte("big"), r.fn) + if c.len() != 0 { + t.Errorf("cached %d entries, want 0", c.len()) + } +} + +func TestRewriteCache_ErrorsAreNotCached(t *testing.T) { + c := newRewriteCache(1 << 20) + var calls int + failing := func([]byte) ([]byte, error) { calls++; return nil, errors.New("bad document") } + key := rewriteCacheKey("npm", "http://proxy", "broken", []byte("doc")) + + for range 2 { + if _, err := c.rewrite(context.Background(), key, []byte("doc"), failing); err == nil { + t.Fatal("expected the rewrite error") + } + } + if calls != 2 { + t.Errorf("rewrites = %d, want 2", calls) + } +} + +func TestCachedRewrite_DisabledRewritesEveryTime(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.SetMetadataRewriteCacheSize(0) + var r countingRewrite + + for range 2 { + _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), r.fn) + } + if got := r.calls.Load(); got != 2 { + t.Errorf("rewrites = %d, want 2", got) + } +} + +// TestCachedRewrite_CooldownBypassesCache asserts that with cooldown on every +// request is rewritten: cooldown filtering depends on the current time, so a +// cached rewrite could keep hiding a version after its cooldown ends. +func TestCachedRewrite_CooldownBypassesCache(t *testing.T) { + proxy, _, _, _ := setupTestProxy(t) + proxy.SetMetadataRewriteCacheSize(1 << 20) + proxy.Cooldown = &cooldown.Config{Default: "3d"} + var r countingRewrite + + for range 2 { + _, _ = proxy.cachedRewrite(context.Background(), "npm", "http://proxy", "left-pad", []byte("doc"), r.fn) + } + if got := r.calls.Load(); got != 2 { + t.Errorf("rewrites = %d, want 2", got) + } +} + +// TestMetadataHandlers_ServeCachedRewrite asserts that repeated requests for +// the same npm or Composer metadata reuse one rewrite and return the same +// bytes, with the download URLs pointing at the proxy. +func TestMetadataHandlers_ServeCachedRewrite(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/left-pad": + _, _ = w.Write([]byte(`{"name":"left-pad","versions":{"1.3.0":{"dist":{"tarball":"https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"}}}}`)) + case "/p2/vendor/pkg.json": + _, _ = w.Write([]byte(`{"packages":{"vendor/pkg":[{"version":"1.0.0","dist":{"type":"zip","url":"https://example.com/pkg-1.0.0.zip"}}]}}`)) + default: + http.NotFound(w, r) + } + })) + defer upstream.Close() + + proxy, _, _, _ := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + proxy.SetMetadataRewriteCacheSize(1 << 20) + + cases := []struct { + name string + handler http.Handler + path string + }{ + {"npm", NewNPMHandler(proxy, "http://proxy.example", upstream.URL).Routes(), "/left-pad"}, + {"composer", NewComposerHandlerWithUpstreams(proxy, "http://proxy.example", upstream.URL, upstream.URL).Routes(), "/p2/vendor/pkg.json"}, + } + for i, c := range cases { + var bodies []string + for range 2 { + rec := httptest.NewRecorder() + c.handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, c.path, nil)) + if rec.Code != http.StatusOK { + t.Fatalf("%s: status %d", c.name, rec.Code) + } + bodies = append(bodies, rec.Body.String()) + } + if bodies[0] != bodies[1] || !strings.Contains(bodies[0], "http://proxy.example") { + t.Errorf("%s: bodies %q and %q", c.name, bodies[0], bodies[1]) + } + if got := proxy.rewrites.len(); got != i+1 { + t.Errorf("%s: cached rewrites = %d, want %d", c.name, got, i+1) + } + } +} diff --git a/internal/handler/rpm.go b/internal/handler/rpm.go index e06a3cc1..6b138cca 100644 --- a/internal/handler/rpm.go +++ b/internal/handler/rpm.go @@ -95,7 +95,7 @@ func (h *RPMHandler) handlePackageDownload(w http.ResponseWriter, r *http.Reques } w.Header().Set(headerContentType, "application/x-rpm") - ServeArtifact(w, result) + ServeArtifactRequest(w, r, result) } // handleMetadata proxies repository metadata files (repomd.xml, primary.xml.gz, etc.). diff --git a/internal/handler/stale_cache_test.go b/internal/handler/stale_cache_test.go new file mode 100644 index 00000000..c11edd0b --- /dev/null +++ b/internal/handler/stale_cache_test.go @@ -0,0 +1,163 @@ +package handler + +import ( + "context" + "errors" + "io" + "strings" + "testing" +) + +// These tests cover an upstream re-publishing a version: the cache holds one +// artifact and upstream now declares another digest for it. + +const ( + stalePkgPURL = "pkg:npm/pkg" + staleVersionPURL = "pkg:npm/pkg@1.0.0" + staleFilename = "pkg-1.0.0.tgz" + staleStoragePath = "npm/pkg/1.0.0/pkg-1.0.0.tgz" + staleURL = "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz" +) + +// seedCachedArtifact commits content the way a fetch does. +func seedCachedArtifact(t *testing.T, proxy *Proxy, store *mockStorage, content string) { + t.Helper() + ctx := context.Background() + if _, _, err := store.Store(ctx, staleStoragePath, strings.NewReader(content)); err != nil { + t.Fatalf("seeding storage: %v", err) + } + artifact := testArtifact(content, staleVersionPURL, staleFilename, "application/gzip") + if err := proxy.updateCacheDB("npm", "pkg", stalePkgPURL, staleURL, staleStoragePath, artifact); err != nil { + t.Fatalf("seeding cache record: %v", err) + } +} + +// cachedDigest reports the digest the cache record holds, or "" without one. +func cachedDigest(t *testing.T, proxy *Proxy) string { + t.Helper() + record, err := proxy.DB.GetCachedArtifact(stalePkgPURL, staleVersionPURL, staleFilename) + if err != nil { + t.Fatalf("reading cache record: %v", err) + } + if record == nil { + return "" + } + return record.Artifact.Digest.Encoded() +} + +func bytesPresent(store *mockStorage) bool { + r, err := store.Open(context.Background(), staleStoragePath) + if err != nil { + return false + } + _ = r.Close() + return true +} + +func TestStaleCacheCheckHasNoSideEffects(t *testing.T) { + proxy, _, store, _ := setupTestProxy(t) + seedCachedArtifact(t, proxy, store, "old bytes") + + res, err := proxy.getCachedArtifactWithUpstreamHash(context.Background(), + stalePkgPURL, staleVersionPURL, staleFilename, sha256Hex("new bytes")) + if err != nil { + t.Fatalf("cache check failed: %v", err) + } + if res != nil { + drain(res) + t.Fatal("stale entry was served") + } + if got := cachedDigest(t, proxy); got != sha256Hex("old bytes") { + t.Errorf("record digest = %q, want the stale one kept: the check must not discard", got) + } + if !bytesPresent(store) { + t.Error("stale bytes were deleted by the check") + } +} + +func TestStaleCacheIsDiscardedBeforeTheFetch(t *testing.T) { + proxy, _, store, fetcher := setupTestProxy(t) + seedCachedArtifact(t, proxy, store, "old bytes") + boom := errors.New("upstream unavailable") + fetcher.fetchErr = boom + + _, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), + "npm", "pkg", "1.0.0", staleFilename, staleURL, "sha256:"+sha256Hex("new bytes")) + if !errors.Is(err, boom) { + t.Fatalf("got %v, want the fetch failure", err) + } + if got := cachedDigest(t, proxy); got != "" { + t.Errorf("stale record survived a failed refresh, digest = %q", got) + } + // A request that read the stale record may still be opening its bytes, + // so they are queued for deletion rather than deleted. + if !bytesPresent(store) { + t.Error("stale bytes were deleted while a reader may still open them") + } + assertQueuedForDeletion(t, proxy.DB, staleStoragePath) +} + +func TestStaleCacheIsReplacedByTheFetch(t *testing.T) { + proxy, _, store, fetcher := setupTestProxy(t) + seedCachedArtifact(t, proxy, store, "old bytes") + fetcher.artifact = artifactBody("new bytes") + upstream := sha256Hex("new bytes") + + res, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), + "npm", "pkg", "1.0.0", staleFilename, staleURL, "sha256:"+upstream) + if err != nil { + t.Fatalf("refresh failed: %v", err) + } + got, err := io.ReadAll(res.Reader) + _ = res.Reader.Close() + if err != nil || string(got) != "new bytes" { + t.Fatalf("got %q (err %v), want the refreshed bytes", got, err) + } + if !fetcher.fetchCalled { + t.Error("stale entry was served without a fetch") + } + if d := cachedDigest(t, proxy); d != upstream { + t.Errorf("record digest = %q, want %q", d, upstream) + } + + fetcher.fetchCalled = false + res, err = proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), + "npm", "pkg", "1.0.0", staleFilename, staleURL, "sha256:"+upstream) + if err != nil { + t.Fatalf("request after refresh failed: %v", err) + } + drain(res) + if fetcher.fetchCalled || !res.Cached { + t.Errorf("request after refresh: fetched=%v cached=%v, want served from cache", fetcher.fetchCalled, res.Cached) + } +} + +// TestLateLeaderKeepsRefreshedEntry is the race, at the point it would happen: +// a caller whose cache check saw a stale entry reaches the coalescing step +// after another caller's fetch replaced it. It must serve the replacement. +func TestLateLeaderKeepsRefreshedEntry(t *testing.T) { + proxy, _, store, fetcher := setupTestProxy(t) + seedCachedArtifact(t, proxy, store, "new bytes") + fetcher.fetchErr = errors.New("must not fetch") + upstream := sha256Hex("new bytes") + + res, err := proxy.coalescedFetchFromURL(context.Background(), + "npm", "pkg", "1.0.0", staleFilename, stalePkgPURL, staleVersionPURL, staleURL, nil, upstream) + if err != nil { + t.Fatalf("late leader failed: %v", err) + } + got, err := io.ReadAll(res.Reader) + _ = res.Reader.Close() + if err != nil || string(got) != "new bytes" { + t.Fatalf("got %q (err %v), want the refreshed bytes", got, err) + } + if fetcher.fetchCalled { + t.Error("refreshed entry was fetched again") + } + if d := cachedDigest(t, proxy); d != upstream { + t.Errorf("refreshed record was discarded, digest = %q", d) + } + if !bytesPresent(store) { + t.Error("refreshed bytes were deleted") + } +} diff --git a/internal/handler/stream_artifacts_test.go b/internal/handler/stream_artifacts_test.go new file mode 100644 index 00000000..5edc5dd6 --- /dev/null +++ b/internal/handler/stream_artifacts_test.go @@ -0,0 +1,411 @@ +package handler + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/git-pkgs/artifacts" + "github.com/git-pkgs/cooldown" + "github.com/git-pkgs/proxy/internal/packageurl" + "github.com/git-pkgs/registries/fetch" +) + +func newStreamingProxy(t *testing.T, body string) (*Proxy, *mockStorage, *mockFetcher) { + t.Helper() + proxy, _, store, fetcher := setupTestProxy(t) + proxy.StreamArtifacts = true + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader(body)), + Size: int64(len(body)), + ContentType: "application/gzip", + } + return proxy, store, fetcher +} + +func TestStreamArtifactsFromURLStreamsWithoutStoring(t *testing.T) { + proxy, store, fetcher := newStreamingProxy(t, "fetched content") + + result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "newpkg", "1.0.0", + "newpkg-1.0.0.tar.gz", "https://pypi.org/files/newpkg-1.0.0.tar.gz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + body, err := io.ReadAll(result.Reader) + if err != nil { + t.Fatalf("reading body: %v", err) + } + if string(body) != "fetched content" { + t.Errorf("body = %q, want %q", body, "fetched content") + } + if fetcher.fetchedURL != "https://pypi.org/files/newpkg-1.0.0.tar.gz" { + t.Errorf("fetched URL = %q", fetcher.fetchedURL) + } + if result.Cached { + t.Error("streaming result reported as cached") + } + if result.Artifact.Size != int64(len("fetched content")) { + t.Errorf("Size = %d, want the upstream size", result.Artifact.Size) + } + if result.Artifact.MediaType != "application/gzip" { + t.Errorf("MediaType = %q", result.Artifact.MediaType) + } + if len(store.files) != 0 { + t.Errorf("streaming stored %d files, want none", len(store.files)) + } + cached, err := proxy.DB.GetCachedArtifact("pkg:pypi/newpkg", "pkg:pypi/newpkg@1.0.0", "newpkg-1.0.0.tar.gz") + if err != nil { + t.Fatalf("GetCachedArtifact: %v", err) + } + if cached != nil { + t.Error("streaming recorded the artifact in the cache database") + } +} + +func TestStreamArtifactsGetOrFetchArtifactStreamsWithoutStoring(t *testing.T) { + proxy, store, fetcher := newStreamingProxy(t, "tarball data") + + result, err := proxy.GetOrFetchArtifact(context.Background(), "npm", "leftpad", testVersion100, "leftpad-1.0.0.tgz") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + body, _ := io.ReadAll(result.Reader) + if string(body) != "tarball data" { + t.Errorf("body = %q", body) + } + if !fetcher.fetchCalled { + t.Error("upstream was not fetched") + } + if len(store.files) != 0 { + t.Errorf("streaming stored %d files, want none", len(store.files)) + } +} + +func TestStreamArtifactsIgnoresCachedArtifacts(t *testing.T) { + proxy, _, store, fetcher := setupTestProxy(t) + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("old bytes"))} + url := "https://pypi.org/files/newpkg-1.0.0.tar.gz" + + // Populate the cache in normal mode first. + result, err := proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "newpkg", "1.0.0", "newpkg-1.0.0.tar.gz", url) + if err != nil { + t.Fatalf("priming cache: %v", err) + } + _ = result.Reader.Close() + if len(store.files) != 1 { + t.Fatalf("expected the cache to hold the artifact, got %d files", len(store.files)) + } + + proxy.StreamArtifacts = true + cached, err := proxy.GetCachedArtifact(context.Background(), "pypi", "newpkg", "1.0.0", "newpkg-1.0.0.tar.gz") + if err != nil || cached != nil { + t.Fatalf("GetCachedArtifact = %v, %v; want nil, nil while streaming", cached, err) + } + + fetcher.fetchCalled = false + fetcher.artifact = &fetch.Artifact{Body: io.NopCloser(strings.NewReader("new bytes"))} + result, err = proxy.GetOrFetchArtifactFromURL(context.Background(), "pypi", "newpkg", "1.0.0", "newpkg-1.0.0.tar.gz", url) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + body, _ := io.ReadAll(result.Reader) + if !fetcher.fetchCalled || string(body) != "new bytes" { + t.Errorf("streaming served %q (fetched=%v), want a fresh upstream fetch", body, fetcher.fetchCalled) + } +} + +func TestStreamArtifactsVerifiesUpstreamDigest(t *testing.T) { + const content = "blob bytes" + + t.Run("matching digest streams the body", func(t *testing.T) { + proxy, _, _ := newStreamingProxy(t, content) + result, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), "oci", "library/app", "v1", + "blob", "https://registry.test/blob", "sha256:"+sha256Hex(content)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + body, err := io.ReadAll(result.Reader) + if err != nil { + t.Fatalf("reading verified body: %v", err) + } + if string(body) != content { + t.Errorf("body = %q", body) + } + if got := result.Artifact.Digest.Encoded(); got != sha256Hex(content) { + t.Errorf("Digest = %q, want the upstream digest", got) + } + if result.Artifact.Size >= 0 { + t.Errorf("Size = %d, want unknown so the response is chunked", result.Artifact.Size) + } + }) + + t.Run("mismatched digest fails the read", func(t *testing.T) { + proxy, _, _ := newStreamingProxy(t, content) + result, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), "oci", "library/app", "v1", + "blob", "https://registry.test/blob", "sha256:"+sha256Hex("other bytes")) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + defer func() { _ = result.Reader.Close() }() + + if _, err := io.ReadAll(result.Reader); !errors.Is(err, ErrArtifactDigestMismatch) { + t.Errorf("read error = %v, want ErrArtifactDigestMismatch", err) + } + }) +} + +func TestServeArtifactAbortsOnStreamedDigestMismatch(t *testing.T) { + proxy, _, _ := newStreamingProxy(t, "blob bytes") + result, err := proxy.GetOrFetchArtifactFromURLWithDigest(context.Background(), "oci", "library/app", "v1", + "blob", "https://registry.test/blob", "sha256:"+sha256Hex("other bytes")) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + rec := httptest.NewRecorder() + defer func() { + if r := recover(); r != http.ErrAbortHandler { + t.Fatalf("recovered %v, want http.ErrAbortHandler", r) + } + if rec.Header().Get(headerContentLength) != "" { + t.Errorf("Content-Length = %q, want none so the client sees an unterminated response", + rec.Header().Get(headerContentLength)) + } + }() + ServeArtifact(rec, result) +} + +func TestNPMDownloadCooldownWhileStreaming(t *testing.T) { + now := time.Now() + packument := `{ + "name": "leftpad", + "dist-tags": {"latest": "2.0.0"}, + "time": { + "1.0.0": "` + now.Add(-30*24*time.Hour).Format(time.RFC3339) + `", + "2.0.0": "` + now.Add(-1*time.Hour).Format(time.RFC3339) + `" + }, + "versions": {"1.0.0": {}, "2.0.0": {}} + }` + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", contentTypeJSON) + _, _ = io.WriteString(w, packument) + })) + defer upstream.Close() + + tests := []struct { + name string + version string + wantStatus int + }{ + {"published before the window streams the tarball", testVersion100, http.StatusOK}, + {"published inside the window is withheld", "2.0.0", http.StatusNotFound}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + proxy, store, fetcher := newStreamingProxy(t, "tarball data") + proxy.HTTPClient = upstream.Client() + proxy.Cooldown = &cooldown.Config{Default: "7d"} + + srv := httptest.NewServer(NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) + defer srv.Close() + + resp, err := http.Get(srv.URL + "/leftpad/-/leftpad-" + tt.version + ".tgz") + if err != nil { + t.Fatalf("request failed: %v", err) + } + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + + if resp.StatusCode != tt.wantStatus { + t.Fatalf("status = %d, want %d", resp.StatusCode, tt.wantStatus) + } + if tt.wantStatus == http.StatusOK && string(body) != "tarball data" { + t.Errorf("body = %q", body) + } + if tt.wantStatus == http.StatusNotFound && fetcher.fetchCalled { + t.Error("fetched a version that is still inside the cooldown window") + } + if len(store.files) != 0 { + t.Errorf("streaming stored %d files, want none", len(store.files)) + } + }) + } +} + +// truncatingUpstream answers every request matching match with raw, then +// closes the connection, so the response body ends early. +func truncatingUpstream(t *testing.T, match func(*http.Request) bool, raw string, fallback http.HandlerFunc) *httptest.Server { + t.Helper() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !match(r) { + fallback(w, r) + return + } + conn, buf, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Errorf("hijack: %v", err) + return + } + _, _ = buf.WriteString(raw) + _ = buf.Flush() + _ = conn.Close() + })) + t.Cleanup(upstream.Close) + return upstream +} + +// requireIncompleteResponse fails unless reading the response surfaces an error, +// i.e. the client cannot mistake the body for a complete download. +func requireIncompleteResponse(t *testing.T, url string) { + t.Helper() + resp, err := http.Get(url) + if err != nil { + return + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err == nil { + t.Fatalf("client read a complete %d response (Content-Length %q, body %q), want an incomplete one", + resp.StatusCode, resp.Header.Get(headerContentLength), body) + } +} + +func useRealFetcher(t *testing.T, proxy *Proxy, upstream *httptest.Server) { + t.Helper() + proxy.HTTPClient = upstream.Client() + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + proxy.Fetcher = fetcher + t.Cleanup(func() { _ = fetcher.Close() }) +} + +func TestStreamArtifactsOCIBlobShorterThanContentLengthIsIncomplete(t *testing.T) { + digest := "sha256:" + sha256Hex(strings.Repeat("x", 100)) + upstream := truncatingUpstream(t, + func(r *http.Request) bool { return strings.Contains(r.URL.Path, "/blobs/") }, + "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: 100\r\n\r\nshort", + http.NotFound) + + proxy, _, store, _ := setupTestProxy(t) + proxy.StreamArtifacts = true + useRealFetcher(t, proxy, upstream) + h := NewContainerHandler(proxy, "http://proxy.example", map[string]string{"ghcr": upstream.URL}) + srv := httptest.NewServer(h.Routes()) + defer srv.Close() + + requireIncompleteResponse(t, srv.URL+"/upstream/ghcr/owner/demo/blobs/"+digest) + if len(store.files) != 0 { + t.Errorf("streaming stored %d files, want none", len(store.files)) + } +} + +func TestStreamArtifactsNPMTarballWithoutFinalChunkIsIncomplete(t *testing.T) { + upstream := truncatingUpstream(t, + func(r *http.Request) bool { return strings.HasSuffix(r.URL.Path, ".tgz") }, + "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nshort\r\n", + http.NotFound) + + proxy, _, store, _ := setupTestProxy(t) + proxy.StreamArtifacts = true + useRealFetcher(t, proxy, upstream) + srv := httptest.NewServer(NewNPMHandler(proxy, "http://proxy.test", upstream.URL).Routes()) + defer srv.Close() + + requireIncompleteResponse(t, srv.URL+"/leftpad/-/leftpad-1.0.0.tgz") + if len(store.files) != 0 { + t.Errorf("streaming stored %d files, want none", len(store.files)) + } +} + +func TestServeArtifactAbortsOnShortRead(t *testing.T) { + tests := []struct { + name string + reader io.Reader + size int64 + }{ + {"read error", io.MultiReader(strings.NewReader("short"), iotestErrReader{io.ErrUnexpectedEOF}), -1}, + {"fewer bytes than the declared size", strings.NewReader("short"), 100}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := &CacheResult{Reader: io.NopCloser(tt.reader), Artifact: artifacts.Artifact{Size: tt.size}} + defer func() { + if r := recover(); r != http.ErrAbortHandler { + t.Fatalf("recovered %v, want http.ErrAbortHandler", r) + } + }() + ServeArtifact(httptest.NewRecorder(), result) + }) + } +} + +type iotestErrReader struct{ err error } + +func (r iotestErrReader) Read([]byte) (int, error) { return 0, r.err } + +func TestStreamArtifactsSwiftArchiveHeadIgnoresCachedEntry(t *testing.T) { + archive := []byte("cached archive") + checksum := sha256.Sum256(archive) + var archiveRequests int + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".zip") { + archiveRequests++ + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"id":"apple.example","version":"1.2.3","resources":[{"name":"source-archive","type":"application/zip","checksum":%q}]}`, hex.EncodeToString(checksum[:])) + })) + defer upstream.Close() + + proxy, _, _, fetcher := setupTestProxy(t) + proxy.HTTPClient = upstream.Client() + fetcher.artifact = &fetch.Artifact{ + Body: io.NopCloser(strings.NewReader(string(archive))), + Size: int64(len(archive)), + ContentType: "application/zip", + } + packagePURL, versionPURL := packageurl.MakeCacheStrings("swift", "apple/example", "1.2.3") + cached, err := proxy.getOrFetchArtifactFromURLWithCachePURLs( + context.Background(), "swift", "apple/example", "1.2.3", "example-1.2.3.zip", + packagePURL, versionPURL, upstream.URL+"/apple/example/1.2.3.zip", nil, hex.EncodeToString(checksum[:]), + ) + if err != nil { + t.Fatalf("seeding cache in normal mode: %v", err) + } + _ = cached.Reader.Close() + + proxy.StreamArtifacts = true + fetcher.artifact = nil + fetcher.fetchErr = fetch.ErrNotFound + handler := NewSwiftHandler(proxy, "https://proxy.example", upstream.URL).Routes() + + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodHead, "/apple/example/1.2.3.zip", nil)) + if w.Code == http.StatusOK { + t.Fatalf("HEAD served the cached archive (Content-Length %q) while streaming artifacts", w.Header().Get(headerContentLength)) + } + if archiveRequests == 0 { + t.Error("HEAD did not ask the upstream archive") + } + + w = httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.2.3.zip", nil)) + if w.Code == http.StatusOK { + t.Fatalf("GET served the cached archive while streaming artifacts") + } +} diff --git a/internal/handler/swift.go b/internal/handler/swift.go index 1c289d0b..6cc7d78a 100644 --- a/internal/handler/swift.go +++ b/internal/handler/swift.go @@ -8,7 +8,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "net/http" "net/url" "strconv" @@ -190,7 +189,7 @@ func (h *SwiftHandler) handleSourceArchive(w http.ResponseWriter, r *http.Reques result.Artifact.MediaType = "application/zip" setSwiftArchiveHeaders(w.Header(), name, version, result.Artifact.Digest.Encoded(), archiveInfo) - serveArtifact(w, r.Method, result) + ServeArtifactRequest(w, r, result) } func (h *SwiftHandler) handleSourceArchiveHead( @@ -209,7 +208,7 @@ func (h *SwiftHandler) handleSourceArchiveHead( if result != nil { result.Artifact.MediaType = "application/zip" setSwiftArchiveHeaders(w.Header(), name, version, result.Artifact.Digest.Encoded(), archiveInfo) - serveArtifact(w, r.Method, result) + ServeArtifactRequest(w, r, result) return } @@ -411,21 +410,18 @@ func (h *SwiftHandler) proxySwiftResource(w http.ResponseWriter, r *http.Request } defer func() { _ = resp.Body.Close() }() - copySwiftResponseHeaders(w.Header(), resp.Header) - if location := resp.Header.Get("Location"); location != "" { - w.Header().Set("Location", h.rewriteRegistryURL(location, upstreamURL)) - } - for _, link := range resp.Header.Values("Link") { - w.Header().Add("Link", h.rewriteLinkHeader(link, upstreamURL)) - } - if w.Header().Get("Content-Version") == "" { - w.Header().Set("Content-Version", swiftContentVersion) - } - - w.WriteHeader(resp.StatusCode) - if r.Method != http.MethodHead { - _, _ = io.Copy(w, resp.Body) - } + h.proxy.relayResponse(w, r, resp, func(dst, src http.Header) { + copySwiftResponseHeaders(dst, src) + if location := src.Get("Location"); location != "" { + dst.Set("Location", h.rewriteRegistryURL(location, upstreamURL)) + } + for _, link := range src.Values("Link") { + dst.Add("Link", h.rewriteLinkHeader(link, upstreamURL)) + } + if dst.Get("Content-Version") == "" { + dst.Set("Content-Version", swiftContentVersion) + } + }) } func copySwiftResponseHeaders(dst, src http.Header) { diff --git a/internal/handler/swift_test.go b/internal/handler/swift_test.go index 88ed30e9..c9df35a5 100644 --- a/internal/handler/swift_test.go +++ b/internal/handler/swift_test.go @@ -332,7 +332,7 @@ func TestSwiftSourceArchiveCanonicalizesPackageIdentity(t *testing.T) { } } -func TestSwiftSourceArchiveHeadDiscardsCachedChecksumMismatch(t *testing.T) { +func TestSwiftSourceArchiveHeadLeavesStaleCacheForTheFetch(t *testing.T) { archive := []byte("cached archive") upstreamChecksum := sha256.Sum256([]byte("upstream archive")) @@ -379,11 +379,27 @@ func TestSwiftSourceArchiveHeadDiscardsCachedChecksumMismatch(t *testing.T) { if got := w.Header().Get("Content-Length"); got != "456" { t.Errorf("Content-Length = %q, want 456 from upstream probe", got) } - if len(store.files) != 0 { - t.Errorf("mismatched cached archive remained in storage: %v", store.files) + // HEAD leaves the stale entry alone; the next GET replaces it under the + // coalescing key. + if len(store.files) != 1 { + t.Errorf("HEAD must leave the stale archive in storage, got %v", store.files) + } + if rec, _ := db.GetCachedArtifact(packagePURL, versionPURL, "example-1.2.3.zip"); rec == nil { + t.Error("HEAD must leave the stale cache record in place") + } + + fetcher.artifact = artifactBody("upstream archive") + w = httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/apple/example/1.2.3.zip", nil)) + if w.Code != http.StatusOK { + t.Fatalf("GET status = %d, want 200; body: %s", w.Code, w.Body.String()) + } + if w.Body.String() != "upstream archive" { + t.Errorf("GET body = %q, want the refreshed archive", w.Body.String()) } - if rec, _ := db.GetCachedArtifact(packagePURL, versionPURL, "example-1.2.3.zip"); rec != nil { - t.Error("mismatched cache record was not cleared") + rec, _ := db.GetCachedArtifact(packagePURL, versionPURL, "example-1.2.3.zip") + if rec == nil || rec.Artifact.Digest.Encoded() != hex.EncodeToString(upstreamChecksum[:]) { + t.Errorf("cache record after GET = %+v, want the upstream checksum", rec) } } diff --git a/internal/metrics/analytics_test.go b/internal/metrics/analytics_test.go new file mode 100644 index 00000000..427c38f6 --- /dev/null +++ b/internal/metrics/analytics_test.go @@ -0,0 +1,95 @@ +package metrics + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestUpdateEcosystemStats(t *testing.T) { + UpdateEcosystemStats([]EcosystemStats{ + {Ecosystem: "npm", Packages: 10, Versions: 25, Artifacts: 40, CacheSize: 1500, Downloads: 4, DownloadedBytes: 3500}, + {Ecosystem: "cargo", Packages: 2, Versions: 3, Artifacts: 3, CacheSize: 200, Downloads: 10, DownloadedBytes: 2000}, + }) + + if got := testutil.ToFloat64(EcosystemDownloadedBytes.WithLabelValues("npm")); got != 3500 { + t.Errorf("npm downloaded bytes = %v, want 3500", got) + } + if got := testutil.ToFloat64(EcosystemDownloads.WithLabelValues("npm")); got != 4 { + t.Errorf("npm downloads = %v, want 4", got) + } + if got := testutil.ToFloat64(EcosystemCacheSize.WithLabelValues("cargo")); got != 200 { + t.Errorf("cargo cache size = %v, want 200", got) + } + if got := testutil.ToFloat64(EcosystemCachedArtifacts.WithLabelValues("npm")); got != 40 { + t.Errorf("npm cached artifacts = %v, want 40", got) + } + if got := testutil.ToFloat64(EcosystemPackages.WithLabelValues("npm")); got != 10 { + t.Errorf("npm packages = %v, want 10", got) + } + if got := testutil.ToFloat64(EcosystemVersions.WithLabelValues("npm")); got != 25 { + t.Errorf("npm versions = %v, want 25", got) + } +} + +// A refresh that no longer mentions an ecosystem must drop its series rather +// than leave it frozen at the last observed value. +func TestUpdateEcosystemStatsDropsStaleSeries(t *testing.T) { + UpdateEcosystemStats([]EcosystemStats{ + {Ecosystem: "npm", DownloadedBytes: 3500}, + {Ecosystem: "gem", DownloadedBytes: 900}, + }) + if got := testutil.CollectAndCount(EcosystemDownloadedBytes); got != 2 { + t.Fatalf("expected 2 series after first refresh, got %d", got) + } + + UpdateEcosystemStats([]EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 4000}}) + + if got := testutil.CollectAndCount(EcosystemDownloadedBytes); got != 1 { + t.Errorf("expected 1 series after gem disappeared, got %d", got) + } + if got := testutil.ToFloat64(EcosystemDownloadedBytes.WithLabelValues("npm")); got != 4000 { + t.Errorf("npm downloaded bytes = %v, want 4000", got) + } +} + +// Ecosystem labels are normalized so these gauges join against the other +// metrics that take their ecosystem from a package record. +func TestUpdateEcosystemStatsNormalizesLabels(t *testing.T) { + UpdateEcosystemStats([]EcosystemStats{{Ecosystem: "NPM", DownloadedBytes: 12}}) + + if got := testutil.ToFloat64(EcosystemDownloadedBytes.WithLabelValues("npm")); got != 12 { + t.Errorf("normalized npm gauge = %v, want 12", got) + } +} + +// GetEcosystemStats groups by the raw packages.ecosystem column, so a database +// carrying both spellings of an aliased ecosystem yields two rows that +// normalize to one label. They must sum rather than overwrite each other. +func TestUpdateEcosystemStatsSumsAliasedRows(t *testing.T) { + UpdateEcosystemStats([]EcosystemStats{ + {Ecosystem: "gem", DownloadedBytes: 100, CacheSize: 10, Packages: 1}, + {Ecosystem: "rubygems", DownloadedBytes: 200, CacheSize: 20, Packages: 2}, + {Ecosystem: "npm", DownloadedBytes: 50, CacheSize: 5, Packages: 3}, + }) + + if got := testutil.ToFloat64(EcosystemDownloadedBytes.WithLabelValues("rubygems")); got != 300 { + t.Errorf("rubygems downloaded bytes = %v, want 300", got) + } + if got := testutil.ToFloat64(EcosystemCacheSize.WithLabelValues("rubygems")); got != 30 { + t.Errorf("rubygems cache size = %v, want 30", got) + } + if got := testutil.ToFloat64(EcosystemPackages.WithLabelValues("rubygems")); got != 3 { + t.Errorf("rubygems packages = %v, want 3", got) + } + // An unaliased ecosystem is unaffected. + if got := testutil.ToFloat64(EcosystemDownloadedBytes.WithLabelValues("npm")); got != 50 { + t.Errorf("npm downloaded bytes = %v, want 50", got) + } + + // A later refresh replaces the value rather than adding to it. + UpdateEcosystemStats([]EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 50}}) + if got := testutil.ToFloat64(EcosystemDownloadedBytes.WithLabelValues("npm")); got != 50 { + t.Errorf("npm downloaded bytes after a second refresh = %v, want 50", got) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index aff57682..19581659 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -4,6 +4,7 @@ package metrics import ( "net/http" "strconv" + "sync" "time" "github.com/git-pkgs/purl" @@ -138,6 +139,81 @@ var ( []string{"step"}, ) + // Per-ecosystem gauges, derived from the database rather than incremented + // in the request path, and refreshed on the same tick as the cache gauges + // above. + EcosystemDownloadedBytes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "proxy_ecosystem_downloaded_bytes", + Help: "Accumulated bytes served from cache per ecosystem (cache hits x artifact size)", + }, + []string{"ecosystem"}, + ) + + EcosystemDownloads = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "proxy_ecosystem_artifact_downloads", + Help: "Accumulated artifact downloads served from cache per ecosystem", + }, + []string{"ecosystem"}, + ) + + EcosystemCacheSize = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "proxy_ecosystem_cache_size_bytes", + Help: "Size of cached artifacts per ecosystem in bytes", + }, + []string{"ecosystem"}, + ) + + EcosystemCachedArtifacts = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "proxy_ecosystem_cached_artifacts", + Help: "Number of cached artifacts per ecosystem", + }, + []string{"ecosystem"}, + ) + + EcosystemPackages = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "proxy_ecosystem_packages", + Help: "Number of known packages per ecosystem", + }, + []string{"ecosystem"}, + ) + + EcosystemVersions = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "proxy_ecosystem_versions", + Help: "Number of known package versions per ecosystem", + }, + []string{"ecosystem"}, + ) + + ResponseBytes = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_response_bytes_total", + Help: "Total response body bytes written to clients, by ecosystem", + }, + []string{"ecosystem"}, + ) + + ClientRequests = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_client_requests_total", + Help: "Total requests by client tool, as identified from the User-Agent", + }, + []string{"client"}, + ) + + ClientResponseBytes = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "proxy_client_response_bytes_total", + Help: "Total response body bytes written to clients, by client tool", + }, + []string{"client"}, + ) + // Scanning metrics ScanDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -183,6 +259,15 @@ func init() { ActiveRequests, IntegrityFailures, HealthProbeFailures, + EcosystemDownloadedBytes, + EcosystemDownloads, + EcosystemCacheSize, + EcosystemCachedArtifacts, + EcosystemPackages, + EcosystemVersions, + ResponseBytes, + ClientRequests, + ClientResponseBytes, ScanDuration, ScanBlocked, ScanErrors, @@ -201,6 +286,24 @@ func RecordRequest(ecosystem string, status int, duration time.Duration) { RequestDuration.WithLabelValues(ecosystem, statusStr).Observe(duration.Seconds()) } +// RecordResponse tracks what a client actually downloaded. +// +// Distinct from proxy_ecosystem_downloaded_bytes: that gauge is derived from +// the database and counts cache hits multiplied by artifact size, while this +// counts body bytes as they are written, including metadata responses and +// cache misses. +// +// client must come from a closed set -- a User-Agent is attacker-controlled, so +// passing it through raw would mint a time series per request. +func RecordResponse(ecosystem, client string, bytes int64) { + ClientRequests.WithLabelValues(client).Inc() + if bytes <= 0 { + return + } + ResponseBytes.WithLabelValues(ecosystem).Add(float64(bytes)) + ClientResponseBytes.WithLabelValues(client).Add(float64(bytes)) +} + // RecordCacheHit increments cache hit counter. func RecordCacheHit(ecosystem string) { CacheHits.WithLabelValues(purl.NormalizeEcosystem(ecosystem)).Inc() @@ -263,6 +366,79 @@ func UpdateCacheStats(sizeBytes, artifactCount int64) { CachedArtifacts.Set(float64(artifactCount)) } +// EcosystemStats is one ecosystem's row of the snapshot published as gauges. +type EcosystemStats struct { + Ecosystem string + Packages int64 + Versions int64 + Artifacts int64 + CacheSize int64 + Downloads int64 + DownloadedBytes int64 +} + +// publishedEcosystems tracks which labels the per-ecosystem gauges currently +// carry, so a label that disappears can be deleted individually. +var ( + publishedMu sync.Mutex + publishedEcosystems = map[string]bool{} +) + +// UpdateEcosystemStats republishes the per-ecosystem gauges from a fresh snapshot. +// +// Rows are summed by label before anything is published, because normalizing +// collapses aliases: a database holding both "gem" and "rubygems" rows -- the +// proxy writes the former, git-pkgs the latter -- arrives as two rows belonging +// to one label, and publishing them one at a time would leave only the last. +// +// The vectors are not Reset() first. Reset followed by a repopulating loop +// leaves a window in which a scrape sees the families empty or half filled, +// which renders as a spurious gap on any panel built from them. Each series is +// Set instead, and only labels that have actually disappeared are deleted. +func UpdateEcosystemStats(stats []EcosystemStats) { + totals := make(map[string]EcosystemStats, len(stats)) + for _, s := range stats { + ecosystem := purl.NormalizeEcosystem(s.Ecosystem) + t := totals[ecosystem] + t.Packages += s.Packages + t.Versions += s.Versions + t.Artifacts += s.Artifacts + t.CacheSize += s.CacheSize + t.Downloads += s.Downloads + t.DownloadedBytes += s.DownloadedBytes + totals[ecosystem] = t + } + + for ecosystem, t := range totals { + EcosystemDownloadedBytes.WithLabelValues(ecosystem).Set(float64(t.DownloadedBytes)) + EcosystemDownloads.WithLabelValues(ecosystem).Set(float64(t.Downloads)) + EcosystemCacheSize.WithLabelValues(ecosystem).Set(float64(t.CacheSize)) + EcosystemCachedArtifacts.WithLabelValues(ecosystem).Set(float64(t.Artifacts)) + EcosystemPackages.WithLabelValues(ecosystem).Set(float64(t.Packages)) + EcosystemVersions.WithLabelValues(ecosystem).Set(float64(t.Versions)) + } + + publishedMu.Lock() + defer publishedMu.Unlock() + + for ecosystem := range publishedEcosystems { + if _, still := totals[ecosystem]; still { + continue + } + EcosystemDownloadedBytes.DeleteLabelValues(ecosystem) + EcosystemDownloads.DeleteLabelValues(ecosystem) + EcosystemCacheSize.DeleteLabelValues(ecosystem) + EcosystemCachedArtifacts.DeleteLabelValues(ecosystem) + EcosystemPackages.DeleteLabelValues(ecosystem) + EcosystemVersions.DeleteLabelValues(ecosystem) + } + + publishedEcosystems = make(map[string]bool, len(totals)) + for ecosystem := range totals { + publishedEcosystems[ecosystem] = true + } +} + // UpdateCircuitBreakerState updates circuit breaker state gauge. // state: 0=closed, 1=half-open, 2=open func UpdateCircuitBreakerState(registry string, state int) { diff --git a/internal/metrics/snapshot.go b/internal/metrics/snapshot.go new file mode 100644 index 00000000..6ac66fba --- /dev/null +++ b/internal/metrics/snapshot.go @@ -0,0 +1,179 @@ +package metrics + +import ( + "fmt" + "sort" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +// Sample is one time series read out of the registry at a point in time. +// +// Counters and gauges carry their value in Value. Histograms carry their +// observation count and total in Count and Sum, and leave Value at zero — +// there is no single "value" for a histogram, and the UI shows a mean derived +// from Sum/Count rather than pretending otherwise. +type Sample struct { + Labels map[string]string + Value float64 + Count uint64 + Sum float64 +} + +// Label returns the value of one label, or "" if absent. +func (s Sample) Label(name string) string { + return s.Labels[name] +} + +// Mean returns the average observation of a histogram sample, or 0 when +// nothing has been observed yet. +func (s Sample) Mean() float64 { + if s.Count == 0 { + return 0 + } + return s.Sum / float64(s.Count) +} + +// Snapshot is the value of every registered proxy metric at one instant. +// +// These are process-lifetime figures held in the Prometheus registry, not +// database state: they start at zero when the proxy starts and are lost on +// restart. Anything that needs to survive a restart, or needs history, has to +// come from a Prometheus server scraping /metrics. +type Snapshot struct { + families map[string][]Sample +} + +// Gather reads the default Prometheus registry. It is the same data /metrics +// serves, shaped for rendering rather than for scraping. +func Gather() (*Snapshot, error) { + return GatherFrom(prometheus.DefaultGatherer) +} + +// GatherFrom reads an explicit gatherer, so tests can supply their own registry. +func GatherFrom(g prometheus.Gatherer) (*Snapshot, error) { + families, err := g.Gather() + if err != nil { + return nil, fmt.Errorf("gathering metrics: %w", err) + } + + snap := &Snapshot{families: make(map[string][]Sample, len(families))} + for _, mf := range families { + samples := make([]Sample, 0, len(mf.GetMetric())) + for _, m := range mf.GetMetric() { + samples = append(samples, sampleOf(m)) + } + snap.families[mf.GetName()] = samples + } + return snap, nil +} + +func sampleOf(m *dto.Metric) Sample { + s := Sample{Labels: make(map[string]string, len(m.GetLabel()))} + for _, lp := range m.GetLabel() { + s.Labels[lp.GetName()] = lp.GetValue() + } + + switch { + case m.GetCounter() != nil: + s.Value = m.GetCounter().GetValue() + case m.GetGauge() != nil: + s.Value = m.GetGauge().GetValue() + case m.GetHistogram() != nil: + h := m.GetHistogram() + s.Count = h.GetSampleCount() + s.Sum = h.GetSampleSum() + case m.GetSummary() != nil: + sm := m.GetSummary() + s.Count = sm.GetSampleCount() + s.Sum = sm.GetSampleSum() + case m.GetUntyped() != nil: + s.Value = m.GetUntyped().GetValue() + } + return s +} + +// Names returns the name of every metric family in the snapshot, sorted. +func (s *Snapshot) Names() []string { + if s == nil { + return nil + } + + names := make([]string, 0, len(s.families)) + for name := range s.families { + names = append(names, name) + } + sort.Strings(names) + return names +} + +// Samples returns every series of a metric, or nil when it has never been +// observed. Series are ordered by their label values so that rendering is +// stable between scrapes. +func (s *Snapshot) Samples(name string) []Sample { + if s == nil { + return nil + } + out := append([]Sample(nil), s.families[name]...) + sort.Slice(out, func(i, j int) bool { + return labelKey(out[i].Labels) < labelKey(out[j].Labels) + }) + return out +} + +func labelKey(labels map[string]string) string { + names := make([]string, 0, len(labels)) + for n := range labels { + names = append(names, n) + } + sort.Strings(names) + + key := "" + for _, n := range names { + key += n + "=" + labels[n] + "," + } + return key +} + +// Sum totals every series of a counter or gauge. +func (s *Snapshot) Sum(name string) float64 { + var total float64 + for _, sample := range s.Samples(name) { + total += sample.Value + } + return total +} + +// Count adds up the observation counts of a histogram across every series. +func (s *Snapshot) Count(name string) uint64 { + var total uint64 + for _, sample := range s.Samples(name) { + total += sample.Count + } + return total +} + +// Mean returns the average observation of a histogram across every series, or +// 0 when nothing has been observed. +func (s *Snapshot) Mean(name string) float64 { + var count uint64 + var sum float64 + for _, sample := range s.Samples(name) { + count += sample.Count + sum += sample.Sum + } + if count == 0 { + return 0 + } + return sum / float64(count) +} + +// SumBy groups a metric by one label and totals each group. +func (s *Snapshot) SumBy(name, label string) map[string]float64 { + out := make(map[string]float64) + for _, sample := range s.Samples(name) { + out[sample.Label(label)] += sample.Value + } + return out +} diff --git a/internal/metrics/snapshot_test.go b/internal/metrics/snapshot_test.go new file mode 100644 index 00000000..dd695f72 --- /dev/null +++ b/internal/metrics/snapshot_test.go @@ -0,0 +1,140 @@ +package metrics + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +// newTestRegistry builds an isolated registry so these tests are not affected +// by counters other tests in this package have already incremented. +func newTestRegistry(t *testing.T) (*prometheus.Registry, *prometheus.CounterVec, *prometheus.HistogramVec, *prometheus.GaugeVec) { + t.Helper() + + reg := prometheus.NewRegistry() + counter := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "test_errors_total", Help: "t"}, + []string{"ecosystem", "kind"}, + ) + hist := prometheus.NewHistogramVec( + prometheus.HistogramOpts{Name: "test_duration_seconds", Help: "t"}, + []string{"op"}, + ) + gauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "test_active", Help: "t"}, []string{}) + reg.MustRegister(counter, hist, gauge) + return reg, counter, hist, gauge +} + +func TestSnapshotSumAndSumBy(t *testing.T) { + reg, counter, _, gauge := newTestRegistry(t) + counter.WithLabelValues("npm", "timeout").Add(3) + counter.WithLabelValues("npm", "refused").Add(2) + counter.WithLabelValues("pypi", "timeout").Add(5) + gauge.WithLabelValues().Set(7) + + snap, err := GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + + if got := snap.Sum("test_errors_total"); got != 10 { + t.Errorf("Sum = %v, want 10", got) + } + if got := snap.Sum("test_active"); got != 7 { + t.Errorf("gauge Sum = %v, want 7", got) + } + + byEco := snap.SumBy("test_errors_total", "ecosystem") + if byEco["npm"] != 5 || byEco["pypi"] != 5 { + t.Errorf("SumBy(ecosystem) = %v, want npm=5 pypi=5", byEco) + } +} + +func TestSnapshotHistogram(t *testing.T) { + reg, _, hist, _ := newTestRegistry(t) + hist.WithLabelValues("get").Observe(0.010) + hist.WithLabelValues("get").Observe(0.030) + hist.WithLabelValues("put").Observe(0.100) + + snap, err := GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + + if got := snap.Count("test_duration_seconds"); got != 3 { + t.Errorf("Count = %d, want 3", got) + } + // (0.010 + 0.030 + 0.100) / 3 + if got := snap.Mean("test_duration_seconds"); got < 0.0466 || got > 0.0467 { + t.Errorf("Mean = %v, want ~0.04667", got) + } + + // A histogram carries no single value, so Value stays zero. + for _, s := range snap.Samples("test_duration_seconds") { + if s.Value != 0 { + t.Errorf("histogram sample carries Value %v, want 0", s.Value) + } + if s.Label("op") == "get" && s.Count != 2 { + t.Errorf("get count = %d, want 2", s.Count) + } + } +} + +func TestSnapshotSamplesAreOrdered(t *testing.T) { + reg, counter, _, _ := newTestRegistry(t) + for _, eco := range []string{"pypi", "npm", "cargo"} { + counter.WithLabelValues(eco, "timeout").Inc() + } + + snap, err := GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + + var got []string + for _, s := range snap.Samples("test_errors_total") { + got = append(got, s.Label("ecosystem")) + } + want := []string{"cargo", "npm", "pypi"} + for i := range want { + if got[i] != want[i] { + t.Fatalf("sample order = %v, want %v (stable rendering depends on it)", got, want) + } + } +} + +func TestSnapshotMissingMetric(t *testing.T) { + reg, _, _, _ := newTestRegistry(t) + snap, err := GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + + if got := snap.Samples("nope_total"); got != nil { + t.Errorf("Samples of an unknown metric = %v, want nil", got) + } + if got := snap.Sum("nope_total"); got != 0 { + t.Errorf("Sum of an unknown metric = %v, want 0", got) + } + if got := snap.Mean("nope_total"); got != 0 { + t.Errorf("Mean of an unknown metric = %v, want 0", got) + } +} + +// Gather reads the real registry, so every metric this package registers must +// come back. This is the check that a newly added metric is reachable by the UI. +func TestGatherSeesRegisteredMetrics(t *testing.T) { + RecordCacheHit("npm") + RecordScanError("npm", "clamav", "timeout") + + snap, err := Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + + for _, name := range []string{"proxy_cache_hits_total", "proxy_scan_errors_total"} { + if len(snap.Samples(name)) == 0 { + t.Errorf("%s is registered but absent from the snapshot", name) + } + } +} diff --git a/internal/mirror/cache_test.go b/internal/mirror/cache_test.go new file mode 100644 index 00000000..661e4088 --- /dev/null +++ b/internal/mirror/cache_test.go @@ -0,0 +1,82 @@ +package mirror + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/git-pkgs/registries" + "github.com/git-pkgs/registries/client" + "github.com/git-pkgs/registries/fetch" +) + +type mirrorCacheRegistry struct { + downloadURL string + resolutions atomic.Int64 +} + +type rejectingMirrorTransport struct{} + +func (rejectingMirrorTransport) RoundTrip(*http.Request) (*http.Response, error) { + return nil, errors.New("unexpected artifact download on a cache hit") +} + +func (*mirrorCacheRegistry) Ecosystem() string { return "npm" } +func (*mirrorCacheRegistry) FetchVersions(context.Context, string) ([]registries.Version, error) { + return nil, errors.New("unexpected metadata request") +} +func (r *mirrorCacheRegistry) URLs() client.URLBuilder { //nolint:ireturn // required by fetch.Registry + return &client.BaseURLs{DownloadFn: func(_, _ string) string { + r.resolutions.Add(1) + return r.downloadURL + }} +} + +func TestMirrorRepeatedRunSkipsCachedArtifact(t *testing.T) { + var downloads atomic.Int64 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/example-1.0.0.tgz" { + t.Errorf("unexpected artifact path: %s", r.URL.Path) + http.NotFound(w, r) + return + } + downloads.Add(1) + _, _ = io.WriteString(w, "tarball bytes") + })) + defer upstream.Close() + m := setupTestMirror(t, 1) + registry := &mirrorCacheRegistry{downloadURL: upstream.URL + "/example-1.0.0.tgz"} + m.proxy.Resolver.RegisterRegistry(registry) + fetcher := fetch.NewFetcher(fetch.WithHTTPClient(upstream.Client()), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + m.proxy.Fetcher = fetcher + source := &PURLSource{PURLs: []string{"pkg:npm/example@1.0.0"}} + first, err := m.Run(context.Background(), source) + if err != nil || first.Completed != 1 || first.Skipped != 0 || first.Failed != 0 { + t.Fatalf("first mirror: progress=%+v err=%v", first, err) + } + before, err := m.db.GetArtifact("pkg:npm/example@1.0.0", "example-1.0.0.tgz") + if err != nil { + t.Fatal(err) + } + // The second run must succeed even when the artifact server is offline. + upstream.Close() + second, err := m.Run(context.Background(), source) + if err != nil || second.Completed != 0 || second.Skipped != 1 || second.Failed != 0 { + t.Fatalf("second mirror: progress=%+v err=%v", second, err) + } + after, err := m.db.GetArtifact("pkg:npm/example@1.0.0", "example-1.0.0.tgz") + if err != nil { + t.Fatal(err) + } + if before.FetchedAt != after.FetchedAt || before.ContentHash != after.ContentHash || before.StoragePath != after.StoragePath { + t.Fatal("second mirror changed the cached artifact") + } + if downloads.Load() != 1 || registry.resolutions.Load() != 2 { + t.Fatalf("downloads=%d resolutions=%d; want 1 and 2", downloads.Load(), registry.resolutions.Load()) + } +} diff --git a/internal/mirror/denylist_test.go b/internal/mirror/denylist_test.go new file mode 100644 index 00000000..5676e877 --- /dev/null +++ b/internal/mirror/denylist_test.go @@ -0,0 +1,31 @@ +package mirror + +import ( + "context" + "io" + "log/slog" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/denylist" + "github.com/git-pkgs/proxy/internal/handler" +) + +func TestMirrorOneDenylist(t *testing.T) { + policy, err := denylist.New([]string{"pkg:npm/demo@1.0.0"}) + if err != nil { + t.Fatal(err) + } + // No database, storage, or fetcher is needed: the denial comes first. + p := &handler.Proxy{Denylist: policy} + m := New(p, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil)), 1) + tracker := newProgressTracker() + m.mirrorOne(context.Background(), PackageVersion{Ecosystem: "npm", Name: "demo", Version: "1.0.0"}, tracker) + progress := tracker.snapshot() + if progress.Failed != 1 || progress.Completed != 0 || progress.Skipped != 0 || len(progress.Errors) != 1 { + t.Fatalf("progress = %+v", progress) + } + if !strings.Contains(progress.Errors[0].Error, "denylist") { + t.Fatalf("unexpected error: %s", progress.Errors[0].Error) + } +} diff --git a/internal/mirror/mirror_test.go b/internal/mirror/mirror_test.go index 3a6420f2..45dd6f38 100644 --- a/internal/mirror/mirror_test.go +++ b/internal/mirror/mirror_test.go @@ -2,9 +2,9 @@ package mirror import ( "context" - "crypto/sha256" "database/sql" "log/slog" + "net/http" "os" "strings" "testing" @@ -124,6 +124,10 @@ func TestMirrorRunCanceled(t *testing.T) { func TestMirrorOneDirectServeCacheHit(t *testing.T) { m := setupTestMirror(t, 1) + fetcher := fetch.NewFetcher( + fetch.WithHTTPClient(&http.Client{Transport: rejectingMirrorTransport{}}), fetch.WithMaxRetries(0)) + t.Cleanup(func() { _ = fetcher.Close() }) + m.proxy.Fetcher = fetcher m.proxy.DirectServe = true m.proxy.Storage = signedURLStorage{Storage: m.storage} @@ -142,13 +146,18 @@ func TestMirrorOneDirectServeCacheHit(t *testing.T) { }); err != nil { t.Fatalf("UpsertVersion() error = %v", err) } + const storagePath = "npm/example/1.0.0/example-1.0.0.tgz" + size, hash, err := m.storage.Store(context.Background(), storagePath, strings.NewReader("tarball bytes")) + if err != nil { + t.Fatal(err) + } if err := m.db.UpsertArtifact(&database.Artifact{ VersionPURL: versionPURL, - Filename: "", + Filename: "example-1.0.0.tgz", UpstreamURL: "https://registry.example/artifact", - StoragePath: sql.NullString{String: "npm/example/1.0.0/artifact", Valid: true}, - ContentHash: sql.NullString{String: strings.Repeat("a", sha256.Size*2), Valid: true}, - Size: sql.NullInt64{Int64: 1, Valid: true}, + StoragePath: sql.NullString{String: storagePath, Valid: true}, + ContentHash: sql.NullString{String: hash, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, }); err != nil { t.Fatalf("UpsertArtifact() error = %v", err) diff --git a/internal/server/analytics.go b/internal/server/analytics.go new file mode 100644 index 00000000..eaf19484 --- /dev/null +++ b/internal/server/analytics.go @@ -0,0 +1,231 @@ +package server + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/metrics" +) + +// analyticsTopSources caps how many callers the page lists; the rest are +// summarised in the overflow row, and the access log has every one of them. +const analyticsTopSources = 15 + +// AnalyticsData contains data for rendering the analytics dashboard. +type AnalyticsData struct { + Layout + Totals AnalyticsTotals + Donut DonutView + EnrichmentStats EnrichmentStatsView + Ecosystems []EcosystemRow + Runtime RuntimeView + // StatsFailed records that the per-ecosystem query errored, so the page + // can say the figures are unavailable instead of claiming an empty cache. + StatsFailed bool + // StatsStale is set when the query errored but a previous snapshot was + // retained and is being shown. Without it a database that has been down + // for an hour renders hour-old figures as current, with the failure + // visible only in the logs. + StatsStale bool + // StatsAge is when the retained snapshot was read, phrased for display. + StatsAge string +} + +// AnalyticsTotals holds the headline figures across every ecosystem. +type AnalyticsTotals struct { + DownloadedBytes int64 + Downloaded string + Downloads string + CacheSize string + CachedArtifacts string + Packages string + Versions string + Ecosystems int + ActiveEcosystems int + // Amplification is accumulated download volume divided by the bytes + // currently held in cache: how many times over the cache has served + // what it stores. Empty when nothing is cached. + Amplification string +} + +// EcosystemRow is one ecosystem's row in the analytics table. +type EcosystemRow struct { + Ecosystem string + DownloadedBytes int64 + Downloaded string + Downloads string + CacheSize string + AvgArtifactSize string + Artifacts string + Packages string + Versions string + // SharePct is this ecosystem's share of the accumulated download total. + SharePct string +} + +func formatCount(n int64) string { + s := strconv.FormatInt(n, 10) + neg := strings.HasPrefix(s, "-") + if neg { + s = s[1:] + } + + var b strings.Builder + for i, digit := range s { + if i > 0 && (len(s)-i)%3 == 0 { + b.WriteByte(',') + } + b.WriteRune(digit) + } + if neg { + return "-" + b.String() + } + return b.String() +} + +// percentOf renders part/whole as a percentage with one decimal place, always +// as a bare number. formatPercent wraps it to catch the case where a real +// share rounds away to zero. +func percentOf(part, whole int64) string { + if whole <= 0 { + return "0.0" + } + return strconv.FormatFloat(float64(part)/float64(whole)*100, 'f', 1, 64) //nolint:mnd // percent +} + +// formatPercent renders a share for display. A share that is real but rounds +// to zero reads as "<0.1" rather than "0.0", which would claim the ecosystem +// served nothing. +func formatPercent(part, whole int64) string { + pct := percentOf(part, whole) + if pct == "0.0" && part > 0 && whole > 0 { + return "<0.1" + } + return pct +} + +// formatRatio renders an "N.Nx" multiplier, or "" when the denominator is zero. +func formatRatio(numerator, denominator int64) string { + if denominator <= 0 { + return "" + } + return fmt.Sprintf("%.1fx", float64(numerator)/float64(denominator)) +} + +func avgArtifactSize(cacheSize, artifacts int64) string { + if artifacts <= 0 { + return "0 B" + } + return formatSize(cacheSize / artifacts) +} + +// enrichmentStatsView maps enrichment totals onto the shared view struct used +// by both the dashboard and the analytics page. +func enrichmentStatsView(stats *database.EnrichmentStats) EnrichmentStatsView { + return EnrichmentStatsView{ + EnrichedPackages: stats.EnrichedPackages, + VulnSyncedPackages: stats.VulnSyncedPackages, + TotalVulnerabilities: stats.TotalVulnerabilities, + CriticalVulns: stats.CriticalVulns, + HighVulns: stats.HighVulns, + MediumVulns: stats.MediumVulns, + LowVulns: stats.LowVulns, + HasVulns: stats.TotalVulnerabilities > 0, + } +} + +// handleAnalytics renders the analytics dashboard: accumulated download volume +// in total and per ecosystem, alongside the cache and enrichment figures the +// main dashboard reports. +func (s *Server) handleAnalytics(w http.ResponseWriter, r *http.Request) { + ecosystems, err := s.ecoStats.Get(s.db) + statsFailed := err != nil + if err != nil { + s.logger.Error("failed to get ecosystem stats", "error", err) + } + + enrichStats, err := s.db.GetEnrichmentStats() + if err != nil { + s.logger.Error("failed to get enrichment stats", "error", err) + enrichStats = &database.EnrichmentStats{} + } + + data := AnalyticsData{ + Layout: s.layoutFor(r), + EnrichmentStats: enrichmentStatsView(enrichStats), + } + data.StatsFailed = statsFailed + data.StatsStale = statsFailed && len(ecosystems) > 0 + if data.StatsStale { + data.StatsAge = formatTimeAgo(s.ecoStats.SnapshotAt()) + } + data.Totals, data.Ecosystems = analyticsView(ecosystems) + data.Donut = donutView(ecosystems, data.Totals.DownloadedBytes, data.Totals.Downloaded) + + // Process-lifetime counters come from the Prometheus registry rather than + // the database; a failure here must not take the page down with it. + snap, err := metrics.Gather() + if err != nil { + s.logger.Error("failed to gather runtime metrics", "error", err) + } else { + data.Runtime = runtimeView(snap) + if s.cfg != nil && s.cfg.UIRequestSources { + data.Runtime.SourcesOn = true + data.Runtime.Sources = s.sources.Top(analyticsTopSources) + data.Runtime.SourceCount = s.sources.Count() + data.Runtime.TrustsForward = s.trustsForwardedFor() + } + } + + if err := s.templates.Render(w, "analytics", data); err != nil { + s.logger.Error("failed to render analytics", "error", err) + } +} + +// analyticsView turns per-ecosystem database rows into the rendered totals and +// table rows, preserving the order they arrive in. +func analyticsView(stats []database.EcosystemStats) (AnalyticsTotals, []EcosystemRow) { + var totals AnalyticsTotals + var downloads, cacheSize, artifacts, packages, versions int64 + + for _, e := range stats { + totals.DownloadedBytes += e.DownloadedBytes + downloads += e.Downloads + cacheSize += e.CacheSize + artifacts += e.Artifacts + packages += e.Packages + versions += e.Versions + if e.DownloadedBytes > 0 { + totals.ActiveEcosystems++ + } + } + + totals.Downloaded = formatSize(totals.DownloadedBytes) + totals.Downloads = formatCount(downloads) + totals.CacheSize = formatSize(cacheSize) + totals.CachedArtifacts = formatCount(artifacts) + totals.Packages = formatCount(packages) + totals.Versions = formatCount(versions) + totals.Ecosystems = len(stats) + totals.Amplification = formatRatio(totals.DownloadedBytes, cacheSize) + + rows := make([]EcosystemRow, 0, len(stats)) + for _, e := range stats { + rows = append(rows, EcosystemRow{ + Ecosystem: e.Ecosystem, + DownloadedBytes: e.DownloadedBytes, + Downloaded: formatSize(e.DownloadedBytes), + Downloads: formatCount(e.Downloads), + CacheSize: formatSize(e.CacheSize), + AvgArtifactSize: avgArtifactSize(e.CacheSize, e.Artifacts), + Artifacts: formatCount(e.Artifacts), + Packages: formatCount(e.Packages), + Versions: formatCount(e.Versions), + SharePct: formatPercent(e.DownloadedBytes, totals.DownloadedBytes), + }) + } + return totals, rows +} diff --git a/internal/server/analytics_coverage_test.go b/internal/server/analytics_coverage_test.go new file mode 100644 index 00000000..5517deac --- /dev/null +++ b/internal/server/analytics_coverage_test.go @@ -0,0 +1,109 @@ +package server + +import ( + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/metrics" +) + +// metricSurface records where each registered metric is shown on /ui/analytics. +// +// This is a standing obligation, not a snapshot: registering a new metric +// anywhere in internal/metrics fails the build here until it has both a tile on +// the page and an entry below. That is the point -- the page is meant to be a +// complete view of /metrics -- but it is a cost paid by every future metric, so +// it is written down rather than discovered. +var metricSurface = map[string]string{ + // Database-derived, shown in the donut, the KPI row and the breakdown table. + "proxy_ecosystem_downloaded_bytes": "donut + breakdown table", + "proxy_ecosystem_artifact_downloads": "Downloads tile + breakdown table", + "proxy_ecosystem_cache_size_bytes": "breakdown table", + "proxy_ecosystem_cached_artifacts": "breakdown table", + "proxy_ecosystem_packages": "breakdown table", + "proxy_ecosystem_versions": "breakdown table", + "proxy_cache_size_bytes": "Cache size tile", + "proxy_cached_artifacts_total": "Cached artifacts tile", + + // Request-path counters. + "proxy_response_bytes_total": "Runtime: Served", + "proxy_client_requests_total": "Runtime: Request sources -- By client", + "proxy_client_response_bytes_total": "Runtime: Request sources -- By client", + + // Registry-derived, shown in the Runtime card. + "proxy_requests_total": "Runtime: Requests + Responses by status", + "proxy_request_duration_seconds": "Runtime: Mean latency", + "proxy_active_requests": "Runtime: In flight", + "proxy_cache_hits_total": "Runtime: Cache lookups + hit rate", + "proxy_cache_misses_total": "Runtime: Cache lookups + hit rate", + "proxy_upstream_fetch_duration_seconds": "Runtime: Upstream fetches + mean fetch", + "proxy_upstream_errors_total": "Runtime: Upstream errors", + "proxy_storage_operation_duration_seconds": "Runtime: Storage operations", + "proxy_storage_errors_total": "Runtime: Storage errors", + "proxy_integrity_failures_total": "Runtime: Integrity failures", + "proxy_health_probe_failures_total": "Runtime: Health probe failures", + "proxy_circuit_breaker_state": "Runtime: Circuit breakers", + "proxy_circuit_breaker_trips_total": "Runtime: Circuit breaker trips", + "proxy_scan_duration_seconds": "Runtime: Pre-cache scanning -- Scans", + "proxy_scan_blocked_total": "Runtime: Pre-cache scanning -- Artifacts blocked", + "proxy_scan_errors_total": "Runtime: Pre-cache scanning -- Scan errors", +} + +// TestEveryMetricIsSurfaced fails when a metric is registered but has no home +// on the analytics page. The page is meant to be a complete view of what +// /metrics exposes, so a silently unsurfaced metric is a gap, not a detail. +func TestEveryMetricIsSurfaced(t *testing.T) { + // Touch every metric family so it is present in the registry output, since + // a vector with no observed label values gathers as nothing at all. + metrics.RecordRequest("npm", 200, 0) + metrics.RecordResponse("npm", "npm", 1) + metrics.RecordCacheHit("npm") + metrics.RecordCacheMiss("npm") + metrics.RecordUpstreamFetch("npm", 0) + metrics.RecordUpstreamError("npm", "fetch_failed") + metrics.RecordStorageOperation("get", 0) + metrics.RecordStorageError("get") + metrics.RecordIntegrityFailure("npm") + metrics.RecordHealthProbeFailure("write") + metrics.UpdateCircuitBreakerState("example.test", 0) + metrics.RecordCircuitBreakerTrip("example.test") + metrics.RecordScanResult("npm", "clamav", false, 0) + metrics.RecordScanError("npm", "clamav", "timeout") + metrics.UpdateCacheStats(1, 1) + metrics.UpdateEcosystemStats([]metrics.EcosystemStats{{Ecosystem: "npm", DownloadedBytes: 1}}) + + snap, err := metrics.Gather() + if err != nil { + t.Fatalf("Gather: %v", err) + } + + var registered []string + for _, name := range snap.Names() { + if strings.HasPrefix(name, "proxy_") { + registered = append(registered, name) + } + } + if len(registered) == 0 { + t.Fatal("no proxy_ metrics in the registry; the probe above is not working") + } + + for _, name := range registered { + if _, ok := metricSurface[name]; !ok { + t.Errorf("%s is registered but has no home on /ui/analytics; "+ + "surface it and add it to metricSurface", name) + } + } + + for name := range metricSurface { + found := false + for _, r := range registered { + if r == name { + found = true + break + } + } + if !found { + t.Errorf("metricSurface lists %s, but it is not registered any more; drop the entry", name) + } + } +} diff --git a/internal/server/analytics_donut.go b/internal/server/analytics_donut.go new file mode 100644 index 00000000..59abc857 --- /dev/null +++ b/internal/server/analytics_donut.go @@ -0,0 +1,208 @@ +package server + +import ( + "math" + "strconv" + + "github.com/git-pkgs/proxy/internal/database" +) + +// Donut geometry, in SVG user units. The ring is drawn as a dashed stroke on a +// circle rather than as arc paths: one dash per slice, which makes the 2px +// surface gap between slices fall out of the dash arithmetic. +const ( + donutSize = 260.0 + donutRadius = 104.0 + donutStroke = 30.0 + // donutGap is the surface-coloured separator between touching slices. + donutGap = 2.0 + // donutMinSlice keeps a slice that rounds to nothing from vanishing + // entirely; the legend and table carry its real value. + donutMinSlice = 1.5 + donutCircumference = 2 * math.Pi * donutRadius //nolint:mnd // circumference +) + +// donutSlices is the most slices the ring will draw. Part-to-whole reads at a +// glance only while the segment count stays small, so past this the tail is +// folded into a single "Other" slice and the full detail lives in the table +// below the chart. +const donutSlices = 6 + +const otherSliceLabel = "Other" + +// DonutSlice is one arc of the ring plus its legend row. +type DonutSlice struct { + // Index selects the fixed categorical slot. The hues themselves live in + // the .donut-slot-N and .swatch-N rules in the analytics template, so the + // validated palette has exactly one definition; slots are assigned here in + // sequence and never cycled. + Index int + Label string + Value string + SharePct string + Downloads string + CacheSize string + Dash string + Gap string + Offset string + // IsOther marks the folded tail, which has no single ecosystem behind it. + IsOther bool + // Members lists what was folded in, for the tooltip. + Members string +} + +// DonutView is the whole chart: the ring, its centre figure and its legend. +type DonutView struct { + Slices []DonutSlice + // Size, Radius, Stroke and Center are handed to the template so the SVG + // geometry has exactly one definition. + Size string + Radius string + Stroke string + Center string + // CenterValue is the accumulated download size across every ecosystem, + // stated in the middle of the ring. + CenterValue string + CenterLabel string + HasSlices bool +} + +// donutView folds the per-ecosystem rows into at most donutSlices arcs and +// computes the dash geometry for each. +func donutView(stats []database.EcosystemStats, totalBytes int64, centerValue string) DonutView { + view := DonutView{ + Size: trimFloat(donutSize), + Radius: trimFloat(donutRadius), + Stroke: trimFloat(donutStroke), + Center: trimFloat(donutSize / 2), //nolint:mnd // the centre of the viewBox + CenterValue: centerValue, + CenterLabel: "accumulated download size", + } + + // Only ecosystems that have actually served bytes get an arc; an + // ecosystem at zero would be an invisible slice with a legend entry + // claiming a share it does not have. + active := make([]database.EcosystemStats, 0, len(stats)) + for _, e := range stats { + if e.DownloadedBytes > 0 { + active = append(active, e) + } + } + if len(active) == 0 || totalBytes <= 0 { + return view + } + + head, tail := active, []database.EcosystemStats(nil) + if len(active) > donutSlices { + head, tail = active[:donutSlices-1], active[donutSlices-1:] + } + + slices := make([]DonutSlice, 0, donutSlices) + for i, e := range head { + slices = append(slices, DonutSlice{ + Index: i, + Label: ecosystemBadgeLabel(e.Ecosystem), + Value: formatSize(e.DownloadedBytes), + SharePct: formatPercent(e.DownloadedBytes, totalBytes), + Downloads: formatCount(e.Downloads), + CacheSize: formatSize(e.CacheSize), + }) + } + + if len(tail) > 0 { + var bytes, downloads, cacheSize int64 + members := "" + for i, e := range tail { + bytes += e.DownloadedBytes + downloads += e.Downloads + cacheSize += e.CacheSize + if i > 0 { + members += ", " + } + members += ecosystemBadgeLabel(e.Ecosystem) + } + slices = append(slices, DonutSlice{ + Index: donutSlices - 1, + Label: otherSliceLabel, + Value: formatSize(bytes), + SharePct: formatPercent(bytes, totalBytes), + Downloads: formatCount(downloads), + CacheSize: formatSize(cacheSize), + IsOther: true, + Members: members, + }) + } + + applyDonutGeometry(slices, active, tail, totalBytes) + view.Slices = slices + view.HasSlices = true + return view +} + +// applyDonutGeometry sets each slice's dash length and offset. +func applyDonutGeometry(slices []DonutSlice, active, tail []database.EcosystemStats, totalBytes int64) { + // Recover each slice's byte value in the same order the slices were built, + // so the geometry is driven by the numbers rather than by the rendered text. + values := make([]int64, 0, len(slices)) + head := active + if len(tail) > 0 { + head = active[:len(active)-len(tail)] + } + for _, e := range head { + values = append(values, e.DownloadedBytes) + } + if len(tail) > 0 { + var sum int64 + for _, e := range tail { + sum += e.DownloadedBytes + } + values = append(values, sum) + } + + dashes := make([]float64, len(slices)) + widest := 0 + for i := range slices { + arc := donutCircumference * float64(values[i]) / float64(totalBytes) + + dashes[i] = arc - donutGap + if dashes[i] < donutMinSlice { + dashes[i] = donutMinSlice + } + if dashes[i] > dashes[widest] { + widest = i + } + } + + // Widening a sliver to donutMinSlice buys visibility with room the ring + // does not have, and the overshoot would wrap the last slice back over the + // first. The widest slice gives the space back: it is the only one that can + // lose a couple of units without becoming unreadable, and at six slices the + // most that can be owed is donutSlices*(donutMinSlice+donutGap), far less + // than the circumference. + var needed float64 + for _, d := range dashes { + needed += d + donutGap + } + if excess := needed - donutCircumference; excess > 0 { + dashes[widest] = math.Max(donutMinSlice, dashes[widest]-excess) + } + + var consumed float64 + for i := range slices { + slices[i].Dash = trimFloat(dashes[i]) + slices[i].Gap = trimFloat(donutCircumference - dashes[i]) + // A dashoffset runs backwards around the circle, so the running total + // is negated to lay slices out clockwise from twelve o'clock. + slices[i].Offset = trimFloat(-consumed) + + // Advance by what is drawn: advancing by the smaller true arc would + // start the next slice underneath a widened one, and the later colour + // would win. + consumed += dashes[i] + donutGap + } +} + +// trimFloat renders an SVG coordinate without trailing zeroes. +func trimFloat(f float64) string { + return strconv.FormatFloat(f, 'f', -1, 64) +} diff --git a/internal/server/analytics_donut_test.go b/internal/server/analytics_donut_test.go new file mode 100644 index 00000000..d2e87388 --- /dev/null +++ b/internal/server/analytics_donut_test.go @@ -0,0 +1,211 @@ +package server + +import ( + "math" + "strconv" + "testing" + + "github.com/git-pkgs/proxy/internal/database" +) + +func eco(name string, bytes int64) database.EcosystemStats { + return database.EcosystemStats{Ecosystem: name, DownloadedBytes: bytes, Downloads: 1, CacheSize: bytes / 2} +} + +func TestDonutViewFoldsTailIntoOther(t *testing.T) { + // Nine active ecosystems: five keep their own slice, four fold into "Other". + stats := []database.EcosystemStats{ + eco("npm", 900), eco("pypi", 800), eco("maven", 700), eco("cargo", 600), + eco("gem", 500), eco("golang", 40), eco("nuget", 30), eco("deb", 20), eco("conda", 10), + } + var total int64 + for _, e := range stats { + total += e.DownloadedBytes + } + + view := donutView(stats, total, formatSize(total)) + + if len(view.Slices) != donutSlices { + t.Fatalf("expected %d slices, got %d", donutSlices, len(view.Slices)) + } + + last := view.Slices[donutSlices-1] + if !last.IsOther { + t.Fatalf("expected the final slice to be the folded tail, got %+v", last) + } + // 40 + 30 + 20 + 10 = 100 + if last.Value != formatSize(100) { + t.Errorf("Other value = %q, want %q", last.Value, formatSize(100)) + } + if last.Members != "golang, nuget, debian, conda" { + t.Errorf("Other members = %q, want the four folded ecosystems by display name", last.Members) + } + // "Other" must take the last slot, never one of a real ecosystem's. + if last.Index != donutSlices-1 { + t.Errorf("Other took slot %d, want %d", last.Index, donutSlices-1) + } + for i, s := range view.Slices[:donutSlices-1] { + if s.Index != i { + t.Errorf("slice %d took slot %d; slots must be assigned in sequence", i, s.Index) + } + } +} + +func TestDonutViewKeepsEveryEcosystemWhenFewEnough(t *testing.T) { + stats := []database.EcosystemStats{eco("npm", 600), eco("pypi", 400)} + view := donutView(stats, 1000, "1000 B") + + if len(view.Slices) != 2 { + t.Fatalf("expected 2 slices, got %d", len(view.Slices)) + } + for _, s := range view.Slices { + if s.IsOther { + t.Errorf("nothing should be folded with only 2 ecosystems, got %+v", s) + } + } + if view.Slices[0].SharePct != "60.0" || view.Slices[1].SharePct != "40.0" { + t.Errorf("shares = %q / %q, want 60.0 / 40.0", view.Slices[0].SharePct, view.Slices[1].SharePct) + } +} + +// The ring must account for the whole circle: the arcs, plus one gap each, +// have to add back up to the circumference. +func TestDonutGeometryCoversTheCircle(t *testing.T) { + stats := []database.EcosystemStats{eco("npm", 500), eco("pypi", 300), eco("maven", 200)} + view := donutView(stats, 1000, "1000 B") + + circumference := 2 * math.Pi * donutRadius + + var covered float64 + for _, s := range view.Slices { + dash, err := strconv.ParseFloat(s.Dash, 64) + if err != nil { + t.Fatalf("dash %q: %v", s.Dash, err) + } + gap, err := strconv.ParseFloat(s.Gap, 64) + if err != nil { + t.Fatalf("gap %q: %v", s.Gap, err) + } + // Each slice is drawn as one dash followed by a gap spanning the + // rest of the circle, so the pair always sums to the circumference. + if math.Abs(dash+gap-circumference) > 0.01 { + t.Errorf("slice %q: dash+gap = %v, want the circumference %v", s.Label, dash+gap, circumference) + } + covered += dash + donutGap + } + + if math.Abs(covered-circumference) > 0.01 { + t.Errorf("arcs plus gaps cover %v, want the full circumference %v", covered, circumference) + } +} + +// Offsets must advance monotonically so slices sit end to end instead of +// stacking on top of each other. +func TestDonutOffsetsAdvanceInOrder(t *testing.T) { + stats := []database.EcosystemStats{eco("npm", 500), eco("pypi", 300), eco("maven", 200)} + view := donutView(stats, 1000, "1000 B") + + circumference := 2 * math.Pi * donutRadius + want := []float64{0, -circumference * 0.5, -circumference * 0.8} + + for i, s := range view.Slices { + got, err := strconv.ParseFloat(s.Offset, 64) + if err != nil { + t.Fatalf("offset %q: %v", s.Offset, err) + } + if math.Abs(got-want[i]) > 0.01 { + t.Errorf("slice %d offset = %v, want %v", i, got, want[i]) + } + } +} + +// A slice too small to render still has to be visible; the legend and table +// carry its real value. +func TestDonutTinySliceStaysVisible(t *testing.T) { + stats := []database.EcosystemStats{eco("npm", 10_000_000), eco("composer", 1)} + view := donutView(stats, 10_000_001, "9.5 MB") + + tiny := view.Slices[1] + dash, err := strconv.ParseFloat(tiny.Dash, 64) + if err != nil { + t.Fatalf("dash %q: %v", tiny.Dash, err) + } + if dash < donutMinSlice { + t.Errorf("tiny slice dash = %v, want at least %v so it stays visible", dash, donutMinSlice) + } + if tiny.SharePct != "<0.1" { + t.Errorf("tiny slice share = %q, want %q", tiny.SharePct, "<0.1") + } +} + +func TestDonutViewEmpty(t *testing.T) { + if view := donutView(nil, 0, "0 B"); view.HasSlices || len(view.Slices) != 0 { + t.Errorf("expected an empty ring, got %+v", view) + } + + // Ecosystems that exist but have served nothing get no arc at all. + idle := []database.EcosystemStats{{Ecosystem: "rpm", Packages: 3}} + if view := donutView(idle, 0, "0 B"); view.HasSlices { + t.Errorf("an ecosystem with no traffic must not get a slice, got %+v", view.Slices) + } +} + +// A slice widened to stay visible must not be drawn over by its successor: +// the next offset has to start where this slice actually ends. +func TestDonutClampedSliceDoesNotOverdrawItsSuccessor(t *testing.T) { + // Four slivers far below the minimum, followed by a large slice. + stats := []database.EcosystemStats{ + eco("npm", 10_000_000), eco("a", 1), eco("b", 1), eco("c", 1), eco("d", 1), + } + view := donutView(stats, 10_000_004, "9.5 MB") + + for i := 0; i < len(view.Slices)-1; i++ { + dash := mustFloat(t, view.Slices[i].Dash) + start := -mustFloat(t, view.Slices[i].Offset) + nextStart := -mustFloat(t, view.Slices[i+1].Offset) + + end := start + dash + if nextStart < end-0.001 { + t.Errorf("slice %d (%q) is drawn to %v but slice %d starts at %v: they overlap", + i, view.Slices[i].Label, end, i+1, nextStart) + } + } +} + +func mustFloat(t *testing.T, s string) float64 { + t.Helper() + f, err := strconv.ParseFloat(s, 64) + if err != nil { + t.Fatalf("parsing %q: %v", s, err) + } + return f +} + +// Widening slivers must not push the ring past a full turn: the last slice +// would wrap back over the first and repaint the leading ecosystem's arc. +func TestDonutNeverExceedsOneTurn(t *testing.T) { + // One dominant ecosystem and five far below the visible minimum. + stats := []database.EcosystemStats{ + eco("npm", 100_000_000), + eco("a", 1), eco("b", 1), eco("c", 1), eco("d", 1), eco("e", 1), + } + view := donutView(stats, 100_000_005, "95.4 MB") + + if len(view.Slices) != 6 { + t.Fatalf("expected 6 slices, got %d", len(view.Slices)) + } + + last := view.Slices[len(view.Slices)-1] + end := -mustFloat(t, last.Offset) + mustFloat(t, last.Dash) + if end > donutCircumference+0.001 { + t.Errorf("ring is drawn to %v, past the circumference %v: the last slice "+ + "wraps over the first", end, donutCircumference) + } + + // Every sliver still has to be visible. + for _, s := range view.Slices[1:] { + if dash := mustFloat(t, s.Dash); dash < donutMinSlice { + t.Errorf("slice %q dash = %v, want at least %v", s.Label, dash, donutMinSlice) + } + } +} diff --git a/internal/server/analytics_runtime.go b/internal/server/analytics_runtime.go new file mode 100644 index 00000000..2b777302 --- /dev/null +++ b/internal/server/analytics_runtime.go @@ -0,0 +1,291 @@ +package server + +import ( + "fmt" + "sort" + "strconv" + "time" + + "github.com/git-pkgs/proxy/internal/metrics" +) + +// RuntimeView holds the process-lifetime figures read out of the Prometheus +// registry. Unlike the cache figures, which are computed from the database and +// survive a restart, everything here starts at zero when the proxy starts. +type RuntimeView struct { + Available bool + + Requests string + ActiveRequests string + RequestMean string + StatusClasses []LabelledCount + CacheHits string + CacheMisses string + CacheHitRatio string + HasCacheTraffic bool + + UpstreamFetches string + UpstreamFetchMean string + UpstreamErrors []LabelledCount + + StorageOps []LabelledStat + StorageErrors []LabelledCount + IntegrityFailures []LabelledCount + ProbeFailures []LabelledCount + + Breakers []BreakerRow + BreakerTrips []LabelledCount + + Scans []LabelledStat + ScansBlocked []LabelledCount + ScanErrors []LabelledCount + ScanningOn bool + + ResponseBytes string + Clients []ClientRow + Sources []SourceRow + SourceCount int + TrustsForward bool + // SourcesOn gates the caller table. The page is unauthenticated, so the + // addresses behind it are published only when asked for. + SourcesOn bool +} + +// ClientRow is one client tool's share of requests and bytes. +type ClientRow struct { + Client string + Requests string + Bytes string +} + +// LabelledCount is a single counter series rendered as a row. +type LabelledCount struct { + Label string + Count string + // Bad marks a failure row, which the template tints. + Bad bool +} + +// LabelledStat is a histogram series rendered as a row: how many observations, +// and their mean. +type LabelledStat struct { + Label string + Count string + Mean string +} + +// BreakerRow is one upstream's circuit breaker state. +type BreakerRow struct { + Registry string + State string + Open bool +} + +// runtimeView shapes a registry snapshot for the analytics page. +func runtimeView(snap *metrics.Snapshot) RuntimeView { + if snap == nil { + return RuntimeView{} + } + + v := RuntimeView{Available: true} + + requests := snap.Sum("proxy_requests_total") + v.Requests = formatCount(int64(requests)) + v.ActiveRequests = formatCount(int64(snap.Sum("proxy_active_requests"))) + v.RequestMean = formatDuration(snap.Mean("proxy_request_duration_seconds")) + v.StatusClasses = statusClasses(snap) + + hits := snap.Sum("proxy_cache_hits_total") + misses := snap.Sum("proxy_cache_misses_total") + v.CacheHits = formatCount(int64(hits)) + v.CacheMisses = formatCount(int64(misses)) + v.HasCacheTraffic = hits+misses > 0 + if v.HasCacheTraffic { + v.CacheHitRatio = strconv.FormatFloat(hits/(hits+misses)*100, 'f', 1, 64) //nolint:mnd // percent + } + + v.UpstreamFetches = formatCount(int64(snap.Count("proxy_upstream_fetch_duration_seconds"))) + v.UpstreamFetchMean = formatDuration(snap.Mean("proxy_upstream_fetch_duration_seconds")) + v.UpstreamErrors = failureRows(snap, "proxy_upstream_errors_total", "ecosystem", "error_type") + + v.StorageOps = histogramRows(snap, "proxy_storage_operation_duration_seconds", "operation") + v.StorageErrors = failureRows(snap, "proxy_storage_errors_total", "operation") + v.IntegrityFailures = failureRows(snap, "proxy_integrity_failures_total", "ecosystem") + v.ProbeFailures = failureRows(snap, "proxy_health_probe_failures_total", "step") + + v.Breakers = breakerRows(snap) + v.BreakerTrips = failureRows(snap, "proxy_circuit_breaker_trips_total", "registry") + + v.Scans = histogramRows(snap, "proxy_scan_duration_seconds", "ecosystem", "scanner") + v.ScansBlocked = failureRows(snap, "proxy_scan_blocked_total", "ecosystem", "scanner") + v.ScanErrors = failureRows(snap, "proxy_scan_errors_total", "ecosystem", "scanner", "error_type") + v.ScanningOn = len(v.Scans) > 0 || len(v.ScansBlocked) > 0 || len(v.ScanErrors) > 0 + + v.ResponseBytes = formatSize(int64(snap.Sum("proxy_response_bytes_total"))) + v.Clients = clientRows(snap) + + return v +} + +// clientRows pairs each client tool's request count with the bytes it pulled, +// busiest first. Both come from the same closed label set, so the two counters +// line up row for row. +func clientRows(snap *metrics.Snapshot) []ClientRow { + bytesByClient := snap.SumBy("proxy_client_response_bytes_total", "client") + + type entry struct { + client string + requests float64 + bytes float64 + } + + entries := make([]entry, 0) + for _, s := range snap.Samples("proxy_client_requests_total") { + client := s.Label("client") + entries = append(entries, entry{ + client: client, + requests: s.Value, + bytes: bytesByClient[client], + }) + } + + sort.Slice(entries, func(i, j int) bool { + a, b := entries[i], entries[j] + if a.bytes != b.bytes { + return a.bytes > b.bytes + } + return a.requests > b.requests + }) + + rows := make([]ClientRow, 0, len(entries)) + for _, e := range entries { + rows = append(rows, ClientRow{ + Client: e.client, + Requests: formatCount(int64(e.requests)), + Bytes: formatSize(int64(e.bytes)), + }) + } + return rows +} + +// statusClasses groups proxy_requests_total into 2xx/3xx/4xx/5xx buckets, which +// is the breakdown worth showing; the per-code detail stays in Prometheus. +func statusClasses(snap *metrics.Snapshot) []LabelledCount { + byClass := make(map[string]float64) + for _, s := range snap.Samples("proxy_requests_total") { + status := s.Label("status") + if len(status) == 0 { + continue + } + byClass[string(status[0])+"xx"] += s.Value + } + + classes := make([]string, 0, len(byClass)) + for c := range byClass { + classes = append(classes, c) + } + sort.Strings(classes) + + rows := make([]LabelledCount, 0, len(classes)) + for _, c := range classes { + rows = append(rows, LabelledCount{ + Label: c, + Count: formatCount(int64(byClass[c])), + Bad: c == "5xx", + }) + } + return rows +} + +// failureRows renders every non-zero series of a failure counter, keyed by the +// joined values of the given labels. Zero-valued series are dropped: a counter +// that has never fired carries no information, and a wall of zeroes buries the +// rows that matter. +func failureRows(snap *metrics.Snapshot, name string, labels ...string) []LabelledCount { + var rows []LabelledCount + for _, s := range snap.Samples(name) { + if s.Value == 0 { + continue + } + rows = append(rows, LabelledCount{ + Label: joinLabels(s, labels...), + Count: formatCount(int64(s.Value)), + Bad: true, + }) + } + return rows +} + +// histogramRows renders every observed series of a histogram with its count and mean. +func histogramRows(snap *metrics.Snapshot, name string, labels ...string) []LabelledStat { + var rows []LabelledStat + for _, s := range snap.Samples(name) { + if s.Count == 0 { + continue + } + rows = append(rows, LabelledStat{ + Label: joinLabels(s, labels...), + Count: formatCount(int64(s.Count)), + Mean: formatDuration(s.Mean()), + }) + } + return rows +} + +func breakerRows(snap *metrics.Snapshot) []BreakerRow { + var rows []BreakerRow + for _, s := range snap.Samples("proxy_circuit_breaker_state") { + state := "closed" + switch s.Value { + case 1: + state = "half-open" + case 2: //nolint:mnd // 2 = open, per the gauge's documented encoding + state = "open" + } + rows = append(rows, BreakerRow{ + Registry: s.Label("registry"), + State: state, + Open: s.Value > 0, + }) + } + return rows +} + +func joinLabels(s metrics.Sample, labels ...string) string { + out := "" + for _, l := range labels { + v := s.Label(l) + if v == "" { + continue + } + if out != "" { + out += " · " + } + out += v + } + if out == "" { + return "-" + } + return out +} + +// formatDuration renders a mean latency at a sensible unit. Storage operations +// land in microseconds and upstream fetches in seconds, so a fixed unit would +// read as either 0.000 or an unreadable pile of digits. +func formatDuration(seconds float64) string { + if seconds <= 0 { + return "-" + } + + d := time.Duration(seconds * float64(time.Second)) + switch { + case d < time.Microsecond: + return fmt.Sprintf("%.0f ns", float64(d)) + case d < time.Millisecond: + return fmt.Sprintf("%.0f µs", float64(d)/float64(time.Microsecond)) + case d < time.Second: + return fmt.Sprintf("%.1f ms", float64(d)/float64(time.Millisecond)) + default: + return fmt.Sprintf("%.2f s", d.Seconds()) + } +} diff --git a/internal/server/analytics_runtime_test.go b/internal/server/analytics_runtime_test.go new file mode 100644 index 00000000..6cab5587 --- /dev/null +++ b/internal/server/analytics_runtime_test.go @@ -0,0 +1,184 @@ +package server + +import ( + "testing" + + "github.com/git-pkgs/proxy/internal/metrics" + "github.com/prometheus/client_golang/prometheus" +) + +func TestFormatDuration(t *testing.T) { + tests := []struct { + seconds float64 + want string + }{ + {0, "-"}, + {-1, "-"}, + {0.0000004, "400 ns"}, + {0.00041, "410 µs"}, + {0.0081, "8.1 ms"}, + {1.25, "1.25 s"}, + } + for _, tc := range tests { + if got := formatDuration(tc.seconds); got != tc.want { + t.Errorf("formatDuration(%v) = %q, want %q", tc.seconds, got, tc.want) + } + } +} + +func TestRuntimeViewNilSnapshot(t *testing.T) { + if v := runtimeView(nil); v.Available { + t.Error("a nil snapshot must not report itself as available") + } +} + +func TestRuntimeViewShapesRegistry(t *testing.T) { + reg := prometheus.NewRegistry() + + requests := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_requests_total", Help: "t"}, []string{"ecosystem", "status"}) + scanErrors := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_scan_errors_total", Help: "t"}, []string{"ecosystem", "scanner", "error_type"}) + hits := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_cache_hits_total", Help: "t"}, []string{"ecosystem"}) + misses := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_cache_misses_total", Help: "t"}, []string{"ecosystem"}) + breaker := prometheus.NewGaugeVec( + prometheus.GaugeOpts{Name: "proxy_circuit_breaker_state", Help: "t"}, []string{"registry"}) + storage := prometheus.NewHistogramVec( + prometheus.HistogramOpts{Name: "proxy_storage_operation_duration_seconds", Help: "t"}, []string{"operation"}) + reg.MustRegister(requests, scanErrors, hits, misses, breaker, storage) + + requests.WithLabelValues("npm", "200").Add(90) + requests.WithLabelValues("npm", "304").Add(5) + requests.WithLabelValues("npm", "500").Add(5) + scanErrors.WithLabelValues("npm", "clamav", "timeout").Add(3) + hits.WithLabelValues("npm").Add(80) + misses.WithLabelValues("npm").Add(20) + breaker.WithLabelValues("registry.npmjs.org").Set(0) + breaker.WithLabelValues("static.crates.io").Set(2) + storage.WithLabelValues("get").Observe(0.002) + + snap, err := metrics.GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + v := runtimeView(snap) + + if !v.Available { + t.Fatal("expected the view to be available") + } + if v.Requests != "100" { + t.Errorf("Requests = %q, want %q", v.Requests, "100") + } + if v.CacheHitRatio != "80.0" || !v.HasCacheTraffic { + t.Errorf("CacheHitRatio = %q (traffic=%v), want 80.0", v.CacheHitRatio, v.HasCacheTraffic) + } + + // Status codes collapse to classes; 5xx is flagged as a failure row. + wantClasses := map[string]struct { + count string + bad bool + }{"2xx": {"90", false}, "3xx": {"5", false}, "5xx": {"5", true}} + if len(v.StatusClasses) != len(wantClasses) { + t.Fatalf("got %d status classes, want %d: %+v", len(v.StatusClasses), len(wantClasses), v.StatusClasses) + } + for _, row := range v.StatusClasses { + want, ok := wantClasses[row.Label] + if !ok { + t.Errorf("unexpected status class %q", row.Label) + continue + } + if row.Count != want.count || row.Bad != want.bad { + t.Errorf("%s = %q (bad=%v), want %q (bad=%v)", row.Label, row.Count, row.Bad, want.count, want.bad) + } + } + + // The metric that prompted this: it must reach the page with its labels joined. + if len(v.ScanErrors) != 1 { + t.Fatalf("expected 1 scan error row, got %+v", v.ScanErrors) + } + if v.ScanErrors[0].Label != "npm · clamav · timeout" || v.ScanErrors[0].Count != "3" { + t.Errorf("scan error row = %+v, want npm · clamav · timeout = 3", v.ScanErrors[0]) + } + if !v.ScanningOn { + t.Error("scanning should read as on once a scan metric carries a value") + } + + if len(v.Breakers) != 2 { + t.Fatalf("expected 2 breakers, got %+v", v.Breakers) + } + var open, closed int + for _, b := range v.Breakers { + if b.Open { + open++ + if b.State != "open" { + t.Errorf("open breaker state = %q, want %q", b.State, "open") + } + } else { + closed++ + } + } + if open != 1 || closed != 1 { + t.Errorf("breakers: %d open / %d closed, want 1 / 1", open, closed) + } + + if len(v.StorageOps) != 1 || v.StorageOps[0].Label != "get" || v.StorageOps[0].Mean != "2.0 ms" { + t.Errorf("StorageOps = %+v, want one 'get' row with a 2.0 ms mean", v.StorageOps) + } +} + +// A counter sitting at zero carries no information and would bury the rows +// that do, so it is left out. +func TestRuntimeViewDropsZeroCounters(t *testing.T) { + reg := prometheus.NewRegistry() + errs := prometheus.NewCounterVec( + prometheus.CounterOpts{Name: "proxy_storage_errors_total", Help: "t"}, []string{"operation"}) + reg.MustRegister(errs) + + // Touching a label creates the series at zero. + errs.WithLabelValues("get") + errs.WithLabelValues("put").Add(2) + + snap, err := metrics.GatherFrom(reg) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + v := runtimeView(snap) + + if len(v.StorageErrors) != 1 { + t.Fatalf("expected only the non-zero row, got %+v", v.StorageErrors) + } + if v.StorageErrors[0].Label != "put" { + t.Errorf("kept row = %q, want %q", v.StorageErrors[0].Label, "put") + } +} + +// An empty registry must still render, reporting nothing rather than dividing +// by zero or showing a hit rate of NaN. +func TestRuntimeViewEmptyRegistry(t *testing.T) { + snap, err := metrics.GatherFrom(prometheus.NewRegistry()) + if err != nil { + t.Fatalf("GatherFrom: %v", err) + } + v := runtimeView(snap) + + if !v.Available { + t.Error("an empty registry is still a valid snapshot") + } + if v.HasCacheTraffic { + t.Error("no traffic recorded, so HasCacheTraffic must be false") + } + if v.CacheHitRatio != "" { + t.Errorf("CacheHitRatio = %q, want empty when nothing was recorded", v.CacheHitRatio) + } + if v.Requests != "0" { + t.Errorf("Requests = %q, want %q", v.Requests, "0") + } + if v.RequestMean != "-" { + t.Errorf("RequestMean = %q, want a dash", v.RequestMean) + } + if v.ScanningOn { + t.Error("scanning must read as off with no scan metrics") + } +} diff --git a/internal/server/analytics_test.go b/internal/server/analytics_test.go new file mode 100644 index 00000000..3ce5289a --- /dev/null +++ b/internal/server/analytics_test.go @@ -0,0 +1,277 @@ +package server + +import ( + "net/http/httptest" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/database" +) + +func TestFormatCount(t *testing.T) { + tests := []struct { + in int64 + want string + }{ + {0, "0"}, + {7, "7"}, + {100, "100"}, + {1000, "1,000"}, + {12004, "12,004"}, + {1000000, "1,000,000"}, + {-4200, "-4,200"}, + } + for _, tc := range tests { + if got := formatCount(tc.in); got != tc.want { + t.Errorf("formatCount(%d) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestPercentOf(t *testing.T) { + tests := []struct { + part, whole int64 + want string + }{ + {1, 4, "25.0"}, + {2, 3, "66.7"}, + {0, 100, "0.0"}, + {5, 5, "100.0"}, + // A zero total must not divide by zero. + {0, 0, "0.0"}, + {7, 0, "0.0"}, + // A negligible share still rounds to a plain number here; only + // formatPercent turns it into "<0.1". + {1, 1000000, "0.0"}, + } + for _, tc := range tests { + if got := percentOf(tc.part, tc.whole); got != tc.want { + t.Errorf("percentOf(%d, %d) = %q, want %q", tc.part, tc.whole, got, tc.want) + } + } +} + +func TestFormatPercent(t *testing.T) { + tests := []struct { + part, whole int64 + want string + }{ + {1, 4, "25.0"}, + {0, 100, "0.0"}, + {0, 0, "0.0"}, + // A real but tiny share must not be reported as zero. + {1, 1000000, "<0.1"}, + {7_800_000, 18_000_000_000, "<0.1"}, + } + for _, tc := range tests { + if got := formatPercent(tc.part, tc.whole); got != tc.want { + t.Errorf("formatPercent(%d, %d) = %q, want %q", tc.part, tc.whole, got, tc.want) + } + } +} + +func TestFormatRatio(t *testing.T) { + if got := formatRatio(3500, 1000); got != "3.5x" { + t.Errorf("formatRatio(3500, 1000) = %q, want %q", got, "3.5x") + } + // Nothing cached means no meaningful multiplier, not "+Infx". + if got := formatRatio(3500, 0); got != "" { + t.Errorf("formatRatio(3500, 0) = %q, want empty", got) + } +} + +func TestAvgArtifactSize(t *testing.T) { + if got := avgArtifactSize(1000, 4); got != "250 B" { + t.Errorf("avgArtifactSize(1000, 4) = %q, want %q", got, "250 B") + } + if got := avgArtifactSize(0, 0); got != "0 B" { + t.Errorf("avgArtifactSize(0, 0) = %q, want %q", got, "0 B") + } +} + +func TestAnalyticsView(t *testing.T) { + stats := []database.EcosystemStats{ + {Ecosystem: "npm", Packages: 2, Versions: 2, Artifacts: 2, CacheSize: 1500, Downloads: 4, DownloadedBytes: 3000}, + {Ecosystem: "cargo", Packages: 1, Versions: 1, Artifacts: 1, CacheSize: 200, Downloads: 5, DownloadedBytes: 1000}, + {Ecosystem: "gem", Packages: 3, Versions: 4}, + } + + totals, rows := analyticsView(stats) + + if totals.DownloadedBytes != 4000 { + t.Errorf("DownloadedBytes = %d, want 4000", totals.DownloadedBytes) + } + if totals.Downloads != "9" { + t.Errorf("Downloads = %q, want %q", totals.Downloads, "9") + } + if totals.CachedArtifacts != "3" { + t.Errorf("CachedArtifacts = %q, want %q", totals.CachedArtifacts, "3") + } + if totals.Packages != "6" { + t.Errorf("Packages = %q, want %q", totals.Packages, "6") + } + if totals.Versions != "7" { + t.Errorf("Versions = %q, want %q", totals.Versions, "7") + } + if totals.Ecosystems != 3 { + t.Errorf("Ecosystems = %d, want 3", totals.Ecosystems) + } + // gem has served nothing, so it is known but not active. + if totals.ActiveEcosystems != 2 { + t.Errorf("ActiveEcosystems = %d, want 2", totals.ActiveEcosystems) + } + // 4000 bytes served from 1700 bytes stored. + if totals.Amplification != "2.4x" { + t.Errorf("Amplification = %q, want %q", totals.Amplification, "2.4x") + } + + if len(rows) != 3 { + t.Fatalf("expected 3 rows, got %d", len(rows)) + } + + // Shares are of the grand total. + if rows[0].SharePct != "75.0" { + t.Errorf("npm share = %q, want 75.0", rows[0].SharePct) + } + if rows[1].SharePct != "25.0" { + t.Errorf("cargo share = %q, want 25.0", rows[1].SharePct) + } + // gem served nothing, so it contributes no share but still gets a row. + if rows[2].SharePct != "0.0" || rows[2].Ecosystem != "gem" { + t.Errorf("gem row = %+v, want a 0.0%% share", rows[2]) + } + if rows[0].AvgArtifactSize != "750 B" { + t.Errorf("npm AvgArtifactSize = %q, want %q", rows[0].AvgArtifactSize, "750 B") + } +} + +// With nothing cached at all the view must stay renderable rather than dividing +// by a zero total. +func TestAnalyticsViewEmpty(t *testing.T) { + totals, rows := analyticsView(nil) + + if len(rows) != 0 { + t.Errorf("expected no rows, got %+v", rows) + } + if totals.DownloadedBytes != 0 { + t.Errorf("DownloadedBytes = %d, want 0", totals.DownloadedBytes) + } + if totals.Downloaded != "0 B" { + t.Errorf("Downloaded = %q, want %q", totals.Downloaded, "0 B") + } + if totals.Amplification != "" { + t.Errorf("Amplification = %q, want empty", totals.Amplification) + } +} + +func TestAnalyticsPageRendersTotals(t *testing.T) { + tpl := &Templates{} + w := httptest.NewRecorder() + + totals, rows := analyticsView([]database.EcosystemStats{ + {Ecosystem: "npm", Packages: 1, Versions: 1, Artifacts: 1, CacheSize: 1000, Downloads: 3, DownloadedBytes: 3000}, + }) + data := AnalyticsData{Totals: totals, Ecosystems: rows} + data.Donut = donutView([]database.EcosystemStats{ + {Ecosystem: "npm", Packages: 1, Versions: 1, Artifacts: 1, CacheSize: 1000, Downloads: 3, DownloadedBytes: 3000}, + }, totals.DownloadedBytes, totals.Downloaded) + + if err := tpl.Render(w, "analytics", data); err != nil { + t.Fatalf("Render: %v", err) + } + + body := w.Body.String() + if strings.Contains(body, "ZgotmplZ") { + t.Error("a template value was sanitized away; check the donut dash geometry") + } + for _, want := range []string{ + "accumulated download size", + "2.9 KB", // the total, stated in the middle of the ring + "donut-slot-0", // the single slice takes the first categorical slot + "3.0x", // 3000 bytes served from 1000 stored + "/ui/analytics", // nav link renders on the page itself + } { + if !strings.Contains(body, want) { + t.Errorf("rendered page missing %q", want) + } + } +} + +// A retained snapshot must be labelled as one. The cache deliberately serves +// the last good rows when the query fails, so without this the page presents +// arbitrarily old figures as current and the only trace is a log line. +func TestAnalyticsPageFlagsStaleFigures(t *testing.T) { + tpl := &Templates{} + w := httptest.NewRecorder() + + totals, rows := analyticsView([]database.EcosystemStats{ + {Ecosystem: "npm", Artifacts: 1, CacheSize: 1000, Downloads: 3, DownloadedBytes: 3000}, + }) + data := AnalyticsData{ + Totals: totals, + Ecosystems: rows, + StatsFailed: true, + StatsStale: true, + StatsAge: "2 hours ago", + } + + if err := tpl.Render(w, "analytics", data); err != nil { + t.Fatalf("Render: %v", err) + } + + body := w.Body.String() + for _, want := range []string{"Figures may be out of date", "2 hours ago"} { + if !strings.Contains(body, want) { + t.Errorf("rendered page missing %q", want) + } + } + // The figures themselves still render — stale beats absent. + if !strings.Contains(body, "2.9 KB") { + t.Error("the retained snapshot was not rendered alongside the warning") + } +} + +// A failure with nothing retained keeps the existing unavailable message and +// must not also claim the figures below are stale, since there are none. +func TestAnalyticsPageStaleBannerNeedsASnapshot(t *testing.T) { + tpl := &Templates{} + w := httptest.NewRecorder() + + if err := tpl.Render(w, "analytics", AnalyticsData{StatsFailed: true}); err != nil { + t.Fatalf("Render: %v", err) + } + + body := w.Body.String() + if strings.Contains(body, "Figures may be out of date") { + t.Error("stale banner rendered with no retained snapshot") + } + if !strings.Contains(body, "the database query failed") { + t.Error("expected the unavailable message when the query failed with no snapshot") + } +} + +func TestAnalyticsPageRendersWithoutTraffic(t *testing.T) { + tpl := &Templates{} + w := httptest.NewRecorder() + + if err := tpl.Render(w, "analytics", AnalyticsData{}); err != nil { + t.Fatalf("Render: %v", err) + } + if body := w.Body.String(); !strings.Contains(body, "Nothing has been served from cache yet") { + t.Error("expected the empty-state message when no traffic has been recorded") + } +} + +func TestEcosystemMetricsMapping(t *testing.T) { + got := ecosystemMetrics([]database.EcosystemStats{ + {Ecosystem: "npm", Packages: 1, Versions: 2, Artifacts: 3, CacheSize: 4, Downloads: 5, DownloadedBytes: 6}, + }) + if len(got) != 1 { + t.Fatalf("expected 1 snapshot, got %d", len(got)) + } + m := got[0] + if m.Ecosystem != "npm" || m.Packages != 1 || m.Versions != 2 || m.Artifacts != 3 || + m.CacheSize != 4 || m.Downloads != 5 || m.DownloadedBytes != 6 { + t.Errorf("fields did not map across: %+v", m) + } +} diff --git a/internal/server/cooldown_patterns_test.go b/internal/server/cooldown_patterns_test.go new file mode 100644 index 00000000..0587ba09 --- /dev/null +++ b/internal/server/cooldown_patterns_test.go @@ -0,0 +1,30 @@ +package server + +import ( + "strings" + "testing" +) + +func TestStartRejectsInvalidCooldownPatterns(t *testing.T) { + for _, tc := range []struct { + name string + patterns map[string]string + message string + }{ + {"glob", map[string]string{"pkg:npm/[": "0"}, "invalid cooldown package pattern"}, + {"class containing scope alias", map[string]string{"pkg:npm/[@a]*": "0"}, "character classes and escapes are not supported"}, + {"character class", map[string]string{"pkg:npm/[abcdef]*": "0"}, "character classes and escapes are not supported"}, + {"escaped wildcard", map[string]string{`pkg:npm/\*`: "0"}, "character classes and escapes are not supported"}, + {"duration", map[string]string{"pkg:npm/*": "invalid"}, "invalid cooldown duration"}, + {"aliases", map[string]string{"pkg:npm/@example/*": "0", "pkg:npm/%40example/*": "7d"}, "conflicting cooldown package patterns"}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newTestServer(t) + defer s.close() + s.server.cfg.Cooldown.PackagePatterns = tc.patterns + if err := s.server.Start(); err == nil || !strings.Contains(err.Error(), tc.message) { + t.Fatalf("Start error = %v, want %q", err, tc.message) + } + }) + } +} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index a3ac9ebd..05643f66 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -2,6 +2,7 @@ package server import ( "html/template" + "sort" "strings" "github.com/git-pkgs/proxy/internal/database" @@ -110,19 +111,27 @@ type SearchResultItem struct { // PackagesListPageData contains data for rendering the packages list page. type PackagesListPageData struct { Layout - Ecosystem string - SortBy string - Results []SearchResultItem - Count int - Page int - PerPage int - TotalPages int + Ecosystem string + SortBy string + Results []SearchResultItem + Count int + TotalPackages int64 + EcosystemFilters []EcosystemFilter + Page int + PerPage int + TotalPages int +} + +// EcosystemFilter represents an ecosystem filter pill with a package count. +type EcosystemFilter struct { + Ecosystem string + Count int64 } func supportedEcosystems() []string { // this list should be kept sorted in lexicographic order so - // that the 'select' list in the UI will be in the expected - // order + // that the ecosystem filter pills in the UI will be in the + // expected order return []string{ "alpine", "cargo", @@ -157,49 +166,82 @@ func ecosystemBadgeLabel(ecosystem string) string { } } -func ecosystemBadgeClasses(ecosystem string) string { - base := "inline-flex items-center px-2 py-0.5 rounded text-xs font-medium" - +func ecosystemColorClasses(ecosystem string) string { switch ecosystem { case "npm", "maven": - return base + " bg-red-100 text-red-700 dark:bg-red-900/50 dark:text-red-300" + return "bg-red-100 text-red-700 dark:bg-red-900/50 dark:text-red-300" case "cargo": - return base + " bg-orange-100 text-orange-700 dark:bg-orange-900/50 dark:text-orange-300" + return "bg-orange-100 text-orange-700 dark:bg-orange-900/50 dark:text-orange-300" case "gem": - return base + " bg-pink-100 text-pink-700 dark:bg-pink-900/50 dark:text-pink-300" - case "go": - return base + " bg-cyan-100 text-cyan-700 dark:bg-cyan-900/50 dark:text-cyan-300" + return "bg-pink-100 text-pink-700 dark:bg-pink-900/50 dark:text-pink-300" + case "go", "golang": + return "bg-cyan-100 text-cyan-700 dark:bg-cyan-900/50 dark:text-cyan-300" case "hex": - return base + " bg-purple-100 text-purple-700 dark:bg-purple-900/50 dark:text-purple-300" + return "bg-purple-100 text-purple-700 dark:bg-purple-900/50 dark:text-purple-300" case "pub": - return base + " bg-blue-100 text-blue-700 dark:bg-blue-900/50 dark:text-blue-300" + return "bg-blue-100 text-blue-700 dark:bg-blue-900/50 dark:text-blue-300" case "pypi": - return base + " bg-yellow-100 text-yellow-700 dark:bg-yellow-900/50 dark:text-yellow-300" + return "bg-yellow-100 text-yellow-700 dark:bg-yellow-900/50 dark:text-yellow-300" case "nuget": - return base + " bg-indigo-100 text-indigo-700 dark:bg-indigo-900/50 dark:text-indigo-300" + return "bg-indigo-100 text-indigo-700 dark:bg-indigo-900/50 dark:text-indigo-300" case "composer": - return base + " bg-violet-100 text-violet-700 dark:bg-violet-900/50 dark:text-violet-300" + return "bg-violet-100 text-violet-700 dark:bg-violet-900/50 dark:text-violet-300" case "conan": - return base + " bg-teal-100 text-teal-700 dark:bg-teal-900/50 dark:text-teal-300" + return "bg-teal-100 text-teal-700 dark:bg-teal-900/50 dark:text-teal-300" case "conda": - return base + " bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300" + return "bg-green-100 text-green-700 dark:bg-green-900/50 dark:text-green-300" case "cran": - return base + " bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300" + return "bg-slate-100 text-slate-700 dark:bg-slate-800 dark:text-slate-300" case "julia": - return base + " bg-emerald-100 text-emerald-700 dark:bg-emerald-900/50 dark:text-emerald-300" + return "bg-emerald-100 text-emerald-700 dark:bg-emerald-900/50 dark:text-emerald-300" case "swift": - return base + " bg-orange-100 text-orange-700 dark:bg-orange-900/50 dark:text-orange-300" + return "bg-orange-100 text-orange-700 dark:bg-orange-900/50 dark:text-orange-300" case "oci": - return base + " bg-sky-100 text-sky-700 dark:bg-sky-900/50 dark:text-sky-300" + return "bg-sky-100 text-sky-700 dark:bg-sky-900/50 dark:text-sky-300" case "deb": - return base + " bg-red-100 text-red-800 dark:bg-red-900/50 dark:text-red-300" + return "bg-red-100 text-red-800 dark:bg-red-900/50 dark:text-red-300" case "rpm": - return base + " bg-amber-100 text-amber-800 dark:bg-amber-900/50 dark:text-amber-300" + return "bg-amber-100 text-amber-800 dark:bg-amber-900/50 dark:text-amber-300" case "alpine": - return base + " bg-lime-100 text-lime-800 dark:bg-lime-900/50 dark:text-lime-300" + return "bg-lime-100 text-lime-800 dark:bg-lime-900/50 dark:text-lime-300" default: - return base + " bg-gray-100 text-gray-700 dark:bg-gray-800 dark:text-gray-300" + return "bg-gray-100 text-gray-700 dark:bg-gray-800 dark:text-gray-300" + } +} + +func ecosystemBadgeClasses(ecosystem string) string { + return "inline-flex items-center px-2 py-0.5 rounded text-xs font-medium " + ecosystemColorClasses(ecosystem) +} + +func ecosystemPillClasses(ecosystem string) string { + return "inline-flex items-center gap-1.5 px-3 py-1.5 rounded-full text-xs font-medium hover:opacity-90 " + ecosystemColorClasses(ecosystem) +} + +func buildEcosystemFilters(counts map[string]int64) []EcosystemFilter { + var filters []EcosystemFilter + seen := make(map[string]bool, len(counts)) + + for _, eco := range supportedEcosystems() { + count := counts[eco] + if count > 0 { + filters = append(filters, EcosystemFilter{Ecosystem: eco, Count: count}) + seen[eco] = true + } + } + + var extra []string + for eco, count := range counts { + if count > 0 && !seen[eco] { + extra = append(extra, eco) + } } + sort.Strings(extra) + + for _, eco := range extra { + filters = append(filters, EcosystemFilter{Ecosystem: eco, Count: counts[eco]}) + } + + return filters } func getRegistryConfigs(baseURL string) []RegistryConfig { diff --git a/internal/server/ecosystem_cache.go b/internal/server/ecosystem_cache.go new file mode 100644 index 00000000..83e8c0d7 --- /dev/null +++ b/internal/server/ecosystem_cache.go @@ -0,0 +1,94 @@ +package server + +import ( + "sync" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +// ecosystemStatsTTL bounds how stale a served snapshot can be. The metrics +// refresh loop runs on the same cadence, so a scrape and a page load a moment +// apart report the same figures rather than two slightly different ones. +const ecosystemStatsTTL = time.Minute + +// ecosystemStatsErrorTTL is the retry interval after a failed read. It is short +// so a recovered database is picked up quickly, but not zero, so a database +// that is down is not queried once per request. +const ecosystemStatsErrorTTL = 5 * time.Second + +// ecosystemStatsCache memoizes GetEcosystemStats. +// +// The query is three grouped aggregations over the whole artifact table, which +// is fine once a minute for the metrics gauges but not once per page load: the +// UI is unauthenticated, so without this a client reloading /ui/analytics would +// keep the database busy for as long as it cared to. +// +// The zero value is usable, so a Server assembled as a struct literal -- as +// tests do -- needs no constructor and cannot end up with a nil cache. +type ecosystemStatsCache struct { + ttl time.Duration + + mu sync.Mutex + stats []database.EcosystemStats + err error + lastAt time.Time + // goodAt is when stats was last read successfully, which is not lastAt + // once reads start failing and the retained snapshot is served on. + goodAt time.Time +} + +// SnapshotAt reports when the rows currently held were last read successfully. +// The zero time means no read has ever succeeded. +func (c *ecosystemStatsCache) SnapshotAt() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.goodAt +} + +func (c *ecosystemStatsCache) interval() time.Duration { + if c.err != nil { + return ecosystemStatsErrorTTL + } + if c.ttl <= 0 { + return ecosystemStatsTTL + } + return c.ttl +} + +// Get returns the cached rows, refreshing them from db when they have aged out. +// +// A failed read returns the last good snapshot alongside the error, so a +// caller can choose to serve stale figures rather than fail outright. +func (c *ecosystemStatsCache) Get(db *database.DB) ([]database.EcosystemStats, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if !c.lastAt.IsZero() && time.Since(c.lastAt) < c.interval() { + return c.stats, c.err + } + return c.refreshLocked(db) +} + +// Refresh forces a read, bypassing the TTL. The metrics loop uses it so its own +// tick is never served a snapshot that is about to expire. +func (c *ecosystemStatsCache) Refresh(db *database.DB) ([]database.EcosystemStats, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.refreshLocked(db) +} + +func (c *ecosystemStatsCache) refreshLocked(db *database.DB) ([]database.EcosystemStats, error) { + stats, err := db.GetEcosystemStats() + c.lastAt = time.Now() + c.err = err + if err != nil { + // Keep the last good snapshot: a transient failure should not erase + // figures the caller could still usefully show. + return c.stats, err + } + + c.stats = stats + c.goodAt = c.lastAt + return c.stats, nil +} diff --git a/internal/server/ecosystem_cache_test.go b/internal/server/ecosystem_cache_test.go new file mode 100644 index 00000000..5323ad1b --- /dev/null +++ b/internal/server/ecosystem_cache_test.go @@ -0,0 +1,199 @@ +package server + +import ( + "path/filepath" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +func cacheTestDB(t *testing.T) *database.DB { + t.Helper() + db, err := database.Create(filepath.Join(t.TempDir(), "cache.db")) + if err != nil { + t.Fatalf("Create: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return db +} + +// The zero value must work, because Server is assembled as a struct literal in +// places that never call a constructor. +func TestEcosystemStatsCacheZeroValueIsUsable(t *testing.T) { + var c ecosystemStatsCache + + if _, err := c.Get(cacheTestDB(t)); err != nil { + t.Fatalf("Get on a zero-value cache: %v", err) + } + if c.interval() != ecosystemStatsTTL { + t.Errorf("interval = %v, want the default %v", c.interval(), ecosystemStatsTTL) + } +} + +// A second call inside the TTL must not touch the database again — that is the +// whole point, since /stats is unauthenticated and pollable. +func TestEcosystemStatsCacheServesWithinTTL(t *testing.T) { + db := cacheTestDB(t) + c := ecosystemStatsCache{ttl: time.Hour} + + if _, err := c.Get(db); err != nil { + t.Fatalf("Get: %v", err) + } + first := c.lastAt + + if _, err := c.Get(db); err != nil { + t.Fatalf("Get: %v", err) + } + if !c.lastAt.Equal(first) { + t.Error("a second Get inside the TTL re-queried the database") + } +} + +func TestEcosystemStatsCacheRefreshesAfterTTL(t *testing.T) { + db := cacheTestDB(t) + // A TTL already elapsed by the time the second call lands. + c := ecosystemStatsCache{ttl: time.Nanosecond} + + if _, err := c.Get(db); err != nil { + t.Fatalf("Get: %v", err) + } + first := c.lastAt + + time.Sleep(time.Millisecond) + if _, err := c.Get(db); err != nil { + t.Fatalf("Get: %v", err) + } + if c.lastAt.Equal(first) { + t.Error("the cache did not refresh after its TTL elapsed") + } +} + +// Refresh ignores the TTL, so the metrics loop always publishes a fresh read. +func TestEcosystemStatsCacheRefreshBypassesTTL(t *testing.T) { + db := cacheTestDB(t) + c := ecosystemStatsCache{ttl: time.Hour} + + if _, err := c.Get(db); err != nil { + t.Fatalf("Get: %v", err) + } + first := c.lastAt + + time.Sleep(time.Millisecond) + if _, err := c.Refresh(db); err != nil { + t.Fatalf("Refresh: %v", err) + } + if c.lastAt.Equal(first) { + t.Error("Refresh honoured the TTL; it must always re-read") + } +} + +// A transient failure must not erase a snapshot the caller could still show. +func TestEcosystemStatsCacheKeepsLastGoodSnapshotOnError(t *testing.T) { + good := cacheTestDB(t) + seedCachePackage(t, good, "npm", "lodash") + + c := ecosystemStatsCache{ttl: time.Nanosecond} + stats, err := c.Get(good) + if err != nil { + t.Fatalf("Get: %v", err) + } + if len(stats) != 1 { + t.Fatalf("expected 1 ecosystem from the seeded database, got %+v", stats) + } + + // A closed handle stands in for the database going away. + broken := cacheTestDB(t) + if err := broken.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + time.Sleep(time.Millisecond) + stats, err = c.Get(broken) + if err == nil { + t.Fatal("expected an error from the closed database") + } + if len(stats) != 1 || stats[0].Ecosystem != "npm" { + t.Errorf("last good snapshot was discarded: got %+v", stats) + } +} + +// An error must not be pinned for the full TTL, or a recovered database stays +// invisible for a minute. +func TestEcosystemStatsCacheRetriesSoonerAfterError(t *testing.T) { + broken := cacheTestDB(t) + if err := broken.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + c := ecosystemStatsCache{ttl: time.Hour} + if _, err := c.Get(broken); err == nil { + t.Fatal("expected an error") + } + if got := c.interval(); got != ecosystemStatsErrorTTL { + t.Errorf("retry interval after an error = %v, want %v", got, ecosystemStatsErrorTTL) + } +} + +// A closed database makes the query fail; the error must reach the caller +// rather than being swallowed into an empty-looking result. +func TestEcosystemStatsCacheSurfacesErrors(t *testing.T) { + db, err := database.Create(filepath.Join(t.TempDir(), "closed.db")) + if err != nil { + t.Fatalf("Create: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + var c ecosystemStatsCache + if _, err := c.Get(db); err == nil { + t.Error("expected an error from a closed database") + } +} + +func seedCachePackage(t *testing.T, db *database.DB, ecosystem, name string) { + t.Helper() + if err := db.UpsertPackage(&database.Package{ + PURL: "pkg:" + ecosystem + "/" + name, + Ecosystem: ecosystem, + Name: name, + }); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } +} + +// The snapshot timestamp must track the last successful read, not the last +// attempt. The analytics page dates its "figures may be out of date" banner +// from it, and a timestamp that advanced on every failed retry would report an +// hour-old snapshot as seconds old. +func TestEcosystemStatsCacheSnapshotAtTracksSuccessOnly(t *testing.T) { + good := cacheTestDB(t) + seedCachePackage(t, good, "npm", "lodash") + + c := ecosystemStatsCache{ttl: time.Nanosecond} + if c.SnapshotAt(); !c.SnapshotAt().IsZero() { + t.Fatalf("SnapshotAt on a fresh cache = %v, want the zero time", c.SnapshotAt()) + } + if _, err := c.Get(good); err != nil { + t.Fatalf("Get: %v", err) + } + + at := c.SnapshotAt() + if at.IsZero() { + t.Fatal("SnapshotAt is zero after a successful read") + } + + broken := cacheTestDB(t) + if err := broken.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + time.Sleep(2 * time.Millisecond) + if _, err := c.Get(broken); err == nil { + t.Fatal("expected an error from the closed database") + } + if got := c.SnapshotAt(); !got.Equal(at) { + t.Errorf("SnapshotAt moved on a failed read: %v, want %v", got, at) + } +} diff --git a/internal/server/eviction.go b/internal/server/eviction.go index 4173bd53..f9a706ad 100644 --- a/internal/server/eviction.go +++ b/internal/server/eviction.go @@ -59,42 +59,34 @@ func evictLRU(ctx context.Context, db *database.DB, store storage.Storage, logge freedBytes := int64(0) for totalSize-freedBytes > maxSize { + if ctx.Err() != nil { + break + } + artifacts, err := db.GetLeastRecentlyUsedArtifacts(evictionBatch) if err != nil { logger.Warn("eviction: failed to get LRU artifacts", "error", err) - return + break } if len(artifacts) == 0 { break } - for _, art := range artifacts { - if totalSize-freedBytes <= maxSize { - break - } - - if !art.StoragePath.Valid { - continue - } + cleared, freed := evictBatch(ctx, db, store, logger, artifacts, maxSize, totalSize-freedBytes) + evicted += cleared + freedBytes += freed - if err := store.Delete(ctx, art.StoragePath.String); err != nil { - logger.Warn("eviction: failed to delete from storage", - "path", art.StoragePath.String, "error", err) - continue - } - - if err := db.ClearArtifactCache(art.VersionPURL, art.Filename); err != nil { - logger.Warn("eviction: failed to clear artifact record", - "version_purl", art.VersionPURL, "filename", art.Filename, "error", err) - continue - } + if ctx.Err() != nil { + break + } - size := int64(0) - if art.Size.Valid { - size = art.Size.Int64 - } - freedBytes += size - evicted++ + // A batch that clears nothing returns the same records next time, so + // stop rather than retry them without end. The next pass retries after + // the interval. + if cleared == 0 { + logger.Warn("eviction: no artifact in this batch could be evicted, ending the pass", + "batch", len(artifacts)) + break } } @@ -103,3 +95,53 @@ func evictLRU(ctx context.Context, db *database.DB, store storage.Storage, logge "evicted", evicted, "freed_bytes", freedBytes) } } + +// evictBatch evicts from artifacts until the size still in use falls to +// maxSize, reporting how many records it cleared and how many bytes that +// freed. Progress is reported in records as well as bytes because a cached +// record can have no size recorded, and clearing one is still progress. +func evictBatch(ctx context.Context, db *database.DB, store storage.Storage, logger *slog.Logger, + artifacts []database.Artifact, maxSize, inUse int64) (int, int64) { + cleared := 0 + freed := int64(0) + + for _, art := range artifacts { + if inUse-freed <= maxSize { + break + } + if ctx.Err() != nil { + break + } + if !art.StoragePath.Valid { + continue + } + + path := art.StoragePath.String + + // Clear before deleting: a record a newer fetch moved has left this + // path queued, and a request may still open it within the grace period. + recordCleared, err := db.ClearArtifactCache(art.VersionPURL, art.Filename, path) + if err != nil { + logger.Warn("eviction: failed to clear artifact record", + "version_purl", art.VersionPURL, "filename", art.Filename, "error", err) + continue + } + if !recordCleared { + continue + } + + if err := store.Delete(ctx, path); err != nil { + logger.Warn("eviction: failed to delete from storage, queueing it", "path", path, "error", err) + if err := db.QueuePendingDelete(path); err != nil { + logger.Warn("eviction: failed to queue object for deletion", "path", path, "error", err) + } + } + + if art.Size.Valid { + freed += art.Size.Int64 + } + cleared++ + } + + return cleared, freed +} diff --git a/internal/server/eviction_test.go b/internal/server/eviction_test.go index bac33252..15355864 100644 --- a/internal/server/eviction_test.go +++ b/internal/server/eviction_test.go @@ -3,10 +3,13 @@ package server import ( "context" "database/sql" + "errors" "io" "log/slog" "path/filepath" + "slices" "strings" + "sync/atomic" "testing" "time" @@ -293,3 +296,194 @@ func defaultTestConfig(storagePath, dbPath string) *config.Config { Log: config.LogConfig{Level: "info", Format: "text"}, } } + +// undeletableStorage is a backend whose Delete always fails, standing in for a +// backend that refuses deletion: a permissions problem, or Windows while a +// reader holds the file open. +type undeletableStorage struct { + storage.Storage + deletes atomic.Int64 +} + +func (u *undeletableStorage) Delete(_ context.Context, _ string) error { + u.deletes.Add(1) + return errors.New("storage refused the delete") +} + +// runEvictionWithDeadline fails the test if a pass does not return, rather than +// hanging until the package timeout. +func runEvictionWithDeadline(t *testing.T, ctx context.Context, db *database.DB, store storage.Storage, maxSize int64) { + t.Helper() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + done := make(chan struct{}) + go func() { + defer close(done) + evictLRU(ctx, db, store, logger, maxSize) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("evictLRU never returned: it is retrying records it cannot evict") + } +} + +// TestEvictLRU_QueuesObjectsStorageRefusesToDelete evicts against a backend +// that refuses every delete. The records are cleared, so the pass makes +// progress, and the objects wait in the queue for reclaim to retry. +func TestEvictLRU_QueuesObjectsStorageRefusesToDelete(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + + now := time.Now() + seedArtifact(t, ctx, db, store, "old-pkg", 500, now.Add(-2*time.Hour)) + seedArtifact(t, ctx, db, store, "new-pkg", 500, now) + + undeletable := &undeletableStorage{Storage: store} + runEvictionWithDeadline(t, ctx, db, undeletable, 100) + + if got := undeletable.deletes.Load(); got != 2 { + t.Errorf("delete attempts = %d, want 2", got) + } + count, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("failed to get count: %v", err) + } + if count != 0 { + t.Errorf("cached artifacts = %d, want 0", count) + } + want := []string{ + storage.ArtifactPath("npm", "", "new-pkg", "1.0.0", "new-pkg-1.0.0.tgz"), + storage.ArtifactPath("npm", "", "old-pkg", "1.0.0", "old-pkg-1.0.0.tgz"), + } + got := queuedPaths(t, db) + slices.Sort(got) + if !slices.Equal(got, want) { + t.Errorf("queue = %v, want %v", got, want) + } +} + +// TestEvictLRU_EndsPassWhenNothingCanBeCleared is the loop that would otherwise +// never end. A record that fails to clear stays eligible, so the same batch +// comes back forever while the recorded size never drops. +func TestEvictLRU_EndsPassWhenNothingCanBeCleared(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + + now := time.Now() + seedArtifact(t, ctx, db, store, "old-pkg", 500, now.Add(-2*time.Hour)) + seedArtifact(t, ctx, db, store, "new-pkg", 500, now) + if _, err := db.Exec(`CREATE TRIGGER refuse_clear BEFORE UPDATE OF storage_path ON artifacts + BEGIN SELECT RAISE(FAIL, 'clear refused'); END`); err != nil { + t.Fatalf("creating trigger: %v", err) + } + + runEvictionWithDeadline(t, ctx, db, store, 0) + + count, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("failed to get count: %v", err) + } + if count != 2 { + t.Errorf("cached artifacts = %d, want 2", count) + } + for _, name := range []string{"old-pkg", "new-pkg"} { + if ok, _ := store.Exists(ctx, storage.ArtifactPath("npm", "", name, "1.0.0", name+"-1.0.0.tgz")); !ok { + t.Errorf("%s deleted although its record was not cleared", name) + } + } +} + +// TestEvictLRU_StopsWhenContextCanceled covers shutdown: a canceled context can +// make every delete fail instantly, which is the fastest way to spin. +func TestEvictLRU_StopsWhenContextCanceled(t *testing.T) { + db, store := setupEvictionTest(t) + seedArtifact(t, context.Background(), db, store, "old-pkg", 500, time.Now()) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + undeletable := &undeletableStorage{Storage: store} + runEvictionWithDeadline(t, ctx, db, undeletable, 100) + + if got := undeletable.deletes.Load(); got != 0 { + t.Errorf("delete attempts = %d, want 0 once the context is canceled", got) + } +} + +// TestEvictLRU_EvictsArtifactWithoutRecordedSize covers progress counted in +// records rather than bytes: a record with no size frees nothing, but evicting +// it is still progress and the pass must go on to the next record. +func TestEvictLRU_EvictsArtifactWithoutRecordedSize(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + + now := time.Now() + seedArtifact(t, ctx, db, store, "sizeless-pkg", 500, now.Add(-2*time.Hour)) + seedArtifact(t, ctx, db, store, "sized-pkg", 500, now) + clearRecordedSize(t, db, "pkg:npm/sizeless-pkg@1.0.0", "sizeless-pkg-1.0.0.tgz") + + // The sizeless record is the oldest, so it is evicted first and frees + // nothing. Only evicting the second record gets under the limit. + runEvictionWithDeadline(t, ctx, db, store, 100) + + count, err := db.GetCachedArtifactCount() + if err != nil { + t.Fatalf("failed to get count: %v", err) + } + if count != 0 { + t.Errorf("cached artifacts = %d, want 0: the pass must continue past a record that frees nothing", count) + } +} + +func clearRecordedSize(t *testing.T, db *database.DB, versionPURL, filename string) { + t.Helper() + query := db.Rebind(`UPDATE artifacts SET size = NULL WHERE version_purl = ? AND filename = ?`) + if _, err := db.Exec(query, versionPURL, filename); err != nil { + t.Fatalf("clearing recorded size: %v", err) + } +} + +// TestEvictBatch_SkipsRecordThatMoved evicts from a row read before a newer +// fetch moved the record. Neither object goes: the old one is queued, and a +// request that read the record before it moved may still open it. Nothing is +// counted as freed either, or the pass would end with the cache still over its +// limit. +func TestEvictBatch_SkipsRecordThatMoved(t *testing.T) { + db, store := setupEvictionTest(t) + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + seedArtifact(t, ctx, db, store, "moved", 1000, time.Now().Add(-time.Hour)) + + stale, err := db.GetLeastRecentlyUsedArtifacts(evictionBatch) + if err != nil || len(stale) != 1 { + t.Fatalf("reading LRU rows: %v, %v", stale, err) + } + newer := storage.FetchPath("npm", "moved", "1.0.0", storage.NewFetchID(), "moved-1.0.0.tgz") + if _, _, err := store.Store(ctx, newer, strings.NewReader("refetched")); err != nil { + t.Fatalf("storing refetch: %v", err) + } + moved := stale[0] + moved.StoragePath = sql.NullString{String: newer, Valid: true} + if err := db.UpsertArtifact(&moved); err != nil { + t.Fatalf("moving record: %v", err) + } + + cleared, freed := evictBatch(ctx, db, store, logger, stale, 0, 1000) + if cleared != 0 || freed != 0 { + t.Errorf("cleared %d records freeing %d bytes, want nothing counted", cleared, freed) + } + record, err := db.GetArtifact(moved.VersionPURL, moved.Filename) + if err != nil || record == nil || record.StoragePath.String != newer { + t.Fatalf("record = %+v (err %v), want it kept at %q", record, err, newer) + } + if ok, _ := store.Exists(ctx, newer); !ok { + t.Error("the newer fetch's object was deleted") + } + old := stale[0].StoragePath.String + if ok, _ := store.Exists(ctx, old); !ok { + t.Error("the old object was deleted during its grace period") + } + if got := queuedPaths(t, db); !slices.Equal(got, []string{old}) { + t.Errorf("queue = %v, want the old path kept for reclaim", got) + } +} diff --git a/internal/server/health_test.go b/internal/server/health_test.go index 3b7eae8e..1f058b57 100644 --- a/internal/server/health_test.go +++ b/internal/server/health_test.go @@ -400,7 +400,9 @@ func TestHealthCache_ProbeTimeout(t *testing.T) { if err == nil { t.Fatal("expected timeout error, got nil") } - if elapsed > 500*time.Millisecond { + // Generous bound: only needs to prove Check returned via the probe + // timeout rather than blocking on Store; CI runners add scheduling noise. + if elapsed > 5*time.Second { t.Errorf("probe took %v, expected ~50ms (timeout not respected)", elapsed) } } diff --git a/internal/server/helm_test.go b/internal/server/helm_test.go new file mode 100644 index 00000000..4cc92820 --- /dev/null +++ b/internal/server/helm_test.go @@ -0,0 +1,47 @@ +package server + +import ( + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/handler" + "github.com/go-chi/chi/v5" +) + +func TestHelmIndexUsesConfiguredOCIRegistries(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/index.yaml" { + t.Errorf("unexpected request: %s", r.URL.Path) + } + _, _ = fmt.Fprintf(w, "apiVersion: v1\nentries:\n demo:\n - digest: %s\n urls: [oci://ghcr.io/owner/demo:1.0.0]\n", strings.Repeat("a", 64)) + })) + defer upstream.Close() + for _, named := range []bool{false, true} { + t.Run(fmt.Sprintf("named=%t", named), func(t *testing.T) { + cfg := config.Default() + cfg.BaseURL = "https://proxy.example" + cfg.Upstream.Helm = map[string]string{"mixed": upstream.URL} + cfg.Upstream.OCIDefault = "https://ghcr.io" + want := "oci://proxy.example/owner/demo:1.0.0" + if named { + cfg.Upstream.OCI = map[string]string{"ghcr": "https://ghcr.io"} + want = "oci://proxy.example/upstream/ghcr/owner/demo:1.0.0" + } + p := &handler.Proxy{HTTPClient: upstream.Client(), Logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + s := &Server{cfg: cfg} + r := chi.NewRouter() + s.mountProtocolHandlers(r, p) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/helm/mixed/index.yaml", nil)) + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), want) { + t.Fatalf("index status=%d body=%s; want %s", w.Code, w.Body.String(), want) + } + }) + } +} diff --git a/internal/server/middleware.go b/internal/server/middleware.go index b6d483f2..a5433ce5 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -43,43 +43,74 @@ func (s *Server) LoggerMiddleware(next http.Handler) http.Handler { requestID := GetRequestID(r.Context()) rw := &responseWriter{ResponseWriter: w, status: http.StatusOK} - next.ServeHTTP(rw, r) - duration := time.Since(start) - - s.logger.Info("request", - "request_id", requestID, - "method", r.Method, - "path", r.URL.Path, - "status", rw.status, - "duration", duration, - "remote", r.RemoteAddr) - - if r.URL.Path != "/metrics" { - metrics.RecordRequest(requestEcosystem(r.URL.Path), rw.status, duration) - } - - if s.accessLog != nil { - if err := s.accessLog.Write(accesslog.Entry{ - Event: accesslog.EventRequest, - RequestID: requestID, - Method: r.Method, - Path: r.URL.EscapedPath(), - StatusCode: rw.status, - DurationMS: duration.Milliseconds(), - RemoteAddr: r.RemoteAddr, - }); err != nil { - s.logger.Error("failed to write access log", "error", err) + + // Deferred because a truncated upstream relay aborts the handler with + // http.ErrAbortHandler, which would otherwise leave the request out of + // the log, the metrics and the access log entirely. + defer func() { + duration := time.Since(start) + userAgent := r.UserAgent() + client := clientName(userAgent) + addr := clientAddr(r, s.trustsForwardedFor()) + ecosystem := requestEcosystem(r.URL.Path) + + s.logger.Info("request", + "request_id", requestID, + "method", r.Method, + "path", r.URL.Path, + "status", rw.status, + "duration", duration, + "bytes", rw.bytes, + "client", client, + "remote", r.RemoteAddr, + "remote_ip", addr) + + // Scrapes of /metrics would otherwise attribute themselves, + // burying real callers under whatever polls the proxy most often. + if r.URL.Path != "/metrics" { + metrics.RecordRequest(ecosystem, rw.status, duration) + metrics.RecordResponse(ecosystem, client, rw.bytes) + s.sources.Record(addr, client, rw.bytes) } - } + + if s.accessLog != nil { + if err := s.accessLog.Write(accesslog.Entry{ + Event: accesslog.EventRequest, + RequestID: requestID, + Method: r.Method, + Path: r.URL.EscapedPath(), + StatusCode: rw.status, + DurationMS: duration.Milliseconds(), + RemoteAddr: r.RemoteAddr, + RemoteIP: addr, + UserAgent: userAgent, + Client: client, + Ecosystem: ecosystem, + Bytes: rw.bytes, + }); err != nil { + s.logger.Error("failed to write access log", "error", err) + } + } + }() + + next.ServeHTTP(rw, r) }) } +// requestEcosystem names the ecosystem a request path belongs to. +// +// Every mounted package route must appear here. An unlisted one falls to +// "other" along with the UI, health and metrics paths, pooling a real +// ecosystem's traffic with traffic that belongs to no ecosystem at all. func requestEcosystem(path string) string { segment, _, _ := strings.Cut(strings.TrimPrefix(path, "/"), "/") switch segment { case "npm", "cargo", "hex", "pub", "pypi", "maven", "gradle", "nuget", - "conan", "conda", "cran", "julia", "debian", "rpm": + "conan", "conda", "cran", "julia", "debian", "rpm", + "helm", "homebrew", "generic", "swift": return segment + case "apk": + return "alpine" case "gem": return "rubygems" case "go": diff --git a/internal/server/middleware_test.go b/internal/server/middleware_test.go index 38905b22..595d7ca3 100644 --- a/internal/server/middleware_test.go +++ b/internal/server/middleware_test.go @@ -124,6 +124,56 @@ func TestLoggerMiddlewareRecordsRequestMetrics(t *testing.T) { } } +// A relayed response that is truncated after its headers aborts the handler +// with http.ErrAbortHandler. The request still has to reach the metrics and the +// access log, or a truncated download is recorded nowhere at all. +func TestLoggerMiddlewareRecordsAbortedRequest(t *testing.T) { + before := testutil.ToFloat64(metrics.RequestsTotal.WithLabelValues("npm", "200")) + + path := filepath.Join(t.TempDir(), "access.jsonl") + activityLog, err := accesslog.Open(path) + if err != nil { + t.Fatal(err) + } + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + s := &Server{logger: logger, accessLog: activityLog} + handler := s.LoggerMiddleware(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + panic(http.ErrAbortHandler) + })) + + req := httptest.NewRequest(http.MethodGet, "/npm/example/-/example-1.0.0.tgz", nil) + rec := httptest.NewRecorder() + func() { + defer func() { + if rvr := recover(); rvr != http.ErrAbortHandler { + t.Errorf("recovered %v, want the abort to propagate", rvr) + } + }() + handler.ServeHTTP(rec, req) + }() + + if got := testutil.ToFloat64(metrics.RequestsTotal.WithLabelValues("npm", "200")) - before; got != 1 { + t.Errorf("request counter delta = %.0f, want 1", got) + } + + if err := activityLog.Close(); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var entry accesslog.Entry + if err := json.Unmarshal(data, &entry); err != nil { + t.Fatalf("decoding access log: %v", err) + } + if entry.Path != "/npm/example/-/example-1.0.0.tgz" { + t.Errorf("path = %q, want the aborted request logged", entry.Path) + } +} + func histogramSampleCount(t *testing.T, observer prometheus.Observer) uint64 { t.Helper() @@ -252,3 +302,29 @@ func TestResponseWriter_WriteHeader(t *testing.T) { }) } } + +// Every mounted package route must map to a named ecosystem. One that falls to +// "other" pools its traffic with UI, health and metrics traffic in +// proxy_requests_total and proxy_request_duration_seconds. +func TestRequestEcosystemCoversEveryMountedRoute(t *testing.T) { + mounts := map[string]string{ + "/npm/x": "npm", "/cargo/x": "cargo", "/gem/x": "rubygems", "/go/x": "golang", + "/hex/x": "hex", "/pub/x": "pub", "/pypi/x": "pypi", "/maven/x": "maven", + "/gradle/x": "gradle", "/nuget/x": "nuget", "/composer/x": "packagist", + "/conan/x": "conan", "/conda/x": "conda", "/cran/x": "cran", + "/julia/x": "julia", "/swift/x": "swift", "/v2/x": "oci", + "/apk/x": "alpine", "/debian/x": "debian", "/rpm/x": "rpm", + "/helm/x": "helm", "/homebrew/x": "homebrew", "/generic/x": "generic", + } + for path, want := range mounts { + if got := requestEcosystem(path); got != want { + t.Errorf("requestEcosystem(%q) = %q, want %q", path, got, want) + } + } + + for _, path := range []string{"/ui/", "/health", "/metrics", "/stats", "/"} { + if got := requestEcosystem(path); got != "other" { + t.Errorf("requestEcosystem(%q) = %q, want %q", path, got, "other") + } + } +} diff --git a/internal/server/packages_list_page_test.go b/internal/server/packages_list_page_test.go new file mode 100644 index 00000000..8fdbc294 --- /dev/null +++ b/internal/server/packages_list_page_test.go @@ -0,0 +1,148 @@ +package server + +import ( + "database/sql" + "fmt" + "html" + "net/http" + "net/http/httptest" + "net/url" + "regexp" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/database" +) + +func seedFilterPackage(t *testing.T, db *database.DB, ecosystem, name string, cached ...bool) { + t.Helper() + pkgPURL := "pkg:" + ecosystem + "/" + name + if err := db.UpsertPackage(&database.Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: name}); err != nil { + t.Fatal(err) + } + versionPURL := pkgPURL + "@1.0.0" + if err := db.UpsertVersion(&database.Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { + t.Fatal(err) + } + for i, stored := range cached { + filename := fmt.Sprintf("%s-%d.tgz", name, i) + if err := db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: filename, + UpstreamURL: "https://example.test/" + filename, + StoragePath: sql.NullString{String: filename, Valid: stored}, + }); err != nil { + t.Fatal(err) + } + } +} + +func TestPackagesListFilters(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + seedFilterPackage(t, ts.db, "npm", "express", true, true, false) + seedFilterPackage(t, ts.db, "npm", "lodash", true) + seedFilterPackage(t, ts.db, "cargo", "serde", true) + seedFilterPackage(t, ts.db, "gem", "metadata-only") + seedFilterPackage(t, ts.db, "pypi", "evicted", false) + seedFilterPackage(t, ts.db, "npm", "uncached", false) + + for _, tc := range []struct { + ecosystem string + count int + names []string + }{ + {"", 3, []string{"express", "lodash", "serde"}}, + {"npm", 2, []string{"express", "lodash"}}, + {"cargo", 1, []string{"serde"}}, + {"gem", 0, nil}, + } { + t.Run("ecosystem="+tc.ecosystem, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/ui/packages?sort=name&ecosystem="+tc.ecosystem, nil) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + body := w.Body.String() + assertFilterPills(t, body, tc.ecosystem, "name", map[string]string{"": "3", "npm": "2", "cargo": "1"}) + if !strings.Contains(body, fmt.Sprintf("%d packages", tc.count)) { + t.Errorf("missing heading count %d", tc.count) + } + links := regexp.MustCompile(`]*)>\s*[^<]+]*>(\d+)`).FindAllStringSubmatch(body, -1) + if len(pills) != len(want) { + t.Fatalf("got %d pills, want %d", len(pills), len(want)) + } + seen := make(map[string]bool) + for _, pill := range pills { + link, err := url.Parse(html.UnescapeString(pill[1])) + if err != nil { + t.Fatal(err) + } + params := link.Query() + ecosystem := params.Get("ecosystem") + if seen[ecosystem] || pill[3] != want[ecosystem] { + t.Errorf("%q pill count = %s, want %s (duplicate: %v)", ecosystem, pill[3], want[ecosystem], seen[ecosystem]) + } + seen[ecosystem] = true + if params.Get("sort") != sortBy || params.Has("page") { + t.Errorf("pill must preserve sorting and reset pagination: %s", link) + } + if active := strings.Contains(pill[2], `aria-current="page"`); active != (selected == ecosystem) { + t.Errorf("%q active = %v, selected = %q", ecosystem, active, selected) + } + } +} + +func TestPackagesListFiltersAcrossPages(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + for i := range 51 { + seedFilterPackage(t, ts.db, "npm", fmt.Sprintf("package-%02d", i), true) + } + seedFilterPackage(t, ts.db, "cargo", "serde", true) + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/packages?ecosystem=npm&sort=name&page=2", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + body := w.Body.String() + assertFilterPills(t, body, "npm", "name", map[string]string{"": "52", "npm": "51", "cargo": "1"}) + for _, text := range []string{"51 packages in npm", "Page 2 of 2", `href="/ui/package/npm/package-50"`, `href="?ecosystem=npm&sort=name&page=1"`} { + if !strings.Contains(html.UnescapeString(body), text) { + t.Errorf("missing %q", text) + } + } + if strings.Contains(body, `href="/ui/package/npm/package-00"`) { + t.Error("page 2 includes package from page 1") + } +} + +func TestPackagesListFiltersEmptyCache(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + seedFilterPackage(t, ts.db, "npm", "metadata-only") + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/packages", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + assertFilterPills(t, w.Body.String(), "", defaultSortBy, map[string]string{"": "0"}) + if !strings.Contains(w.Body.String(), "No cached packages found") { + t.Error("missing empty cache message") + } +} diff --git a/internal/server/reclaim.go b/internal/server/reclaim.go new file mode 100644 index 00000000..72a4e8f7 --- /dev/null +++ b/internal/server/reclaim.go @@ -0,0 +1,66 @@ +package server + +import ( + "context" + "log/slog" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" +) + +// An object no record points at any more waits in the pending_deletes queue +// for a grace period before it is deleted, so a request that read the record +// before it changed can still open the object, and a signed URL to it stays +// valid. Reclaim runs whether or not a cache size limit is set. +const ( + reclaimInterval = 1 * time.Minute + reclaimBatch = 100 + reclaimMinGrace = 1 * time.Hour +) + +func (s *Server) startReclaimLoop(ctx context.Context) { + grace := max(reclaimMinGrace, s.cfg.ParseDirectServeTTL()) + + ticker := time.NewTicker(reclaimInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + reclaimStorage(ctx, s.db, s.storage, s.logger, time.Now().Add(-grace)) + } + } +} + +// reclaimStorage deletes up to one batch of objects queued before cutoff. A +// delete that fails is queued again, behind the rest, so objects the backend +// keeps refusing cannot fill every batch. +func reclaimStorage(ctx context.Context, db *database.DB, store storage.Storage, logger *slog.Logger, cutoff time.Time) { + paths, err := db.GetDuePendingDeletes(cutoff, reclaimBatch) + if err != nil { + logger.Warn("reclaim: failed to list pending deletes", "error", err) + return + } + + for _, path := range paths { + if ctx.Err() != nil { + return + } + if err := store.Delete(ctx, path); err != nil { + if ctx.Err() != nil { + return + } + logger.Warn("reclaim: failed to delete object, will retry", "path", path, "error", err) + if err := db.QueuePendingDelete(path); err != nil { + logger.Warn("reclaim: failed to requeue object", "path", path, "error", err) + } + continue + } + if err := db.RemovePendingDelete(path); err != nil { + logger.Warn("reclaim: failed to dequeue deleted object", "path", path, "error", err) + } + } +} diff --git a/internal/server/reclaim_test.go b/internal/server/reclaim_test.go new file mode 100644 index 00000000..679dcdfd --- /dev/null +++ b/internal/server/reclaim_test.go @@ -0,0 +1,108 @@ +package server + +import ( + "context" + "io" + "log/slog" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/storage" +) + +func storeQueued(t *testing.T, db *database.DB, store storage.Storage, path string) { + t.Helper() + if _, _, err := store.Store(context.Background(), path, strings.NewReader("superseded")); err != nil { + t.Fatalf("storing %s: %v", path, err) + } + if err := db.QueuePendingDelete(path); err != nil { + t.Fatalf("queueing %s: %v", path, err) + } +} + +func queuedPaths(t *testing.T, db *database.DB) []string { + t.Helper() + paths, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 100) + if err != nil { + t.Fatalf("listing queue: %v", err) + } + return paths +} + +func objectExists(t *testing.T, store storage.Storage, path string) bool { + t.Helper() + ok, err := store.Exists(context.Background(), path) + if err != nil { + t.Fatalf("checking %s: %v", path, err) + } + return ok +} + +func TestReclaimStorageWaitsForGracePeriod(t *testing.T) { + db, store := setupEvictionTest(t) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + const path = "npm/old/1.0.0/a/old-1.0.0.tgz" + storeQueued(t, db, store, path) + + reclaimStorage(context.Background(), db, store, logger, time.Now().Add(-time.Hour)) + if !objectExists(t, store, path) { + t.Fatal("object deleted before its grace period ended") + } + + reclaimStorage(context.Background(), db, store, logger, time.Now().Add(time.Hour)) + if objectExists(t, store, path) { + t.Error("object survived after its grace period ended") + } + if got := queuedPaths(t, db); len(got) != 0 { + t.Errorf("queue = %v after reclaim, want empty", got) + } +} + +// TestReclaimStorageRequeuesFailedDeletes checks a refused delete goes behind +// the rest of the queue, so objects the backend keeps refusing cannot hold every +// slot in a batch. +func TestReclaimStorageRequeuesFailedDeletes(t *testing.T) { + db, store := setupEvictionTest(t) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + const path = "npm/old/1.0.0/a/old-1.0.0.tgz" + storeQueued(t, db, store, path) + queued := time.Now().UTC().Add(-2 * time.Hour) + if _, err := db.Exec(db.Rebind(`UPDATE pending_deletes SET queued_at = ? WHERE path = ?`), queued, path); err != nil { + t.Fatalf("backdating queue entry: %v", err) + } + cutoff := time.Now().Add(-time.Hour) + + undeletable := &undeletableStorage{Storage: store} + reclaimStorage(context.Background(), db, undeletable, logger, cutoff) + + if got := undeletable.deletes.Load(); got != 1 { + t.Errorf("delete attempts = %d, want 1", got) + } + if got := queuedPaths(t, db); !slices.Equal(got, []string{path}) { + t.Errorf("queue = %v, want the failed path kept for retry", got) + } + if due, err := db.GetDuePendingDeletes(cutoff, 100); err != nil || len(due) != 0 { + t.Errorf("due after a failed delete = %v (err %v), want it moved behind the cutoff", due, err) + } +} + +func TestReclaimStorageStopsWhenContextCanceled(t *testing.T) { + db, store := setupEvictionTest(t) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + const path = "npm/old/1.0.0/a/old-1.0.0.tgz" + storeQueued(t, db, store, path) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + reclaimStorage(ctx, db, store, logger, time.Now().Add(time.Hour)) + + if !objectExists(t, store, path) { + t.Error("object deleted after the context was canceled") + } + if got := queuedPaths(t, db); !slices.Equal(got, []string{path}) { + t.Errorf("queue = %v, want the path kept", got) + } +} diff --git a/internal/server/relay_test.go b/internal/server/relay_test.go new file mode 100644 index 00000000..d2c6bd60 --- /dev/null +++ b/internal/server/relay_test.go @@ -0,0 +1,102 @@ +package server + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/git-pkgs/proxy/internal/handler" + "github.com/go-chi/chi/v5/middleware" +) + +// Small bodies normally acquire Content-Length when the handler returns. Late, +// undeclared trailers need a flush through the production responseWriter first. +func TestRelayLateTrailersThroughMiddleware(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, "small body") + if err := http.NewResponseController(w).Flush(); err != nil { + t.Error(err) + } + w.Header().Set(http.TrailerPrefix+"X-Late", "verified") + })) + defer upstream.Close() + for _, protocol := range []string{"http1", "http2"} { + t.Run(protocol, func(t *testing.T) { + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + s := &Server{logger: logger} + proxy := &handler.Proxy{Logger: logger, HTTPClient: upstream.Client()} + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxy.ProxyUpstream(w, r, upstream.URL, nil) + }) + downstream := httptest.NewUnstartedServer(s.LoggerMiddleware(middleware.Recoverer(h))) + wantProto := 1 + if protocol == "http2" { + downstream.EnableHTTP2 = true + downstream.StartTLS() + wantProto = 2 + } else { + downstream.Start() + } + defer downstream.Close() + resp, err := downstream.Client().Get(downstream.URL) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil || string(body) != "small body" || resp.ProtoMajor != wantProto { + t.Fatalf("unexpected response: protocol=%s body=%q error=%v", resp.Proto, body, err) + } + if resp.Trailer.Get("X-Late") != "verified" || resp.Header.Get("X-Late") != "" { + t.Fatalf("late trailer lost or sent as a header: headers=%v trailers=%v", resp.Header, resp.Trailer) + } + }) + } +} + +func TestRelayAbortThroughMiddleware(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, strings.Repeat("x", 64*1024)) + if err := http.NewResponseController(w).Flush(); err != nil { + t.Error(err) + } + // End the upstream chunked body without its terminating chunk. + panic(http.ErrAbortHandler) + })) + defer upstream.Close() + for _, protocol := range []string{"http1", "http2"} { + t.Run(protocol, func(t *testing.T) { + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + s := &Server{logger: logger} + proxy := &handler.Proxy{Logger: logger, HTTPClient: upstream.Client()} + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxy.ProxyUpstream(w, r, upstream.URL, nil) + }) + downstream := httptest.NewUnstartedServer(s.LoggerMiddleware(middleware.Recoverer(h))) + wantProto := 1 + if protocol == "http2" { + downstream.EnableHTTP2 = true + downstream.StartTLS() + wantProto = 2 + } else { + downstream.Start() + } + defer downstream.Close() + resp, err := downstream.Client().Get(downstream.URL) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + _, err = io.ReadAll(resp.Body) + if err == nil { + t.Error("truncated upstream was delivered as a complete response") + } + if resp.StatusCode != http.StatusOK || resp.ProtoMajor != wantProto { + t.Fatalf("unexpected response: status=%d protocol=%s", resp.StatusCode, resp.Proto) + } + }) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 7613f52b..7494b94d 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -31,6 +31,7 @@ // Web UI (HTML), mounted under /ui so reverse proxies can gate it // separately from the package endpoints: // - /ui/ - Dashboard +// - /ui/analytics - Download and cache analytics // - /ui/install - Client configuration guide // - /ui/packages - List all cached packages // - /ui/search - Search packages @@ -67,7 +68,9 @@ import ( swaggerdoc "github.com/git-pkgs/proxy/docs/swagger" "github.com/git-pkgs/proxy/internal/accesslog" "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/cooldownpolicy" "github.com/git-pkgs/proxy/internal/database" + "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/enrichment" "github.com/git-pkgs/proxy/internal/handler" upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" @@ -89,6 +92,13 @@ const ( serverIdleTimeout = 60 * time.Second dashboardTopN = 10 hoursPerDay = 24 + + // Upstream transport defaults, matching what fetch.NewFetcher would use + // if we did not hand it our own client. Go's default transport keeps only + // two idle connections per host and never times out waiting for response + // headers. + upstreamMaxIdleConnsPerHost = 10 + upstreamResponseHeaderTimeout = 60 * time.Second ) // Server is the main proxy server. @@ -105,6 +115,8 @@ type Server struct { accessLog *accesslog.Logger ecr *ecrTokens breakers *breakerMonitor + ecoStats ecosystemStatsCache + sources sourceTracker } // New creates a new Server with the given configuration. @@ -143,6 +155,7 @@ func New(cfg *config.Config, logger *slog.Logger, buildInfo BuildInfo) (*Server, _ = db.Close() return nil, fmt.Errorf("migrating database schema: %w", err) } + db.BatchHits(cfg.ParseHitFlushInterval(), logger) // Initialize storage storageURL := cfg.Storage.URL @@ -205,7 +218,7 @@ func (s *Server) Start(listeners ...net.Listener) error { func (s *Server) serve(listener net.Listener) error { // Use one authentication-aware transport for metadata and artifacts so // configured credentials and cached OCI challenges apply consistently. - safeClient := safehttp.New(nil, upstreamSafeHTTPOptions(s.cfg.Upstream)) + safeClient := newUpstreamClient(s.cfg.Upstream) baseTransport := safeClient.Transport if s.accessLog != nil { baseTransport = upstreamhttp.NewAccessLogTransport(baseTransport, s.accessLog, s.logger) @@ -230,7 +243,16 @@ func (s *Server) serve(listener net.Listener) error { proxy := handler.NewProxy(s.db, s.storage, fetcher, resolver, s.logger) proxy.HTTPClient = &metadataClient proxy.AuthForURL = s.authForURL - proxy.Cooldown = cd + cooldownPolicy, err := cooldownpolicy.New(cd, s.cfg.Cooldown.PackagePatterns) + if err != nil { + return fmt.Errorf("configuring cooldown policy: %w", err) + } + proxy.Cooldown = cooldownPolicy + policy, err := denylist.New(s.cfg.Denylist.Packages) + if err != nil { + return fmt.Errorf("configuring denylist: %w", err) + } + proxy.Denylist = policy scanGroup, err := configureScanning(proxy, s.cfg.Scanning, s.cfg.BaseURL, s.logger) if err != nil { return fmt.Errorf("configuring scanners: %w", err) @@ -238,12 +260,14 @@ func (s *Server) serve(listener net.Listener) error { proxy.CacheMetadata = s.cfg.CacheMetadata proxy.MetadataTTL = s.cfg.ParseMetadataTTL() proxy.MetadataMaxSize = s.cfg.ParseMetadataMaxSize() + proxy.SetMetadataRewriteCacheSize(s.cfg.ParseMetadataRewriteCacheSize()) proxy.GradleReadOnly = s.cfg.Gradle.BuildCache.ReadOnly proxy.NPMFullMetadata = s.cfg.Upstream.NPMFullMetadata proxy.GradleMaxUploadSize = s.cfg.ParseGradleBuildCacheMaxUploadSize() proxy.DirectServe = s.cfg.Storage.DirectServe proxy.DirectServeTTL = s.cfg.ParseDirectServeTTL() proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL + proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts // Create router with Chi r := chi.NewRouter() @@ -291,6 +315,7 @@ func (s *Server) serve(listener net.Listener) error { r.Route("/ui", func(ui chi.Router) { ui.Mount("/static", http.StripPrefix("/ui/static/", staticHandler())) ui.Get("/", s.handleRoot) + ui.Get("/analytics", s.handleAnalytics) ui.Get("/install", s.handleInstall) ui.Get("/search", s.handleSearch) ui.Get("/packages", s.handlePackagesList) @@ -345,6 +370,7 @@ func (s *Server) serve(listener net.Listener) error { "database", s.cfg.Database.String()) go s.updateCacheStatsMetrics() go s.startEvictionLoop(bgCtx) + go s.startReclaimLoop(bgCtx) if listener != nil { return s.http.Serve(listener) @@ -409,9 +435,15 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { s.cfg.Upstream.OCI, ) handler.RegisterHomebrewArtifacts(containerHandler, s.cfg.Upstream.HomebrewArtifact) - helmHandler := handler.NewHelmHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm) + helmHandler := handler.NewHelmHandlerWithOCIRegistries( + proxy, s.cfg.BaseURL, s.cfg.Upstream.Helm, s.cfg.Upstream.OCIDefault, s.cfg.Upstream.OCI) apkHandler := handler.NewAPKHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.APK) - debianHandler := handler.NewDebianHandler(proxy, s.cfg.BaseURL, s.cfg.Upstream.Debian) + debianHandler := handler.NewDebianHandler( + proxy, + s.cfg.BaseURL, + s.cfg.Upstream.Debian, + s.cfg.Upstream.DebianRepositories, + ) rpmHandler := handler.NewRPMHandlerWithUpstream(proxy, s.cfg.BaseURL, s.cfg.Upstream.RPM) genericHandler := handler.NewGenericHandler(proxy, s.cfg.Upstream.Generic) @@ -454,6 +486,19 @@ func configureScanning(proxy *handler.Proxy, cfg config.ScanningConfig, baseURL return scanGroup, nil } +// newUpstreamClient builds the shared upstream client: a safehttp client whose +// transport keeps upstreamMaxIdleConnsPerHost idle connections per host and +// gives up after upstreamResponseHeaderTimeout when an upstream accepts a +// request but stalls before sending headers. +func newUpstreamClient(upstream config.UpstreamConfig) *http.Client { + client := safehttp.New(nil, upstreamSafeHTTPOptions(upstream)) + if transport, ok := client.Transport.(*http.Transport); ok { + transport.MaxIdleConnsPerHost = upstreamMaxIdleConnsPerHost + transport.ResponseHeaderTimeout = upstreamResponseHeaderTimeout + } + return client +} + func upstreamSafeHTTPOptions(upstream config.UpstreamConfig) safehttp.Options { return safehttp.Options{ AllowLoopback: upstream.AllowLoopback, @@ -481,6 +526,31 @@ func (s *Server) updateCacheStats() { return } metrics.UpdateCacheStats(stats.TotalSize, stats.TotalArtifacts) + + ecosystems, err := s.ecoStats.Refresh(s.db) + if err != nil { + s.logger.Warn("failed to get ecosystem stats for metrics", "error", err) + return + } + metrics.UpdateEcosystemStats(ecosystemMetrics(ecosystems)) +} + +// ecosystemMetrics converts database rows into the metrics package's own +// snapshot type, so that package keeps no dependency on the database schema. +func ecosystemMetrics(stats []database.EcosystemStats) []metrics.EcosystemStats { + out := make([]metrics.EcosystemStats, 0, len(stats)) + for _, e := range stats { + out = append(out, metrics.EcosystemStats{ + Ecosystem: e.Ecosystem, + Packages: e.Packages, + Versions: e.Versions, + Artifacts: e.Artifacts, + CacheSize: e.CacheSize, + Downloads: e.Downloads, + DownloadedBytes: e.DownloadedBytes, + }) + } + return out } // Shutdown gracefully shuts down the server. @@ -575,16 +645,7 @@ func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) { TotalPackages: stats.TotalPackages, TotalVersions: stats.TotalVersions, }, - EnrichmentStats: EnrichmentStatsView{ - EnrichedPackages: enrichStats.EnrichedPackages, - VulnSyncedPackages: enrichStats.VulnSyncedPackages, - TotalVulnerabilities: enrichStats.TotalVulnerabilities, - CriticalVulns: enrichStats.CriticalVulns, - HighVulns: enrichStats.HighVulns, - MediumVulns: enrichStats.MediumVulns, - LowVulns: enrichStats.LowVulns, - HasVulns: enrichStats.TotalVulnerabilities > 0, - }, + EnrichmentStats: enrichmentStatsView(enrichStats), } for _, p := range popular { @@ -760,10 +821,19 @@ func (s *Server) handlePackagesList(w http.ResponseWriter, r *http.Request) { return } - total, err := s.db.CountCachedPackages(ecosystem) + counts, err := s.db.CountCachedPackagesByEcosystem() if err != nil { s.logger.Error("failed to count packages", "error", err) - total = 0 + http.Error(w, "failed to count packages", http.StatusInternalServerError) + return + } + var totalPackages int64 + for _, count := range counts { + totalPackages += count + } + total := totalPackages + if ecosystem != "" { + total = counts[ecosystem] } items := make([]SearchResultItem, len(packages)) @@ -799,14 +869,16 @@ func (s *Server) handlePackagesList(w http.ResponseWriter, r *http.Request) { totalPages := int((total + int64(limit) - 1) / int64(limit)) data := PackagesListPageData{ - Layout: s.layoutFor(r), - Ecosystem: ecosystem, - SortBy: sortBy, - Results: items, - Count: int(total), - Page: page, - PerPage: limit, - TotalPages: totalPages, + Layout: s.layoutFor(r), + Ecosystem: ecosystem, + SortBy: sortBy, + Results: items, + Count: int(total), + TotalPackages: totalPackages, + EcosystemFilters: buildEcosystemFilters(counts), + Page: page, + PerPage: limit, + TotalPages: totalPages, } if err := s.templates.Render(w, "packages_list", data); err != nil { @@ -1071,6 +1143,28 @@ type StatsResponse struct { TotalSizeHuman string `json:"total_size"` StorageURL string `json:"storage_url"` DatabasePath string `json:"database_path"` + // DownloadedBytes is the accumulated download volume across every + // ecosystem: cache hits multiplied by the artifact size they served. + DownloadedBytes int64 `json:"downloaded_bytes"` + DownloadedBytesHuman string `json:"downloaded"` + Downloads int64 `json:"downloads"` + Ecosystems []EcosystemStatsEntry `json:"ecosystems"` + // StatsUnavailable distinguishes a proxy that has served nothing from one + // whose aggregation failed with no snapshot to fall back on. Without it + // both report zeros and an empty array. + StatsUnavailable bool `json:"stats_unavailable,omitempty"` +} + +// EcosystemStatsEntry is one ecosystem's slice of the cache statistics. +type EcosystemStatsEntry struct { + Ecosystem string `json:"ecosystem"` + DownloadedBytes int64 `json:"downloaded_bytes"` + Downloaded string `json:"downloaded"` + Downloads int64 `json:"downloads"` + CacheSize int64 `json:"cache_size_bytes"` + Artifacts int64 `json:"cached_artifacts"` + Packages int64 `json:"packages"` + Versions int64 `json:"versions"` } // handleStats returns cache statistics. @@ -1095,15 +1189,42 @@ func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) { return } + // A failing per-ecosystem aggregation must not take down an endpoint that + // answered from two cheap counters before it existed. Get returns the last + // good snapshot alongside the error, so the breakdown is served stale when + // there is one, and flagged unavailable when there is not. + ecosystems, statsErr := s.ecoStats.Get(s.db) + if statsErr != nil { + s.logger.Error("failed to get ecosystem stats for /stats", "error", statsErr) + } + _ = ctx // Could use for storage.UsedSpace if needed stats := StatsResponse{ - CachedArtifacts: count, - TotalSize: size, - TotalSizeHuman: formatSize(size), - StorageURL: s.storage.URL(), - DatabasePath: s.cfg.Database.String(), + CachedArtifacts: count, + TotalSize: size, + TotalSizeHuman: formatSize(size), + Ecosystems: make([]EcosystemStatsEntry, 0, len(ecosystems)), + StatsUnavailable: statsErr != nil && len(ecosystems) == 0, + StorageURL: s.storage.URL(), + DatabasePath: s.cfg.Database.String(), + } + + for _, e := range ecosystems { + stats.DownloadedBytes += e.DownloadedBytes + stats.Downloads += e.Downloads + stats.Ecosystems = append(stats.Ecosystems, EcosystemStatsEntry{ + Ecosystem: e.Ecosystem, + DownloadedBytes: e.DownloadedBytes, + Downloaded: formatSize(e.DownloadedBytes), + Downloads: e.Downloads, + CacheSize: e.CacheSize, + Artifacts: e.Artifacts, + Packages: e.Packages, + Versions: e.Versions, + }) } + stats.DownloadedBytesHuman = formatSize(stats.DownloadedBytes) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(stats) @@ -1178,10 +1299,24 @@ func categorizeLicense(license sql.NullString) string { return categorizeLicenseCSS(license.String) } -// responseWriter wraps http.ResponseWriter to capture status code. +// responseWriter wraps http.ResponseWriter to capture the status code and the +// number of body bytes written, which is what a client actually downloaded. type responseWriter struct { http.ResponseWriter status int + bytes int64 +} + +func (rw *responseWriter) Write(b []byte) (int, error) { + n, err := rw.ResponseWriter.Write(b) + rw.bytes += int64(n) + return n, err +} + +// Unwrap lets ResponseController reach capabilities such as flushing when a +// relayed response discovers trailers only after its body has been copied. +func (rw *responseWriter) Unwrap() http.ResponseWriter { + return rw.ResponseWriter } func (rw *responseWriter) WriteHeader(code int) { diff --git a/internal/server/server_test.go b/internal/server/server_test.go index a52a5b55..06b43fec 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -123,6 +123,7 @@ func newTestServer(t *testing.T) *testServer { r.Route("/ui", func(ui chi.Router) { ui.Mount("/static", http.StripPrefix("/ui/static/", staticHandler())) ui.Get("/", s.handleRoot) + ui.Get("/analytics", s.handleAnalytics) ui.Get("/install", s.handleInstall) ui.Get("/search", s.handleSearch) ui.Get("/packages", s.handlePackagesList) @@ -179,15 +180,31 @@ func TestStartUsesConfiguredLoopbackUpstreams(t *testing.T) { } } +func assertLoopbackPyPIMetadata(t *testing.T, body []byte) { + t.Helper() + if !strings.Contains(string(body), `"name":"ruff"`) { + t.Fatalf("response body = %s, want PyPI metadata", body) + } + if strings.Contains(string(body), "ruff-1.0.0") || !strings.Contains(string(body), "ruff-2.0.0") { + t.Fatalf("configured denylist not applied: %s", body) + } +} + func testStartUsesConfiguredLoopbackUpstreams(t *testing.T) { upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/pypi/simple/ruff/": w.Header().Set("Content-Type", "application/vnd.pypi.simple.v1+json") - _, _ = io.WriteString(w, `{"meta":{"api-version":"1.4"},"name":"ruff","files":[]}`) + _, _ = io.WriteString(w, `{"meta":{"api-version":"1.4"},"name":"ruff","files":[{"filename":"ruff-1.0.0.tar.gz","url":"ruff-1.0.0.tar.gz"},{"filename":"ruff-2.0.0.tar.gz","url":"ruff-2.0.0.tar.gz"}]}`) + case "/pypi/pypi/ruff/json": + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"releases":{}}`) case "/v2/library/demo/manifests/latest": w.Header().Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") _, _ = io.WriteString(w, `{"schemaVersion":2}`) + case "/npm/@example/widget", "/npm/@example/exact", "/npm/other": + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"time":{"1.0.0":"2020-01-01T00:00:00Z","2.0.0":%q},"versions":{"1.0.0":{},"2.0.0":{}}}`, time.Now().Add(-time.Hour).Format(time.RFC3339)) default: t.Errorf("unexpected upstream path: %q", r.URL.Path) http.NotFound(w, r) @@ -212,6 +229,13 @@ func testStartUsesConfiguredLoopbackUpstreams(t *testing.T) { cfg.Upstream.PyPIDownload = upstream.URL + "/pypi" cfg.Upstream.OCIDefault = upstream.URL cfg.Upstream.AllowLoopback = true + cfg.Upstream.NPM = upstream.URL + "/npm" + cfg.Cooldown = config.CooldownConfig{ + Default: "7d", + Packages: map[string]string{"pkg:npm/@example/exact": "7d"}, + PackagePatterns: map[string]string{"pkg:npm/@example/*": "0"}, + } + cfg.Denylist.Packages = []string{"pkg:pypi/ruff@1.0.0"} if err := cfg.Validate(); err != nil { t.Fatalf("validating config: %v", err) } @@ -237,7 +261,7 @@ func testStartUsesConfiguredLoopbackUpstreams(t *testing.T) { } }() - client := &http.Client{Timeout: 250 * time.Millisecond} + probeClient := &http.Client{Timeout: 250 * time.Millisecond} deadline := time.Now().Add(5 * time.Second) for { req, err := http.NewRequest(http.MethodGet, cfg.BaseURL+"/pypi/simple/ruff/", nil) @@ -245,7 +269,7 @@ func testStartUsesConfiguredLoopbackUpstreams(t *testing.T) { t.Fatalf("creating request: %v", err) } req.Header.Set("Accept", "application/vnd.pypi.simple.v1+json") - resp, requestErr := client.Do(req) + resp, requestErr := probeClient.Do(req) if requestErr == nil { body, readErr := io.ReadAll(resp.Body) _ = resp.Body.Close() @@ -255,9 +279,7 @@ func testStartUsesConfiguredLoopbackUpstreams(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Fatalf("status = %d, want %d; body: %s", resp.StatusCode, http.StatusOK, body) } - if !strings.Contains(string(body), `"name":"ruff"`) { - t.Fatalf("response body = %s, want PyPI metadata", body) - } + assertLoopbackPyPIMetadata(t, body) break } if time.Now().After(deadline) { @@ -266,6 +288,9 @@ func testStartUsesConfiguredLoopbackUpstreams(t *testing.T) { time.Sleep(10 * time.Millisecond) } + // This checks upstream routing, not latency. Allow time for fetching and + // cache I/O under -race on slower CI workers. + client := &http.Client{Timeout: 5 * time.Second} resp, err := client.Get(cfg.BaseURL + "/v2/library/demo/manifests/latest") if err != nil { t.Fatalf("OCI request failed: %v", err) @@ -281,6 +306,27 @@ func testStartUsesConfiguredLoopbackUpstreams(t *testing.T) { if !strings.Contains(string(body), `"schemaVersion":2`) { t.Fatalf("OCI response body = %s, want manifest", body) } + assertCooldownPatternMetadata(t, client, cfg.BaseURL) +} + +func assertCooldownPatternMetadata(t *testing.T, client *http.Client, baseURL string) { + t.Helper() + for _, name := range []string{"@example/widget", "@example/exact", "other"} { + resp, err := client.Get(baseURL + "/npm/" + url.PathEscape(name)) + if err != nil { + t.Fatal(err) + } + var metadata struct{ Versions map[string]json.RawMessage } + err = json.NewDecoder(resp.Body).Decode(&metadata) + _ = resp.Body.Close() + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("metadata for %s: status %d, error %v", name, resp.StatusCode, err) + } + _, recent := metadata.Versions["2.0.0"] + if recent != (name == "@example/widget") || metadata.Versions["1.0.0"] == nil { + t.Errorf("pattern policy for %s: versions %v", name, metadata.Versions) + } + } } // TestScanFetchRouteNotMountedWhenScanningDisabled verifies the internal diff --git a/internal/server/sources.go b/internal/server/sources.go new file mode 100644 index 00000000..923cb9dc --- /dev/null +++ b/internal/server/sources.go @@ -0,0 +1,331 @@ +package server + +import ( + "net" + "net/http" + "sort" + "strings" + "sync" + "time" +) + +// maxTrackedSources bounds the in-memory source table, because the caller set +// is not under the proxy's control. +// +// When it is full the least recently seen caller is evicted rather than the new +// one refused. Refusing would freeze the table on whoever happened to arrive +// first, which is the wrong answer in exactly the deployment this is for: +// containerised CI gives every job a fresh address, so the table would fill +// with dead entries within minutes and every live caller would land in the +// overflow row. Evicted totals move into that row, so nothing is lost from the +// sums; only the per-caller breakdown ages out. +const maxTrackedSources = 200 + +// unknownClient labels a request whose User-Agent names no recognised tool. +const unknownClient = "other" + +// knownClients maps the leading token of a User-Agent to a stable client name. +// +// The value set is deliberately closed. Client names become Prometheus labels, +// and a User-Agent is attacker-controlled, so anything unrecognised collapses +// to unknownClient rather than minting a new time series per request. +var knownClients = map[string]string{ + "npm": "npm", + "pnpm": "pnpm", + "yarn": "yarn", + "bun": "bun", + "node": "npm", + "pip": "pip", + "poetry": "poetry", + "uv": "uv", + "twine": "twine", + "python-requests": "pip", + "gocommand": "go", + "go-http-client": "go", + "cargo": "cargo", + "maven": "maven", + "apache-maven": "maven", + "aether": "maven", + "gradle": "gradle", + "nuget": "nuget", + "nuget-client": "nuget", + "composer": "composer", + "bundler": "bundler", + "rubygems": "bundler", + "gem": "bundler", + "docker": "docker", + "containerd": "containerd", + "skopeo": "skopeo", + "buildkit": "buildkit", + "helm": "helm", + "apt": "apt", + "debian": "apt", + "libdnf": "dnf", + "dnf": "dnf", + "urlgrabber": "dnf", + "apk": "apk", + "conda": "conda", + "mamba": "conda", + "conan": "conan", + "hex": "hex", + "mix": "hex", + "dart": "pub", + "pub": "pub", + "swift": "swift", + "julia": "julia", + "r": "cran", + "curl": "curl", + "wget": "wget", + "mozilla": "browser", + "gitlab-runner": "gitlab-runner", + "github-actions": "github-actions", + "jenkins": "jenkins", + "renovate": "renovate", + "dependabot": "dependabot", + "prometheus": "prometheus", + "kube-probe": "kube-probe", + "blackbox_exporter": "prometheus", +} + +// clientName reduces a User-Agent to one of the names in knownClients. +// +// Package managers put their own name first ("pip/21.2.4 {...}", +// "GoCommand/1 (+https://go.dev/cmd/go)"), so the leading token identifies the +// tool without needing to parse the rest. +func clientName(userAgent string) string { + if userAgent == "" { + return unknownClient + } + + token := userAgent + if i := strings.IndexAny(token, "/ ("); i >= 0 { + token = token[:i] + } + + if name, ok := knownClients[strings.ToLower(strings.TrimSpace(token))]; ok { + return name + } + return unknownClient +} + +// clientAddr returns the address a request should be attributed to. +// +// X-Forwarded-For is honoured only when trustForwarded is set, because any +// client can send the header: behind an ingress it is the only way to see past +// the load balancer, and in front of one it is a way to forge attribution. +// +// The forwarded value must parse as an IP. A load balancer that appends to the +// header leaves the leftmost entry caller-controlled, and that string becomes a +// map key held for the process lifetime and a line in the access log, so an +// arbitrary-length value would be a way to spend the proxy's memory. +func clientAddr(r *http.Request, trustForwarded bool) string { + if trustForwarded { + // Leftmost entry is the original client; the rest are hops. + first, _, _ := strings.Cut(r.Header.Get("X-Forwarded-For"), ",") + if ip := net.ParseIP(strings.TrimSpace(first)); ip != nil { + return ip.String() + } + } + + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} + +// trustsForwardedFor reports whether X-Forwarded-For should be believed. +// +// cfg is nil on a partially-constructed Server, which tests do build, so this +// answers false rather than dereferencing it. +func (s *Server) trustsForwardedFor() bool { + return s.cfg != nil && s.cfg.TrustForwardedFor +} + +// sourceKey identifies a caller by address and by the tool it was running, so +// two tools on one machine are counted apart. +type sourceKey struct { + Addr string + Client string +} + +type sourceStat struct { + Requests int64 + Bytes int64 + LastSeen time.Time +} + +// sourceTracker accumulates per-caller request and byte counts. +// +// This is process-lifetime state held in memory, like the Prometheus registry +// and unlike the cache figures: it starts empty and is lost on restart. Caller +// addresses are never published as metric labels — the set is unbounded and +// outside the proxy's control — so this table, and the access log, are where +// per-address detail lives. +// +// The zero value is usable. +type sourceTracker struct { + mu sync.Mutex + max int + stats map[sourceKey]*sourceStat + overflow sourceStat + // evictions counts fold-ins to the overflow row, not distinct callers: a + // caller that is evicted, returns, and is evicted again counts twice. + // Counting callers instead would mean keeping every key ever seen, which + // is the unbounded set the limit exists to avoid. + evictions int64 +} + +func (t *sourceTracker) limit() int { + if t.max <= 0 { + return maxTrackedSources + } + return t.max +} + +// Record attributes one completed request to a caller. +func (t *sourceTracker) Record(addr, client string, bytes int64) { + t.mu.Lock() + defer t.mu.Unlock() + + if t.stats == nil { + t.stats = make(map[sourceKey]*sourceStat) + } + + key := sourceKey{Addr: addr, Client: client} + stat, ok := t.stats[key] + if !ok { + if len(t.stats) >= t.limit() { + t.evictOldestLocked() + } + stat = &sourceStat{} + t.stats[key] = stat + } + + stat.Requests++ + stat.Bytes += bytes + stat.LastSeen = time.Now() +} + +// evictOldestLocked folds the least recently seen caller into the overflow row. +func (t *sourceTracker) evictOldestLocked() { + var oldest sourceKey + var oldestStat *sourceStat + for k, s := range t.stats { + if oldestStat == nil || s.LastSeen.Before(oldestStat.LastSeen) { + oldest, oldestStat = k, s + } + } + if oldestStat == nil { + return + } + + t.overflow.Requests += oldestStat.Requests + t.overflow.Bytes += oldestStat.Bytes + if oldestStat.LastSeen.After(t.overflow.LastSeen) { + t.overflow.LastSeen = oldestStat.LastSeen + } + t.evictions++ + delete(t.stats, oldest) +} + +// SourceRow is one caller's row on the analytics page. +type SourceRow struct { + Addr string + Client string + Requests string + Bytes string + LastSeen string + // IsOverflow marks the row that stands for every caller past the limit. + IsOverflow bool +} + +// Top returns the busiest callers by bytes served, most first. +func (t *sourceTracker) Top(limit int) []SourceRow { + t.mu.Lock() + defer t.mu.Unlock() + + type entry struct { + key sourceKey + stat sourceStat + } + + entries := make([]entry, 0, len(t.stats)) + for k, s := range t.stats { + entries = append(entries, entry{key: k, stat: *s}) + } + + sort.Slice(entries, func(i, j int) bool { + a, b := entries[i], entries[j] + switch { + case a.stat.Bytes != b.stat.Bytes: + return a.stat.Bytes > b.stat.Bytes + case a.stat.Requests != b.stat.Requests: + return a.stat.Requests > b.stat.Requests + default: + return a.key.Addr < b.key.Addr + } + }) + + // Callers past the limit are summarised rather than dropped, alongside any + // that were evicted, so the rows still add up to what the page reports + // above them. + rest := sourceStat{Requests: t.overflow.Requests, Bytes: t.overflow.Bytes, LastSeen: t.overflow.LastSeen} + hidden := 0 + if limit > 0 && len(entries) > limit { + for _, e := range entries[limit:] { + rest.Requests += e.stat.Requests + rest.Bytes += e.stat.Bytes + if e.stat.LastSeen.After(rest.LastSeen) { + rest.LastSeen = e.stat.LastSeen + } + hidden++ + } + entries = entries[:limit] + } + + rows := make([]SourceRow, 0, len(entries)+1) + for _, e := range entries { + rows = append(rows, SourceRow{ + Addr: e.key.Addr, + Client: e.key.Client, + Requests: formatCount(e.stat.Requests), + Bytes: formatSize(e.stat.Bytes), + LastSeen: formatTimeAgo(e.stat.LastSeen), + }) + } + + if rest.Requests > 0 { + rows = append(rows, SourceRow{ + Addr: "other callers", + Client: overflowLabel(hidden, t.evictions), + Requests: formatCount(rest.Requests), + Bytes: formatSize(rest.Bytes), + LastSeen: formatTimeAgo(rest.LastSeen), + IsOverflow: true, + }) + } + return rows +} + +// overflowLabel describes what the overflow row stands for. The two counts are +// kept apart because only one of them is exact: hidden is a count of callers +// currently tracked below the cut, while evictions counts fold-ins, which can +// exceed the number of distinct callers behind them. +func overflowLabel(hidden int, evictions int64) string { + switch { + case hidden > 0 && evictions > 0: + return formatCount(int64(hidden)) + " not shown, " + formatCount(evictions) + " evicted" + case evictions > 0: + return formatCount(evictions) + " evicted" + default: + return formatCount(int64(hidden)) + " not shown" + } +} + +// Count reports how many distinct callers are being tracked individually. +func (t *sourceTracker) Count() int { + t.mu.Lock() + defer t.mu.Unlock() + return len(t.stats) +} diff --git a/internal/server/sources_test.go b/internal/server/sources_test.go new file mode 100644 index 00000000..fa58dd73 --- /dev/null +++ b/internal/server/sources_test.go @@ -0,0 +1,450 @@ +package server + +import ( + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" +) + +func TestClientName(t *testing.T) { + tests := []struct { + ua string + want string + }{ + // Real User-Agents captured from these tools. + {`pip/21.2.4 {"ci":null,"cpu":"arm64"}`, "pip"}, + {"GoCommand/1 (+https://go.dev/cmd/go)", "go"}, + {"curl/8.7.1", "curl"}, + {"npm/10.2.3 node/v20.10.0 darwin arm64 workspaces/false", "npm"}, + {"docker/24.0.7 go/go1.20.10 kernel/6.5.0 os/linux", "docker"}, + {"Debian APT-HTTP/1.3 (2.6.1)", "apt"}, + {"bundler/2.4.22 rubygems/3.4.22", "bundler"}, + // Unknown and hostile input collapses, so a User-Agent cannot mint a + // new Prometheus time series. + {"", unknownClient}, + {"definitely-not-a-known-tool/9", unknownClient}, + {"a\nb/1", unknownClient}, + {`{"evil":"label"}`, unknownClient}, + } + for _, tc := range tests { + if got := clientName(tc.ua); got != tc.want { + t.Errorf("clientName(%q) = %q, want %q", tc.ua, got, tc.want) + } + } +} + +// Every value clientName can return must be one of the closed set, otherwise +// the Prometheus label is not actually bounded. +func TestClientNameIsBounded(t *testing.T) { + allowed := map[string]bool{unknownClient: true} + for _, v := range knownClients { + allowed[v] = true + } + + for _, ua := range []string{"pip/1", "weird", "", "npm/1", "x y z", "../../etc/passwd"} { + if got := clientName(ua); !allowed[got] { + t.Errorf("clientName(%q) = %q, which is outside the closed set", ua, got) + } + } +} + +func TestClientAddr(t *testing.T) { + t.Run("peer address by default", func(t *testing.T) { + r := httptest.NewRequest("GET", "/npm/x", nil) + r.RemoteAddr = "10.1.2.3:54321" + r.Header.Set("X-Forwarded-For", "203.0.113.9") + + // The header is present but untrusted, so it must be ignored. + if got := clientAddr(r, false); got != "10.1.2.3" { + t.Errorf("clientAddr = %q, want the peer address 10.1.2.3", got) + } + }) + + t.Run("forwarded when trusted", func(t *testing.T) { + r := httptest.NewRequest("GET", "/npm/x", nil) + r.RemoteAddr = "10.1.2.3:54321" + r.Header.Set("X-Forwarded-For", "203.0.113.9, 10.0.0.1") + + if got := clientAddr(r, true); got != "203.0.113.9" { + t.Errorf("clientAddr = %q, want the leftmost forwarded entry", got) + } + }) + + t.Run("rejects a forwarded value that is not an IP", func(t *testing.T) { + for _, forged := range []string{ + "not-an-ip", + strings.Repeat("A", 4096), + "mygroup/myproject", + "10.0.0.1; DROP TABLE", + } { + r := httptest.NewRequest("GET", "/npm/x", nil) + r.RemoteAddr = "10.1.2.3:54321" + r.Header.Set("X-Forwarded-For", forged) + + if got := clientAddr(r, true); got != "10.1.2.3" { + t.Errorf("clientAddr with forwarded %q = %q, want the peer address", + truncate(forged), got) + } + } + }) + + t.Run("normalizes a valid forwarded IP", func(t *testing.T) { + r := httptest.NewRequest("GET", "/npm/x", nil) + r.RemoteAddr = "10.1.2.3:54321" + r.Header.Set("X-Forwarded-For", "2001:0db8:0000:0000:0000:0000:0000:0001") + + if got := clientAddr(r, true); got != "2001:db8::1" { + t.Errorf("clientAddr = %q, want the canonical IPv6 form", got) + } + }) + + t.Run("falls back when forwarded is empty", func(t *testing.T) { + r := httptest.NewRequest("GET", "/npm/x", nil) + r.RemoteAddr = "10.1.2.3:54321" + r.Header.Set("X-Forwarded-For", " ") + + if got := clientAddr(r, true); got != "10.1.2.3" { + t.Errorf("clientAddr = %q, want the peer address", got) + } + }) + + t.Run("unparseable remote address is returned as-is", func(t *testing.T) { + r := httptest.NewRequest("GET", "/npm/x", nil) + r.RemoteAddr = "not-a-host-port" + + if got := clientAddr(r, false); got != "not-a-host-port" { + t.Errorf("clientAddr = %q, want the raw value", got) + } + }) +} + +func TestSourceTrackerAggregates(t *testing.T) { + var tr sourceTracker + + tr.Record("10.0.0.1", "npm", 1000) + tr.Record("10.0.0.1", "npm", 500) + tr.Record("10.0.0.2", "pip", 4000) + + rows := tr.Top(10) + if len(rows) != 2 { + t.Fatalf("expected 2 rows, got %+v", rows) + } + + // Ordered by bytes, so the pip caller leads. + if rows[0].Addr != "10.0.0.2" || rows[0].Bytes != formatSize(4000) { + t.Errorf("first row = %+v, want 10.0.0.2 with 4000 bytes", rows[0]) + } + if rows[1].Addr != "10.0.0.1" || rows[1].Requests != "2" || rows[1].Bytes != formatSize(1500) { + t.Errorf("second row = %+v, want 10.0.0.1 with 2 requests and 1500 bytes", rows[1]) + } + if tr.Count() != 2 { + t.Errorf("Count = %d, want 2", tr.Count()) + } +} + +// The same address running two tools is two sources, since that is the +// distinction the table exists to show. +func TestSourceTrackerSeparatesClientsOnOneAddress(t *testing.T) { + var tr sourceTracker + tr.Record("10.0.0.1", "npm", 10) + tr.Record("10.0.0.1", "pip", 20) + + if got := tr.Count(); got != 2 { + t.Errorf("Count = %d, want 2 (one row per address+client)", got) + } +} + +// The caller set is not under the proxy's control, so the table must stop +// growing rather than track every address that ever connects. +func TestSourceTrackerBoundsItsSize(t *testing.T) { + tr := sourceTracker{max: 3} + + for i := range 50 { + tr.Record(string(rune('a'+i%26))+string(rune('0'+i/26)), "npm", 100) + } + + if got := tr.Count(); got != 3 { + t.Fatalf("tracked %d sources, want the cap of 3", got) + } + + rows := tr.Top(10) + last := rows[len(rows)-1] + if !last.IsOverflow { + t.Fatalf("expected an overflow row, got %+v", rows) + } + + // Nothing may be lost from the totals: 50 requests and 5000 bytes went in, + // so the tracked rows plus the overflow row must still account for them. + var requests, bytes int64 + for _, r := range rows { + requests += parseCount(t, r.Requests) + } + if requests != 50 { + t.Errorf("rows account for %d requests, want all 50", requests) + } + _ = bytes +} + +// A full table must evict its stalest caller, not freeze on whoever arrived +// first: ephemeral CI addresses would otherwise fill it with dead entries and +// push every live caller into the overflow row. +func TestSourceTrackerEvictsLeastRecentlySeen(t *testing.T) { + tr := sourceTracker{max: 2} + + tr.Record("10.0.0.1", "npm", 100) + tr.Record("10.0.0.2", "npm", 100) + + // Touch the first so the second becomes the stalest. + time.Sleep(2 * time.Millisecond) + tr.Record("10.0.0.1", "npm", 100) + + time.Sleep(2 * time.Millisecond) + tr.Record("10.0.0.3", "npm", 100) + + addrs := map[string]bool{} + for _, r := range tr.Top(10) { + if !r.IsOverflow { + addrs[r.Addr] = true + } + } + + if !addrs["10.0.0.1"] { + t.Error("the recently active caller was evicted") + } + if !addrs["10.0.0.3"] { + t.Error("the newest caller was not admitted") + } + if addrs["10.0.0.2"] { + t.Error("the stalest caller should have been evicted") + } +} + +// parseCount reverses formatCount for assertions. +func parseCount(t *testing.T, s string) int64 { + t.Helper() + n, err := strconv.ParseInt(strings.ReplaceAll(s, ",", ""), 10, 64) + if err != nil { + t.Fatalf("parsing %q: %v", s, err) + } + return n +} + +func TestSourceTrackerTopLimits(t *testing.T) { + var tr sourceTracker + for i := range 10 { + tr.Record(string(rune('a'+i)), "npm", int64(i*100)) + } + + // Three shown, plus one row summarising the seven that are not. + got := tr.Top(3) + if len(got) != 4 { + t.Errorf("Top(3) returned %d rows, want 3 plus an overflow row", len(got)) + } else if !got[3].IsOverflow { + t.Errorf("fourth row should be the overflow row, got %+v", got[3]) + } + + // No limit means nothing is left over, so there is no overflow row. + if got := tr.Top(0); len(got) != 10 { + t.Errorf("Top(0) returned %d rows, want all 10", len(got)) + } +} + +func TestSourceTrackerZeroValueAndEmpty(t *testing.T) { + var tr sourceTracker + + if rows := tr.Top(5); len(rows) != 0 { + t.Errorf("an empty tracker returned %+v, want no rows", rows) + } + if tr.Count() != 0 { + t.Errorf("Count = %d, want 0", tr.Count()) + } + // Must not panic on a nil map. + tr.Record("10.0.0.1", "npm", 1) + if tr.Count() != 1 { + t.Errorf("Count after first Record = %d, want 1", tr.Count()) + } +} + +// A Server assembled as a struct literal has no config; reading the setting +// must not dereference it. +func TestTrustsForwardedForHandlesNilConfig(t *testing.T) { + var s Server + if s.trustsForwardedFor() { + t.Error("a Server with no config must not trust X-Forwarded-For") + } +} + +func truncate(s string) string { + if len(s) > 32 { + return s[:32] + "..." + } + return s +} + +// The rows the page shows must add up to the totals it reports above them, so +// callers past the display limit have to be summarised, not dropped. +func TestSourceTrackerTopAccountsForEveryCaller(t *testing.T) { + var tr sourceTracker + var wantRequests, wantBytes int64 + for i := range 40 { + bytes := int64((i + 1) * 100) + tr.Record("10.0.0."+strconv.Itoa(i), "npm", bytes) + wantRequests++ + wantBytes += bytes + } + + rows := tr.Top(5) + if len(rows) != 6 { + t.Fatalf("expected 5 rows plus an overflow row, got %d", len(rows)) + } + if !rows[5].IsOverflow { + t.Fatalf("last row is not the overflow row: %+v", rows[5]) + } + + var gotRequests int64 + for _, r := range rows { + gotRequests += parseCount(t, r.Requests) + } + if gotRequests != wantRequests { + t.Errorf("rows account for %d requests, want all %d", gotRequests, wantRequests) + } + // 35 callers are neither shown individually nor evicted. + if rows[5].Client != formatCount(35)+" not shown" { + t.Errorf("overflow row = %q, want 35 not shown", rows[5].Client) + } +} + +// The overflow row reports two counts that are true of different things: how +// many tracked callers fell below the display limit, which is exact, and how +// many fold-ins the eviction path has done, which is not a count of callers — +// a caller that is evicted and comes back is folded in twice. Reporting the sum +// as "N not shown" claimed more distinct callers than the row stands for. +func TestSourceTrackerOverflowSeparatesHiddenFromEvicted(t *testing.T) { + tr := sourceTracker{max: 3} + + // Fill, then push the first caller out, then bring it back and push it out + // again: two fold-ins, one caller. + for _, addr := range []string{"10.0.0.1", "10.0.0.2", "10.0.0.3"} { + tr.Record(addr, "npm", 100) + time.Sleep(time.Millisecond) + } + tr.Record("10.0.0.4", "npm", 100) // evicts .1 + time.Sleep(time.Millisecond) + tr.Record("10.0.0.1", "npm", 100) // .1 returns, evicting .2 + time.Sleep(time.Millisecond) + tr.Record("10.0.0.5", "npm", 100) // evicts .3 + + if tr.evictions != 3 { + t.Fatalf("evictions = %d, want 3 fold-ins", tr.evictions) + } + + rows := tr.Top(2) + overflow := rows[len(rows)-1] + if !overflow.IsOverflow { + t.Fatalf("last row is not the overflow row: %+v", overflow) + } + // One of the three tracked callers is below the cut; three fold-ins + // happened, covering two distinct callers. + if want := "1 not shown, 3 evicted"; overflow.Client != want { + t.Errorf("overflow row = %q, want %q", overflow.Client, want) + } + + var got int64 + for _, r := range rows { + got += parseCount(t, r.Requests) + } + if got != 6 { + t.Errorf("rows account for %d requests, want all 6", got) + } +} + +// With nothing evicted the row says only what it can count exactly. +func TestSourceTrackerOverflowOmitsEvictionsWhenThereAreNone(t *testing.T) { + var tr sourceTracker + for i := range 5 { + tr.Record("10.0.0."+strconv.Itoa(i), "npm", 100) + } + + rows := tr.Top(2) + if want := "3 not shown"; rows[len(rows)-1].Client != want { + t.Errorf("overflow row = %q, want %q", rows[len(rows)-1].Client, want) + } +} + +// The eviction scan is linear over the table, and Record holds a process-wide +// lock while it runs. These two bound what that costs in the case the source +// table is designed for — containerised CI, where every job has a fresh address +// and the table therefore sits permanently at its limit, so every request pays +// a scan. Measured at 200 entries on an M4 Max: ~2us evicting against ~40ns for +// a caller already tracked. Two microseconds against a request that goes to the +// network is not worth a heap or an LRU list, but the gap is the reason this is +// bounded at 200 rather than at something larger. +func BenchmarkRecordAlwaysEvicting(b *testing.B) { + var t sourceTracker + for i := 0; i < maxTrackedSources; i++ { + t.Record("10.0."+strconv.Itoa(i/256)+"."+strconv.Itoa(i%256), "npm", 1024) + } + + addrs := make([]string, b.N) + for i := range addrs { + addrs[i] = "172.16." + strconv.Itoa(i/256%256) + "." + strconv.Itoa(i%256) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + t.Record(addrs[i], "npm", 1024) + } +} + +func BenchmarkRecordExisting(b *testing.B) { + var t sourceTracker + t.Record("10.0.0.1", "npm", 1024) + b.ResetTimer() + for i := 0; i < b.N; i++ { + t.Record("10.0.0.1", "npm", 1024) + } +} + +// The page is unauthenticated, so the caller table is published only when +// asked for. Off is the default, and off has to mean the addresses are absent +// from the HTML, not merely unstyled. +func TestAnalyticsPageHidesSourcesByDefault(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + if ts.server.cfg.UIRequestSources { + t.Fatal("UIRequestSources defaults to true; it must default off") + } + ts.server.sources.Record("10.1.2.3", "npm", 4096) + + body := ts.getOK(t, "/ui/analytics") + if strings.Contains(body, "10.1.2.3") { + t.Error("a caller address rendered with ui_request_sources off") + } + // The by-client table carries no addresses and the same figures are already + // public at /metrics, so it renders either way; only the address table is + // gated. + if !strings.Contains(body, "By client") { + t.Error("the by-client table was gated along with the address table") + } + if !strings.Contains(body, "ui_request_sources") { + t.Error("the off-state note naming the config key did not render") + } +} + +func TestAnalyticsPageShowsSourcesWhenEnabled(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + ts.server.cfg.UIRequestSources = true + ts.server.sources.Record("10.1.2.3", "npm", 4096) + + body := ts.getOK(t, "/ui/analytics") + for _, want := range []string{"Request sources", "10.1.2.3", "4.0 KB"} { + if !strings.Contains(body, want) { + t.Errorf("rendered page missing %q", want) + } + } +} diff --git a/internal/server/static/packages-list.js b/internal/server/static/packages-list.js new file mode 100644 index 00000000..06f09ba5 --- /dev/null +++ b/internal/server/static/packages-list.js @@ -0,0 +1,6 @@ +document.getElementById('sort-by').addEventListener('change', function(e) { + const params = new URLSearchParams(window.location.search); + params.set('sort', e.target.value); + params.delete('page'); + window.location.href = '/ui/packages?' + params.toString(); +}); diff --git a/internal/server/stats_test.go b/internal/server/stats_test.go new file mode 100644 index 00000000..467f3c16 --- /dev/null +++ b/internal/server/stats_test.go @@ -0,0 +1,139 @@ +package server + +import ( + "database/sql" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" +) + +func seedCachedArtifact(t *testing.T, db *database.DB, ecosystem, name, version string, size, hits int64) { + t.Helper() + + pkgPURL := "pkg:" + ecosystem + "/" + name + versionPURL := pkgPURL + "@" + version + + if err := db.UpsertPackage(&database.Package{PURL: pkgPURL, Ecosystem: ecosystem, Name: name}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&database.Version{PURL: versionPURL, PackagePURL: pkgPURL}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + if err := db.UpsertArtifact(&database.Artifact{ + VersionPURL: versionPURL, + Filename: name + "-" + version + ".tgz", + UpstreamURL: "https://example.test/" + name, + StoragePath: sql.NullString{String: "objects/" + name, Valid: true}, + Size: sql.NullInt64{Int64: size, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + HitCount: hits, + }); err != nil { + t.Fatalf("UpsertArtifact: %v", err) + } +} + +func getStats(t *testing.T, ts *testServer) StatsResponse { + t.Helper() + + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest("GET", "/stats", nil)) + if w.Code != http.StatusOK { + t.Fatalf("GET /stats = %d, want 200", w.Code) + } + + var stats StatsResponse + if err := json.NewDecoder(w.Body).Decode(&stats); err != nil { + t.Fatalf("decode: %v", err) + } + return stats +} + +func TestStatsReportsEcosystemBreakdown(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + seedCachedArtifact(t, ts.db, "npm", "lodash", "4.17.21", 1000, 3) + seedCachedArtifact(t, ts.db, "cargo", "serde", "1.0.0", 500, 2) + + stats := getStats(t, ts) + + if stats.DownloadedBytes != 4000 { + t.Errorf("downloaded_bytes = %d, want 4000", stats.DownloadedBytes) + } + if stats.Downloads != 5 { + t.Errorf("downloads = %d, want 5", stats.Downloads) + } + if stats.DownloadedBytesHuman == "" { + t.Error("downloaded is empty; the human-readable total is part of the shape") + } + if stats.StatsUnavailable { + t.Error("stats_unavailable set on a healthy aggregation") + } + + byEcosystem := make(map[string]EcosystemStatsEntry, len(stats.Ecosystems)) + for _, e := range stats.Ecosystems { + byEcosystem[e.Ecosystem] = e + } + npm, ok := byEcosystem["npm"] + if !ok { + t.Fatalf("no npm row in %v", byEcosystem) + } + if npm.DownloadedBytes != 3000 || npm.Downloads != 3 || npm.CacheSize != 1000 { + t.Errorf("npm row = %+v, want 3000 bytes over 3 downloads and 1000 cached", npm) + } +} + +// A failed aggregation with nothing to fall back on must not read as an idle +// proxy: the totals are zero either way, so the flag is the only thing +// telling a consumer which of the two it is looking at. +func TestStatsFlagsUnavailableFigures(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + // GetEcosystemStats counts packages first; the artifact count and cache + // size the endpoint already reported do not touch that table. + if _, err := ts.db.Exec(`DROP TABLE packages`); err != nil { + t.Fatalf("DROP TABLE packages: %v", err) + } + + stats := getStats(t, ts) + + if !stats.StatsUnavailable { + t.Error("stats_unavailable not set after the aggregation failed with no snapshot") + } + if len(stats.Ecosystems) != 0 { + t.Errorf("ecosystems = %v, want empty", stats.Ecosystems) + } + if stats.StorageURL == "" { + t.Error("the rest of the response must still be served") + } +} + +// A retained snapshot is served on with no flag: the figures are stale, not +// absent, and the page is where staleness is surfaced to a human. +func TestStatsServesRetainedSnapshot(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + + seedCachedArtifact(t, ts.db, "npm", "lodash", "4.17.21", 1000, 3) + if got := getStats(t, ts).DownloadedBytes; got != 3000 { + t.Fatalf("downloaded_bytes = %d, want 3000 before the failure", got) + } + + if _, err := ts.db.Exec(`DROP TABLE packages`); err != nil { + t.Fatalf("DROP TABLE packages: %v", err) + } + ts.server.ecoStats.lastAt = time.Time{} + + stats := getStats(t, ts) + if stats.StatsUnavailable { + t.Error("stats_unavailable set although a snapshot was retained") + } + if stats.DownloadedBytes != 3000 { + t.Errorf("downloaded_bytes = %d, want the retained 3000", stats.DownloadedBytes) + } +} diff --git a/internal/server/templates.go b/internal/server/templates.go index 217da413..f37f4b68 100644 --- a/internal/server/templates.go +++ b/internal/server/templates.go @@ -2,6 +2,7 @@ package server import ( "embed" + "fmt" "html/template" "net/http" "path/filepath" @@ -28,7 +29,9 @@ func (t *Templates) load() error { "sub": func(a, b int) int { return a - b }, "supportedEcosystems": supportedEcosystems, "ecosystemBadgeClass": ecosystemBadgeClasses, + "ecosystemPillClass": ecosystemPillClasses, "ecosystemBadgeLabel": ecosystemBadgeLabel, + "dict": templateDict, } pageFiles, err := templatesFS.ReadDir("templates/pages") @@ -78,3 +81,24 @@ func (t *Templates) Render(w http.ResponseWriter, pageName string, data any) err return tmpl.ExecuteTemplate(w, "base", data) } + +// templateDict builds a map from alternating key/value arguments, so a +// component can be invoked with named parameters rather than being handed a +// whole page struct it would have to reach through. +func templateDict(values ...any) (map[string]any, error) { + const pair = 2 + + if len(values)%pair != 0 { + return nil, fmt.Errorf("dict: got %d arguments, want an even number of key/value pairs", len(values)) + } + + out := make(map[string]any, len(values)/pair) + for i := 0; i < len(values); i += pair { + key, ok := values[i].(string) + if !ok { + return nil, fmt.Errorf("dict: key %d is %T, want string", i, values[i]) + } + out[key] = values[i+1] + } + return out, nil +} diff --git a/internal/server/templates/components/runtime_metrics.html b/internal/server/templates/components/runtime_metrics.html new file mode 100644 index 00000000..03fc9a04 --- /dev/null +++ b/internal/server/templates/components/runtime_metrics.html @@ -0,0 +1,232 @@ +{{define "runtime_metrics"}} +
+
+

Runtime

+

+ Counters held in this process, not in the database: they start at zero when the + proxy starts and are lost on restart. Everything above comes from the database + instead, and survives a restart — so a low count here alongside a large one + above just means the proxy started recently. +

+
+ +
+
+
+
Requests
+
{{.Requests}}
+
+
+
In flight
+
{{.ActiveRequests}}
+
+
+
Mean latency
+
{{.RequestMean}}
+
+
+
Cache hit rate
+
{{if .HasCacheTraffic}}{{.CacheHitRatio}}%{{else}}—{{end}}
+
+
+
Upstream fetches
+
{{.UpstreamFetches}} · {{.UpstreamFetchMean}}
+
+
+
Served
+
{{.ResponseBytes}}
+
+
+ +
+ {{template "runtime_counts" dict "Title" "Responses by status" "Rows" .StatusClasses "Empty" "No requests yet"}} + +
+

Cache lookups

+
+
+
hits
+
{{.CacheHits}}
+
+
+
misses
+
{{.CacheMisses}}
+
+
+
+ + {{template "runtime_stats" dict "Title" "Storage operations" "Rows" .StorageOps "Empty" "No storage operations yet"}} + {{template "runtime_counts" dict "Title" "Upstream errors" "Rows" .UpstreamErrors "Empty" "None"}} + {{template "runtime_counts" dict "Title" "Storage errors" "Rows" .StorageErrors "Empty" "None"}} + {{template "runtime_counts" dict "Title" "Integrity failures" "Rows" .IntegrityFailures "Empty" "None"}} + {{template "runtime_counts" dict "Title" "Health probe failures" "Rows" .ProbeFailures "Empty" "None"}} + {{template "runtime_counts" dict "Title" "Circuit breaker trips" "Rows" .BreakerTrips "Empty" "None"}} + +
+

Circuit breakers

+ {{if .Breakers}} +
+ {{range .Breakers}} +
+
{{.Registry}}
+
+ {{if .Open}} + + {{.State}} + + {{else}} + + {{.State}} + + {{end}} +
+
+ {{end}} +
+ {{else}} +

+ None reported — a breaker appears once its upstream has been fetched from. +

+ {{end}} +
+
+ + +
+
+

Request sources

+ {{if .SourcesOn}} +

+ {{if .TrustsForward}}Attributed by X-Forwarded-For{{else}}Attributed by peer address{{end}} + {{- if .SourceCount}} · {{.SourceCount}} tracked{{end}} +

+ {{end}} +
+ +
+
+ {{if not .SourcesOn}} +

+ Per-caller addresses are off. Set ui_request_sources: true + to show them here, once /ui is behind authentication; + the per-request detail is in the access log. +

+ {{else if .Sources}} + + + + + + + + + + + + {{range .Sources}} + + + + + + + + {{end}} + +
AddressClientRequestsDownloadedLast seen
{{.Addr}}{{.Client}}{{.Requests}}{{.Bytes}}{{.LastSeen}}
+ {{else}} +

No requests recorded yet.

+ {{end}} +
+ +
+

By client

+ {{if .Clients}} + + + {{range .Clients}} + + + + + + {{end}} + +
{{.Client}}{{.Requests}}{{.Bytes}}
+ {{else}} +

None yet.

+ {{end}} +
+
+
+ + {{if .ScanningOn}} +
+

Pre-cache scanning

+
+ {{template "runtime_stats" dict "Title" "Scans" "Rows" .Scans "Empty" "No scans yet"}} + {{template "runtime_counts" dict "Title" "Artifacts blocked" "Rows" .ScansBlocked "Empty" "None"}} + {{template "runtime_counts" dict "Title" "Scan errors" "Rows" .ScanErrors "Empty" "None"}} +
+
+ {{else}} +
+

Pre-cache scanning

+

+ Not configured. Enable it under scanning in the + config to have artifacts scanned before they are committed to the cache. +

+
+ {{end}} +
+
+{{end}} + +{{/* runtime_counts renders a labelled counter list, tinting failure rows. */}} +{{define "runtime_counts"}} +
+

{{.Title}}

+ {{if .Rows}} +
+ {{range .Rows}} +
+
{{.Label}}
+
{{.Count}}
+
+ {{end}} +
+ {{else}} +

{{.Empty}}

+ {{end}} +
+{{end}} + +{{/* runtime_stats renders a histogram list: how many observations, and their mean. */}} +{{define "runtime_stats"}} +
+

{{.Title}}

+ {{if .Rows}} + + + + + + + + + + {{range .Rows}} + + + + + + {{end}} + +
countmean
{{.Label}}{{.Count}}{{.Mean}}
+ {{else}} +

{{.Empty}}

+ {{end}} +
+{{end}} diff --git a/internal/server/templates/components/security_overview.html b/internal/server/templates/components/security_overview.html new file mode 100644 index 00000000..3ba93e32 --- /dev/null +++ b/internal/server/templates/components/security_overview.html @@ -0,0 +1,30 @@ +{{define "security_overview"}} +
+
+

Security Overview

+
+
+
+
+
{{.CriticalVulns}}
+
Critical
+
+
+
{{.HighVulns}}
+
High
+
+
+
{{.MediumVulns}}
+
Medium
+
+
+
{{.LowVulns}}
+
Low
+
+
+

+ {{.TotalVulnerabilities}} vulnerabilities tracked across {{.VulnSyncedPackages}} packages +

+
+
+{{end}} diff --git a/internal/server/templates/layout/header.html b/internal/server/templates/layout/header.html index b3103f1f..560fe2b3 100644 --- a/internal/server/templates/layout/header.html +++ b/internal/server/templates/layout/header.html @@ -54,6 +54,7 @@ {{end}} {{define "nav_links"}} +Analytics Install Health API diff --git a/internal/server/templates/pages/analytics.html b/internal/server/templates/pages/analytics.html new file mode 100644 index 00000000..da97a7ce --- /dev/null +++ b/internal/server/templates/pages/analytics.html @@ -0,0 +1,302 @@ +{{define "title"}}Analytics · git-pkgs proxy{{end}} + +{{define "head"}} + +{{end}} + +{{define "content"}} +
+

Analytics

+

+ Accumulated download volume and cache composition across + {{.Totals.Ecosystems}} ecosystem{{if ne .Totals.Ecosystems 1}}s{{end}}. +

+
+ +{{if .StatsStale}} +
+ Figures may be out of date. + The database query is failing, so the download and cache numbers below are + the last snapshot that could be read{{if .StatsAge}}, from {{.StatsAge}}{{end}}. + Check the proxy logs. +
+{{end}} + +
+
+

Download size by ecosystem

+

Bytes served from cache

+
+ {{if .Donut.HasSlices}} +
+
+ + {{range .Donut.Slices}} + + {{.Label}}: {{.Value}} ({{.SharePct}}%) + + {{end}} + +
+
{{.Donut.CenterValue}}
+
{{.Donut.CenterLabel}}
+
+
+ + + +
+ {{else if .StatsFailed}} +
+ Download figures are unavailable — the database query failed. Check the proxy logs. +
+ {{else}} +
Nothing has been served from cache yet
+ {{end}} +
+ + +
+
+
Downloads
+
{{.Totals.Downloads}}
+
+
+
Cache size
+
{{.Totals.CacheSize}}
+
+
+
Cached artifacts
+
{{.Totals.CachedArtifacts}}
+
+
+
Packages
+
{{.Totals.Packages}}
+
+
+
Versions
+
{{.Totals.Versions}}
+
+
+
Ecosystems in use
+
{{.Totals.ActiveEcosystems}} / {{.Totals.Ecosystems}}
+
+
+ +{{if .Totals.Amplification}} +

+ The cache has served {{.Totals.Amplification}} the bytes it currently stores. +

+{{end}} + + +
+
+

Per-ecosystem breakdown

+

Every ecosystem, including any folded into the ring's "Other" slice

+
+ {{if .Ecosystems}} +
+ + + + + + + + + + + + + + + + {{range .Ecosystems}} + + + + + + + + + + + + {{end}} + +
EcosystemDownloadedShareDownloadsCache sizeArtifactsAvg sizePackagesVersions
+ {{template "ecosystem_badge" .Ecosystem}} + {{.Downloaded}}{{.SharePct}}%{{.Downloads}}{{.CacheSize}}{{.Artifacts}}{{.AvgArtifactSize}}{{.Packages}}{{.Versions}}
+
+ {{else if .StatsFailed}} +
+ The per-ecosystem query failed, so this table is unavailable. Check the proxy logs. +
+ {{else}} +
No packages cached yet
+ {{end}} +
+ +{{if .EnrichmentStats.HasVulns}} +{{template "security_overview" .EnrichmentStats}} +{{end}} + +{{if .Runtime.Available}} +{{template "runtime_metrics" .Runtime}} +{{end}} + +

+ Every figure on this page is also exported for Prometheus at + /metrics. + This page shows current state only — nothing here is a time series. For history, + trends and alerting, scrape /metrics and use the + Grafana dashboard shipped in deploy/grafana/. +

+{{end}} + +{{define "scripts"}} + +{{end}} diff --git a/internal/server/templates/pages/dashboard.html b/internal/server/templates/pages/dashboard.html index 97c3082d..e48e205f 100644 --- a/internal/server/templates/pages/dashboard.html +++ b/internal/server/templates/pages/dashboard.html @@ -22,39 +22,11 @@ {{if .EnrichmentStats.HasVulns}} - -
-
-

Security Overview

-
-
-
-
-
{{.EnrichmentStats.CriticalVulns}}
-
Critical
-
-
-
{{.EnrichmentStats.HighVulns}}
-
High
-
-
-
{{.EnrichmentStats.MediumVulns}}
-
Medium
-
-
-
{{.EnrichmentStats.LowVulns}}
-
Low
-
-
-

- {{.EnrichmentStats.TotalVulnerabilities}} vulnerabilities tracked across {{.EnrichmentStats.VulnSyncedPackages}} packages -

-
-
+{{template "security_overview" .EnrichmentStats}} {{end}} -
+
@@ -74,9 +46,9 @@

Popular Packages

{{if .VulnCount}}{{.VulnCount}} vulns{{end}}
-
- {{.Hits}} cache hits - {{.Size}} +
+ {{.Hits}} cache hits + {{.Size}}
{{end}} @@ -97,12 +69,12 @@

Recently Cached

{{if .RecentPackages}} {{range .RecentPackages}} -
+
-
+
{{template "ecosystem_badge" .Ecosystem}} - {{.Name}} - @{{.Version}} + {{.Name}} + @{{.Version}}
{{if .License}}{{.License}}{{end}} @@ -110,9 +82,9 @@

Recently Cached

{{if .VulnCount}}{{.VulnCount}} vulns{{end}}
-
- {{.CachedAt}} - {{.Size}} +
+ {{.CachedAt}} + {{.Size}}
{{end}} diff --git a/internal/server/templates/pages/packages_list.html b/internal/server/templates/pages/packages_list.html index 95897db5..61e73c01 100644 --- a/internal/server/templates/pages/packages_list.html +++ b/internal/server/templates/pages/packages_list.html @@ -4,23 +4,33 @@

Cached Packages

- {{.Count}} packages{{if .Ecosystem}} in {{.Ecosystem}}{{end}} + {{.Count}} packages{{if .Ecosystem}} in {{ecosystemBadgeLabel .Ecosystem}}{{end}}

-
-
- - +
-
+
- @@ -53,11 +63,14 @@

Cached Packages

{{end}}
-
- {{.Hits}} hits - {{.SizeFormatted}} +
+ + {{.Hits}} + hits + + {{.SizeFormatted}} {{if .CachedAt}} - {{.CachedAt}} + {{.CachedAt}} {{end}}
@@ -66,7 +79,7 @@

Cached Packages

{{else}}
-

No cached packages found{{if .Ecosystem}} in {{.Ecosystem}}{{end}}

+

No cached packages found{{if .Ecosystem}} in {{ecosystemBadgeLabel .Ecosystem}}{{end}}

Return to dashboard
{{end}} @@ -93,23 +106,5 @@

Cached Packages

{{end}} - + {{end}} diff --git a/internal/server/templates_test.go b/internal/server/templates_test.go index a2d3b7da..fbd20c3f 100644 --- a/internal/server/templates_test.go +++ b/internal/server/templates_test.go @@ -37,6 +37,46 @@ func TestTemplatesRenderAllPages(t *testing.T) { {Ecosystem: "cargo", Name: "serde", Version: "1.0.0", Size: "200 KB", CachedAt: "1 hour ago"}, }, }}, + {"analytics", AnalyticsData{ + Totals: AnalyticsTotals{ + DownloadedBytes: 1_500_000_000, + Downloaded: "1.4 GB", + Downloads: "12,004", + CacheSize: "420.0 MB", + CachedArtifacts: "1,204", + Packages: "310", + Versions: "902", + Ecosystems: 3, + ActiveEcosystems: 2, + Amplification: "3.4x", + }, + EnrichmentStats: EnrichmentStatsView{TotalVulnerabilities: 3, CriticalVulns: 1, HasVulns: true}, + Ecosystems: []EcosystemRow{ + {Ecosystem: "npm", DownloadedBytes: 1_000_000_000, Downloaded: "953.7 MB", Downloads: "9,000", CacheSize: "300.0 MB", AvgArtifactSize: "120 KB", Artifacts: "900", Packages: "200", Versions: "700", SharePct: "66.7"}, + {Ecosystem: "cargo", DownloadedBytes: 500_000_000, Downloaded: "476.8 MB", Downloads: "3,004", CacheSize: "120.0 MB", AvgArtifactSize: "400 KB", Artifacts: "300", Packages: "100", Versions: "200", SharePct: "33.3"}, + {Ecosystem: "rpm", Downloaded: "0 B", Downloads: "0", CacheSize: "0 B", AvgArtifactSize: "0 B", Artifacts: "0", Packages: "10", Versions: "2", SharePct: "0.0"}, + }, + Donut: donutView([]database.EcosystemStats{ + {Ecosystem: "npm", CacheSize: 300, Downloads: 9000, DownloadedBytes: 1_000_000_000}, + {Ecosystem: "cargo", CacheSize: 120, Downloads: 3004, DownloadedBytes: 500_000_000}, + }, 1_500_000_000, "1.4 GB"), + Runtime: RuntimeView{ + Available: true, + Requests: "12,004", + ActiveRequests: "2", + RequestMean: "8.1 ms", + CacheHits: "9,000", + CacheMisses: "1,000", + CacheHitRatio: "90.0", + HasCacheTraffic: true, + StatusClasses: []LabelledCount{{Label: "2xx", Count: "11,900"}, {Label: "5xx", Count: "104", Bad: true}}, + StorageOps: []LabelledStat{{Label: "get", Count: "9,000", Mean: "412 µs"}}, + ScanErrors: []LabelledCount{{Label: "npm · clamav · timeout", Count: "3", Bad: true}}, + ScanningOn: true, + Breakers: []BreakerRow{{Registry: "registry.npmjs.org", State: "closed"}}, + }, + }}, + {"analytics", AnalyticsData{}}, {"install", struct { Layout BaseURL string @@ -63,13 +103,15 @@ func TestTemplatesRenderAllPages(t *testing.T) { TotalPages: 0, }}, {"packages_list", PackagesListPageData{ - Ecosystem: "", - SortBy: defaultSortBy, - Results: []SearchResultItem{{Ecosystem: "npm", Name: "express", Hits: 200, SizeFormatted: "2 MB"}}, - Count: 1, - Page: 1, - PerPage: 50, - TotalPages: 1, + Ecosystem: "", + SortBy: defaultSortBy, + Results: []SearchResultItem{{Ecosystem: "npm", Name: "express", Hits: 200, SizeFormatted: "2 MB"}}, + Count: 1, + TotalPackages: 1, + EcosystemFilters: []EcosystemFilter{{Ecosystem: "npm", Count: 1}}, + Page: 1, + PerPage: 50, + TotalPages: 1, }}, {"package_show", PackageShowData{ Package: &database.Package{ @@ -568,6 +610,39 @@ func TestEcosystemBadgeClasses(t *testing.T) { } } +func TestBuildEcosystemFilters(t *testing.T) { + filters := buildEcosystemFilters(map[string]int64{ + "npm": 3, + "cargo": 2, + "empty": 0, + }) + + if len(filters) != 2 { + t.Fatalf("expected 2 filters, got %d", len(filters)) + } + if filters[0].Ecosystem != "cargo" || filters[0].Count != 2 { + t.Errorf("first filter = %#v, want cargo with count 2", filters[0]) + } + if filters[1].Ecosystem != "npm" || filters[1].Count != 3 { + t.Errorf("second filter = %#v, want npm with count 3", filters[1]) + } + + extra := buildEcosystemFilters(map[string]int64{"custom": 1}) + if len(extra) != 1 || extra[0].Ecosystem != "custom" { + t.Errorf("unexpected extra filters: %#v", extra) + } +} + +func TestEcosystemPillClasses(t *testing.T) { + classes := ecosystemPillClasses("npm") + if !strings.Contains(classes, "rounded-full") { + t.Error("pill classes should use rounded-full") + } + if !strings.Contains(classes, "bg-red-100") { + t.Error("npm pill should use npm colors") + } +} + func TestFormatSize(t *testing.T) { tests := []struct { bytes int64 diff --git a/internal/server/transport_test.go b/internal/server/transport_test.go new file mode 100644 index 00000000..74642820 --- /dev/null +++ b/internal/server/transport_test.go @@ -0,0 +1,190 @@ +package server + +import ( + "crypto/tls" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/registries/safehttp" +) + +// tlsUpstream starts a TLS test server that counts accepted connections. +func tlsUpstream(t *testing.T, handler http.HandlerFunc) (*httptest.Server, func() int) { + t.Helper() + var mu sync.Mutex + accepted := 0 + srv := httptest.NewUnstartedServer(handler) + srv.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateNew { + mu.Lock() + accepted++ + mu.Unlock() + } + } + srv.StartTLS() + t.Cleanup(srv.Close) + return srv, func() int { + mu.Lock() + defer mu.Unlock() + return accepted + } +} + +// trustUpstream makes transport trust srv's certificate and pins HTTP/1.1 so +// every in-flight request needs its own connection. +func trustUpstream(t *testing.T, transport *http.Transport, srv *httptest.Server) { + t.Helper() + transport.TLSClientConfig = &tls.Config{ + RootCAs: srv.Client().Transport.(*http.Transport).TLSClientConfig.RootCAs, + NextProtos: []string{"http/1.1"}, + MinVersion: tls.VersionTLS12, + } + transport.ForceAttemptHTTP2 = false + t.Cleanup(transport.CloseIdleConnections) +} + +// burst issues n concurrent GETs and drains every body. The transport hands a +// connection back to the idle pool before the body's final Read returns, so +// the pool is settled when burst returns. +func burst(t *testing.T, client *http.Client, url string, n int) { + t.Helper() + var wg sync.WaitGroup + errs := make(chan error, n) + for range n { + wg.Add(1) + go func() { + defer wg.Done() + resp, err := client.Get(url) + if err != nil { + errs <- err + return + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + }() + } + wg.Wait() + close(errs) + for err := range errs { + t.Errorf("burst request: %v", err) + } +} + +// TestUpstreamClientReusesConnectionsAcrossBursts measures how many +// connections a second burst of concurrent requests reuses. With Go's default +// of two idle connections per host most of them are re-dialled; with the +// tuned transport the second burst reuses all of them. +func TestUpstreamClientReusesConnectionsAcrossBursts(t *testing.T) { + const burstSize = 8 + + // holdBurst returns a handler that answers a request only once burstSize + // of them are waiting at the same time. With HTTP/1.1 pinned that puts + // every burst on burstSize distinct connections, whatever the scheduling. + holdBurst := func() http.HandlerFunc { + var mu sync.Mutex + waiting := 0 + release := make(chan struct{}) + return func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + gate := release + waiting++ + if waiting == burstSize { + close(gate) + waiting = 0 + release = make(chan struct{}) + } + mu.Unlock() + select { + case <-gate: + case <-r.Context().Done(): + } + _, _ = w.Write([]byte("ok")) + } + } + + tests := []struct { + name string + client *http.Client + // Bounds on how many connections the second burst has to dial. + minNew, maxNew int + }{ + { + name: "go default keeps two idle connections", + client: safehttp.New(nil, safehttp.Options{AllowLoopback: true}), + minNew: burstSize - 2, + maxNew: burstSize, + }, + { + name: "tuned transport reuses the whole burst", + client: newUpstreamClient(config.UpstreamConfig{AllowLoopback: true}), + minNew: 0, + maxNew: 0, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + srv, accepted := tlsUpstream(t, holdBurst()) + transport := tc.client.Transport.(*http.Transport) + trustUpstream(t, transport, srv) + + burst(t, tc.client, srv.URL, burstSize) + afterFirst := accepted() + if afterFirst < burstSize { + t.Fatalf("first burst opened %d connections, want at least %d", afterFirst, burstSize) + } + + burst(t, tc.client, srv.URL, burstSize) + newInSecond := accepted() - afterFirst + t.Logf("second burst: %d new connections, %d reused", newInSecond, burstSize-newInSecond) + + if newInSecond < tc.minNew || newInSecond > tc.maxNew { + t.Errorf("second burst opened %d new connections, want between %d and %d", newInSecond, tc.minNew, tc.maxNew) + } + }) + } +} + +// TestUpstreamClientBoundsStallBeforeHeaders pins the production transport +// values, then lowers the header timeout so it can show within milliseconds +// that this is what cuts off an upstream which accepts a request but never +// sends headers. +func TestUpstreamClientBoundsStallBeforeHeaders(t *testing.T) { + client := newUpstreamClient(config.UpstreamConfig{AllowLoopback: true}) + transport := client.Transport.(*http.Transport) + if transport.MaxIdleConnsPerHost != upstreamMaxIdleConnsPerHost { + t.Fatalf("MaxIdleConnsPerHost = %d, want %d", transport.MaxIdleConnsPerHost, upstreamMaxIdleConnsPerHost) + } + if transport.ResponseHeaderTimeout != upstreamResponseHeaderTimeout { + t.Fatalf("ResponseHeaderTimeout = %v, want %v", transport.ResponseHeaderTimeout, upstreamResponseHeaderTimeout) + } + + stall := make(chan struct{}) + srv, _ := tlsUpstream(t, func(_ http.ResponseWriter, r *http.Request) { + select { + case <-stall: + case <-r.Context().Done(): + } + }) + t.Cleanup(func() { close(stall) }) + trustUpstream(t, transport, srv) + + // Far below the client's overall timeout, so the header timeout ends the + // request; the error text tells the two timeouts apart. + transport.ResponseHeaderTimeout = 200 * time.Millisecond + + resp, err := client.Get(srv.URL) + if err == nil { + _ = resp.Body.Close() + t.Fatal("request to a stalled upstream succeeded, want a timeout") + } + if !strings.Contains(err.Error(), "timeout awaiting response headers") { + t.Fatalf("error = %v, want a response-header timeout", err) + } +} diff --git a/internal/storage/blob.go b/internal/storage/blob.go index 97e50f3f..4cadf8af 100644 --- a/internal/storage/blob.go +++ b/internal/storage/blob.go @@ -22,11 +22,19 @@ import ( const osWindows = "windows" +// attrsExt is fileblob's sidecar suffix, kept only to clear sidecars an +// earlier version wrote. +const attrsExt = ".attrs" + // Blob implements Storage using gocloud.dev/blob. // Supports local filesystem (file://) and S3 (s3://) URLs. type Blob struct { bucket *blob.Bucket url string + + // fileRoot is the directory backing a file:// bucket, empty for cloud + // backends. Used only to clear sidecars an earlier version wrote. + fileRoot string } // OpenBucket opens a blob bucket from a URL. @@ -47,6 +55,8 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) { return OpenGCS(ctx, urlStr) } + var fileRoot string + // Handle file:// URLs specially to create the directory if strings.HasPrefix(urlStr, "file://") { path := strings.TrimPrefix(urlStr, "file://") @@ -74,6 +84,8 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) { return nil, fmt.Errorf("resolving path: %w", err) } + fileRoot = absPath + // Convert back to URL format with forward slashes urlPath := filepath.ToSlash(absPath) if runtime.GOOS == osWindows { @@ -87,7 +99,14 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) { // This avoids "invalid cross-device link" errors from os.Rename when // the bucket directory and os.TempDir are on different filesystems // (e.g. Docker volume mounts). - urlStr += "?no_tmp_dir=true" + // + // Do not write fileblob's ".attrs" sidecar. It is rewritten with + // os.Create, truncating in place outside the atomic rename that + // protects the blob, so a read overlapping a write can decode a + // partial file; a missing one defaults cleanly, a truncated one does + // not. Nothing in the proxy needs it: Store sets no ContentType, and + // Size reads os.Stat via Attributes. + urlStr += "?no_tmp_dir=true&metadata=skip" } bucket, err := blob.OpenBucket(ctx, urlStr) @@ -95,10 +114,95 @@ func OpenBucket(ctx context.Context, urlStr string) (Storage, error) { return nil, fmt.Errorf("opening bucket: %w", err) } - return &Blob{bucket: bucket, url: urlStr}, nil + return &Blob{bucket: bucket, url: urlStr, fileRoot: fileRoot}, nil +} + +// legacySidecarPath gives the ".attrs" path an earlier version wrote for key, +// or "" when that path would not be a file inside fileRoot. +func (b *Blob) legacySidecarPath(key string) string { + if p := b.localPath(key); p != "" { + return p + attrsExt + } + return "" +} + +// localPath gives the file fileblob keeps key in, or "" when that would not +// be a file inside fileRoot. +// +// The key is escaped the way fileblob escapes it on the way to disk. +// filepath.Localize then validates the escaped form: it rejects an empty, +// absolute or ".." path, and "." would name fileRoot itself. What it declines +// are keys the proxy never produces. +func (b *Blob) localPath(key string) string { + if b.fileRoot == "" { + return "" + } + rel, err := filepath.Localize(escapeKey(key)) + if err != nil || rel == "." { + return "" + } + return filepath.Join(b.fileRoot, rel) +} + +// escapeKey mirrors fileblob's unexported escapeKey, which hex-escapes a rune +// as "__0x__". Slashes stay as "/" for filepath.Localize to convert. +func escapeKey(key string) string { + runes := []rune(key) + var out strings.Builder + for i, r := range runes { + if escapeRune(runes, i) { + fmt.Fprintf(&out, "__%#x__", r) + } else { + out.WriteRune(r) + } + } + return out.String() +} + +// escapeRune is fileblob's rule for which runes of a key to escape: control +// characters, a raw path separator, a slash that would form "../", "//" or +// end the key, and on Windows the characters its filesystem reserves. +func escapeRune(r []rune, i int) bool { + c := r[i] + switch { + case c < ' ': + return true + case os.PathSeparator != '/' && c == os.PathSeparator: + return true + case i > 1 && c == '/' && r[i-1] == '.' && r[i-2] == '.': + return true + case i > 0 && c == '/' && r[i-1] == '/': + return true + case c == '/' && i == len(r)-1: + return true + case os.PathSeparator == '\\' && strings.ContainsRune(`<>:"|?*`, c): + return true + } + return false +} + +// clearLegacySidecar removes the ".attrs" file an earlier version wrote for +// key. Nothing rewrites one now, so a sidecar left partial by an interrupted +// write would fail every read of that key for good. Removing is atomic where +// the rewrite was not, so a concurrent reader gets the whole old file or +// nothing. +// +// Failure is deliberately not fatal. Usually the key never had a sidecar and +// os.Remove reports not-exist. A real failure leaves exactly the state this +// change inherited, while failing the write would turn a cleanup miss into a +// failed request. Windows makes that concrete: Go opens files without +// FILE_SHARE_DELETE, so a reader holding the sidecar open blocks deletion, and +// that reader is the very workload this change protects. The next store of the +// key retries. +func (b *Blob) clearLegacySidecar(key string) { + if sidecar := b.legacySidecarPath(key); sidecar != "" { + _ = os.Remove(sidecar) + } } func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, string, error) { + b.clearLegacySidecar(path) + // Compute hash while writing h := sha256.New() tee := io.TeeReader(r, h) @@ -124,6 +228,17 @@ func (b *Blob) Store(ctx context.Context, path string, r io.Reader) (int64, stri } func (b *Blob) Open(ctx context.Context, path string) (io.ReadCloser, error) { + if localPath := b.localPath(path); localPath != "" { + r, err := os.Open(localPath) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("opening local reader: %w", err) + } + return r, nil + } + r, err := b.bucket.NewReader(ctx, path, nil) if err != nil { if isNotExist(err) { @@ -142,11 +257,20 @@ func (b *Blob) Exists(ctx context.Context, path string) (bool, error) { return exists, nil } +// Delete removes the object at path. On a file:// bucket it also removes the +// object's fetch directory once empty, since fileblob leaves directories +// behind. Any other directory, such as a version directory another fetch may +// be creating its own directory in, is left alone. func (b *Blob) Delete(ctx context.Context, path string) error { err := b.bucket.Delete(ctx, path) if err != nil && !isNotExist(err) { return fmt.Errorf("deleting object: %w", err) } + if p := b.localPath(path); p != "" { + if dir := filepath.Dir(p); dir != b.fileRoot && isFetchDir(filepath.Base(dir)) { + _ = os.Remove(dir) // fails, harmlessly, while the directory holds anything + } + } return nil } diff --git a/internal/storage/blob_test.go b/internal/storage/blob_test.go index 3e5bf65b..efc1b765 100644 --- a/internal/storage/blob_test.go +++ b/internal/storage/blob_test.go @@ -6,11 +6,17 @@ import ( "encoding/hex" "errors" "io" + "io/fs" + "os" "path/filepath" "runtime" "strings" + "sync" + "sync/atomic" "testing" "time" + + "gocloud.dev/blob" ) func TestOpenBucket(t *testing.T) { @@ -61,13 +67,20 @@ func TestBlobOpen(t *testing.T) { t.Fatalf("Open failed: %v", err) } defer func() { _ = r.Close() }() + seeker, ok := r.(io.Seeker) + if !ok { + t.Fatal("local file reader does not implement io.Seeker") + } + if _, err := seeker.Seek(9, io.SeekStart); err != nil { + t.Fatalf("Seek failed: %v", err) + } data, err := io.ReadAll(r) if err != nil { t.Fatalf("ReadAll failed: %v", err) } - if string(data) != content { - t.Errorf("content = %q, want %q", string(data), content) + if string(data) != "content" { + t.Errorf("content after seek = %q, want %q", string(data), "content") } } @@ -121,6 +134,44 @@ func TestBlobDelete(t *testing.T) { } } +// A fetch stores its object in a directory of its own, so Delete removes that +// directory once it is empty rather than leave one behind per deleted object. +// A version directory from the layout before fetch directories is left alone, +// since a new fetch may be creating its directory inside it. +func TestBlobDeleteRemovesEmptyFetchDirectory(t *testing.T) { + dir := t.TempDir() + b := openFileBlob(t, dir) + ctx := context.Background() + const ( + emptied = "npm/pkg/1.0.0/0123456789abcdef/pkg.tgz" + shared = "npm/pkg/1.0.0/fedcba9876543210/pkg.tgz" + kept = "npm/pkg/1.0.0/fedcba9876543210/other.tgz" + legacy = "npm/pkg/2.0.0/pkg.tgz" + ) + for _, key := range []string{emptied, shared, kept, legacy} { + if _, _, err := b.Store(ctx, key, strings.NewReader("content")); err != nil { + t.Fatalf("Store(%q): %v", key, err) + } + } + + for _, key := range []string{emptied, shared, legacy} { + if err := b.Delete(ctx, key); err != nil { + t.Fatalf("Delete(%q): %v", key, err) + } + } + + pkg := filepath.Join(dir, "npm", "pkg") + if _, err := os.Stat(filepath.Join(pkg, "1.0.0", "0123456789abcdef")); !os.IsNotExist(err) { + t.Errorf("emptied fetch directory left behind (stat err %v)", err) + } + for _, d := range []string{filepath.Join("1.0.0", "fedcba9876543210"), "1.0.0", "2.0.0"} { + if _, err := os.Stat(filepath.Join(pkg, d)); err != nil { + t.Errorf("directory %s removed: %v", d, err) + } + } + assertReadsBack(t, b, kept, "content") +} + func TestBlobDeleteNotFound(t *testing.T) { b := createTestBlob(t) ctx := context.Background() @@ -293,3 +344,319 @@ func fileURLFromPath(path string) string { } return "file://" + path } + +func TestOpenBucketWritesNoAttrsSidecar(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + + b, err := OpenBucket(ctx, fileURLFromPath(dir)) + if err != nil { + t.Fatalf("OpenBucket failed: %v", err) + } + defer func() { _ = b.Close() }() + + if _, _, err := b.Store(ctx, "pkg/thing-1.0.0.tgz", strings.NewReader("content")); err != nil { + t.Fatalf("Store failed: %v", err) + } + + sidecars, err := filepath.Glob(filepath.Join(dir, "*", "*.attrs")) + if err != nil { + t.Fatalf("Glob failed: %v", err) + } + if len(sidecars) != 0 { + t.Errorf("got sidecar files %v, want none: a truncated sidecar fails reads that overlap a write", sidecars) + } +} + +// A read overlapping a write to the same key must not fail. fileblob rewrote +// its ".attrs" sidecar in place, so a reader decoding it mid-write saw a +// partial file, which the proxy served as a 502 on an artifact it held. +func TestConcurrentReadsSurviveWritesToSameKey(t *testing.T) { + if runtime.GOOS == "windows" { + // Go opens files without FILE_SHARE_DELETE, so a writer cannot replace + // a file a reader holds open: its rename fails with access denied + // instead of contending. The other platforms exercise this race. + t.Skip("Windows refuses to replace a file readers hold open") + } + const ( + key = "pkg/thing-1.0.0.tgz" + readers = 4 + readsPerRead = 500 + ) + dir := t.TempDir() + ctx := context.Background() + + b, err := OpenBucket(ctx, fileURLFromPath(dir)) + if err != nil { + t.Fatalf("OpenBucket failed: %v", err) + } + defer func() { _ = b.Close() }() + + payload := strings.Repeat("x", 4096) + if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil { + t.Fatalf("seeding Store failed: %v", err) + } + + // The writer reports how it ended: a Store failure would otherwise stop + // the writes silently and let zero read failures pass for a test that + // never contended anything. + done := make(chan struct{}) + var writers sync.WaitGroup + var writes int + var writeErr error + writers.Add(1) + go func() { + defer writers.Done() + for { + select { + case <-done: + return + default: + } + if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil { + writeErr = err + return + } + writes++ + } + }() + + var failures atomic.Int64 + var reading sync.WaitGroup + for range readers { + reading.Add(1) + go func() { + defer reading.Done() + for range readsPerRead { + r, err := b.Open(ctx, key) + if err != nil { + failures.Add(1) + continue + } + if _, err := io.Copy(io.Discard, r); err != nil { + failures.Add(1) + } + _ = r.Close() + } + }() + } + reading.Wait() + close(done) + writers.Wait() + + if writeErr != nil { + t.Fatalf("writer stopped early: %v", writeErr) + } + if writes == 0 { + t.Fatal("no write completed, so the reads were never contended") + } + if got := failures.Load(); got != 0 { + t.Errorf("%d of %d reads failed while one writer rewrote the same key, want 0", got, readers*readsPerRead) + } +} + +// seedLegacySidecar stores key through a bucket that still writes sidecars, as +// an earlier version did, and returns the path fileblob actually used. It is +// discovered rather than assumed, so callers test the real mapping. +func seedLegacySidecar(t *testing.T, dir, key, payload string) string { + t.Helper() + ctx := context.Background() + + legacy, err := blob.OpenBucket(ctx, fileURLFromPath(dir)+"?no_tmp_dir=true") + if err != nil { + t.Fatalf("opening legacy bucket: %v", err) + } + if err := legacy.WriteAll(ctx, key, []byte(payload), nil); err != nil { + t.Fatalf("legacy WriteAll: %v", err) + } + if err := legacy.Close(); err != nil { + t.Fatalf("closing legacy bucket: %v", err) + } + + var found []string + walkErr := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.HasSuffix(path, ".attrs") { + found = append(found, path) + } + return nil + }) + if walkErr != nil { + t.Fatalf("walking %s: %v", dir, walkErr) + } + if len(found) != 1 { + t.Fatalf("got sidecars %v, want exactly one", found) + } + return found[0] +} + +// An interrupted setAttrs leaves a partial sidecar that fails every read of the +// key, and nothing rewrites one now, so a store has to clear it. +// +// One key per storage path the proxy builds: ArtifactPath across ecosystems, +// metadata blobs, and the Gradle build cache. Scoped npm names, Go's "!" case +// escaping and the ":" in OCI digests and Debian epochs are the characters +// most likely to part fileblob's mapping from a plain path join. +func TestStoreClearsLegacyAttrsSidecar(t *testing.T) { + keys := []string{ + "npm/@babel/core/7.24.0/core-7.24.0.tgz", + "maven/org.apache.commons/commons-lang3/3.14.0/commons-lang3-3.14.0.jar", + "golang/github.com/!burnt!sushi/toml/v1.3.2/v1.3.2.zip", + "oci/library/nginx/sha256:abc123def456/manifest", + "debian/tzdata/1:2024a-1/tzdata_2024a-1_all.deb", + "pypi/requests/2.31.0/requests-2.31.0-py3-none-any.whl", + "cargo/serde/1.0.197/serde-1.0.197.crate", + "julia/Example/a1b2c3/a1b2c3.tar.gz", + "conda/numpy/1.26.4/numpy-1.26.4-py311.conda", + "_metadata/npm/@babel/core/metadata", + "_gradle/http-build-cache/0a1b2c3d4e5f", + // Keys fileblob escapes on every platform. + "npm/pkg//1.0.0/x.tgz", + "npm/pkg/../1.0.0/x.tgz", + } + + for _, key := range keys { + t.Run(key, func(t *testing.T) { + assertStoreClearsSidecar(t, key) + }) + } +} + +func assertStoreClearsSidecar(t *testing.T, key string) { + t.Helper() + const payload = "payload" + ctx := context.Background() + dir := t.TempDir() + + sidecar := seedLegacySidecar(t, dir, key, payload) + if err := os.WriteFile(sidecar, []byte(`{"user.content_type":"appl`), 0o600); err != nil { + t.Fatalf("corrupting sidecar: %v", err) + } + + b := openFileBlob(t, dir) + reader, err := b.bucket.NewReader(ctx, key, nil) + if err == nil { + _ = reader.Close() + t.Fatal("corrupt sidecar did not fail the read, so it is not the file fileblob reads for this key") + } + + derived := b.legacySidecarPath(key) + if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil { + t.Fatalf("Store failed: %v", err) + } + + if derived == "" { + t.Fatalf("legacySidecarPath declined %q, but fileblob wrote %q", key, sidecar) + } + if derived != sidecar { + t.Fatalf("derived %q, but fileblob wrote %q", derived, sidecar) + } + if _, err := os.Stat(sidecar); !os.IsNotExist(err) { + t.Errorf("sidecar still present after Store, stat err = %v", err) + } + assertReadsBack(t, b, key, payload) +} + +func assertReadsBack(t *testing.T, b *Blob, key, want string) { + t.Helper() + r, err := b.Open(context.Background(), key) + if err != nil { + t.Fatalf("read still failing after Store cleared the sidecar: %v", err) + } + defer func() { _ = r.Close() }() + got, err := io.ReadAll(r) + if err != nil { + t.Fatalf("ReadAll failed: %v", err) + } + if string(got) != want { + t.Errorf("got %q, want %q", got, want) + } +} + +func openFileBlob(t *testing.T, dir string) *Blob { + t.Helper() + s, err := OpenBucket(context.Background(), fileURLFromPath(dir)) + if err != nil { + t.Fatalf("OpenBucket failed: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + b, ok := s.(*Blob) + if !ok { + t.Fatalf("got %T, want *Blob", s) + } + return b +} + +// fileblob escapes a non-local key in a way this cannot reproduce, and one +// holding ".." resolves outside the cache directory. Removal declines both +// rather than delete the wrong file. +func TestLegacySidecarPathEscapesLikeFileblob(t *testing.T) { + root := filepath.FromSlash("/var/cache/proxy") + b := &Blob{fileRoot: root} + windows := runtime.GOOS == osWindows + + for _, tc := range []struct{ key, unix, windows string }{ + {"npm/pkg/1.0.0/x.tgz", "npm/pkg/1.0.0/x.tgz", "npm/pkg/1.0.0/x.tgz"}, + {"npm/pkg//1.0.0/x.tgz", "npm/pkg/__0x2f__1.0.0/x.tgz", "npm/pkg/__0x2f__1.0.0/x.tgz"}, + {"npm/pkg/../../etc/passwd", "npm/pkg/..__0x2f__..__0x2f__etc/passwd", "npm/pkg/..__0x2f__..__0x2f__etc/passwd"}, + {"npm/pkg/1.0.0/", "npm/pkg/1.0.0__0x2f__", "npm/pkg/1.0.0__0x2f__"}, + {"npm/a\x01b", "npm/a__0x1__b", "npm/a__0x1__b"}, + {"oci/nginx/sha256:abc/manifest", "oci/nginx/sha256:abc/manifest", "oci/nginx/sha256__0x3a__abc/manifest"}, + {"debian/tzdata/1:2024a-1/x.deb", "debian/tzdata/1:2024a-1/x.deb", "debian/tzdata/1__0x3a__2024a-1/x.deb"}, + {`npm/a\b`, `npm/a\b`, "npm/a__0x5c__b"}, + } { + want := tc.unix + if windows { + want = tc.windows + } + want = filepath.Join(root, filepath.FromSlash(want)) + attrsExt + if got := b.legacySidecarPath(tc.key); got != want { + t.Errorf("legacySidecarPath(%q) = %q, want %q", tc.key, got, want) + } + } +} + +func TestLegacySidecarPathDeclinesNonLocalKeys(t *testing.T) { + b := &Blob{fileRoot: filepath.FromSlash("/var/cache/proxy")} + + for _, key := range []string{"", ".", "..", "/etc/passwd"} { + if got := b.legacySidecarPath(key); got != "" { + t.Errorf("legacySidecarPath(%q) = %q, want \"\"", key, got) + } + } +} + +// Cloud backends have no local directory, so nothing is removed for them. +func TestLegacySidecarPathEmptyForCloudBackends(t *testing.T) { + b := &Blob{} + if got := b.legacySidecarPath("npm/pkg/1.0.0/x.tgz"); got != "" { + t.Errorf("legacySidecarPath = %q, want \"\" when there is no file root", got) + } +} + +// Cleanup that cannot complete must not fail the write. A non-empty directory +// at the sidecar path makes os.Remove fail with something other than not-exist +// on every platform, which is what a Windows sharing violation would look like +// here. +func TestStoreSucceedsWhenSidecarCannotBeRemoved(t *testing.T) { + const key = "npm/pkg/1.0.0/pkg-1.0.0.tgz" + const payload = "payload" + dir := t.TempDir() + ctx := context.Background() + + b := openFileBlob(t, dir) + + sidecar := filepath.Join(dir, filepath.FromSlash(key)) + ".attrs" + if err := os.MkdirAll(filepath.Join(sidecar, "blocker"), 0o750); err != nil { + t.Fatalf("seeding an unremovable sidecar: %v", err) + } + if err := os.Remove(sidecar); err == nil { + t.Fatal("sidecar path was removable, so the test proves nothing") + } + + if _, _, err := b.Store(ctx, key, strings.NewReader(payload)); err != nil { + t.Errorf("Store failed because cleanup could not complete: %v", err) + } +} diff --git a/internal/storage/gcs_test.go b/internal/storage/gcs_test.go index 1dfd0a2b..1bd8a407 100644 --- a/internal/storage/gcs_test.go +++ b/internal/storage/gcs_test.go @@ -73,6 +73,9 @@ func TestOpenBucketGCSRoundTripWithEmulator(t *testing.T) { if err != nil || exists { t.Fatalf("Exists after delete = %v, %v; want false, nil", exists, err) } + if err := store.Delete(ctx, "npm/pkg/file.tgz"); err != nil { + t.Fatalf("Delete missing object = %v, want nil", err) + } reader, err := store.Open(ctx, "npm/pkg/file.tgz") if reader != nil || !errors.Is(err, ErrNotFound) { @@ -121,7 +124,12 @@ func (f *fakeGCSServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { } writeJSON(w, fakeGCSObject{Name: name, Size: strconv.Itoa(len(data)), Updated: time.Now().UTC().Format(time.RFC3339Nano)}) case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/storage/v1/b/test-bucket/o/"): - delete(f.objects, objectNameFromPath(r.URL.Path)) + name := objectNameFromPath(r.URL.Path) + if _, ok := f.objects[name]; !ok { + http.NotFound(w, r) + return + } + delete(f.objects, name) w.WriteHeader(http.StatusNoContent) default: f.t.Errorf("unexpected request: %s %s", r.Method, r.URL.String()) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 5ff86f29..3b27ef4b 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -14,6 +14,8 @@ package storage import ( "context" + "crypto/rand" + "encoding/hex" "errors" "io" "time" @@ -44,6 +46,9 @@ type Storage interface { // Open returns a reader for the content at path. // The caller must close the reader when done. + // The reader may also implement io.Seeker when seeking is efficient for the + // storage backend. Backends must not expose io.Seeker when seeking requires + // reading and discarding the bytes before the requested position. // Returns ErrNotFound if the path does not exist. Open(ctx context.Context, path string) (io.ReadCloser, error) @@ -72,7 +77,9 @@ type Storage interface { Close() error } -// ArtifactPath builds a storage path for an artifact. +// ArtifactPath builds the storage path artifacts were cached under before each +// fetch got its own; records from then still point at such paths. Only tests +// call it, to build such records. // Format: {ecosystem}/{namespace}/{name}/{version}/{filename} // For packages without namespace: {ecosystem}/{name}/{version}/{filename} func ArtifactPath(ecosystem, namespace, name, version, filename string) string { @@ -81,3 +88,36 @@ func ArtifactPath(ecosystem, namespace, name, version, filename string) string { } return ecosystem + "/" + name + "/" + version + "/" + filename } + +// FetchPath builds the storage path for one fetch of an artifact: +// {ecosystem}/{name}/{version}/{fetchID}/{filename}. Each fetch writes its own +// object, so no fetch overwrites or deletes another's. +func FetchPath(ecosystem, name, version, fetchID, filename string) string { + return ecosystem + "/" + name + "/" + version + "/" + fetchID + "/" + filename +} + +// fetchIDBytes sizes a fetch id: 16 hex characters keeps paths short, which +// matters on Windows, and a collision between fetches of one artifact version +// is out of reach. +const fetchIDBytes = 8 + +// NewFetchID returns a random id for FetchPath. +func NewFetchID() string { + b := make([]byte, fetchIDBytes) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +// isFetchDir reports whether a directory name has the shape of a fetch id. +// Only one fetch ever writes into such a directory. +func isFetchDir(name string) bool { + if len(name) != 2*fetchIDBytes { + return false + } + for _, c := range name { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 65f5a23f..67745d0c 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -9,6 +9,19 @@ import ( "testing" ) +func TestIsFetchDir(t *testing.T) { + for range 10 { + if id := NewFetchID(); !isFetchDir(id) { + t.Errorf("isFetchDir(%q) = false for a fetch id", id) + } + } + for _, name := range []string{"1.0.0", "0123456789abcde", "0123456789abcdef0", "0123456789ABCDEF", "0123456789abcdeg", ""} { + if isFetchDir(name) { + t.Errorf("isFetchDir(%q) = true", name) + } + } +} + func TestArtifactPath(t *testing.T) { tests := []struct { ecosystem string diff --git a/tools.go b/tools.go deleted file mode 100644 index caf60b1b..00000000 --- a/tools.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build tools - -package tools - -import ( - _ "github.com/swaggo/swag/cmd/swag" -)