feat(data): show the OSM extract's age and let Nominatim follow its feed

.changeset/osm-extract-dataset-fields.md

@@ -0,0 +1,8 @@
+---
+"@openmapx/extension-cli": patch
+---
+
+Refresh the extension CLI's bundled core inputs following the data-manager dataset
+record's two new optional fields for an OSM extract's replication timestamp and diff
+feed. This covers its bundled workspace dependency artifact; the extension scaffolding,
+validation and packaging command interfaces remain unchanged.

apps/api/openapi.json

@@ -42238,6 +42238,25 @@
                       ],
                       "additionalProperties": false
                     },
+                    "osmExtract": {
+                      "type": "object",
+                      "properties": {
+                        "timestamp": {
+                          "type": "string",
+                          "format": "date-time",
+                          "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
+                        },
+                        "replicationUrl": {
+                          "type": "string",
+                          "maxLength": 2048,
+                          "format": "uri"
+                        }
+                      },
+                      "required": [
+                        "timestamp"
+                      ],
+                      "additionalProperties": false
+                    },
                     "fetchedAt": {
                       "type": "string",
                       "format": "date-time",

apps/api/src/routes/admin-services.ts

@@ -27,6 +27,7 @@ import {
 } from "../services/direct-ops-idempotency";
 import { jobRunner } from "../services/job-runner";
 import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "../services/ops-client";
+import { osmExtractFor, readDataManagerDatasets } from "../services/osm-extract";
 import { isSecretsConfigured } from "../services/secrets";
 import { resolveServiceConfigWithSources } from "../services/service-config-resolver";
 import { persistServiceConfigPatch } from "../services/service-config-writer";
@@ -727,14 +728,19 @@ export async function adminServicesRoute(app: FastifyInstance): Promise<void> {
 
   // GET /admin/services/data — data inventory (OSM, builds, MOTIS health)
   registerServiceAdminRoute(app, "getAdminServiceData", async () => {
-    const inventory = await executeAndWait(
-      createApiOpsClient(),
-      { kind: "data.inspect" },
-      createDurableOpsKey("admin.data.inspect", "current"),
-      { signal: AbortSignal.timeout(30_000) },
-    );
+    const [inventory, datasets] = await Promise.all([
+      executeAndWait(
+        createApiOpsClient(),
+        { kind: "data.inspect" },
+        createDurableOpsKey("admin.data.inspect", "current"),
+        { signal: AbortSignal.timeout(30_000) },
+      ),
+      readDataManagerDatasets(),
+    ]);
+    const osmExtract = osmExtractFor(inventory.osm, datasets);
     return {
       ...inventory,
+      ...(osmExtract ? { osmExtract } : {}),
       fetchedAt: new Date().toISOString(),
     };
   });

apps/api/src/services/osm-extract.test.ts

@@ -0,0 +1,54 @@
+import type { DatasetMetadata } from "@openmapx/core/data-manager-contracts";
+import { describe, expect, it } from "vitest";
+import { osmExtractFor } from "./osm-extract";
+
+const osm = {
+  found: true,
+  filename: "europe-germany-berlin.osm.pbf",
+  sizeBytes: 1000,
+  modifiedAt: "2026-11-01T03:00:00.000Z",
+  region: "europe-germany-berlin",
+};
+
+function dataset(over: Partial<DatasetMetadata> = {}): DatasetMetadata {
+  return {
+    type: "osm-pbf",
+    id: "europe/germany/berlin",
+    path: "/data/osm/europe-germany-berlin.osm.pbf",
+    sizeBytes: 1000,
+    downloadedAt: "2026-11-01T03:00:00.000Z",
+    osmTimestamp: "2026-10-31T20:21:42Z",
+    osmReplicationUrl: "https://download.geofabrik.de/europe/germany/berlin-updates",
+    ...over,
+  };
+}
+
+describe("osmExtractFor", () => {
+  it("reports the extract time recorded for the file the inventory found", () => {
+    expect(osmExtractFor(osm, [dataset()])).toEqual({
+      timestamp: "2026-10-31T20:21:42.000Z",
+      replicationUrl: "https://download.geofabrik.de/europe/germany/berlin-updates",
+    });
+  });
+
+  it("does not show a record of another or an earlier file as this one's", () => {
+    expect(
+      osmExtractFor(osm, [dataset({ path: "/data/osm/europe-germany.osm.pbf" })]),
+    ).toBeUndefined();
+    expect(osmExtractFor(osm, [dataset({ sizeBytes: 999 })])).toBeUndefined();
+    expect(osmExtractFor(osm, [dataset({ type: "osm-pbf-bz2" })])).toBeUndefined();
+  });
+
+  it("reports nothing without a recorded time, a file or a usable record", () => {
+    expect(osmExtractFor(osm, [dataset({ osmTimestamp: undefined })])).toBeUndefined();
+    expect(osmExtractFor(osm, [dataset({ osmTimestamp: "yesterday-ish" })])).toBeUndefined();
+    expect(osmExtractFor({ found: false }, [dataset()])).toBeUndefined();
+    expect(osmExtractFor(osm, [])).toBeUndefined();
+  });
+
+  it("drops a feed that is not https rather than failing the response", () => {
+    expect(
+      osmExtractFor(osm, [dataset({ osmReplicationUrl: "http://example.org/updates" })]),
+    ).toEqual({ timestamp: "2026-10-31T20:21:42.000Z" });
+  });
+});

apps/api/src/services/osm-extract.ts

@@ -0,0 +1,78 @@
+import type { DatasetMetadata } from "@openmapx/core/data-manager-contracts";
+import * as dm from "@openmapx/core/data-manager-sdk";
+import type { OpsResultFor } from "@openmapx/core/ops";
+import { envString } from "@openmapx/core/server-env";
+
+const DATA_MANAGER_URL_DEFAULT = "http://localhost:4000";
+const DATA_MANAGER_TIMEOUT_MS = 5_000;
+const DATA_MANAGER_MAX_BYTES = 1024 * 1024;
+
+type OsmInventory = OpsResultFor<"data.inspect">["osm"];
+
+/** How current the OSM extract on disk is, from its own header. */
+export interface OsmExtract {
+  /** The time the extract's data is current to. */
+  timestamp: string;
+  /** The diff feed that continues the extract. */
+  replicationUrl?: string;
+}
+
+function basename(path: string): string {
+  return path.slice(path.lastIndexOf("/") + 1);
+}
+
+/** An https URL short enough to show, or nothing, so a bad record cannot fail the response. */
+function httpsUrl(value: string | undefined): string | undefined {
+  if (!value || value.length > 2048) return undefined;
+  try {
+    return new URL(value).protocol === "https:" ? value : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function isTimestamp(value: string | undefined): value is string {
+  return value !== undefined && !Number.isNaN(Date.parse(value));
+}
+
+/**
+ * The extract time data-manager recorded for the PBF the inventory found:
+ * the same file name and size, so a record of an earlier or another file is
+ * never shown as this one's.
+ */
+export function osmExtractFor(
+  osm: OsmInventory,
+  datasets: readonly DatasetMetadata[],
+): OsmExtract | undefined {
+  if (!osm.found || !osm.filename) return undefined;
+  const dataset = datasets.find(
+    (d) =>
+      d.type === "osm-pbf" &&
+      basename(d.path) === osm.filename &&
+      d.sizeBytes === osm.sizeBytes &&
+      isTimestamp(d.osmTimestamp),
+  );
+  if (!dataset?.osmTimestamp) return undefined;
+  const replicationUrl = httpsUrl(dataset.osmReplicationUrl);
+  return {
+    timestamp: new Date(dataset.osmTimestamp).toISOString(),
+    ...(replicationUrl ? { replicationUrl } : {}),
+  };
+}
+
+/** The datasets data-manager has recorded, or none when it cannot be reached. */
+export async function readDataManagerDatasets(): Promise<DatasetMetadata[]> {
+  try {
+    const response = await dm.getDatasets({
+      transport: dm.createDataManagerTransport({
+        baseUrl: envString("DATA_MANAGER_URL", DATA_MANAGER_URL_DEFAULT),
+        authToken: envString("DATA_MANAGER_AUTH_TOKEN", ""),
+        requestTimeoutMs: DATA_MANAGER_TIMEOUT_MS,
+        maxJsonBytes: DATA_MANAGER_MAX_BYTES,
+      }),
+    });
+    return dm.unwrapDataManagerResponse(response).datasets;
+  } catch {
+    return [];
+  }
+}

apps/web/src/components/admin/services/DataWorkflowsPage.tsx

@@ -41,6 +41,7 @@ import { AdminPageHeader } from "../shared/AdminPageHeader";
 import { useAdminToast } from "../shared/AdminToast";
 import { AmbientPlacesMaintenance } from "./AmbientPlacesMaintenance";
 import { NotablePlacesMaintenance, notablePlacesStatusQueryKey } from "./NotablePlacesMaintenance";
+import { OsmExtractDetails } from "./OsmExtractDetails";
 import { OvertureMaintenance, overtureStatusQueryKey } from "./OvertureMaintenance";
 import { SearchIndexMaintenance, searchIndexStatusQueryKey } from "./SearchIndexMaintenance";
 import { TransitSourcesSection } from "./TransitSourcesSection";
@@ -174,7 +175,7 @@ function DataOperationsSection({ apiUrl }: { apiUrl: string }) {
   );
 }
 
-function OsmSection({ osm }: { osm: OsmInfo }) {
+function OsmSection({ osm, extract }: { osm: OsmInfo; extract: AdminServiceData["osmExtract"] }) {
   return (
     <Paper variant="outlined" sx={{ p: 2.5 }}>
       <Stack
@@ -262,6 +263,7 @@ function OsmSection({ osm }: { osm: OsmInfo }) {
                 <Typography variant="body2">{formatDate(osm.modifiedAt)}</Typography>
               </Box>
             )}
+            <OsmExtractDetails extract={extract} />
           </Stack>
           <Chip
             icon={<CheckCircleIcon />}
@@ -642,7 +644,7 @@ function DataWorkflowsView() {
       </Box>
       <Stack spacing={2}>
         <DataOperationsSection apiUrl={apiUrl} />
-        <OsmSection osm={data.osm} />
+        <OsmSection osm={data.osm} extract={data.osmExtract} />
         <OvertureMaintenance apiUrl={apiUrl} />
         <SearchIndexMaintenance apiUrl={apiUrl} />
         <AmbientPlacesMaintenance apiUrl={apiUrl} />

apps/web/src/components/admin/services/OsmExtractDetails.test.tsx

@@ -0,0 +1,53 @@
+import { cleanup, render, screen } from "@testing-library/react";
+import { afterEach, describe, expect, it } from "vitest";
+import { extractAge, OsmExtractDetails } from "./OsmExtractDetails";
+
+const NOW = Date.parse("2026-11-01T12:00:00Z");
+const hoursAgo = (hours: number) => new Date(NOW - hours * 3_600_000).toISOString();
+
+afterEach(cleanup);
+
+describe("extractAge", () => {
+  it("counts hours within two days, then days", () => {
+    expect(extractAge(hoursAgo(0.5), NOW)).toBe("less than an hour ago");
+    expect(extractAge(hoursAgo(5), NOW)).toBe("5 hours ago");
+    expect(extractAge(hoursAgo(47), NOW)).toBe("47 hours ago");
+    expect(extractAge(hoursAgo(48), NOW)).toBe("2 days ago");
+    expect(extractAge(hoursAgo(24 * 40), NOW)).toBe("40 days ago");
+  });
+
+  it("does not report a future time as negative age", () => {
+    expect(extractAge(hoursAgo(-3), NOW)).toBe("less than an hour ago");
+  });
+});
+
+describe("OsmExtractDetails", () => {
+  it("shows when the data is current to, its age and the feed that continues it", () => {
+    render(
+      <OsmExtractDetails
+        now={NOW}
+        extract={{
+          timestamp: hoursAgo(20),
+          replicationUrl: "https://download.geofabrik.de/europe/germany/berlin-updates",
+        }}
+      />,
+    );
+    expect(screen.getByText("Data as of")).toBeTruthy();
+    expect(screen.getByText(/20 hours ago/)).toBeTruthy();
+    expect(
+      screen.getByText("https://download.geofabrik.de/europe/germany/berlin-updates"),
+    ).toBeTruthy();
+    expect(screen.queryByText(/Older than/)).toBeNull();
+  });
+
+  it("flags an extract older than thirty days", () => {
+    render(<OsmExtractDetails now={NOW} extract={{ timestamp: hoursAgo(24 * 31) }} />);
+    expect(screen.getByText("Older than 30 days")).toBeTruthy();
+    expect(screen.queryByText(/Update feed/)).toBeNull();
+  });
+
+  it("says the time is unknown when the file records none", () => {
+    render(<OsmExtractDetails now={NOW} extract={undefined} />);
+    expect(screen.getByText("Unknown: the file records no replication time")).toBeTruthy();
+  });
+});

apps/web/src/components/admin/services/OsmExtractDetails.tsx

@@ -0,0 +1,90 @@
+"use client";
+
+import Box from "@mui/material/Box";
+import Chip from "@mui/material/Chip";
+import Typography from "@mui/material/Typography";
+import type { AdminServiceData } from "@openmapx/core/api-contracts";
+
+type OsmExtract = NonNullable<AdminServiceData["osmExtract"]>;
+
+/** Past this age the card flags the extract: Geofabrik publishes a new one every day. */
+export const STALE_EXTRACT_DAYS = 30;
+
+const HOUR_MS = 3_600_000;
+const DAY_MS = 24 * HOUR_MS;
+
+/** How long ago the extract's data was current, coarsely: hours within two days, else days. */
+export function extractAge(timestamp: string, now: number): string {
+  const elapsed = Math.max(0, now - Date.parse(timestamp));
+  const format = new Intl.RelativeTimeFormat("en", { numeric: "auto" });
+  if (elapsed < HOUR_MS) return "less than an hour ago";
+  if (elapsed < 2 * DAY_MS) return format.format(-Math.floor(elapsed / HOUR_MS), "hour");
+  return format.format(-Math.floor(elapsed / DAY_MS), "day");
+}
+
+function formatDate(iso: string): string {
+  return new Date(iso).toLocaleDateString(undefined, {
+    year: "numeric",
+    month: "short",
+    day: "numeric",
+    hour: "2-digit",
+    minute: "2-digit",
+  });
+}
+
+/**
+ * When the OSM extract's data is current to, from the file's own header, and
+ * the diff feed that continues it. Download time says nothing about this: a
+ * mirror can serve a weeks-old file today.
+ */
+export function OsmExtractDetails({
+  extract,
+  now = Date.now(),
+}: {
+  extract: OsmExtract | undefined;
+  now?: number;
+}) {
+  if (!extract) {
+    return (
+      <Box>
+        <Typography variant="caption" sx={{ color: "text.secondary" }}>
+          Data as of
+        </Typography>
+        <Typography variant="body2" sx={{ color: "text.secondary" }}>
+          Unknown: the file records no replication time
+        </Typography>
+      </Box>
+    );
+  }
+  const stale = now - Date.parse(extract.timestamp) > STALE_EXTRACT_DAYS * DAY_MS;
+  return (
+    <>
+      <Box>
+        <Typography variant="caption" sx={{ color: "text.secondary" }}>
+          Data as of
+        </Typography>
+        <Typography variant="body2">
+          {formatDate(extract.timestamp)} ({extractAge(extract.timestamp, now)})
+        </Typography>
+        {stale && (
+          <Chip
+            label={`Older than ${STALE_EXTRACT_DAYS} days`}
+            color="warning"
+            size="small"
+            sx={{ mt: 0.5 }}
+          />
+        )}
+      </Box>
+      {extract.replicationUrl && (
+        <Box>
+          <Typography variant="caption" sx={{ color: "text.secondary" }}>
+            Update feed (for NOMINATIM_REPLICATION_URL)
+          </Typography>
+          <Typography variant="body2" sx={{ fontFamily: "monospace", wordBreak: "break-all" }}>
+            {extract.replicationUrl}
+          </Typography>
+        </Box>
+      )}
+    </>
+  );
+}