From 21286f923e2661564a1ca5ef89c5d6e126f83cc7 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 1 Nov 2026 19:11:58 +0000 Subject: [PATCH 1/2] feat(data): show the OSM extract's age and let Nominatim follow its feed An OSM extract's header says when its data is current to and which diff feed continues it. data-manager now reads both when it downloads a PBF, and once at startup for a PBF downloaded before, and records them with the dataset. The admin data route reports them for the file the inventory found, and the Data workflows OSM card shows the time, its age, a flag past 30 days and the feed. Nominatim's replication URL, update mode and intervals were fixed in its manifest: the planet feed with the image's default mode `none`, so a regional import never applied diffs. They are now NOMINATIM_REPLICATION_URL, NOMINATIM_UPDATE_MODE, NOMINATIM_REPLICATION_UPDATE_INTERVAL and NOMINATIM_REPLICATION_RECHECK_INTERVAL, with the same defaults, documented with the regional Geofabrik feed. Part of #303 --- apps/api/openapi.json | 19 +++ apps/api/src/routes/admin-services.ts | 18 +- apps/api/src/services/osm-extract.test.ts | 54 ++++++ apps/api/src/services/osm-extract.ts | 78 +++++++++ .../admin/services/DataWorkflowsPage.tsx | 6 +- .../admin/services/OsmExtractDetails.test.tsx | 53 ++++++ .../admin/services/OsmExtractDetails.tsx | 90 ++++++++++ docs/docs/install/configuration.md | 4 + docs/docs/install/preparing-data.md | 15 ++ infra/docker/.env.example | 12 ++ .../src/api/contracts/service-admin/data.ts | 13 ++ .../service-models/getAdminServiceData200.ts | 2 + .../getAdminServiceData200OsmExtract.ts | 9 + .../generated/admin/service-models/index.ts | 1 + .../data-manager/contracts/datasets.ts | 4 + .../models/getDatasets200DatasetsItem.ts | 2 + services/data-manager/__tests__/api.test.ts | 5 +- .../__tests__/download-osm.test.ts | 89 +++++++++- .../__tests__/utils/osm-pbf-header.test.ts | 70 ++++++++ services/data-manager/openapi.json | 6 + .../osm-pbf-header/replication-raw.osm.pbf | Bin 0 -> 482 bytes .../osm-pbf-header/replication-zlib.osm.pbf | Bin 0 -> 411 bytes services/data-manager/src/api.ts | 4 +- .../data-manager/src/jobs/download-osm.ts | 29 ++++ .../data-manager/src/utils/osm-pbf-header.ts | 156 ++++++++++++++++++ services/nominatim/service.json | 6 +- 26 files changed, 732 insertions(+), 13 deletions(-) create mode 100644 apps/api/src/services/osm-extract.test.ts create mode 100644 apps/api/src/services/osm-extract.ts create mode 100644 apps/web/src/components/admin/services/OsmExtractDetails.test.tsx create mode 100644 apps/web/src/components/admin/services/OsmExtractDetails.tsx create mode 100644 packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmExtract.ts create mode 100644 services/data-manager/__tests__/utils/osm-pbf-header.test.ts create mode 100644 services/data-manager/src/__tests__/fixtures/osm-pbf-header/replication-raw.osm.pbf create mode 100644 services/data-manager/src/__tests__/fixtures/osm-pbf-header/replication-zlib.osm.pbf create mode 100644 services/data-manager/src/utils/osm-pbf-header.ts diff --git a/apps/api/openapi.json b/apps/api/openapi.json index ac8aeb102..2d1be520c 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -42238,6 +42238,25 @@ ], "additionalProperties": false }, + "osmExtract": { + "type": "object", + "properties": { + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "replicationUrl": { + "type": "string", + "maxLength": 2048, + "format": "uri" + } + }, + "required": [ + "timestamp" + ], + "additionalProperties": false + }, "fetchedAt": { "type": "string", "format": "date-time", diff --git a/apps/api/src/routes/admin-services.ts b/apps/api/src/routes/admin-services.ts index 0e209932e..6623135dc 100644 --- a/apps/api/src/routes/admin-services.ts +++ b/apps/api/src/routes/admin-services.ts @@ -27,6 +27,7 @@ import { } from "../services/direct-ops-idempotency"; import { jobRunner } from "../services/job-runner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "../services/ops-client"; +import { osmExtractFor, readDataManagerDatasets } from "../services/osm-extract"; import { isSecretsConfigured } from "../services/secrets"; import { resolveServiceConfigWithSources } from "../services/service-config-resolver"; import { persistServiceConfigPatch } from "../services/service-config-writer"; @@ -727,14 +728,19 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { // GET /admin/services/data — data inventory (OSM, builds, MOTIS health) registerServiceAdminRoute(app, "getAdminServiceData", async () => { - const inventory = await executeAndWait( - createApiOpsClient(), - { kind: "data.inspect" }, - createDurableOpsKey("admin.data.inspect", "current"), - { signal: AbortSignal.timeout(30_000) }, - ); + const [inventory, datasets] = await Promise.all([ + executeAndWait( + createApiOpsClient(), + { kind: "data.inspect" }, + createDurableOpsKey("admin.data.inspect", "current"), + { signal: AbortSignal.timeout(30_000) }, + ), + readDataManagerDatasets(), + ]); + const osmExtract = osmExtractFor(inventory.osm, datasets); return { ...inventory, + ...(osmExtract ? { osmExtract } : {}), fetchedAt: new Date().toISOString(), }; }); diff --git a/apps/api/src/services/osm-extract.test.ts b/apps/api/src/services/osm-extract.test.ts new file mode 100644 index 000000000..5585a3352 --- /dev/null +++ b/apps/api/src/services/osm-extract.test.ts @@ -0,0 +1,54 @@ +import type { DatasetMetadata } from "@openmapx/core/data-manager-contracts"; +import { describe, expect, it } from "vitest"; +import { osmExtractFor } from "./osm-extract"; + +const osm = { + found: true, + filename: "europe-germany-berlin.osm.pbf", + sizeBytes: 1000, + modifiedAt: "2026-11-01T03:00:00.000Z", + region: "europe-germany-berlin", +}; + +function dataset(over: Partial = {}): DatasetMetadata { + return { + type: "osm-pbf", + id: "europe/germany/berlin", + path: "/data/osm/europe-germany-berlin.osm.pbf", + sizeBytes: 1000, + downloadedAt: "2026-11-01T03:00:00.000Z", + osmTimestamp: "2026-10-31T20:21:42Z", + osmReplicationUrl: "https://download.geofabrik.de/europe/germany/berlin-updates", + ...over, + }; +} + +describe("osmExtractFor", () => { + it("reports the extract time recorded for the file the inventory found", () => { + expect(osmExtractFor(osm, [dataset()])).toEqual({ + timestamp: "2026-10-31T20:21:42.000Z", + replicationUrl: "https://download.geofabrik.de/europe/germany/berlin-updates", + }); + }); + + it("does not show a record of another or an earlier file as this one's", () => { + expect( + osmExtractFor(osm, [dataset({ path: "/data/osm/europe-germany.osm.pbf" })]), + ).toBeUndefined(); + expect(osmExtractFor(osm, [dataset({ sizeBytes: 999 })])).toBeUndefined(); + expect(osmExtractFor(osm, [dataset({ type: "osm-pbf-bz2" })])).toBeUndefined(); + }); + + it("reports nothing without a recorded time, a file or a usable record", () => { + expect(osmExtractFor(osm, [dataset({ osmTimestamp: undefined })])).toBeUndefined(); + expect(osmExtractFor(osm, [dataset({ osmTimestamp: "yesterday-ish" })])).toBeUndefined(); + expect(osmExtractFor({ found: false }, [dataset()])).toBeUndefined(); + expect(osmExtractFor(osm, [])).toBeUndefined(); + }); + + it("drops a feed that is not https rather than failing the response", () => { + expect( + osmExtractFor(osm, [dataset({ osmReplicationUrl: "http://example.org/updates" })]), + ).toEqual({ timestamp: "2026-10-31T20:21:42.000Z" }); + }); +}); diff --git a/apps/api/src/services/osm-extract.ts b/apps/api/src/services/osm-extract.ts new file mode 100644 index 000000000..4335d1694 --- /dev/null +++ b/apps/api/src/services/osm-extract.ts @@ -0,0 +1,78 @@ +import type { DatasetMetadata } from "@openmapx/core/data-manager-contracts"; +import * as dm from "@openmapx/core/data-manager-sdk"; +import type { OpsResultFor } from "@openmapx/core/ops"; +import { envString } from "@openmapx/core/server-env"; + +const DATA_MANAGER_URL_DEFAULT = "http://localhost:4000"; +const DATA_MANAGER_TIMEOUT_MS = 5_000; +const DATA_MANAGER_MAX_BYTES = 1024 * 1024; + +type OsmInventory = OpsResultFor<"data.inspect">["osm"]; + +/** How current the OSM extract on disk is, from its own header. */ +export interface OsmExtract { + /** The time the extract's data is current to. */ + timestamp: string; + /** The diff feed that continues the extract. */ + replicationUrl?: string; +} + +function basename(path: string): string { + return path.slice(path.lastIndexOf("/") + 1); +} + +/** An https URL short enough to show, or nothing, so a bad record cannot fail the response. */ +function httpsUrl(value: string | undefined): string | undefined { + if (!value || value.length > 2048) return undefined; + try { + return new URL(value).protocol === "https:" ? value : undefined; + } catch { + return undefined; + } +} + +function isTimestamp(value: string | undefined): value is string { + return value !== undefined && !Number.isNaN(Date.parse(value)); +} + +/** + * The extract time data-manager recorded for the PBF the inventory found: + * the same file name and size, so a record of an earlier or another file is + * never shown as this one's. + */ +export function osmExtractFor( + osm: OsmInventory, + datasets: readonly DatasetMetadata[], +): OsmExtract | undefined { + if (!osm.found || !osm.filename) return undefined; + const dataset = datasets.find( + (d) => + d.type === "osm-pbf" && + basename(d.path) === osm.filename && + d.sizeBytes === osm.sizeBytes && + isTimestamp(d.osmTimestamp), + ); + if (!dataset?.osmTimestamp) return undefined; + const replicationUrl = httpsUrl(dataset.osmReplicationUrl); + return { + timestamp: new Date(dataset.osmTimestamp).toISOString(), + ...(replicationUrl ? { replicationUrl } : {}), + }; +} + +/** The datasets data-manager has recorded, or none when it cannot be reached. */ +export async function readDataManagerDatasets(): Promise { + try { + const response = await dm.getDatasets({ + transport: dm.createDataManagerTransport({ + baseUrl: envString("DATA_MANAGER_URL", DATA_MANAGER_URL_DEFAULT), + authToken: envString("DATA_MANAGER_AUTH_TOKEN", ""), + requestTimeoutMs: DATA_MANAGER_TIMEOUT_MS, + maxJsonBytes: DATA_MANAGER_MAX_BYTES, + }), + }); + return dm.unwrapDataManagerResponse(response).datasets; + } catch { + return []; + } +} diff --git a/apps/web/src/components/admin/services/DataWorkflowsPage.tsx b/apps/web/src/components/admin/services/DataWorkflowsPage.tsx index 3abc22ba2..cc1468f49 100644 --- a/apps/web/src/components/admin/services/DataWorkflowsPage.tsx +++ b/apps/web/src/components/admin/services/DataWorkflowsPage.tsx @@ -41,6 +41,7 @@ import { AdminPageHeader } from "../shared/AdminPageHeader"; import { useAdminToast } from "../shared/AdminToast"; import { AmbientPlacesMaintenance } from "./AmbientPlacesMaintenance"; import { NotablePlacesMaintenance, notablePlacesStatusQueryKey } from "./NotablePlacesMaintenance"; +import { OsmExtractDetails } from "./OsmExtractDetails"; import { OvertureMaintenance, overtureStatusQueryKey } from "./OvertureMaintenance"; import { SearchIndexMaintenance, searchIndexStatusQueryKey } from "./SearchIndexMaintenance"; import { TransitSourcesSection } from "./TransitSourcesSection"; @@ -174,7 +175,7 @@ function DataOperationsSection({ apiUrl }: { apiUrl: string }) { ); } -function OsmSection({ osm }: { osm: OsmInfo }) { +function OsmSection({ osm, extract }: { osm: OsmInfo; extract: AdminServiceData["osmExtract"] }) { return ( {formatDate(osm.modifiedAt)} )} + } @@ -642,7 +644,7 @@ function DataWorkflowsView() { - + diff --git a/apps/web/src/components/admin/services/OsmExtractDetails.test.tsx b/apps/web/src/components/admin/services/OsmExtractDetails.test.tsx new file mode 100644 index 000000000..74e0de20e --- /dev/null +++ b/apps/web/src/components/admin/services/OsmExtractDetails.test.tsx @@ -0,0 +1,53 @@ +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it } from "vitest"; +import { extractAge, OsmExtractDetails } from "./OsmExtractDetails"; + +const NOW = Date.parse("2026-11-01T12:00:00Z"); +const hoursAgo = (hours: number) => new Date(NOW - hours * 3_600_000).toISOString(); + +afterEach(cleanup); + +describe("extractAge", () => { + it("counts hours within two days, then days", () => { + expect(extractAge(hoursAgo(0.5), NOW)).toBe("less than an hour ago"); + expect(extractAge(hoursAgo(5), NOW)).toBe("5 hours ago"); + expect(extractAge(hoursAgo(47), NOW)).toBe("47 hours ago"); + expect(extractAge(hoursAgo(48), NOW)).toBe("2 days ago"); + expect(extractAge(hoursAgo(24 * 40), NOW)).toBe("40 days ago"); + }); + + it("does not report a future time as negative age", () => { + expect(extractAge(hoursAgo(-3), NOW)).toBe("less than an hour ago"); + }); +}); + +describe("OsmExtractDetails", () => { + it("shows when the data is current to, its age and the feed that continues it", () => { + render( + , + ); + expect(screen.getByText("Data as of")).toBeTruthy(); + expect(screen.getByText(/20 hours ago/)).toBeTruthy(); + expect( + screen.getByText("https://download.geofabrik.de/europe/germany/berlin-updates"), + ).toBeTruthy(); + expect(screen.queryByText(/Older than/)).toBeNull(); + }); + + it("flags an extract older than thirty days", () => { + render(); + expect(screen.getByText("Older than 30 days")).toBeTruthy(); + expect(screen.queryByText(/Update feed/)).toBeNull(); + }); + + it("says the time is unknown when the file records none", () => { + render(); + expect(screen.getByText("Unknown: the file records no replication time")).toBeTruthy(); + }); +}); diff --git a/apps/web/src/components/admin/services/OsmExtractDetails.tsx b/apps/web/src/components/admin/services/OsmExtractDetails.tsx new file mode 100644 index 000000000..20673da89 --- /dev/null +++ b/apps/web/src/components/admin/services/OsmExtractDetails.tsx @@ -0,0 +1,90 @@ +"use client"; + +import Box from "@mui/material/Box"; +import Chip from "@mui/material/Chip"; +import Typography from "@mui/material/Typography"; +import type { AdminServiceData } from "@openmapx/core/api-contracts"; + +type OsmExtract = NonNullable; + +/** Past this age the card flags the extract: Geofabrik publishes a new one every day. */ +export const STALE_EXTRACT_DAYS = 30; + +const HOUR_MS = 3_600_000; +const DAY_MS = 24 * HOUR_MS; + +/** How long ago the extract's data was current, coarsely: hours within two days, else days. */ +export function extractAge(timestamp: string, now: number): string { + const elapsed = Math.max(0, now - Date.parse(timestamp)); + const format = new Intl.RelativeTimeFormat("en", { numeric: "auto" }); + if (elapsed < HOUR_MS) return "less than an hour ago"; + if (elapsed < 2 * DAY_MS) return format.format(-Math.floor(elapsed / HOUR_MS), "hour"); + return format.format(-Math.floor(elapsed / DAY_MS), "day"); +} + +function formatDate(iso: string): string { + return new Date(iso).toLocaleDateString(undefined, { + year: "numeric", + month: "short", + day: "numeric", + hour: "2-digit", + minute: "2-digit", + }); +} + +/** + * When the OSM extract's data is current to, from the file's own header, and + * the diff feed that continues it. Download time says nothing about this: a + * mirror can serve a weeks-old file today. + */ +export function OsmExtractDetails({ + extract, + now = Date.now(), +}: { + extract: OsmExtract | undefined; + now?: number; +}) { + if (!extract) { + return ( + + + Data as of + + + Unknown: the file records no replication time + + + ); + } + const stale = now - Date.parse(extract.timestamp) > STALE_EXTRACT_DAYS * DAY_MS; + return ( + <> + + + Data as of + + + {formatDate(extract.timestamp)} ({extractAge(extract.timestamp, now)}) + + {stale && ( + + )} + + {extract.replicationUrl && ( + + + Update feed (for NOMINATIM_REPLICATION_URL) + + + {extract.replicationUrl} + + + )} + + ); +} diff --git a/docs/docs/install/configuration.md b/docs/docs/install/configuration.md index d119861ab..35b1a726d 100644 --- a/docs/docs/install/configuration.md +++ b/docs/docs/install/configuration.md @@ -381,6 +381,10 @@ workload; a manifest default is not a hardware sizing guarantee. | `NOMINATIM_IMPORT_WIKIPEDIA` | Import Wikipedia importance scores during the Nominatim build. | Default `true` | | `NOMINATIM_IMPORT_GB_POSTCODES` | Import GB postcodes. | Default `false` | | `NOMINATIM_IMPORT_US_POSTCODES` | Import US postcodes. | Default `false` | +| `NOMINATIM_REPLICATION_URL` | Diff feed Nominatim follows. The default suits a planet import; a regional import must follow its extract's own feed (for Germany `https://download.geofabrik.de/europe/germany-updates/`; the admin panel's OSM card shows the feed the downloaded extract names), or the planet diffs pull changes from everywhere into the regional database. Takes effect when Nominatim is recreated. | Default `https://planet.openstreetmap.org/replication/day` | +| `NOMINATIM_UPDATE_MODE` | Whether Nominatim applies that feed: `continuous` keeps applying it, `catch-up` applies everything published and stops, `once` applies one batch per start, `none` applies nothing, so search stays at the import's date. | Default `none` | +| `NOMINATIM_REPLICATION_UPDATE_INTERVAL` | How often the feed publishes diffs, in seconds. | Default `86400` | +| `NOMINATIM_REPLICATION_RECHECK_INTERVAL` | How long Nominatim waits before asking again when the feed has nothing new, in seconds. | Default `900` | | `OVERPASS_SPACE` | Overpass dispatcher available-RAM budget in bytes, not database disk capacity. The 100 GiB default exceeds the 32 GiB container ceiling; tune together with worker concurrency. | Default `107374182400` (~100 GiB) | | `OVERPASS_FASTCGI_PROCESSES` | Number of Overpass FastCGI worker processes. | Default `4` | | `OVERPASS_DIFF_URL` | Replication feed Overpass applies daily. The default suits a planet import; a regional import must follow its extract's own feed (for Germany `https://download.geofabrik.de/europe/germany-updates/`), or the planet diffs pull changes from everywhere into the regional database. | Default `https://planet.openstreetmap.org/replication/day/` | diff --git a/docs/docs/install/preparing-data.md b/docs/docs/install/preparing-data.md index e3ab175b1..cc5ddaa6f 100644 --- a/docs/docs/install/preparing-data.md +++ b/docs/docs/install/preparing-data.md @@ -57,6 +57,12 @@ do not replace the previous good file. Check disk space before downloading a larger extract; serving multiple PBFs does not automatically give each consumer the correct region. Inspect the generated hardlink plan when several inputs exist. +The extract's own header says when its data is current to, which can be well +before the download. Data-manager records it, and the admin panel's +**Data workflows** page shows it with its age on the OSM card, flagged once it +is more than 30 days old. The card also shows the diff feed the extract names, +which is the feed to give Nominatim (below). + ### Transit schedule sources For a configured MOTIS deployment, sync the requested country subset: @@ -172,6 +178,15 @@ runs the runtime import. It is distinct from transactional live transit sync. Valhalla, Nominatim, and Overpass import within their service containers; they have no CLI build hook. +Nominatim and Overpass can then keep themselves current from a diff feed +instead of a new extract. Both default to the planet feed, which suits only a +planet import: a regional import must follow its extract's feed, or the planet +diffs pull changes from everywhere into the regional database. For Nominatim +set `NOMINATIM_REPLICATION_URL` to that feed and `NOMINATIM_UPDATE_MODE` to +`continuous`; with the default mode `none` it applies no diffs and search stays +at the import's date. For Overpass set `OVERPASS_DIFF_URL`. See +[Configuration](./configuration.md) for each setting. + `services build-all --region europe/germany` currently plans OSRM, OTP, MOTIS, Pelias, and TileServer GL even when only one is selected. It continues after individual failures unless given `--fail-fast`. Explicit `services build ` diff --git a/infra/docker/.env.example b/infra/docker/.env.example index 11b0542e7..d955aa0c7 100644 --- a/infra/docker/.env.example +++ b/infra/docker/.env.example @@ -403,6 +403,18 @@ NOMINATIM_THREADS=8 NOMINATIM_IMPORT_WIKIPEDIA=true NOMINATIM_IMPORT_GB_POSTCODES=false NOMINATIM_IMPORT_US_POSTCODES=false +# Diff feed Nominatim follows and whether it applies it. With the default mode +# `none` it never does, so search stays at the import's date. A regional +# import must follow its extract's own feed, which the admin panel's OSM card +# shows, or the planet diffs pull changes from everywhere into the regional +# database. Modes: `continuous` keeps applying, `catch-up` applies everything +# published and stops, `once` applies one batch per start, `none` applies +# nothing. Takes effect when Nominatim is recreated. +# NOMINATIM_REPLICATION_URL=https://download.geofabrik.de/europe/germany-updates/ +# NOMINATIM_UPDATE_MODE=continuous +# How often the feed publishes, and how long to wait when it has nothing new (seconds). +# NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400 +# NOMINATIM_REPLICATION_RECHECK_INTERVAL=900 # Overpass API # OVERPASS_SPACE is the dispatcher's query RAM budget in bytes, not disk space. diff --git a/packages/core/src/api/contracts/service-admin/data.ts b/packages/core/src/api/contracts/service-admin/data.ts index 5ce33a02d..7e1c60528 100644 --- a/packages/core/src/api/contracts/service-admin/data.ts +++ b/packages/core/src/api/contracts/service-admin/data.ts @@ -4,6 +4,19 @@ import { adminCountSchema, adminTimestampSchema } from "../admin"; import { adminServiceIdListSchema } from "./common"; export const adminServiceDataSchema = OPS_RESULT_SCHEMAS["data.inspect"].extend({ + /** + * How current the OSM extract on disk is, from its header as data-manager + * recorded it; absent when the file names no time or data-manager has none. + */ + osmExtract: z + .strictObject({ + timestamp: adminTimestampSchema, + replicationUrl: z + .url({ protocol: /^https$/ }) + .max(2048) + .optional(), + }) + .optional(), fetchedAt: adminTimestampSchema, }); export const adminServiceSelectionSchema = z.strictObject({ diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200.ts index def60cf87..6ad269dd7 100644 --- a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200.ts +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200.ts @@ -3,12 +3,14 @@ import type { GetAdminServiceData200BuildsItem } from "./getAdminServiceData200BuildsItem"; import type { GetAdminServiceData200MotisTransitous } from "./getAdminServiceData200MotisTransitous"; import type { GetAdminServiceData200Osm } from "./getAdminServiceData200Osm"; +import type { GetAdminServiceData200OsmExtract } from "./getAdminServiceData200OsmExtract"; export type GetAdminServiceData200 = { osm: GetAdminServiceData200Osm; /** @maxItems 10 */ builds: GetAdminServiceData200BuildsItem[]; motisTransitous: GetAdminServiceData200MotisTransitous; + osmExtract?: GetAdminServiceData200OsmExtract; /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ fetchedAt: string; }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmExtract.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmExtract.ts new file mode 100644 index 000000000..cf0d8e197 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmExtract.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceData200OsmExtract = { + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + timestamp: string; + /** @maxLength 2048 */ + replicationUrl?: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index 71a117c35..e14cd0c40 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -172,6 +172,7 @@ export * from "./getAdminServiceData200MotisTransitousOperationsProfile"; export * from "./getAdminServiceData200MotisTransitousPreflightState"; export * from "./getAdminServiceData200MotisTransitousPreviousHealthySlot"; export * from "./getAdminServiceData200Osm"; +export * from "./getAdminServiceData200OsmExtract"; export * from "./getAdminServiceData400"; export * from "./getAdminServiceData401"; export * from "./getAdminServiceData403"; diff --git a/packages/core/src/services/data-manager/contracts/datasets.ts b/packages/core/src/services/data-manager/contracts/datasets.ts index 8b81311ea..dfd78507e 100644 --- a/packages/core/src/services/data-manager/contracts/datasets.ts +++ b/packages/core/src/services/data-manager/contracts/datasets.ts @@ -24,6 +24,10 @@ export const datasetMetadataSchema = z.strictObject({ sha256: z.string().optional(), md5: z.string().optional(), path: z.string().min(1), + /** An OSM PBF's replication timestamp: the time its data is current to. */ + osmTimestamp: z.string().optional(), + /** The diff feed an OSM PBF's header names as continuing it. */ + osmReplicationUrl: z.string().optional(), }); export const datasetsSchema = z.strictObject({ datasets: z.array(datasetMetadataSchema) }); export const reloadDatasetsSchema = z.strictObject({ ok: z.literal(true), datasets: countSchema }); diff --git a/packages/core/src/services/data-manager/generated/models/getDatasets200DatasetsItem.ts b/packages/core/src/services/data-manager/generated/models/getDatasets200DatasetsItem.ts index 785cab7f3..af30acbea 100644 --- a/packages/core/src/services/data-manager/generated/models/getDatasets200DatasetsItem.ts +++ b/packages/core/src/services/data-manager/generated/models/getDatasets200DatasetsItem.ts @@ -16,4 +16,6 @@ export type GetDatasets200DatasetsItem = { md5?: string; /** @minLength 1 */ path: string; + osmTimestamp?: string; + osmReplicationUrl?: string; }; diff --git a/services/data-manager/__tests__/api.test.ts b/services/data-manager/__tests__/api.test.ts index bb9b7487c..3fc8cf26a 100644 --- a/services/data-manager/__tests__/api.test.ts +++ b/services/data-manager/__tests__/api.test.ts @@ -17,7 +17,10 @@ const mockDownloadOsm = vi.hoisted(() => })), ); -vi.mock("../src/jobs/download-osm.js", () => ({ downloadOsm: mockDownloadOsm })); +vi.mock("../src/jobs/download-osm.js", () => ({ + backfillOsmHeaders: () => 0, + downloadOsm: mockDownloadOsm, +})); describe("data-manager API", () => { let fixtureRoot: string; diff --git a/services/data-manager/__tests__/download-osm.test.ts b/services/data-manager/__tests__/download-osm.test.ts index 3f48b6d42..4ddd59bf7 100644 --- a/services/data-manager/__tests__/download-osm.test.ts +++ b/services/data-manager/__tests__/download-osm.test.ts @@ -1,8 +1,9 @@ -import { mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { copyFileSync, mkdirSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { + backfillOsmHeaders, type DownloadOsmOptions, downloadOsm, resolveOsmMd5Url, @@ -102,3 +103,89 @@ describe("downloadOsm publication", () => { } }); }); + +const STAMPED_PBF = join( + import.meta.dirname, + "../src/__tests__/fixtures/osm-pbf-header/replication-zlib.osm.pbf", +); + +describe("downloadOsm extract time", () => { + it("records the time the extract is current to and its diff feed", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "openmapx-osm-header-")); + const store = new StateStore(dataDir); + try { + await downloadOsm({ + region: "europe/monaco", + dataDir, + store, + verifyChecksum: false, + downloadImpl: async (_url, targetPath) => { + copyFileSync(STAMPED_PBF, targetPath); + return { published: true }; + }, + }); + + expect(store.getAll()[0]).toMatchObject({ + type: "osm-pbf", + id: "europe/monaco", + osmTimestamp: "2026-10-31T20:21:42.000Z", + osmReplicationUrl: "https://download.geofabrik.de/europe/monaco-updates", + }); + } finally { + rmSync(dataDir, { recursive: true, force: true }); + } + }); + + it("records none for a file whose header carries no replication state", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "openmapx-osm-header-")); + const store = new StateStore(dataDir); + try { + await downloadOsm({ + region: "europe/monaco", + dataDir, + store, + verifyChecksum: false, + downloadImpl: async (_url, targetPath) => { + writeFileSync(targetPath, "not a pbf"); + return { published: true }; + }, + }); + + expect(store.getAll()[0]?.osmTimestamp).toBeUndefined(); + expect(store.getAll()[0]?.osmReplicationUrl).toBeUndefined(); + } finally { + rmSync(dataDir, { recursive: true, force: true }); + } + }); +}); + +describe("backfillOsmHeaders", () => { + it("fills the extract time of a PBF downloaded before it was recorded, and only that", () => { + const dataDir = mkdtempSync(join(tmpdir(), "openmapx-osm-backfill-")); + try { + mkdirSync(join(dataDir, "osm")); + const path = join(dataDir, "osm", "europe-monaco.osm.pbf"); + copyFileSync(STAMPED_PBF, path); + const store = new StateStore(dataDir); + const base = { sizeBytes: 411, downloadedAt: "2026-10-01T00:00:00.000Z" }; + store.upsert({ type: "osm-pbf", id: "europe/monaco", path, ...base }); + store.upsert({ + type: "osm-pbf", + id: "gone", + path: join(dataDir, "osm", "gone.osm.pbf"), + ...base, + }); + store.upsert({ type: "tile-mbtiles", id: "tiles", path, ...base }); + + expect(backfillOsmHeaders(store)).toBe(1); + + const byId = new Map(new StateStore(dataDir).getAll().map((d) => [d.id, d])); + expect(byId.get("europe/monaco")?.osmTimestamp).toBe("2026-10-31T20:21:42.000Z"); + expect(byId.get("gone")?.osmTimestamp).toBeUndefined(); + expect(byId.get("tiles")?.osmTimestamp).toBeUndefined(); + expect(backfillOsmHeaders(store)).toBe(0); + } finally { + rmSync(dataDir, { recursive: true, force: true }); + } + }); +}); diff --git a/services/data-manager/__tests__/utils/osm-pbf-header.test.ts b/services/data-manager/__tests__/utils/osm-pbf-header.test.ts new file mode 100644 index 000000000..18c9e5ca0 --- /dev/null +++ b/services/data-manager/__tests__/utils/osm-pbf-header.test.ts @@ -0,0 +1,70 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { readOsmPbfHeader, tryReadOsmPbfHeader } from "../../src/utils/osm-pbf-header.js"; + +// Written by libosmium (pyosmium) from the traffic-engine fixture, with the +// replication header Geofabrik sets on its extracts. +const FIXTURES = join(import.meta.dirname, "../../src/__tests__/fixtures"); +const ZLIB = join(FIXTURES, "osm-pbf-header/replication-zlib.osm.pbf"); +const RAW = join(FIXTURES, "osm-pbf-header/replication-raw.osm.pbf"); +const NO_REPLICATION = join(FIXTURES, "traffic-engine/roads.osm.pbf"); + +const dirs: string[] = []; +function scratch(name: string, bytes: Buffer): string { + const dir = mkdtempSync(join(tmpdir(), "openmapx-pbf-header-")); + dirs.push(dir); + const path = join(dir, name); + writeFileSync(path, bytes); + return path; +} + +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +describe("readOsmPbfHeader", () => { + it("reads the replication timestamp and diff feed from a compressed header block", () => { + expect(readOsmPbfHeader(ZLIB)).toEqual({ + timestamp: "2026-10-31T20:21:42.000Z", + replicationUrl: "https://download.geofabrik.de/europe/monaco-updates", + }); + }); + + it("reads an uncompressed header block too", () => { + expect(readOsmPbfHeader(RAW)).toEqual(readOsmPbfHeader(ZLIB)); + }); + + it("reports nothing for a PBF whose header names no replication state", () => { + expect(readOsmPbfHeader(NO_REPLICATION)).toEqual({}); + }); + + it("rejects a file that does not start with an OSM header block", () => { + expect(() => + readOsmPbfHeader(scratch("text.osm.pbf", Buffer.from("not a pbf at all"))), + ).toThrow(/not an OSM PBF/); + }); + + it("rejects a file cut off inside the header block", () => { + const truncated = scratch("cut.osm.pbf", readFileSync(ZLIB).subarray(0, 40)); + expect(() => readOsmPbfHeader(truncated)).toThrow(); + }); + + it("drops a diff feed that is not https", () => { + const bytes = readFileSync(RAW); + const at = bytes.indexOf("https://"); + const patched = Buffer.from(bytes); + patched.write("http:/", at, "latin1"); + expect(readOsmPbfHeader(scratch("http.osm.pbf", patched))).toEqual({ + timestamp: "2026-10-31T20:21:42.000Z", + }); + }); +}); + +describe("tryReadOsmPbfHeader", () => { + it("reports nothing for a missing or unreadable file instead of throwing", () => { + expect(tryReadOsmPbfHeader("/nonexistent/region.osm.pbf")).toEqual({}); + expect(tryReadOsmPbfHeader(scratch("empty.osm.pbf", Buffer.alloc(0)))).toEqual({}); + }); +}); diff --git a/services/data-manager/openapi.json b/services/data-manager/openapi.json index 4ee373ca6..63a389fe2 100644 --- a/services/data-manager/openapi.json +++ b/services/data-manager/openapi.json @@ -14841,6 +14841,12 @@ "path": { "type": "string", "minLength": 1 + }, + "osmTimestamp": { + "type": "string" + }, + "osmReplicationUrl": { + "type": "string" } }, "required": [ diff --git a/services/data-manager/src/__tests__/fixtures/osm-pbf-header/replication-raw.osm.pbf b/services/data-manager/src/__tests__/fixtures/osm-pbf-header/replication-raw.osm.pbf new file mode 100644 index 0000000000000000000000000000000000000000..630789983bb41f47b648b215f721fcb12fda9eea GIT binary patch literal 482 zcmZ9H&q@MO6vpq&*q!lcAyW&H3b7UQdQ6E$d6I*lS*r$atzlD_FL3lfmO&LXyP&W{ GG5!S)@@I^DoW~PR>ZpP1FrD&@)rwa!JiA zPW8)ANiA+-6v)X;$}i5%EX~z7(lavFGi+eGHhboEwhpH2%9EIkGfGMdimmkZQ}WC6 za`F>X^wLxF(-M=4GPCtkQuR|yi}DLn^>g#{5|i_FOAAsGOHzvgTkR*u0000B3I|V9 zO+;aIVHi^oX&O&>oa18QV&GyBN)(FcVq*dWEe$Rh$<@#x#)5+fMrlX=S~shr;)|0XrSTDXq-Vzx*$$vvXn8 zEt2bKhwPK@6f#a3&v~*C2PnEC8&Z%`UN~bc@H(=Zv!MA_?ISP$nL!3Eqhh8)d;w6` FAbngmw=n { + return { + ...(header.timestamp ? { osmTimestamp: header.timestamp } : {}), + ...(header.replicationUrl ? { osmReplicationUrl: header.replicationUrl } : {}), + }; +} + +/** + * Record the header of every downloaded OSM PBF that has none recorded yet, + * so a file fetched before the header was read still reports how current it + * is. A file that is gone or unreadable is left as it is. + */ +export function backfillOsmHeaders(store: StateStore): number { + let filled = 0; + for (const dataset of store.getAll()) { + if (dataset.type !== "osm-pbf" || dataset.osmTimestamp) continue; + const fields = headerFields(tryReadOsmPbfHeader(dataset.path)); + if (!fields.osmTimestamp) continue; + store.upsert({ ...dataset, ...fields }); + filled += 1; + } + return filled; +} + export async function downloadOsm(opts: DownloadOsmOptions): Promise { const url = resolveOsmUrl(opts.region); const fileName = osmPbfName(opts.region); @@ -151,6 +179,7 @@ export async function downloadOsm(opts: DownloadOsmOptions): Promise { + let offset = 0; + const varint = (): number => { + let result = 0; + let scale = 1; + for (let i = 0; i < 10; i += 1) { + if (offset >= bytes.length) throw new Error("truncated varint"); + const byte = bytes[offset++] as number; + result += (byte & 0x7f) * scale; + if ((byte & 0x80) === 0) { + if (!Number.isSafeInteger(result)) throw new Error("varint out of range"); + return result; + } + scale *= 128; + } + throw new Error("varint too long"); + }; + const skip = (count: number): Uint8Array => { + if (count < 0 || offset + count > bytes.length) throw new Error("truncated field"); + const slice = bytes.subarray(offset, offset + count); + offset += count; + return slice; + }; + while (offset < bytes.length) { + const key = varint(); + const number = Math.floor(key / 8); + const wire = key % 8; + if (wire === WIRE_VARINT) yield { number, value: varint() }; + else if (wire === WIRE_BYTES) yield { number, value: skip(varint()) }; + else if (wire === WIRE_64BIT) skip(8); + else if (wire === WIRE_32BIT) skip(4); + else throw new Error(`unsupported wire type ${wire}`); + } +} + +function readPrefix(path: string, length: number, position: number): Buffer { + const descriptor = openSync(path, "r"); + try { + const buffer = Buffer.alloc(length); + let read = 0; + while (read < length) { + const count = readSync(descriptor, buffer, read, length - read, position + read); + if (count === 0) throw new Error("file ends inside the header block"); + read += count; + } + return buffer; + } finally { + closeSync(descriptor); + } +} + +function text(value: number | Uint8Array): string | undefined { + return value instanceof Uint8Array ? new TextDecoder().decode(value) : undefined; +} + +function httpsUrl(value: string | undefined): string | undefined { + if (!value || value.length > MAX_URL_LENGTH) return undefined; + try { + return new URL(value).protocol === "https:" ? value : undefined; + } catch { + return undefined; + } +} + +/** The header block's bytes from its Blob: stored raw or zlib-compressed. */ +function headerBlockBytes(blob: Uint8Array): Uint8Array { + let raw: Uint8Array | undefined; + let zlib: Uint8Array | undefined; + let rawSize: number | undefined; + for (const field of fields(blob)) { + if (field.number === 1 && field.value instanceof Uint8Array) raw = field.value; + else if (field.number === 2 && typeof field.value === "number") rawSize = field.value; + else if (field.number === 3 && field.value instanceof Uint8Array) zlib = field.value; + } + if (raw) return raw; + if (zlib && rawSize !== undefined && rawSize <= MAX_HEADER_BLOB_BYTES) { + return inflateSync(zlib, { maxOutputLength: MAX_HEADER_BLOB_BYTES }); + } + throw new Error("header block is neither raw nor zlib-compressed"); +} + +/** + * Read the replication timestamp and diff feed from an OSM PBF's header + * block, the file's first blob. Only that blob is read, so this is cheap on + * a planet file. Throws when the file does not start with a header block. + */ +export function readOsmPbfHeader(path: string): OsmPbfHeader { + const headerLength = readPrefix(path, 4, 0).readUInt32BE(0); + if (headerLength < 1 || headerLength > MAX_BLOB_HEADER_BYTES) { + throw new Error("not an OSM PBF: implausible blob header length"); + } + let type: string | undefined; + let dataSize: number | undefined; + for (const field of fields(readPrefix(path, headerLength, 4))) { + if (field.number === 1) type = text(field.value); + else if (field.number === 3 && typeof field.value === "number") dataSize = field.value; + } + if (type !== "OSMHeader") throw new Error("not an OSM PBF: first blob is not OSMHeader"); + if (dataSize === undefined || dataSize < 1 || dataSize > MAX_HEADER_BLOB_BYTES) { + throw new Error("not an OSM PBF: implausible header blob size"); + } + const block = headerBlockBytes(readPrefix(path, dataSize, 4 + headerLength)); + + const header: OsmPbfHeader = {}; + for (const field of fields(block)) { + if (field.number === 32 && typeof field.value === "number" && field.value > 0) { + const at = new Date(field.value * 1000); + if (!Number.isNaN(at.getTime())) header.timestamp = at.toISOString(); + } else if (field.number === 34) { + const url = httpsUrl(text(field.value)); + if (url) header.replicationUrl = url; + } + } + return header; +} + +/** `readOsmPbfHeader`, or nothing when the file is missing or not readable as one. */ +export function tryReadOsmPbfHeader(path: string): OsmPbfHeader { + try { + return readOsmPbfHeader(path); + } catch { + return {}; + } +} diff --git a/services/nominatim/service.json b/services/nominatim/service.json index fe6f7bb6c..b16041bdb 100644 --- a/services/nominatim/service.json +++ b/services/nominatim/service.json @@ -19,8 +19,10 @@ "expose": [8080], "environment": { "PBF_PATH": "/nominatim/data/data.osm.pbf", - "REPLICATION_URL": "https://planet.openstreetmap.org/replication/day", - "REPLICATION_UPDATE_INTERVAL": "86400", + "REPLICATION_URL": "${NOMINATIM_REPLICATION_URL:-https://planet.openstreetmap.org/replication/day}", + "REPLICATION_UPDATE_INTERVAL": "${NOMINATIM_REPLICATION_UPDATE_INTERVAL:-86400}", + "REPLICATION_RECHECK_INTERVAL": "${NOMINATIM_REPLICATION_RECHECK_INTERVAL:-900}", + "UPDATE_MODE": "${NOMINATIM_UPDATE_MODE:-none}", "NOMINATIM_PASSWORD": "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set}", "THREADS": "${NOMINATIM_THREADS:-8}", "IMPORT_STYLE": "full", From 493082cf2788310e9cb2b6f001c3277c807cd4bd Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 1 Nov 2026 19:12:32 +0000 Subject: [PATCH 2/2] chore(changeset): note the extension CLI's refreshed bundled dataset schema --- .changeset/osm-extract-dataset-fields.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/osm-extract-dataset-fields.md diff --git a/.changeset/osm-extract-dataset-fields.md b/.changeset/osm-extract-dataset-fields.md new file mode 100644 index 000000000..1c4011368 --- /dev/null +++ b/.changeset/osm-extract-dataset-fields.md @@ -0,0 +1,8 @@ +--- +"@openmapx/extension-cli": patch +--- + +Refresh the extension CLI's bundled core inputs following the data-manager dataset +record's two new optional fields for an OSM extract's replication timestamp and diff +feed. This covers its bundled workspace dependency artifact; the extension scaffolding, +validation and packaging command interfaces remain unchanged.