diff --git a/.changeset/osm-extract-dataset-fields.md b/.changeset/osm-extract-dataset-fields.md new file mode 100644 index 000000000..1c4011368 --- /dev/null +++ b/.changeset/osm-extract-dataset-fields.md @@ -0,0 +1,8 @@ +--- +"@openmapx/extension-cli": patch +--- + +Refresh the extension CLI's bundled core inputs following the data-manager dataset +record's two new optional fields for an OSM extract's replication timestamp and diff +feed. This covers its bundled workspace dependency artifact; the extension scaffolding, +validation and packaging command interfaces remain unchanged. diff --git a/apps/api/openapi.json b/apps/api/openapi.json index ac8aeb102..2d1be520c 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -42238,6 +42238,25 @@ ], "additionalProperties": false }, + "osmExtract": { + "type": "object", + "properties": { + "timestamp": { + "type": "string", + "format": "date-time", + "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d:[0-5]\\d(?:\\.\\d+)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$" + }, + "replicationUrl": { + "type": "string", + "maxLength": 2048, + "format": "uri" + } + }, + "required": [ + "timestamp" + ], + "additionalProperties": false + }, "fetchedAt": { "type": "string", "format": "date-time", diff --git a/apps/api/src/routes/admin-services.ts b/apps/api/src/routes/admin-services.ts index 0e209932e..6623135dc 100644 --- a/apps/api/src/routes/admin-services.ts +++ b/apps/api/src/routes/admin-services.ts @@ -27,6 +27,7 @@ import { } from "../services/direct-ops-idempotency"; import { jobRunner } from "../services/job-runner"; import { createApiOpsClient, createDurableOpsKey, executeAndWait } from "../services/ops-client"; +import { osmExtractFor, readDataManagerDatasets } from "../services/osm-extract"; import { isSecretsConfigured } from "../services/secrets"; import { resolveServiceConfigWithSources } from "../services/service-config-resolver"; import { persistServiceConfigPatch } from "../services/service-config-writer"; @@ -727,14 +728,19 @@ export async function adminServicesRoute(app: FastifyInstance): Promise { // GET /admin/services/data — data inventory (OSM, builds, MOTIS health) registerServiceAdminRoute(app, "getAdminServiceData", async () => { - const inventory = await executeAndWait( - createApiOpsClient(), - { kind: "data.inspect" }, - createDurableOpsKey("admin.data.inspect", "current"), - { signal: AbortSignal.timeout(30_000) }, - ); + const [inventory, datasets] = await Promise.all([ + executeAndWait( + createApiOpsClient(), + { kind: "data.inspect" }, + createDurableOpsKey("admin.data.inspect", "current"), + { signal: AbortSignal.timeout(30_000) }, + ), + readDataManagerDatasets(), + ]); + const osmExtract = osmExtractFor(inventory.osm, datasets); return { ...inventory, + ...(osmExtract ? { osmExtract } : {}), fetchedAt: new Date().toISOString(), }; }); diff --git a/apps/api/src/services/osm-extract.test.ts b/apps/api/src/services/osm-extract.test.ts new file mode 100644 index 000000000..5585a3352 --- /dev/null +++ b/apps/api/src/services/osm-extract.test.ts @@ -0,0 +1,54 @@ +import type { DatasetMetadata } from "@openmapx/core/data-manager-contracts"; +import { describe, expect, it } from "vitest"; +import { osmExtractFor } from "./osm-extract"; + +const osm = { + found: true, + filename: "europe-germany-berlin.osm.pbf", + sizeBytes: 1000, + modifiedAt: "2026-11-01T03:00:00.000Z", + region: "europe-germany-berlin", +}; + +function dataset(over: Partial = {}): DatasetMetadata { + return { + type: "osm-pbf", + id: "europe/germany/berlin", + path: "/data/osm/europe-germany-berlin.osm.pbf", + sizeBytes: 1000, + downloadedAt: "2026-11-01T03:00:00.000Z", + osmTimestamp: "2026-10-31T20:21:42Z", + osmReplicationUrl: "https://download.geofabrik.de/europe/germany/berlin-updates", + ...over, + }; +} + +describe("osmExtractFor", () => { + it("reports the extract time recorded for the file the inventory found", () => { + expect(osmExtractFor(osm, [dataset()])).toEqual({ + timestamp: "2026-10-31T20:21:42.000Z", + replicationUrl: "https://download.geofabrik.de/europe/germany/berlin-updates", + }); + }); + + it("does not show a record of another or an earlier file as this one's", () => { + expect( + osmExtractFor(osm, [dataset({ path: "/data/osm/europe-germany.osm.pbf" })]), + ).toBeUndefined(); + expect(osmExtractFor(osm, [dataset({ sizeBytes: 999 })])).toBeUndefined(); + expect(osmExtractFor(osm, [dataset({ type: "osm-pbf-bz2" })])).toBeUndefined(); + }); + + it("reports nothing without a recorded time, a file or a usable record", () => { + expect(osmExtractFor(osm, [dataset({ osmTimestamp: undefined })])).toBeUndefined(); + expect(osmExtractFor(osm, [dataset({ osmTimestamp: "yesterday-ish" })])).toBeUndefined(); + expect(osmExtractFor({ found: false }, [dataset()])).toBeUndefined(); + expect(osmExtractFor(osm, [])).toBeUndefined(); + }); + + it("drops a feed that is not https rather than failing the response", () => { + expect( + osmExtractFor(osm, [dataset({ osmReplicationUrl: "http://example.org/updates" })]), + ).toEqual({ timestamp: "2026-10-31T20:21:42.000Z" }); + }); +}); diff --git a/apps/api/src/services/osm-extract.ts b/apps/api/src/services/osm-extract.ts new file mode 100644 index 000000000..4335d1694 --- /dev/null +++ b/apps/api/src/services/osm-extract.ts @@ -0,0 +1,78 @@ +import type { DatasetMetadata } from "@openmapx/core/data-manager-contracts"; +import * as dm from "@openmapx/core/data-manager-sdk"; +import type { OpsResultFor } from "@openmapx/core/ops"; +import { envString } from "@openmapx/core/server-env"; + +const DATA_MANAGER_URL_DEFAULT = "http://localhost:4000"; +const DATA_MANAGER_TIMEOUT_MS = 5_000; +const DATA_MANAGER_MAX_BYTES = 1024 * 1024; + +type OsmInventory = OpsResultFor<"data.inspect">["osm"]; + +/** How current the OSM extract on disk is, from its own header. */ +export interface OsmExtract { + /** The time the extract's data is current to. */ + timestamp: string; + /** The diff feed that continues the extract. */ + replicationUrl?: string; +} + +function basename(path: string): string { + return path.slice(path.lastIndexOf("/") + 1); +} + +/** An https URL short enough to show, or nothing, so a bad record cannot fail the response. */ +function httpsUrl(value: string | undefined): string | undefined { + if (!value || value.length > 2048) return undefined; + try { + return new URL(value).protocol === "https:" ? value : undefined; + } catch { + return undefined; + } +} + +function isTimestamp(value: string | undefined): value is string { + return value !== undefined && !Number.isNaN(Date.parse(value)); +} + +/** + * The extract time data-manager recorded for the PBF the inventory found: + * the same file name and size, so a record of an earlier or another file is + * never shown as this one's. + */ +export function osmExtractFor( + osm: OsmInventory, + datasets: readonly DatasetMetadata[], +): OsmExtract | undefined { + if (!osm.found || !osm.filename) return undefined; + const dataset = datasets.find( + (d) => + d.type === "osm-pbf" && + basename(d.path) === osm.filename && + d.sizeBytes === osm.sizeBytes && + isTimestamp(d.osmTimestamp), + ); + if (!dataset?.osmTimestamp) return undefined; + const replicationUrl = httpsUrl(dataset.osmReplicationUrl); + return { + timestamp: new Date(dataset.osmTimestamp).toISOString(), + ...(replicationUrl ? { replicationUrl } : {}), + }; +} + +/** The datasets data-manager has recorded, or none when it cannot be reached. */ +export async function readDataManagerDatasets(): Promise { + try { + const response = await dm.getDatasets({ + transport: dm.createDataManagerTransport({ + baseUrl: envString("DATA_MANAGER_URL", DATA_MANAGER_URL_DEFAULT), + authToken: envString("DATA_MANAGER_AUTH_TOKEN", ""), + requestTimeoutMs: DATA_MANAGER_TIMEOUT_MS, + maxJsonBytes: DATA_MANAGER_MAX_BYTES, + }), + }); + return dm.unwrapDataManagerResponse(response).datasets; + } catch { + return []; + } +} diff --git a/apps/web/src/components/admin/services/DataWorkflowsPage.tsx b/apps/web/src/components/admin/services/DataWorkflowsPage.tsx index 3abc22ba2..cc1468f49 100644 --- a/apps/web/src/components/admin/services/DataWorkflowsPage.tsx +++ b/apps/web/src/components/admin/services/DataWorkflowsPage.tsx @@ -41,6 +41,7 @@ import { AdminPageHeader } from "../shared/AdminPageHeader"; import { useAdminToast } from "../shared/AdminToast"; import { AmbientPlacesMaintenance } from "./AmbientPlacesMaintenance"; import { NotablePlacesMaintenance, notablePlacesStatusQueryKey } from "./NotablePlacesMaintenance"; +import { OsmExtractDetails } from "./OsmExtractDetails"; import { OvertureMaintenance, overtureStatusQueryKey } from "./OvertureMaintenance"; import { SearchIndexMaintenance, searchIndexStatusQueryKey } from "./SearchIndexMaintenance"; import { TransitSourcesSection } from "./TransitSourcesSection"; @@ -174,7 +175,7 @@ function DataOperationsSection({ apiUrl }: { apiUrl: string }) { ); } -function OsmSection({ osm }: { osm: OsmInfo }) { +function OsmSection({ osm, extract }: { osm: OsmInfo; extract: AdminServiceData["osmExtract"] }) { return ( {formatDate(osm.modifiedAt)} )} + } @@ -642,7 +644,7 @@ function DataWorkflowsView() { - + diff --git a/apps/web/src/components/admin/services/OsmExtractDetails.test.tsx b/apps/web/src/components/admin/services/OsmExtractDetails.test.tsx new file mode 100644 index 000000000..74e0de20e --- /dev/null +++ b/apps/web/src/components/admin/services/OsmExtractDetails.test.tsx @@ -0,0 +1,53 @@ +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it } from "vitest"; +import { extractAge, OsmExtractDetails } from "./OsmExtractDetails"; + +const NOW = Date.parse("2026-11-01T12:00:00Z"); +const hoursAgo = (hours: number) => new Date(NOW - hours * 3_600_000).toISOString(); + +afterEach(cleanup); + +describe("extractAge", () => { + it("counts hours within two days, then days", () => { + expect(extractAge(hoursAgo(0.5), NOW)).toBe("less than an hour ago"); + expect(extractAge(hoursAgo(5), NOW)).toBe("5 hours ago"); + expect(extractAge(hoursAgo(47), NOW)).toBe("47 hours ago"); + expect(extractAge(hoursAgo(48), NOW)).toBe("2 days ago"); + expect(extractAge(hoursAgo(24 * 40), NOW)).toBe("40 days ago"); + }); + + it("does not report a future time as negative age", () => { + expect(extractAge(hoursAgo(-3), NOW)).toBe("less than an hour ago"); + }); +}); + +describe("OsmExtractDetails", () => { + it("shows when the data is current to, its age and the feed that continues it", () => { + render( + , + ); + expect(screen.getByText("Data as of")).toBeTruthy(); + expect(screen.getByText(/20 hours ago/)).toBeTruthy(); + expect( + screen.getByText("https://download.geofabrik.de/europe/germany/berlin-updates"), + ).toBeTruthy(); + expect(screen.queryByText(/Older than/)).toBeNull(); + }); + + it("flags an extract older than thirty days", () => { + render(); + expect(screen.getByText("Older than 30 days")).toBeTruthy(); + expect(screen.queryByText(/Update feed/)).toBeNull(); + }); + + it("says the time is unknown when the file records none", () => { + render(); + expect(screen.getByText("Unknown: the file records no replication time")).toBeTruthy(); + }); +}); diff --git a/apps/web/src/components/admin/services/OsmExtractDetails.tsx b/apps/web/src/components/admin/services/OsmExtractDetails.tsx new file mode 100644 index 000000000..20673da89 --- /dev/null +++ b/apps/web/src/components/admin/services/OsmExtractDetails.tsx @@ -0,0 +1,90 @@ +"use client"; + +import Box from "@mui/material/Box"; +import Chip from "@mui/material/Chip"; +import Typography from "@mui/material/Typography"; +import type { AdminServiceData } from "@openmapx/core/api-contracts"; + +type OsmExtract = NonNullable; + +/** Past this age the card flags the extract: Geofabrik publishes a new one every day. */ +export const STALE_EXTRACT_DAYS = 30; + +const HOUR_MS = 3_600_000; +const DAY_MS = 24 * HOUR_MS; + +/** How long ago the extract's data was current, coarsely: hours within two days, else days. */ +export function extractAge(timestamp: string, now: number): string { + const elapsed = Math.max(0, now - Date.parse(timestamp)); + const format = new Intl.RelativeTimeFormat("en", { numeric: "auto" }); + if (elapsed < HOUR_MS) return "less than an hour ago"; + if (elapsed < 2 * DAY_MS) return format.format(-Math.floor(elapsed / HOUR_MS), "hour"); + return format.format(-Math.floor(elapsed / DAY_MS), "day"); +} + +function formatDate(iso: string): string { + return new Date(iso).toLocaleDateString(undefined, { + year: "numeric", + month: "short", + day: "numeric", + hour: "2-digit", + minute: "2-digit", + }); +} + +/** + * When the OSM extract's data is current to, from the file's own header, and + * the diff feed that continues it. Download time says nothing about this: a + * mirror can serve a weeks-old file today. + */ +export function OsmExtractDetails({ + extract, + now = Date.now(), +}: { + extract: OsmExtract | undefined; + now?: number; +}) { + if (!extract) { + return ( + + + Data as of + + + Unknown: the file records no replication time + + + ); + } + const stale = now - Date.parse(extract.timestamp) > STALE_EXTRACT_DAYS * DAY_MS; + return ( + <> + + + Data as of + + + {formatDate(extract.timestamp)} ({extractAge(extract.timestamp, now)}) + + {stale && ( + + )} + + {extract.replicationUrl && ( + + + Update feed (for NOMINATIM_REPLICATION_URL) + + + {extract.replicationUrl} + + + )} + + ); +} diff --git a/docs/docs/install/configuration.md b/docs/docs/install/configuration.md index d119861ab..35b1a726d 100644 --- a/docs/docs/install/configuration.md +++ b/docs/docs/install/configuration.md @@ -381,6 +381,10 @@ workload; a manifest default is not a hardware sizing guarantee. | `NOMINATIM_IMPORT_WIKIPEDIA` | Import Wikipedia importance scores during the Nominatim build. | Default `true` | | `NOMINATIM_IMPORT_GB_POSTCODES` | Import GB postcodes. | Default `false` | | `NOMINATIM_IMPORT_US_POSTCODES` | Import US postcodes. | Default `false` | +| `NOMINATIM_REPLICATION_URL` | Diff feed Nominatim follows. The default suits a planet import; a regional import must follow its extract's own feed (for Germany `https://download.geofabrik.de/europe/germany-updates/`; the admin panel's OSM card shows the feed the downloaded extract names), or the planet diffs pull changes from everywhere into the regional database. Takes effect when Nominatim is recreated. | Default `https://planet.openstreetmap.org/replication/day` | +| `NOMINATIM_UPDATE_MODE` | Whether Nominatim applies that feed: `continuous` keeps applying it, `catch-up` applies everything published and stops, `once` applies one batch per start, `none` applies nothing, so search stays at the import's date. | Default `none` | +| `NOMINATIM_REPLICATION_UPDATE_INTERVAL` | How often the feed publishes diffs, in seconds. | Default `86400` | +| `NOMINATIM_REPLICATION_RECHECK_INTERVAL` | How long Nominatim waits before asking again when the feed has nothing new, in seconds. | Default `900` | | `OVERPASS_SPACE` | Overpass dispatcher available-RAM budget in bytes, not database disk capacity. The 100 GiB default exceeds the 32 GiB container ceiling; tune together with worker concurrency. | Default `107374182400` (~100 GiB) | | `OVERPASS_FASTCGI_PROCESSES` | Number of Overpass FastCGI worker processes. | Default `4` | | `OVERPASS_DIFF_URL` | Replication feed Overpass applies daily. The default suits a planet import; a regional import must follow its extract's own feed (for Germany `https://download.geofabrik.de/europe/germany-updates/`), or the planet diffs pull changes from everywhere into the regional database. | Default `https://planet.openstreetmap.org/replication/day/` | diff --git a/docs/docs/install/preparing-data.md b/docs/docs/install/preparing-data.md index e3ab175b1..cc5ddaa6f 100644 --- a/docs/docs/install/preparing-data.md +++ b/docs/docs/install/preparing-data.md @@ -57,6 +57,12 @@ do not replace the previous good file. Check disk space before downloading a larger extract; serving multiple PBFs does not automatically give each consumer the correct region. Inspect the generated hardlink plan when several inputs exist. +The extract's own header says when its data is current to, which can be well +before the download. Data-manager records it, and the admin panel's +**Data workflows** page shows it with its age on the OSM card, flagged once it +is more than 30 days old. The card also shows the diff feed the extract names, +which is the feed to give Nominatim (below). + ### Transit schedule sources For a configured MOTIS deployment, sync the requested country subset: @@ -172,6 +178,15 @@ runs the runtime import. It is distinct from transactional live transit sync. Valhalla, Nominatim, and Overpass import within their service containers; they have no CLI build hook. +Nominatim and Overpass can then keep themselves current from a diff feed +instead of a new extract. Both default to the planet feed, which suits only a +planet import: a regional import must follow its extract's feed, or the planet +diffs pull changes from everywhere into the regional database. For Nominatim +set `NOMINATIM_REPLICATION_URL` to that feed and `NOMINATIM_UPDATE_MODE` to +`continuous`; with the default mode `none` it applies no diffs and search stays +at the import's date. For Overpass set `OVERPASS_DIFF_URL`. See +[Configuration](./configuration.md) for each setting. + `services build-all --region europe/germany` currently plans OSRM, OTP, MOTIS, Pelias, and TileServer GL even when only one is selected. It continues after individual failures unless given `--fail-fast`. Explicit `services build ` diff --git a/infra/docker/.env.example b/infra/docker/.env.example index 11b0542e7..d955aa0c7 100644 --- a/infra/docker/.env.example +++ b/infra/docker/.env.example @@ -403,6 +403,18 @@ NOMINATIM_THREADS=8 NOMINATIM_IMPORT_WIKIPEDIA=true NOMINATIM_IMPORT_GB_POSTCODES=false NOMINATIM_IMPORT_US_POSTCODES=false +# Diff feed Nominatim follows and whether it applies it. With the default mode +# `none` it never does, so search stays at the import's date. A regional +# import must follow its extract's own feed, which the admin panel's OSM card +# shows, or the planet diffs pull changes from everywhere into the regional +# database. Modes: `continuous` keeps applying, `catch-up` applies everything +# published and stops, `once` applies one batch per start, `none` applies +# nothing. Takes effect when Nominatim is recreated. +# NOMINATIM_REPLICATION_URL=https://download.geofabrik.de/europe/germany-updates/ +# NOMINATIM_UPDATE_MODE=continuous +# How often the feed publishes, and how long to wait when it has nothing new (seconds). +# NOMINATIM_REPLICATION_UPDATE_INTERVAL=86400 +# NOMINATIM_REPLICATION_RECHECK_INTERVAL=900 # Overpass API # OVERPASS_SPACE is the dispatcher's query RAM budget in bytes, not disk space. diff --git a/packages/core/src/api/contracts/service-admin/data.ts b/packages/core/src/api/contracts/service-admin/data.ts index 5ce33a02d..7e1c60528 100644 --- a/packages/core/src/api/contracts/service-admin/data.ts +++ b/packages/core/src/api/contracts/service-admin/data.ts @@ -4,6 +4,19 @@ import { adminCountSchema, adminTimestampSchema } from "../admin"; import { adminServiceIdListSchema } from "./common"; export const adminServiceDataSchema = OPS_RESULT_SCHEMAS["data.inspect"].extend({ + /** + * How current the OSM extract on disk is, from its header as data-manager + * recorded it; absent when the file names no time or data-manager has none. + */ + osmExtract: z + .strictObject({ + timestamp: adminTimestampSchema, + replicationUrl: z + .url({ protocol: /^https$/ }) + .max(2048) + .optional(), + }) + .optional(), fetchedAt: adminTimestampSchema, }); export const adminServiceSelectionSchema = z.strictObject({ diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200.ts index def60cf87..6ad269dd7 100644 --- a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200.ts +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200.ts @@ -3,12 +3,14 @@ import type { GetAdminServiceData200BuildsItem } from "./getAdminServiceData200BuildsItem"; import type { GetAdminServiceData200MotisTransitous } from "./getAdminServiceData200MotisTransitous"; import type { GetAdminServiceData200Osm } from "./getAdminServiceData200Osm"; +import type { GetAdminServiceData200OsmExtract } from "./getAdminServiceData200OsmExtract"; export type GetAdminServiceData200 = { osm: GetAdminServiceData200Osm; /** @maxItems 10 */ builds: GetAdminServiceData200BuildsItem[]; motisTransitous: GetAdminServiceData200MotisTransitous; + osmExtract?: GetAdminServiceData200OsmExtract; /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ fetchedAt: string; }; diff --git a/packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmExtract.ts b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmExtract.ts new file mode 100644 index 000000000..cf0d8e197 --- /dev/null +++ b/packages/core/src/api/generated/admin/service-models/getAdminServiceData200OsmExtract.ts @@ -0,0 +1,9 @@ +// biome-ignore-all lint: Generated by Orval; edit API contracts and regenerate. +// Generated by pnpm sdk:generate. Do not edit. + +export type GetAdminServiceData200OsmExtract = { + /** @pattern ^(?:(?:\d\d[2468][048]|\d\d[13579][26]|\d\d0[48]|[02468][048]00|[13579][26]00)-02-29|\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\d|30)|(?:02)-(?:0[1-9]|1\d|2[0-8])))T(?:(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|([+-](?:[01]\d|2[0-3]):[0-5]\d)))$ */ + timestamp: string; + /** @maxLength 2048 */ + replicationUrl?: string; +}; diff --git a/packages/core/src/api/generated/admin/service-models/index.ts b/packages/core/src/api/generated/admin/service-models/index.ts index 71a117c35..e14cd0c40 100644 --- a/packages/core/src/api/generated/admin/service-models/index.ts +++ b/packages/core/src/api/generated/admin/service-models/index.ts @@ -172,6 +172,7 @@ export * from "./getAdminServiceData200MotisTransitousOperationsProfile"; export * from "./getAdminServiceData200MotisTransitousPreflightState"; export * from "./getAdminServiceData200MotisTransitousPreviousHealthySlot"; export * from "./getAdminServiceData200Osm"; +export * from "./getAdminServiceData200OsmExtract"; export * from "./getAdminServiceData400"; export * from "./getAdminServiceData401"; export * from "./getAdminServiceData403"; diff --git a/packages/core/src/services/data-manager/contracts/datasets.ts b/packages/core/src/services/data-manager/contracts/datasets.ts index 8b81311ea..dfd78507e 100644 --- a/packages/core/src/services/data-manager/contracts/datasets.ts +++ b/packages/core/src/services/data-manager/contracts/datasets.ts @@ -24,6 +24,10 @@ export const datasetMetadataSchema = z.strictObject({ sha256: z.string().optional(), md5: z.string().optional(), path: z.string().min(1), + /** An OSM PBF's replication timestamp: the time its data is current to. */ + osmTimestamp: z.string().optional(), + /** The diff feed an OSM PBF's header names as continuing it. */ + osmReplicationUrl: z.string().optional(), }); export const datasetsSchema = z.strictObject({ datasets: z.array(datasetMetadataSchema) }); export const reloadDatasetsSchema = z.strictObject({ ok: z.literal(true), datasets: countSchema }); diff --git a/packages/core/src/services/data-manager/generated/models/getDatasets200DatasetsItem.ts b/packages/core/src/services/data-manager/generated/models/getDatasets200DatasetsItem.ts index 785cab7f3..af30acbea 100644 --- a/packages/core/src/services/data-manager/generated/models/getDatasets200DatasetsItem.ts +++ b/packages/core/src/services/data-manager/generated/models/getDatasets200DatasetsItem.ts @@ -16,4 +16,6 @@ export type GetDatasets200DatasetsItem = { md5?: string; /** @minLength 1 */ path: string; + osmTimestamp?: string; + osmReplicationUrl?: string; }; diff --git a/services/data-manager/__tests__/api.test.ts b/services/data-manager/__tests__/api.test.ts index bb9b7487c..3fc8cf26a 100644 --- a/services/data-manager/__tests__/api.test.ts +++ b/services/data-manager/__tests__/api.test.ts @@ -17,7 +17,10 @@ const mockDownloadOsm = vi.hoisted(() => })), ); -vi.mock("../src/jobs/download-osm.js", () => ({ downloadOsm: mockDownloadOsm })); +vi.mock("../src/jobs/download-osm.js", () => ({ + backfillOsmHeaders: () => 0, + downloadOsm: mockDownloadOsm, +})); describe("data-manager API", () => { let fixtureRoot: string; diff --git a/services/data-manager/__tests__/download-osm.test.ts b/services/data-manager/__tests__/download-osm.test.ts index 3f48b6d42..4ddd59bf7 100644 --- a/services/data-manager/__tests__/download-osm.test.ts +++ b/services/data-manager/__tests__/download-osm.test.ts @@ -1,8 +1,9 @@ -import { mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { copyFileSync, mkdirSync, mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { + backfillOsmHeaders, type DownloadOsmOptions, downloadOsm, resolveOsmMd5Url, @@ -102,3 +103,89 @@ describe("downloadOsm publication", () => { } }); }); + +const STAMPED_PBF = join( + import.meta.dirname, + "../src/__tests__/fixtures/osm-pbf-header/replication-zlib.osm.pbf", +); + +describe("downloadOsm extract time", () => { + it("records the time the extract is current to and its diff feed", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "openmapx-osm-header-")); + const store = new StateStore(dataDir); + try { + await downloadOsm({ + region: "europe/monaco", + dataDir, + store, + verifyChecksum: false, + downloadImpl: async (_url, targetPath) => { + copyFileSync(STAMPED_PBF, targetPath); + return { published: true }; + }, + }); + + expect(store.getAll()[0]).toMatchObject({ + type: "osm-pbf", + id: "europe/monaco", + osmTimestamp: "2026-10-31T20:21:42.000Z", + osmReplicationUrl: "https://download.geofabrik.de/europe/monaco-updates", + }); + } finally { + rmSync(dataDir, { recursive: true, force: true }); + } + }); + + it("records none for a file whose header carries no replication state", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "openmapx-osm-header-")); + const store = new StateStore(dataDir); + try { + await downloadOsm({ + region: "europe/monaco", + dataDir, + store, + verifyChecksum: false, + downloadImpl: async (_url, targetPath) => { + writeFileSync(targetPath, "not a pbf"); + return { published: true }; + }, + }); + + expect(store.getAll()[0]?.osmTimestamp).toBeUndefined(); + expect(store.getAll()[0]?.osmReplicationUrl).toBeUndefined(); + } finally { + rmSync(dataDir, { recursive: true, force: true }); + } + }); +}); + +describe("backfillOsmHeaders", () => { + it("fills the extract time of a PBF downloaded before it was recorded, and only that", () => { + const dataDir = mkdtempSync(join(tmpdir(), "openmapx-osm-backfill-")); + try { + mkdirSync(join(dataDir, "osm")); + const path = join(dataDir, "osm", "europe-monaco.osm.pbf"); + copyFileSync(STAMPED_PBF, path); + const store = new StateStore(dataDir); + const base = { sizeBytes: 411, downloadedAt: "2026-10-01T00:00:00.000Z" }; + store.upsert({ type: "osm-pbf", id: "europe/monaco", path, ...base }); + store.upsert({ + type: "osm-pbf", + id: "gone", + path: join(dataDir, "osm", "gone.osm.pbf"), + ...base, + }); + store.upsert({ type: "tile-mbtiles", id: "tiles", path, ...base }); + + expect(backfillOsmHeaders(store)).toBe(1); + + const byId = new Map(new StateStore(dataDir).getAll().map((d) => [d.id, d])); + expect(byId.get("europe/monaco")?.osmTimestamp).toBe("2026-10-31T20:21:42.000Z"); + expect(byId.get("gone")?.osmTimestamp).toBeUndefined(); + expect(byId.get("tiles")?.osmTimestamp).toBeUndefined(); + expect(backfillOsmHeaders(store)).toBe(0); + } finally { + rmSync(dataDir, { recursive: true, force: true }); + } + }); +}); diff --git a/services/data-manager/__tests__/utils/osm-pbf-header.test.ts b/services/data-manager/__tests__/utils/osm-pbf-header.test.ts new file mode 100644 index 000000000..18c9e5ca0 --- /dev/null +++ b/services/data-manager/__tests__/utils/osm-pbf-header.test.ts @@ -0,0 +1,70 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { readOsmPbfHeader, tryReadOsmPbfHeader } from "../../src/utils/osm-pbf-header.js"; + +// Written by libosmium (pyosmium) from the traffic-engine fixture, with the +// replication header Geofabrik sets on its extracts. +const FIXTURES = join(import.meta.dirname, "../../src/__tests__/fixtures"); +const ZLIB = join(FIXTURES, "osm-pbf-header/replication-zlib.osm.pbf"); +const RAW = join(FIXTURES, "osm-pbf-header/replication-raw.osm.pbf"); +const NO_REPLICATION = join(FIXTURES, "traffic-engine/roads.osm.pbf"); + +const dirs: string[] = []; +function scratch(name: string, bytes: Buffer): string { + const dir = mkdtempSync(join(tmpdir(), "openmapx-pbf-header-")); + dirs.push(dir); + const path = join(dir, name); + writeFileSync(path, bytes); + return path; +} + +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +describe("readOsmPbfHeader", () => { + it("reads the replication timestamp and diff feed from a compressed header block", () => { + expect(readOsmPbfHeader(ZLIB)).toEqual({ + timestamp: "2026-10-31T20:21:42.000Z", + replicationUrl: "https://download.geofabrik.de/europe/monaco-updates", + }); + }); + + it("reads an uncompressed header block too", () => { + expect(readOsmPbfHeader(RAW)).toEqual(readOsmPbfHeader(ZLIB)); + }); + + it("reports nothing for a PBF whose header names no replication state", () => { + expect(readOsmPbfHeader(NO_REPLICATION)).toEqual({}); + }); + + it("rejects a file that does not start with an OSM header block", () => { + expect(() => + readOsmPbfHeader(scratch("text.osm.pbf", Buffer.from("not a pbf at all"))), + ).toThrow(/not an OSM PBF/); + }); + + it("rejects a file cut off inside the header block", () => { + const truncated = scratch("cut.osm.pbf", readFileSync(ZLIB).subarray(0, 40)); + expect(() => readOsmPbfHeader(truncated)).toThrow(); + }); + + it("drops a diff feed that is not https", () => { + const bytes = readFileSync(RAW); + const at = bytes.indexOf("https://"); + const patched = Buffer.from(bytes); + patched.write("http:/", at, "latin1"); + expect(readOsmPbfHeader(scratch("http.osm.pbf", patched))).toEqual({ + timestamp: "2026-10-31T20:21:42.000Z", + }); + }); +}); + +describe("tryReadOsmPbfHeader", () => { + it("reports nothing for a missing or unreadable file instead of throwing", () => { + expect(tryReadOsmPbfHeader("/nonexistent/region.osm.pbf")).toEqual({}); + expect(tryReadOsmPbfHeader(scratch("empty.osm.pbf", Buffer.alloc(0)))).toEqual({}); + }); +}); diff --git a/services/data-manager/openapi.json b/services/data-manager/openapi.json index 4ee373ca6..63a389fe2 100644 --- a/services/data-manager/openapi.json +++ b/services/data-manager/openapi.json @@ -14841,6 +14841,12 @@ "path": { "type": "string", "minLength": 1 + }, + "osmTimestamp": { + "type": "string" + }, + "osmReplicationUrl": { + "type": "string" } }, "required": [ diff --git a/services/data-manager/src/__tests__/fixtures/osm-pbf-header/replication-raw.osm.pbf b/services/data-manager/src/__tests__/fixtures/osm-pbf-header/replication-raw.osm.pbf new file mode 100644 index 000000000..630789983 Binary files /dev/null and b/services/data-manager/src/__tests__/fixtures/osm-pbf-header/replication-raw.osm.pbf differ diff --git a/services/data-manager/src/__tests__/fixtures/osm-pbf-header/replication-zlib.osm.pbf b/services/data-manager/src/__tests__/fixtures/osm-pbf-header/replication-zlib.osm.pbf new file mode 100644 index 000000000..1aa1f18cd Binary files /dev/null and b/services/data-manager/src/__tests__/fixtures/osm-pbf-header/replication-zlib.osm.pbf differ diff --git a/services/data-manager/src/api.ts b/services/data-manager/src/api.ts index 562577f49..23e33346d 100644 --- a/services/data-manager/src/api.ts +++ b/services/data-manager/src/api.ts @@ -18,7 +18,7 @@ import { db, sql } from "./db/index.js"; import { registerAmbientPlacesApi } from "./jobs/ambient-places/api.js"; import { convertPbfToBz2ForRegion } from "./jobs/convert-overpass.js"; import { downloadFonts } from "./jobs/download-fonts.js"; -import { downloadOsm } from "./jobs/download-osm.js"; +import { backfillOsmHeaders, downloadOsm } from "./jobs/download-osm.js"; import { applyHardlinkPlan } from "./jobs/link.js"; import { type BuildNotablePlacesOptions, @@ -504,6 +504,8 @@ export function registerApi(app: FastifyInstance, opts: ApiOptions = {}): void { const dataDir = opts.dataDir ?? process.env.DATA_DIR ?? "/data"; const singleFlight = opts.singleFlight ?? getSingleFlightController(); const store = new StateStore(dataDir); + // A PBF downloaded before its header was recorded still reports how current it is. + backfillOsmHeaders(store); const searchIndexSql = opts.searchIndexSql ?? sql; registerAmbientPlacesApi(app, searchIndexSql); const searchIndexRuntimeState = createSearchIndexRuntimeState(); diff --git a/services/data-manager/src/jobs/download-osm.ts b/services/data-manager/src/jobs/download-osm.ts index 9631bdfbe..0e6e93ffb 100644 --- a/services/data-manager/src/jobs/download-osm.ts +++ b/services/data-manager/src/jobs/download-osm.ts @@ -3,6 +3,7 @@ import { createReadStream, mkdirSync, statSync } from "node:fs"; import { join } from "node:path"; import { execa } from "execa"; import type { DatasetMetadata, StateStore } from "../state.js"; +import { type OsmPbfHeader, tryReadOsmPbfHeader } from "../utils/osm-pbf-header.js"; import { type CurlAtomicOptions, type CurlAtomicResult, @@ -101,6 +102,33 @@ export interface DownloadOsmResult { md5?: string; } +/** The dataset fields an OSM PBF's header supplies. */ +function headerFields( + header: OsmPbfHeader, +): Pick { + return { + ...(header.timestamp ? { osmTimestamp: header.timestamp } : {}), + ...(header.replicationUrl ? { osmReplicationUrl: header.replicationUrl } : {}), + }; +} + +/** + * Record the header of every downloaded OSM PBF that has none recorded yet, + * so a file fetched before the header was read still reports how current it + * is. A file that is gone or unreadable is left as it is. + */ +export function backfillOsmHeaders(store: StateStore): number { + let filled = 0; + for (const dataset of store.getAll()) { + if (dataset.type !== "osm-pbf" || dataset.osmTimestamp) continue; + const fields = headerFields(tryReadOsmPbfHeader(dataset.path)); + if (!fields.osmTimestamp) continue; + store.upsert({ ...dataset, ...fields }); + filled += 1; + } + return filled; +} + export async function downloadOsm(opts: DownloadOsmOptions): Promise { const url = resolveOsmUrl(opts.region); const fileName = osmPbfName(opts.region); @@ -151,6 +179,7 @@ export async function downloadOsm(opts: DownloadOsmOptions): Promise { + let offset = 0; + const varint = (): number => { + let result = 0; + let scale = 1; + for (let i = 0; i < 10; i += 1) { + if (offset >= bytes.length) throw new Error("truncated varint"); + const byte = bytes[offset++] as number; + result += (byte & 0x7f) * scale; + if ((byte & 0x80) === 0) { + if (!Number.isSafeInteger(result)) throw new Error("varint out of range"); + return result; + } + scale *= 128; + } + throw new Error("varint too long"); + }; + const skip = (count: number): Uint8Array => { + if (count < 0 || offset + count > bytes.length) throw new Error("truncated field"); + const slice = bytes.subarray(offset, offset + count); + offset += count; + return slice; + }; + while (offset < bytes.length) { + const key = varint(); + const number = Math.floor(key / 8); + const wire = key % 8; + if (wire === WIRE_VARINT) yield { number, value: varint() }; + else if (wire === WIRE_BYTES) yield { number, value: skip(varint()) }; + else if (wire === WIRE_64BIT) skip(8); + else if (wire === WIRE_32BIT) skip(4); + else throw new Error(`unsupported wire type ${wire}`); + } +} + +function readPrefix(path: string, length: number, position: number): Buffer { + const descriptor = openSync(path, "r"); + try { + const buffer = Buffer.alloc(length); + let read = 0; + while (read < length) { + const count = readSync(descriptor, buffer, read, length - read, position + read); + if (count === 0) throw new Error("file ends inside the header block"); + read += count; + } + return buffer; + } finally { + closeSync(descriptor); + } +} + +function text(value: number | Uint8Array): string | undefined { + return value instanceof Uint8Array ? new TextDecoder().decode(value) : undefined; +} + +function httpsUrl(value: string | undefined): string | undefined { + if (!value || value.length > MAX_URL_LENGTH) return undefined; + try { + return new URL(value).protocol === "https:" ? value : undefined; + } catch { + return undefined; + } +} + +/** The header block's bytes from its Blob: stored raw or zlib-compressed. */ +function headerBlockBytes(blob: Uint8Array): Uint8Array { + let raw: Uint8Array | undefined; + let zlib: Uint8Array | undefined; + let rawSize: number | undefined; + for (const field of fields(blob)) { + if (field.number === 1 && field.value instanceof Uint8Array) raw = field.value; + else if (field.number === 2 && typeof field.value === "number") rawSize = field.value; + else if (field.number === 3 && field.value instanceof Uint8Array) zlib = field.value; + } + if (raw) return raw; + if (zlib && rawSize !== undefined && rawSize <= MAX_HEADER_BLOB_BYTES) { + return inflateSync(zlib, { maxOutputLength: MAX_HEADER_BLOB_BYTES }); + } + throw new Error("header block is neither raw nor zlib-compressed"); +} + +/** + * Read the replication timestamp and diff feed from an OSM PBF's header + * block, the file's first blob. Only that blob is read, so this is cheap on + * a planet file. Throws when the file does not start with a header block. + */ +export function readOsmPbfHeader(path: string): OsmPbfHeader { + const headerLength = readPrefix(path, 4, 0).readUInt32BE(0); + if (headerLength < 1 || headerLength > MAX_BLOB_HEADER_BYTES) { + throw new Error("not an OSM PBF: implausible blob header length"); + } + let type: string | undefined; + let dataSize: number | undefined; + for (const field of fields(readPrefix(path, headerLength, 4))) { + if (field.number === 1) type = text(field.value); + else if (field.number === 3 && typeof field.value === "number") dataSize = field.value; + } + if (type !== "OSMHeader") throw new Error("not an OSM PBF: first blob is not OSMHeader"); + if (dataSize === undefined || dataSize < 1 || dataSize > MAX_HEADER_BLOB_BYTES) { + throw new Error("not an OSM PBF: implausible header blob size"); + } + const block = headerBlockBytes(readPrefix(path, dataSize, 4 + headerLength)); + + const header: OsmPbfHeader = {}; + for (const field of fields(block)) { + if (field.number === 32 && typeof field.value === "number" && field.value > 0) { + const at = new Date(field.value * 1000); + if (!Number.isNaN(at.getTime())) header.timestamp = at.toISOString(); + } else if (field.number === 34) { + const url = httpsUrl(text(field.value)); + if (url) header.replicationUrl = url; + } + } + return header; +} + +/** `readOsmPbfHeader`, or nothing when the file is missing or not readable as one. */ +export function tryReadOsmPbfHeader(path: string): OsmPbfHeader { + try { + return readOsmPbfHeader(path); + } catch { + return {}; + } +} diff --git a/services/nominatim/service.json b/services/nominatim/service.json index fe6f7bb6c..b16041bdb 100644 --- a/services/nominatim/service.json +++ b/services/nominatim/service.json @@ -19,8 +19,10 @@ "expose": [8080], "environment": { "PBF_PATH": "/nominatim/data/data.osm.pbf", - "REPLICATION_URL": "https://planet.openstreetmap.org/replication/day", - "REPLICATION_UPDATE_INTERVAL": "86400", + "REPLICATION_URL": "${NOMINATIM_REPLICATION_URL:-https://planet.openstreetmap.org/replication/day}", + "REPLICATION_UPDATE_INTERVAL": "${NOMINATIM_REPLICATION_UPDATE_INTERVAL:-86400}", + "REPLICATION_RECHECK_INTERVAL": "${NOMINATIM_REPLICATION_RECHECK_INTERVAL:-900}", + "UPDATE_MODE": "${NOMINATIM_UPDATE_MODE:-none}", "NOMINATIM_PASSWORD": "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set}", "THREADS": "${NOMINATIM_THREADS:-8}", "IMPORT_STYLE": "full",