From 15b65b59a3326e3577c1f0f1b17187b80cfc3bd7 Mon Sep 17 00:00:00 2001 From: "Ronald A. Richardson" Date: Wed, 23 Sep 2026 14:01:13 +0800 Subject: [PATCH 01/24] Add telematics data retention settings and prune sweep Telematics polling writes a device event, a position and a delivery envelope per reporting device per poll, and nothing bounded device_events or positions. Add a per-company retention policy (Fleet-Ops > Settings > Telematics Data, with admin system defaults) enforced by fleetops:prune-telematics-data every fifteen minutes in bounded hard-delete batches, with raw payload compaction, inbox tables resolved through connections, an orphan sweep, dry-run/filter/lock options, supporting indexes, a storage usage endpoint and an on-demand cleanup job. Ingestion fixes: device_events.meta no longer duplicates the raw provider unit, and telemetry-driven saves skip the activity log unless the company opts in. The drain command now only recovers deliveries and finishes interrupted runs. --- RELEASE.md | 23 +- .../components/admin/telematics-settings.hbs | 44 +++ addon/components/admin/telematics-settings.js | 76 +++++ addon/components/layout/fleet-ops-sidebar.js | 6 +- addon/controllers/settings/index.js | 8 + addon/controllers/settings/telematics.js | 171 ++++++++++ addon/extension.js | 6 + addon/routes.js | 1 + addon/routes/settings/telematics.js | 16 + addon/templates/settings/telematics.hbs | 128 +++++++ app/components/admin/telematics-settings.js | 1 + app/controllers/settings/telematics.js | 1 + app/routes/settings/telematics.js | 1 + app/templates/settings/telematics.js | 1 + composer.json | 2 +- docs/AFAQY.md | 2 +- docs/TELEMATICS_QUEUES.md | 2 +- docs/TELEMETRY_ARCHITECTURE.md | 49 ++- extension.json | 2 +- package.json | 2 +- server/config/telemetry.php | 9 + ...1_add_retention_index_to_device_events.php | 40 +++ ...00002_add_retention_index_to_positions.php | 40 +++ ...retention_index_to_telematic_sync_runs.php | 40 +++ server/src/Auth/Schemas/FleetOps.php | 20 ++ .../Console/Commands/DrainTelematicInbox.php | 16 +- .../Console/Commands/PruneTelematicsData.php | 304 +++++++++++++++++ .../Internal/v1/SettingController.php | 199 +++++++++++ server/src/Jobs/PruneTelematicsDataJob.php | 63 ++++ .../src/Providers/FleetOpsServiceProvider.php | 3 + server/src/Support/Database/TableIndexes.php | 66 ++++ .../Telematics/Retention/RetentionPolicy.php | 232 +++++++++++++ .../Retention/TelemetryActivity.php | 40 +++ .../Support/Telematics/TelematicService.php | 14 +- .../Support/Telematics/Telemetry/Ingestor.php | 6 +- server/src/routes.php | 6 + .../Http/AfaqyRealtimeIngestionTest.php | 17 +- .../TelematicsRetentionIndexMigrationTest.php | 115 +++++++ server/tests/ProviderContractsTest.php | 10 + .../tests/SettingControllerContractsTest.php | 240 +++++++++++++ .../Support/PruneTelematicsDataProbe.php | 55 +++ .../PruneTelematicsDataCommandTest.php | 314 ++++++++++++++++++ .../Unit/Jobs/PruneTelematicsDataJobTest.php | 48 +++ .../Retention/RetentionPolicyTest.php | 130 ++++++++ .../Retention/TelemetryActivityTest.php | 60 ++++ .../admin/telematics-settings-test.js | 54 +++ .../controllers/settings/telematics-test.js | 117 +++++++ tests/unit/routes/settings/telematics-test.js | 58 ++++ translations/en-us.yaml | 43 +++ 49 files changed, 2866 insertions(+), 35 deletions(-) create mode 100644 addon/components/admin/telematics-settings.hbs create mode 100644 addon/components/admin/telematics-settings.js create mode 100644 addon/controllers/settings/telematics.js create mode 100644 addon/routes/settings/telematics.js create mode 100644 addon/templates/settings/telematics.hbs create mode 100644 app/components/admin/telematics-settings.js create mode 100644 app/controllers/settings/telematics.js create mode 100644 app/routes/settings/telematics.js create mode 100644 app/templates/settings/telematics.js create mode 100644 server/migrations/2026_09_23_000001_add_retention_index_to_device_events.php create mode 100644 server/migrations/2026_09_23_000002_add_retention_index_to_positions.php create mode 100644 server/migrations/2026_09_23_000003_add_retention_index_to_telematic_sync_runs.php create mode 100644 server/src/Console/Commands/PruneTelematicsData.php create mode 100644 server/src/Jobs/PruneTelematicsDataJob.php create mode 100644 server/src/Support/Database/TableIndexes.php create mode 100644 server/src/Support/Telematics/Retention/RetentionPolicy.php create mode 100644 server/src/Support/Telematics/Retention/TelemetryActivity.php create mode 100644 server/tests/Feature/Http/Api/TelematicsRetentionIndexMigrationTest.php create mode 100644 server/tests/Support/PruneTelematicsDataProbe.php create mode 100644 server/tests/Unit/Console/PruneTelematicsDataCommandTest.php create mode 100644 server/tests/Unit/Jobs/PruneTelematicsDataJobTest.php create mode 100644 server/tests/Unit/Support/Telematics/Retention/RetentionPolicyTest.php create mode 100644 server/tests/Unit/Support/Telematics/Retention/TelemetryActivityTest.php create mode 100644 tests/integration/components/admin/telematics-settings-test.js create mode 100644 tests/unit/controllers/settings/telematics-test.js create mode 100644 tests/unit/routes/settings/telematics-test.js diff --git a/RELEASE.md b/RELEASE.md index 32ebd138d..56b6d7d34 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,23 +1,28 @@ -> v0.6.69 ~ "Fleet-Ops tools for Fleetbase AI" +> v0.6.70 ~ "Telematics data retention" --- ## What's New -- **Fleet-Ops works with Fleetbase AI tool calling.** Creating orders from the AI prompt works again with the tool-calling assistant, and the assistant can also propose an optimized waypoint sequence for an order and explain what an order or resource import needs. Every change is a preview card the user confirms. -- **Fleet-Ops console actions.** The assistant can offer to open Fleet-Ops pages and dialogs, such as **Operations › Orders** and **New Order**, as confirmation cards. -- **Resource search for the assistant.** A `fleetops_search` tool finds orders, vehicles, drivers, work orders, maintenances, devices, sensors and telematics records by id, name, plate, VIN, email or phone. +- **Telematics Data settings.** A new Fleet-Ops → Settings → Telematics Data page lets each organization decide how long telemetry is kept: device events, raw event payloads, positions, processed and quarantined delivery envelopes, and sync run diagnostics. Administrators set system-wide defaults from the admin console (Fleet-Ops Config → Telematics Data). A value of 0 keeps rows forever. +- **Scheduled retention sweep.** `fleetops:prune-telematics-data` runs every fifteen minutes and applies each organization's policy in bounded batches, so a large backlog is drained gradually without stalling ingestion. Defaults: events 30 days, raw payloads stripped after 7 days, positions 90 days, processed deliveries 24 hours, quarantined deliveries 7 days, sync runs 7 days. +- **Storage usage and on-demand cleanup.** The settings page shows rows, oldest row and estimated size per table for your organization, and "Run cleanup now" queues an immediate retention run. --- ## Fixes -- AI resource search no longer fails on every call with `Unknown column 'sensor_type'`, and database errors are no longer passed to the model. +- Device events no longer store the raw provider unit twice. `payload` keeps the raw unit; `meta` holds only the normalized block (speed, heading, odometer, ignition, fuel level, timing). New event rows are roughly half the size. +- Telemetry-driven saves (device, event, position, vehicle, sensors) no longer write to the activity log on every poll. Organizations that want them can enable "Log telemetry activity" in Telematics Data settings. Manual edits are logged as before. +- Delivery inbox and sync run retention moved from the drain command into the retention sweep; `fleetops:drain-telematic-inbox` now only recovers deliveries and finishes interrupted runs. --- -## Dependencies -- `@fleetbase/fleetops-data` upgraded to `^0.2.2`, which adds the read-only login state (`is_staff_linked`, `login_status`) to the driver and contact models. -- `@fleetbase/ember-ui` upgraded to `^0.4.3`, which provides the `btn-auth` style the Track Order button on the login page uses. +## Upgrade notes +- New indexes on `device_events (company_uuid, created_at)`, `positions (company_uuid, created_at)` and `telematic_sync_runs (telematic_uuid, updated_at)`; on very large tables run `php artisan migrate` in a maintenance window. +- Retention is enabled by default. Existing rows older than the defaults are removed over successive runs after upgrading; set the values to 0 before upgrading if you need to keep history indefinitely. +- Deleting rows frees space inside the database files, not on disk. Run `OPTIMIZE TABLE` off-peak to return space to the operating system. +- Run `php artisan fleetbase:create-permissions` to register the `telematics-settings` permission and the Telematics Settings Manager policy and role. --- ## Testing -- Unit tests cover the new AI tools and console commands, and the AI capability registration. +- Backend tests cover the retention policy layering and clamping, the prune command (per-company policies, compaction, inbox tables via connections, orphans, batch caps, dry runs, filters, locking), the settings and storage usage endpoints, the index migrations, the on-demand job, and the meta and activity-log changes to ingestion. +- Ember tests cover the settings route guard, the settings controller, and the admin defaults component. --- ## Need help? diff --git a/addon/components/admin/telematics-settings.hbs b/addon/components/admin/telematics-settings.hbs new file mode 100644 index 000000000..6c225c74b --- /dev/null +++ b/addon/components/admin/telematics-settings.hbs @@ -0,0 +1,44 @@ +
+ {{#unless @embedded}} + + Configure system-wide telematics data retention defaults here. Organizations can override these values from Fleet-Ops › Settings › Telematics Data. A value of 0 keeps rows forever. + + {{/unless}} + + +
+ + + + + + + + + + + + + + + + + + +
+ + + +
+
+
+
diff --git a/addon/components/admin/telematics-settings.js b/addon/components/admin/telematics-settings.js new file mode 100644 index 000000000..c030990f9 --- /dev/null +++ b/addon/components/admin/telematics-settings.js @@ -0,0 +1,76 @@ +import Component from '@glimmer/component'; +import { tracked } from '@glimmer/tracking'; +import { inject as service } from '@ember/service'; +import { task } from 'ember-concurrency'; + +/** + * System-wide telematics retention defaults, applied to every company that + * has not set its own values from Fleet-Ops > Settings > Telematics Data. + */ +export default class AdminTelematicsSettingsComponent extends Component { + @service fetch; + @service notifications; + @tracked eventRetentionDays = 30; + @tracked eventCompactAfterDays = 7; + @tracked positionRetentionDays = 90; + @tracked processedRetentionHours = 24; + @tracked quarantineRetentionDays = 7; + @tracked syncRunRetentionDays = 7; + @tracked logTelemetryActivity = false; + + constructor() { + super(...arguments); + this.loadSettings.perform(); + } + + @task *loadSettings() { + try { + const settings = yield this.fetch.get('fleet-ops/settings/admin-telematics-settings'); + this.applySettings(settings); + } catch (error) { + this.notifications.serverError(error); + } + } + + @task *saveSettings() { + try { + const settings = yield this.fetch.post('fleet-ops/settings/admin-telematics-settings', this.settingsPayload); + this.applySettings(settings); + this.notifications.success('Telematics retention defaults saved.'); + } catch (error) { + this.notifications.serverError(error); + } + } + + get settingsPayload() { + return { + event_retention_days: this.toInteger(this.eventRetentionDays), + event_compact_after_days: this.toInteger(this.eventCompactAfterDays), + position_retention_days: this.toInteger(this.positionRetentionDays), + processed_retention_hours: this.toInteger(this.processedRetentionHours), + quarantine_retention_days: this.toInteger(this.quarantineRetentionDays), + sync_run_retention_days: this.toInteger(this.syncRunRetentionDays), + log_telemetry_activity: Boolean(this.logTelemetryActivity), + }; + } + + applySettings(settings = {}) { + if (!settings) { + return; + } + + this.eventRetentionDays = settings.event_retention_days ?? this.eventRetentionDays; + this.eventCompactAfterDays = settings.event_compact_after_days ?? this.eventCompactAfterDays; + this.positionRetentionDays = settings.position_retention_days ?? this.positionRetentionDays; + this.processedRetentionHours = settings.processed_retention_hours ?? this.processedRetentionHours; + this.quarantineRetentionDays = settings.quarantine_retention_days ?? this.quarantineRetentionDays; + this.syncRunRetentionDays = settings.sync_run_retention_days ?? this.syncRunRetentionDays; + this.logTelemetryActivity = settings.log_telemetry_activity ?? this.logTelemetryActivity; + } + + toInteger(value) { + const number = parseInt(value, 10); + + return Number.isFinite(number) && number > 0 ? number : 0; + } +} diff --git a/addon/components/layout/fleet-ops-sidebar.js b/addon/components/layout/fleet-ops-sidebar.js index 0abf9056f..c7394f079 100644 --- a/addon/components/layout/fleet-ops-sidebar.js +++ b/addon/components/layout/fleet-ops-sidebar.js @@ -210,6 +210,7 @@ export default class LayoutFleetOpsSidebarComponent extends Component { this.createItem('menu.routing', 'route', 'settings.routing', 'fleet-ops view routing-settings', 'fleet-ops see routing-settings'), this.createItem('menu.orchestrator', 'circle-nodes', 'settings.orchestrator', 'fleet-ops view routing-settings', 'fleet-ops see routing-settings'), this.createItem('menu.scheduling', 'calendar-days', 'settings.scheduling', 'fleet-ops view scheduling-settings', 'fleet-ops see scheduling-settings'), + this.createItem('menu.telematics-settings', 'database', 'settings.telematics', 'fleet-ops view telematics-settings', 'fleet-ops see telematics-settings'), this.createItem('menu.custom-fields', 'pen-to-square', 'settings.custom-fields', 'fleet-ops view custom-field', 'fleet-ops see custom-field'), this.createItem('menu.avatars', 'icons', 'settings.avatars', 'fleet-ops view avatar', 'fleet-ops see avatar'), ]); @@ -387,8 +388,9 @@ export default class LayoutFleetOpsSidebarComponent extends Component { 'settings.routing': 5, 'settings.orchestrator': 6, 'settings.scheduling': 7, - 'settings.custom-fields': 8, - 'settings.avatars': 9, + 'settings.telematics': 8, + 'settings.custom-fields': 9, + 'settings.avatars': 10, }; return priorities[route] ?? 0; diff --git a/addon/controllers/settings/index.js b/addon/controllers/settings/index.js index 9ec9284b8..771f45f0c 100644 --- a/addon/controllers/settings/index.js +++ b/addon/controllers/settings/index.js @@ -28,6 +28,7 @@ export default class SettingsIndexController extends Controller { description: 'Keep commerce, metadata, and visual conventions aligned.', links: [ { label: 'Payments', route: 'settings.payments', icon: 'cash-register', description: 'Payment setup for operational commerce workflows.' }, + { label: 'Telematics Data', route: 'settings.telematics', icon: 'database', description: 'Retention and storage for telemetry events, positions, and delivery history.' }, { label: 'Custom Fields', route: 'settings.custom-fields', icon: 'pen-to-square', description: 'Operational metadata fields for Fleet-Ops records.' }, { label: 'Avatars', route: 'settings.avatars', icon: 'icons', description: 'Visual assets for driver, vehicle, and map displays.' }, ], @@ -113,6 +114,13 @@ export default class SettingsIndexController extends Controller { title: 'Scheduling settings', description: 'Manage schedule templates and timing rules for planned work.', }, + { + label: 'Telematics Data', + icon: 'database', + slug: 'fleet-ops/settings/telematics', + title: 'Telematics data settings', + description: 'Control how long telemetry events, positions, and delivery history are kept.', + }, { label: 'Custom Fields', icon: 'pen-to-square', diff --git a/addon/controllers/settings/telematics.js b/addon/controllers/settings/telematics.js new file mode 100644 index 000000000..712f9c6ba --- /dev/null +++ b/addon/controllers/settings/telematics.js @@ -0,0 +1,171 @@ +import Controller from '@ember/controller'; +import { tracked } from '@glimmer/tracking'; +import { inject as service } from '@ember/service'; +import { action } from '@ember/object'; +import { task, timeout } from 'ember-concurrency'; + +/** + * Settings::TelematicsController + * + * Company-level data retention for telematics ingestion: + * - How long device events are kept, and when their raw provider payloads are stripped + * - How long positions are kept + * - How long processed and quarantined delivery envelopes and sync runs are kept + * - Whether telemetry-driven saves are written to the activity log + * plus a read-only view of the rows each table holds and an on-demand cleanup run. + */ +export default class SettingsTelematicsController extends Controller { + @service fetch; + @service notifications; + @service intl; + @service currentUser; + @service modalsManager; + + /** Days to keep device events; 0 keeps them forever. */ + @tracked eventRetentionDays = 30; + + /** Days after which raw payload/meta blobs are stripped from kept events; 0 disables compaction. */ + @tracked eventCompactAfterDays = 7; + + /** Days to keep positions; 0 keeps them forever. */ + @tracked positionRetentionDays = 90; + + /** Hours to keep processed delivery envelopes. */ + @tracked processedRetentionHours = 24; + + /** Days to keep quarantined delivery envelopes. */ + @tracked quarantineRetentionDays = 7; + + /** Days to keep sync run diagnostics. */ + @tracked syncRunRetentionDays = 7; + + /** Whether telemetry-driven saves are written to the activity log. */ + @tracked logTelemetryActivity = false; + + /** System defaults and clamp limits reported by the API. */ + @tracked defaults = {}; + @tracked limits = {}; + + /** Per-table storage usage reported by the API. */ + @tracked usage = null; + + get isAdmin() { + return this.currentUser.isAdmin === true; + } + + /** Compaction is pointless once events are deleted at the same age or sooner. */ + get compactionIneffective() { + const retention = Number(this.eventRetentionDays); + const compactAfter = Number(this.eventCompactAfterDays); + + return retention > 0 && compactAfter > 0 && compactAfter >= retention; + } + + get usageRows() { + const tables = this.usage?.tables ?? {}; + + return ['device_events', 'positions', 'telematic_deliveries', 'telematic_sync_runs'] + .filter((table) => tables[table]) + .map((table) => ({ table, label: this.intl.t(`settings.telematics.tables.${table}`), ...tables[table] })); + } + + get settingsPayload() { + return { + event_retention_days: this.toInteger(this.eventRetentionDays), + event_compact_after_days: this.toInteger(this.eventCompactAfterDays), + position_retention_days: this.toInteger(this.positionRetentionDays), + processed_retention_hours: this.toInteger(this.processedRetentionHours), + quarantine_retention_days: this.toInteger(this.quarantineRetentionDays), + sync_run_retention_days: this.toInteger(this.syncRunRetentionDays), + log_telemetry_activity: Boolean(this.logTelemetryActivity), + }; + } + + constructor() { + super(...arguments); + this.getSettings.perform(); + this.loadUsage.perform(); + } + + /** + * Load retention settings from the backend. + */ + @task *getSettings() { + try { + const settings = yield this.fetch.get('fleet-ops/settings/telematics-settings'); + this.applySettings(settings); + } catch { + // Settings may not exist yet — use defaults silently + } + } + + /** + * Save retention settings to the backend. + */ + @task *saveSettings() { + try { + const settings = yield this.fetch.post('fleet-ops/settings/telematics-settings', this.settingsPayload); + this.applySettings(settings); + this.notifications.success(this.intl.t('settings.telematics.settings-saved')); + } catch (error) { + this.notifications.serverError(error); + } + } + + /** + * Load per-table storage usage for the current company. + */ + @task *loadUsage() { + try { + this.usage = yield this.fetch.get('fleet-ops/settings/telematics-storage-usage'); + } catch { + this.usage = null; + } + } + + /** + * Queue an immediate retention run for the current company, then refresh usage. + */ + @task *runCleanup() { + try { + yield this.fetch.post('fleet-ops/settings/telematics-retention/run'); + this.notifications.success(this.intl.t('settings.telematics.cleanup-queued')); + yield timeout(5000); + yield this.loadUsage.perform(); + } catch (error) { + this.notifications.serverError(error); + } + } + + @action confirmRunCleanup() { + return this.modalsManager.confirm({ + title: this.intl.t('settings.telematics.run-cleanup'), + body: this.intl.t('settings.telematics.run-cleanup-confirm'), + acceptButtonText: this.intl.t('settings.telematics.run-cleanup'), + acceptButtonIcon: 'broom', + onConfirm: () => this.runCleanup.perform(), + }); + } + + applySettings(settings = {}) { + if (!settings) { + return; + } + + this.eventRetentionDays = settings.event_retention_days ?? this.eventRetentionDays; + this.eventCompactAfterDays = settings.event_compact_after_days ?? this.eventCompactAfterDays; + this.positionRetentionDays = settings.position_retention_days ?? this.positionRetentionDays; + this.processedRetentionHours = settings.processed_retention_hours ?? this.processedRetentionHours; + this.quarantineRetentionDays = settings.quarantine_retention_days ?? this.quarantineRetentionDays; + this.syncRunRetentionDays = settings.sync_run_retention_days ?? this.syncRunRetentionDays; + this.logTelemetryActivity = settings.log_telemetry_activity ?? this.logTelemetryActivity; + this.defaults = settings.defaults ?? this.defaults; + this.limits = settings.limits ?? this.limits; + } + + toInteger(value) { + const number = parseInt(value, 10); + + return Number.isFinite(number) && number > 0 ? number : 0; + } +} diff --git a/addon/extension.js b/addon/extension.js index bcf5b4602..b69ee334a 100644 --- a/addon/extension.js +++ b/addon/extension.js @@ -101,6 +101,11 @@ export default { icon: 'location-arrow', component: new ExtensionComponent('@fleetbase/fleetops-engine', 'admin/navigator-app'), }), + new MenuItem({ + title: 'Telematics Data', + icon: 'database', + component: new ExtensionComponent('@fleetbase/fleetops-engine', 'admin/telematics-settings'), + }), ], { slug: 'fleet-ops', @@ -456,6 +461,7 @@ export default { 'fleet-ops:template:settings:routing', 'fleet-ops:template:settings:orchestrator', 'fleet-ops:component:admin:routing-settings', + 'fleet-ops:component:admin:telematics-settings', ]); }, }; diff --git a/addon/routes.js b/addon/routes.js index 3454717b5..9596e2885 100644 --- a/addon/routes.js +++ b/addon/routes.js @@ -323,6 +323,7 @@ export default buildRoutes(function () { this.route('map'); this.route('orchestrator'); this.route('scheduling'); + this.route('telematics'); this.route('payments', function () { this.route('index', { path: '/' }); this.route('onboard'); diff --git a/addon/routes/settings/telematics.js b/addon/routes/settings/telematics.js new file mode 100644 index 000000000..73e817fd3 --- /dev/null +++ b/addon/routes/settings/telematics.js @@ -0,0 +1,16 @@ +import Route from '@ember/routing/route'; +import { inject as service } from '@ember/service'; + +export default class SettingsTelematicsRoute extends Route { + @service notifications; + @service hostRouter; + @service abilities; + @service intl; + + beforeModel() { + if (this.abilities.cannot('fleet-ops view telematics-settings')) { + this.notifications.warning(this.intl.t('common.unauthorized-access')); + return this.hostRouter.transitionTo('console.fleet-ops'); + } + } +} diff --git a/addon/templates/settings/telematics.hbs b/addon/templates/settings/telematics.hbs new file mode 100644 index 000000000..f8595f9f4 --- /dev/null +++ b/addon/templates/settings/telematics.hbs @@ -0,0 +1,128 @@ + +