From 5860a2148f23ef801d916ff43ffb9d8cb74c4a4b Mon Sep 17 00:00:00 2001 From: Wasabules <39313803+Wasabules@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:03:46 +0100 Subject: [PATCH] feat(sources): Settings > Devices, every sender and how it is read The devices that could use a source rule were found by noticing that their messages looked wrong. Settings > Devices lists every address messages arrive from, and marks them. - Engine: sources.Inventory counts what the reader reads as messages arrive - not what is read again or tried: per address, the messages since the engine started and in the last minute, the last heard, the protocol, the hostname and application of the last message and the different hostnames read lately, how many had a header that bends the RFCs, no priority, or no hostname once read, the rule that read the last one, and that last message cut to 400 bytes. Bounded to 5,000 addresses. GetSourceDevices and ClearSourceDevices are on Backend, so the service's inventory is shown when the window drives it. - The tab adds the addresses the history holds (QueryMessageGroups by source), with their counts. A device that bends the RFCs or sends no hostname and that no rule reads is marked, and can be listed alone; the button on each line opens the editor on the rule that reads it, or a new one for its address, tried on its latest live messages or at least its last one. Several hostnames behind one address are counted and explained: a relay, or a device that sends none. - The 8 languages, the user guide (Devices), the README and the documentation page. --- README.md | 3 +- docs/USER_GUIDE.md | 18 ++ docs/documentation.html | 4 +- frontend/screenshots/bridge/App.js | 4 +- frontend/screenshots/fixtures.js | 15 ++ frontend/src/components/Settings.svelte | 7 +- frontend/src/components/SourceDevices.svelte | 233 +++++++++++++++++++ frontend/src/lib/i18n/de.json | 29 +++ frontend/src/lib/i18n/en.json | 29 +++ frontend/src/lib/i18n/es.json | 29 +++ frontend/src/lib/i18n/fr.json | 29 +++ frontend/src/lib/i18n/it.json | 29 +++ frontend/src/lib/i18n/ja.json | 29 +++ frontend/src/lib/i18n/pt.json | 29 +++ frontend/src/lib/i18n/zh.json | 29 +++ frontend/src/lib/sources.ts | 34 +++ frontend/wailsjs/go/main/App.d.ts | 4 + frontend/wailsjs/go/main/App.js | 8 + frontend/wailsjs/go/models.ts | 60 +++++ internal/engine/engine.go | 4 + internal/engine/sources.go | 17 ++ internal/engine/sources_test.go | 20 ++ internal/models/sources.go | 34 +++ internal/remote/backend.go | 3 + internal/remote/client_gen.go | 10 + internal/remote/switch_gen.go | 8 + internal/sources/inventory.go | 142 +++++++++++ internal/sources/inventory_test.go | 88 +++++++ internal/sources/sources.go | 24 +- 29 files changed, 964 insertions(+), 8 deletions(-) create mode 100644 frontend/src/components/SourceDevices.svelte create mode 100644 internal/sources/inventory.go create mode 100644 internal/sources/inventory_test.go diff --git a/README.md b/README.md index 154f575..ab60e61 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,8 @@ starts with the computer, before anyone signs in, and that the window then drive - **Source rules** for the devices that bend it further: right-click a message, say how that device is read (any import format, a regex), fix its hostname, application, clock, severity or encoding, and watch its latest messages change before saving — then read the stored - history again with it. Start from a template, save your own, and export them to share + history again with it. Start from a template, save your own, and export them to share. + **Settings > Devices** lists every sender, and marks those that could use a rule - **Mutual TLS** where senders must prove who they are - **Allowed sources** by IP literal or CIDR (hygiene, not authentication — see the note below) - **Per-IP connection cap** so one host cannot occupy every slot diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 6f2a505..1991b5f 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -196,6 +196,24 @@ that read a message. Rules belong to the engine the window drives: while it drives the system service, they are the service's. Installing the service copies the window's rules to it. +#### Devices + +**Settings > Devices** lists every address messages arrive from: those heard since SyslogStudio +started — or the service, while the window drives it — and those kept in the history. + +| Column | What it says | +|---|---| +| Device | The address and protocol, and the hostname of its last message. *N hostnames*: several behind one address — a relay, or a device that sends none and has the first word of each message taken for one | +| Last heard, Since start, /min | When its last message arrived, how many since the start, how many in the last minute | +| Stored | How many of its messages the history holds | +| Reading | The rule that reads it; the share of its messages whose header bends the RFCs; *no priority*; *no hostname*; or *standard* | + +A device whose messages bend the RFCs or come without a hostname, and that no rule reads, is marked +in the margin: it could use a rule. **Only those that could use a rule** lists just them, and the +button on each line opens the editor on the rule that reads it, or on a new one for its address, +tried on its latest messages — its last one at least, even when the live view holds none. +**Count again** starts the counts from now. + #### Templates, and sharing rules **Start from** in the rule editor fills the format and the corrections from a template, keeping the diff --git a/docs/documentation.html b/docs/documentation.html index 1344542..e5ed8b1 100644 --- a/docs/documentation.html +++ b/docs/documentation.html @@ -165,7 +165,9 @@

Messages that bend the standard

start from a template — a device that sends no hostname, a camera, JSON behind a syslog header, garbled accents, a clock nobody set — and its way of reading, a regular expression of your own included, can be saved as a template for the next rule or a file - import. The messages already stored can be read + import. Settings › Devices lists every address messages arrive from, + how much each sends and how it is read, and marks the devices that bend the RFCs or + send no hostname with no rule to read them. The messages already stored can be read again from their raw text with the rules as they are now — from every device or from one rule's addresses, over the last day, week, month or all of it.

diff --git a/frontend/screenshots/bridge/App.js b/frontend/screenshots/bridge/App.js index eb6a6cb..b228fdc 100644 --- a/frontend/screenshots/bridge/App.js +++ b/frontend/screenshots/bridge/App.js @@ -33,7 +33,7 @@ import { IMPORT_PREVIEW, SIMULATOR_CONFIG, SIMULATOR_STATUS, NETWORK_INTERFACES, CERT_INFO, - BACKGROUND_SETTINGS, SYSTEM_SERVICE, SERVICE_LINK, SOURCE_RULES, SOURCE_TEMPLATES, + BACKGROUND_SETTINGS, SYSTEM_SERVICE, SERVICE_LINK, SOURCE_RULES, SOURCE_TEMPLATES, SOURCE_DEVICES, BACKUP_CONFIG, BACKUP_STATUS, BACKUP_FILES, BACKUP_LISTING, RECOVERY_KEY, } from '../fixtures.js'; @@ -363,6 +363,8 @@ const HANDLERS = { return Promise.resolve(null); }, TrySourceRule: (rule, samples) => Promise.resolve(trySource(rule, samples || [])), + GetSourceDevices: () => Promise.resolve(clone(SOURCE_DEVICES)), + ClearSourceDevices: () => Promise.resolve(null), GetSourceTemplates: () => Promise.resolve(clone(sourceTemplates)), SaveSourceTemplate: (t) => { const saved = { ...t, id: t.id || `mine-${Date.now()}` }; diff --git a/frontend/screenshots/fixtures.js b/frontend/screenshots/fixtures.js index 6e0d567..39579b6 100644 --- a/frontend/screenshots/fixtures.js +++ b/frontend/screenshots/fixtures.js @@ -656,6 +656,21 @@ export const BACKUP_LISTING = { files: BACKUP_FILES, problems: [] }; * sends no hostname. Read the way the engine reads them, as far as a page can * pretend to (bridge/App.js, TrySourceRule). */ +/** + * The devices heard from since the engine started, as GetSourceDevices says + * them: one its rule reads, a switch that bends the RFCs with no rule yet, a + * relay, and plain ones. + */ +const ago = (s) => new Date(Date.now() - s * 1000).toISOString(); +export const SOURCE_DEVICES = [ + { address: '10.10.0.1', hostname: 'FGT60F', hostnames: ['FGT60F'], appName: '', protocol: 'UDP', firstSeen: ago(86400), lastSeen: ago(2), messages: 182340, perMinute: 214, nonStandard: 182340, noPriority: 0, noHostname: 0, ruled: 182340, readBy: 'Edge firewall', lastRaw: '<189>date=2026-10-27 time=09:14:02 devname="FGT60F" level="warning" msg="Admin login failed"' }, + { address: '10.10.0.23', hostname: '', hostnames: [], appName: '%LINK-3-UPDOWN', protocol: 'UDP', firstSeen: ago(86400), lastSeen: ago(6), messages: 4821, perMinute: 12, nonStandard: 4821, noPriority: 0, noHostname: 4821, ruled: 0, lastRaw: '<187>123: *Oct 27 09:14:02.123: %LINK-3-UPDOWN: Interface Gi0/1, changed state to down' }, + { address: '10.10.0.2', hostname: 'sw-core-01', hostnames: ['sw-core-01'], appName: 'stp', protocol: 'UDP', firstSeen: ago(86400), lastSeen: ago(9), messages: 9120, perMinute: 7, nonStandard: 0, noPriority: 0, noHostname: 0, ruled: 0, lastRaw: '<190>Oct 27 09:14:00 sw-core-01 stp: topology change on vlan 20' }, + { address: '10.10.7.21', hostname: '10.10.7.21', hostnames: ['10.10.7.21'], appName: 'camera', protocol: 'UDP', firstSeen: ago(7200), lastSeen: ago(31), messages: 640, perMinute: 2, nonStandard: 640, noPriority: 640, noHostname: 0, ruled: 640, readBy: 'Lobby cameras', lastRaw: '2026-10-27 09:13:31 ERROR motion detection failed on channel 2' }, + { address: '10.10.1.15', hostname: 'app-prod-02', hostnames: ['app-prod-02', 'jump-01', 'db-prod-01', 'backup-01'], appName: 'sshd', protocol: 'TCP', firstSeen: ago(86400), lastSeen: ago(44), messages: 25310, perMinute: 31, nonStandard: 0, noPriority: 0, noHostname: 0, ruled: 0, lastRaw: '<38>Oct 27 09:13:18 app-prod-02 sshd[4410]: Accepted publickey for deploy' }, + { address: '10.10.3.40', hostname: 'printsrv', hostnames: ['printsrv'], appName: 'spooler', protocol: 'UDP', firstSeen: ago(5400), lastSeen: ago(380), messages: 58, perMinute: 0, nonStandard: 0, noPriority: 0, noHostname: 0, ruled: 0, lastRaw: '<13>Oct 27 09:07:42 printsrv spooler: Impression termin\uFFFDe' }, +]; + /** The templates that come with SyslogStudio (sources.Templates), and one saved. */ export const SOURCE_TEMPLATES = [ { id: 'no-hostname', rule: { format: { mode: 'auto' }, noHostname: true } }, diff --git a/frontend/src/components/Settings.svelte b/frontend/src/components/Settings.svelte index 7c7d6d3..8080127 100644 --- a/frontend/src/components/Settings.svelte +++ b/frontend/src/components/Settings.svelte @@ -29,6 +29,7 @@ import BackgroundSettings from './BackgroundSettings.svelte'; import BackupSettings from './BackupSettings.svelte'; import SourceRules from './SourceRules.svelte'; + import SourceDevices from './SourceDevices.svelte'; import { ruleDraft } from '../lib/sources'; import RecoveryKeySettings from './RecoveryKeySettings.svelte'; import { serviceLink } from '../lib/serviceLink'; @@ -38,7 +39,7 @@ // Opens Settings on a given tab — the service banner opens Background. export let initialTab: Tab | '' = ''; - type Tab = 'general' | 'storage' | 'backups' | 'background' | 'sources' | 'about'; + type Tab = 'general' | 'storage' | 'backups' | 'background' | 'sources' | 'devices' | 'about'; let activeTab: Tab = 'general'; const TABS: { id: Tab; label: string }[] = [ { id: 'general', label: 'settings.general' }, @@ -46,6 +47,7 @@ { id: 'backups', label: 'backup.tab' }, { id: 'background', label: 'background.tab' }, { id: 'sources', label: 'sources.tab' }, + { id: 'devices', label: 'sources.devicesTab' }, { id: 'about', label: 'settings.about' }, ]; let tabButtons: HTMLButtonElement[] = []; @@ -873,6 +875,9 @@ {:else if activeTab === 'sources'} + {:else if activeTab === 'devices'} + + {:else if activeTab === 'about'}
SyslogStudio
diff --git a/frontend/src/components/SourceDevices.svelte b/frontend/src/components/SourceDevices.svelte new file mode 100644 index 0000000..e4c9ac1 --- /dev/null +++ b/frontend/src/components/SourceDevices.svelte @@ -0,0 +1,233 @@ + + +

{$_('sources.devicesIntro')}

+ +
+ + + + + +
+ +

{$_('sources.devicesSummary', { values: { devices: n(rows.length), attention: n(needing) } })}

+ +{#if rows.length === 0} +

{$_('sources.devicesEmpty')}

+{:else} +
+ + + + + + + + + + + + + + {#each rows as row (row.address)} + {@const d = row.device} + + + + + + + + + + {/each} + +
{$_('sources.colDevice')}{$_('sources.colHeard')}{$_('sources.colMessages')}{$_('sources.colRate')}{$_('sources.colStored')}{$_('sources.colReading')}
+
{row.address}{#if d}{d.protocol}{/if}
+ {#if d} +
+ {d.hostname || '—'} + {#if d.hostnames.length > 1} + + {$_('sources.badgeHosts', { values: { count: d.hostnames.length } })} + + {/if} +
+ {/if} + 		{d ? heard(d.lastSeen, $activeZone) : $_('sources.devNotHeard')}{d ? n(d.messages) : ''}{d ? n(d.perMinute) : ''}{row.stored ? n(row.stored) : ''} + {#if d}
+ {#if d.readBy}{$_('sources.badgeRule', { values: { name: d.readBy } })}{/if} + {#if d.nonStandard > 0}{$_('sources.badgeBends', { values: { pct: pct(d.nonStandard, d.messages) } })}{/if} + {#if d.noPriority > 0}{$_('sources.badgeNoPri')}{/if} + {#if d.noHostname > 0}{$_('sources.badgeNoHost')}{/if} + {#if !d.readBy && !d.nonStandard && !d.noHostname}{$_('sources.badgeStandard')}{/if} +
{/if} + 		+ +
+
+{/if} + + diff --git a/frontend/src/lib/i18n/de.json b/frontend/src/lib/i18n/de.json index 4db7107..d6c8957 100644 --- a/frontend/src/lib/i18n/de.json +++ b/frontend/src/lib/i18n/de.json @@ -703,6 +703,35 @@ }, "sources": { "tab": "Quellen", + "devicesTab": "Geräte", + "devicesIntro": "Jede Adresse, von der Meldungen eintreffen: gehört seit dem Start von SyslogStudio — oder des Dienstes, wenn dieses Fenster ihn steuert — und im Verlauf gespeichert. Ein Gerät, dessen Meldungen von den RFCs abweichen oder ohne Hostnamen kommen und das keine Regel liest, wird markiert: Eine Regel würde ihm helfen.", + "devicesFilter": "Nach Adresse oder Hostname filtern", + "devicesAttention": "Nur solche, denen eine Regel helfen würde", + "devicesSort": "Sortieren nach", + "sortLast": "Zuletzt gehört", + "sortMessages": "Meldungen", + "sortRate": "Pro Minute", + "sortStored": "Gespeichert", + "sortAddress": "Adresse", + "devicesClear": "Neu zählen", + "devicesSummary": "{devices} Adressen · {attention} würde eine Regel helfen", + "devicesEmpty": "Es ist noch keine Meldung eingetroffen.", + "colDevice": "Gerät", + "colHeard": "Zuletzt gehört", + "colMessages": "Seit Start", + "colRate": "/min", + "colStored": "Gespeichert", + "colReading": "Lesart", + "devNotHeard": "nicht seit Start", + "badgeStandard": "Standard", + "badgeBends": "{pct} % weichen von den RFCs ab", + "badgeNoPri": "keine Priorität", + "badgeNoHost": "kein Hostname", + "badgeHosts": "{count} Hostnamen", + "badgeHostsHint": "Mehrere Hostnamen für eine Adresse: ein Relay, oder ein Gerät, das keinen sendet und bei dem das erste Wort jeder Meldung dafür gehalten wird.", + "badgeRule": "Regel: {name}", + "devRead": "Anders lesen…", + "devEditRule": "Seine Regel…", "intro": "Manche Geräte weichen vom Syslog-Standard ab: kein Hostname, ein eigenes Format, überall dieselbe Priorität, eine Uhr in einer anderen Zeitzone, eine andere Kodierung als UTF-8. Eine Regel legt fest, wie die Meldungen eines solchen Geräts beim Eintreffen gelesen werden, bevor Live-Ansicht, Verlauf, Alarmregeln und Routing sie sehen. Die Rohmeldung bleibt immer erhalten.", "rules": "Regeln", "add": "Regel hinzufügen…", diff --git a/frontend/src/lib/i18n/en.json b/frontend/src/lib/i18n/en.json index 16a3927..3b26b67 100644 --- a/frontend/src/lib/i18n/en.json +++ b/frontend/src/lib/i18n/en.json @@ -703,6 +703,35 @@ }, "sources": { "tab": "Sources", + "devicesTab": "Devices", + "devicesIntro": "Every address messages arrive from: heard since SyslogStudio started — or the service, when this window drives it — and kept in the history. A device whose messages bend the RFCs or come without a hostname, and that no rule reads, is marked: it could use one.", + "devicesFilter": "Filter by address or hostname", + "devicesAttention": "Only those that could use a rule", + "devicesSort": "Sort by", + "sortLast": "Last heard from", + "sortMessages": "Messages", + "sortRate": "Per minute", + "sortStored": "Stored", + "sortAddress": "Address", + "devicesClear": "Count again", + "devicesSummary": "{devices} addresses · {attention} could use a rule", + "devicesEmpty": "No message has arrived yet.", + "colDevice": "Device", + "colHeard": "Last heard", + "colMessages": "Since start", + "colRate": "/min", + "colStored": "Stored", + "colReading": "Reading", + "devNotHeard": "not since start", + "badgeStandard": "standard", + "badgeBends": "{pct}% bend the RFCs", + "badgeNoPri": "no priority", + "badgeNoHost": "no hostname", + "badgeHosts": "{count} hostnames", + "badgeHostsHint": "Several hostnames from one address: a relay, or a device that sends none and has the first word of each message taken for one.", + "badgeRule": "rule: {name}", + "devRead": "Read differently…", + "devEditRule": "Its rule…", "intro": "Some devices bend the syslog standard: no hostname, a format of their own, the same priority on everything, a clock in another zone, an encoding other than UTF-8. A rule says how the messages of such a device are read as they arrive, before the live view, the history, the alert rules and the routing see them. The raw message is always kept.", "rules": "Rules", "add": "Add a rule…", diff --git a/frontend/src/lib/i18n/es.json b/frontend/src/lib/i18n/es.json index 3737e3c..60bb4b9 100644 --- a/frontend/src/lib/i18n/es.json +++ b/frontend/src/lib/i18n/es.json @@ -703,6 +703,35 @@ }, "sources": { "tab": "Fuentes", + "devicesTab": "Dispositivos", + "devicesIntro": "Cada dirección de la que llegan mensajes: oída desde que se inició SyslogStudio — o el servicio, cuando esta ventana lo maneja — y guardada en el historial. Un dispositivo cuyos mensajes se apartan de las RFC o llegan sin nombre de host, y que ninguna regla lee, queda marcado: una regla le vendría bien.", + "devicesFilter": "Filtrar por dirección o nombre de host", + "devicesAttention": "Solo los que necesitarían una regla", + "devicesSort": "Ordenar por", + "sortLast": "Último mensaje", + "sortMessages": "Mensajes", + "sortRate": "Por minuto", + "sortStored": "Guardados", + "sortAddress": "Dirección", + "devicesClear": "Volver a contar", + "devicesSummary": "{devices} direcciones · {attention} necesitarían una regla", + "devicesEmpty": "Aún no ha llegado ningún mensaje.", + "colDevice": "Dispositivo", + "colHeard": "Último mensaje", + "colMessages": "Desde el inicio", + "colRate": "/min", + "colStored": "Guardados", + "colReading": "Lectura", + "devNotHeard": "no desde el inicio", + "badgeStandard": "estándar", + "badgeBends": "{pct} % fuera de las RFC", + "badgeNoPri": "sin prioridad", + "badgeNoHost": "sin nombre de host", + "badgeHosts": "{count} nombres de host", + "badgeHostsHint": "Varios nombres de host para una dirección: un relé, o un dispositivo que no envía ninguno y del que se toma la primera palabra de cada mensaje por uno.", + "badgeRule": "regla: {name}", + "devRead": "Leer de otra forma…", + "devEditRule": "Su regla…", "intro": "Algunos dispositivos se apartan del estándar syslog: sin nombre de host, un formato propio, la misma prioridad en todo, un reloj en otra zona, una codificación distinta de UTF-8. Una regla indica cómo leer los mensajes de un dispositivo así al llegar, antes de que la vista en directo, el historial, las alertas y el enrutamiento los vean. El mensaje bruto siempre se conserva.", "rules": "Reglas", "add": "Añadir una regla…", diff --git a/frontend/src/lib/i18n/fr.json b/frontend/src/lib/i18n/fr.json index 7c92c08..9cebd81 100644 --- a/frontend/src/lib/i18n/fr.json +++ b/frontend/src/lib/i18n/fr.json @@ -703,6 +703,35 @@ }, "sources": { "tab": "Sources", + "devicesTab": "Appareils", + "devicesIntro": "Toutes les adresses d’où arrivent des messages : entendues depuis le démarrage de SyslogStudio — ou du service, quand cette fenêtre le pilote — et conservées dans l’historique. Un appareil dont les messages s’écartent des RFC ou arrivent sans nom d’hôte, et qu’aucune règle ne lit, est signalé : une règle lui serait utile.", + "devicesFilter": "Filtrer par adresse ou nom d’hôte", + "devicesAttention": "Seulement ceux à qui une règle serait utile", + "devicesSort": "Trier par", + "sortLast": "Dernier message", + "sortMessages": "Messages", + "sortRate": "Par minute", + "sortStored": "Enregistrés", + "sortAddress": "Adresse", + "devicesClear": "Recompter", + "devicesSummary": "{devices} adresses · {attention} à qui une règle serait utile", + "devicesEmpty": "Aucun message n’est encore arrivé.", + "colDevice": "Appareil", + "colHeard": "Dernier message", + "colMessages": "Depuis le démarrage", + "colRate": "/min", + "colStored": "Enregistrés", + "colReading": "Lecture", + "devNotHeard": "pas depuis le démarrage", + "badgeStandard": "standard", + "badgeBends": "{pct} % hors RFC", + "badgeNoPri": "sans priorité", + "badgeNoHost": "sans nom d’hôte", + "badgeHosts": "{count} noms d’hôte", + "badgeHostsHint": "Plusieurs noms d’hôte pour une adresse : un relais, ou un appareil qui n’en envoie pas et dont le premier mot de chaque message est pris pour un.", + "badgeRule": "règle : {name}", + "devRead": "Lire autrement…", + "devEditRule": "Sa règle…", "intro": "Certains appareils s’écartent du standard syslog : pas de nom d’hôte, un format à eux, la même priorité sur tout, une horloge dans un autre fuseau, un encodage autre qu’UTF-8. Une règle dit comment lire les messages d’un tel appareil à leur arrivée, avant que la vue en direct, l’historique, les alertes et le routage ne les voient. Le message brut est toujours conservé.", "rules": "Règles", "add": "Ajouter une règle…", diff --git a/frontend/src/lib/i18n/it.json b/frontend/src/lib/i18n/it.json index b1d933d..8d65c6c 100644 --- a/frontend/src/lib/i18n/it.json +++ b/frontend/src/lib/i18n/it.json @@ -703,6 +703,35 @@ }, "sources": { "tab": "Sorgenti", + "devicesTab": "Dispositivi", + "devicesIntro": "Ogni indirizzo da cui arrivano messaggi: ascoltato dall’avvio di SyslogStudio — o del servizio, quando questa finestra lo pilota — e conservato nella cronologia. Un dispositivo i cui messaggi si discostano dalle RFC o arrivano senza nome host, e che nessuna regola legge, viene segnalato: una regola gli sarebbe utile.", + "devicesFilter": "Filtra per indirizzo o nome host", + "devicesAttention": "Solo quelli a cui servirebbe una regola", + "devicesSort": "Ordina per", + "sortLast": "Ultimo messaggio", + "sortMessages": "Messaggi", + "sortRate": "Al minuto", + "sortStored": "Salvati", + "sortAddress": "Indirizzo", + "devicesClear": "Ricomincia a contare", + "devicesSummary": "{devices} indirizzi · {attention} a cui servirebbe una regola", + "devicesEmpty": "Non è ancora arrivato alcun messaggio.", + "colDevice": "Dispositivo", + "colHeard": "Ultimo messaggio", + "colMessages": "Dall’avvio", + "colRate": "/min", + "colStored": "Salvati", + "colReading": "Lettura", + "devNotHeard": "non dall’avvio", + "badgeStandard": "standard", + "badgeBends": "{pct}% fuori dalle RFC", + "badgeNoPri": "senza priorità", + "badgeNoHost": "senza nome host", + "badgeHosts": "{count} nomi host", + "badgeHostsHint": "Più nomi host per un indirizzo: un relay, o un dispositivo che non ne invia e di cui la prima parola di ogni messaggio viene presa per uno.", + "badgeRule": "regola: {name}", + "devRead": "Leggi diversamente…", + "devEditRule": "La sua regola…", "intro": "Alcuni dispositivi si discostano dallo standard syslog: nessun nome host, un formato proprio, la stessa priorità su tutto, un orologio in un altro fuso, una codifica diversa da UTF-8. Una regola dice come leggere i messaggi di un tale dispositivo all’arrivo, prima che la vista in diretta, la cronologia, gli avvisi e l’instradamento li vedano. Il messaggio grezzo viene sempre conservato.", "rules": "Regole", "add": "Aggiungi una regola…", diff --git a/frontend/src/lib/i18n/ja.json b/frontend/src/lib/i18n/ja.json index 6688fe1..9b7b5ca 100644 --- a/frontend/src/lib/i18n/ja.json +++ b/frontend/src/lib/i18n/ja.json @@ -703,6 +703,35 @@ }, "sources": { "tab": "ソース", + "devicesTab": "デバイス", + "devicesIntro": "メッセージの送信元となるすべてのアドレス: SyslogStudio (このウィンドウがサービスを操作している場合はサービス) の起動以降に受信したものと、履歴に保存されているもの。RFC から外れたメッセージやホスト名のないメッセージを送り、どのルールにも読まれていないデバイスには印が付きます。ルールを作ると役立ちます。", + "devicesFilter": "アドレスまたはホスト名で絞り込み", + "devicesAttention": "ルールが役立つものだけ", + "devicesSort": "並べ替え", + "sortLast": "最終受信", + "sortMessages": "メッセージ数", + "sortRate": "毎分", + "sortStored": "保存済み", + "sortAddress": "アドレス", + "devicesClear": "数え直す", + "devicesSummary": "{devices} 件のアドレス · ルールが役立つもの {attention} 件", + "devicesEmpty": "まだメッセージが届いていません。", + "colDevice": "デバイス", + "colHeard": "最終受信", + "colMessages": "起動以降", + "colRate": "/分", + "colStored": "保存済み", + "colReading": "読み方", + "devNotHeard": "起動以降なし", + "badgeStandard": "標準", + "badgeBends": "{pct}% が RFC 外", + "badgeNoPri": "優先度なし", + "badgeNoHost": "ホスト名なし", + "badgeHosts": "ホスト名 {count} 件", + "badgeHostsHint": "1 つのアドレスに複数のホスト名: リレー、またはホスト名を送らず各メッセージの最初の語がホスト名と見なされているデバイスです。", + "badgeRule": "ルール: {name}", + "devRead": "別の方法で読む…", + "devEditRule": "そのルール…", "intro": "一部の機器は syslog 標準から外れています：ホスト名がない、独自の形式、すべて同じ優先度、別のタイムゾーンの時計、UTF-8 以外のエンコーディング。ルールは、そのような機器のメッセージを受信時にどう読むかを決めます。ライブビュー、履歴、アラート、ルーティングより前に適用され、生のメッセージは常に保持されます。", "rules": "ルール", "add": "ルールを追加…", diff --git a/frontend/src/lib/i18n/pt.json b/frontend/src/lib/i18n/pt.json index 0d6609a..e864500 100644 --- a/frontend/src/lib/i18n/pt.json +++ b/frontend/src/lib/i18n/pt.json @@ -703,6 +703,35 @@ }, "sources": { "tab": "Fontes", + "devicesTab": "Dispositivos", + "devicesIntro": "Cada endereço de onde chegam mensagens: ouvido desde o arranque do SyslogStudio — ou do serviço, quando esta janela o comanda — e guardado no histórico. Um dispositivo cujas mensagens fogem às RFC ou chegam sem nome de host, e que nenhuma regra lê, fica assinalado: uma regra ser-lhe-ia útil.", + "devicesFilter": "Filtrar por endereço ou nome de host", + "devicesAttention": "Só os que precisariam de uma regra", + "devicesSort": "Ordenar por", + "sortLast": "Última mensagem", + "sortMessages": "Mensagens", + "sortRate": "Por minuto", + "sortStored": "Guardadas", + "sortAddress": "Endereço", + "devicesClear": "Contar de novo", + "devicesSummary": "{devices} endereços · {attention} precisariam de uma regra", + "devicesEmpty": "Ainda não chegou nenhuma mensagem.", + "colDevice": "Dispositivo", + "colHeard": "Última mensagem", + "colMessages": "Desde o arranque", + "colRate": "/min", + "colStored": "Guardadas", + "colReading": "Leitura", + "devNotHeard": "não desde o arranque", + "badgeStandard": "padrão", + "badgeBends": "{pct} % fora das RFC", + "badgeNoPri": "sem prioridade", + "badgeNoHost": "sem nome de host", + "badgeHosts": "{count} nomes de host", + "badgeHostsHint": "Vários nomes de host para um endereço: um relay, ou um dispositivo que não envia nenhum e cuja primeira palavra de cada mensagem é tomada por um.", + "badgeRule": "regra: {name}", + "devRead": "Ler de outra forma…", + "devEditRule": "A sua regra…", "intro": "Alguns dispositivos afastam-se da norma syslog: sem nome de host, um formato próprio, a mesma prioridade em tudo, um relógio noutro fuso, uma codificação diferente de UTF-8. Uma regra diz como ler as mensagens de um dispositivo assim à chegada, antes de a vista em direto, o histórico, os alertas e o encaminhamento as verem. A mensagem bruta é sempre conservada.", "rules": "Regras", "add": "Adicionar uma regra…", diff --git a/frontend/src/lib/i18n/zh.json b/frontend/src/lib/i18n/zh.json index 581b02d..23f491a 100644 --- a/frontend/src/lib/i18n/zh.json +++ b/frontend/src/lib/i18n/zh.json @@ -703,6 +703,35 @@ }, "sources": { "tab": "来源", + "devicesTab": "设备", + "devicesIntro": "所有发送消息的地址：自 SyslogStudio（或此窗口控制服务时的服务）启动以来收到的，以及保存在历史记录中的。消息不符合 RFC 或没有主机名、且没有规则读取的设备会被标记：为它编写规则会有帮助。", + "devicesFilter": "按地址或主机名筛选", + "devicesAttention": "只显示需要规则的设备", + "devicesSort": "排序方式", + "sortLast": "最近收到", + "sortMessages": "消息数", + "sortRate": "每分钟", + "sortStored": "已存储", + "sortAddress": "地址", + "devicesClear": "重新计数", + "devicesSummary": "{devices} 个地址 · {attention} 个需要规则", + "devicesEmpty": "尚未收到任何消息。", + "colDevice": "设备", + "colHeard": "最近收到", + "colMessages": "启动以来", + "colRate": "/分", + "colStored": "已存储", + "colReading": "读取", + "devNotHeard": "启动以来未收到", + "badgeStandard": "标准", + "badgeBends": "{pct}% 不符合 RFC", + "badgeNoPri": "无优先级", + "badgeNoHost": "无主机名", + "badgeHosts": "{count} 个主机名", + "badgeHostsHint": "一个地址对应多个主机名：可能是中继，或者设备不发送主机名，每条消息的第一个词被当作主机名。", + "badgeRule": "规则：{name}", + "devRead": "换一种方式读取…", + "devEditRule": "其规则…", "intro": "有些设备偏离 syslog 标准：没有主机名、自有格式、所有消息同一优先级、时钟在其他时区、编码不是 UTF-8。规则规定此类设备的消息到达时如何读取，先于实时视图、历史记录、告警规则和路由。原始消息始终保留。", "rules": "规则", "add": "添加规则…", diff --git a/frontend/src/lib/sources.ts b/frontend/src/lib/sources.ts index fb10969..22f1703 100644 --- a/frontend/src/lib/sources.ts +++ b/frontend/src/lib/sources.ts @@ -113,6 +113,40 @@ export function withTemplate(rule: SourceRule, t: SourceTemplate): SourceRule { }; } +/** One address messages arrive from, as the engine has heard it since it started. */ +export interface SourceDevice { + address: string; + hostname: string; + /** The different hostnames read lately, the last first. */ + hostnames: string[]; + appName: string; + protocol: string; + firstSeen: string; + lastSeen: string; + messages: number; + perMinute: number; + /** With a header that is not the RFCs' (no priority included). */ + nonStandard: number; + noPriority: number; + /** Left without a hostname once read. */ + noHostname: number; + ruled: number; + readBy?: string; + /** Its last message as it arrived, cut short. */ + lastRaw: string; +} + +export const getSourceDevices = async (): Promise => (await callGo('GetSourceDevices')) || []; +export const clearSourceDevices = (): Promise => callGo('ClearSourceDevices'); + +/** A new rule for a device of the inventory. */ +export function ruleForDevice(address: string, d?: SourceDevice): SourceRule { + const rule = emptyRule(); + rule.match = { sources: [address] }; + rule.name = d?.hostname && d.hostname !== address ? `${d.hostname} (${address})` : address; + return rule; +} + /** Which stored messages to read again with the rules. */ export interface RereadOptions { /** Received at or after this instant (RFC 3339); '' is every one. */ diff --git a/frontend/wailsjs/go/main/App.d.ts b/frontend/wailsjs/go/main/App.d.ts index 79b2ed1..8dbf630 100644 --- a/frontend/wailsjs/go/main/App.d.ts +++ b/frontend/wailsjs/go/main/App.d.ts @@ -30,6 +30,8 @@ export function ClearMessages():Promise; export function ClearNotifyLog():Promise; +export function ClearSourceDevices():Promise; + export function CloseBackup():Promise; export function CompactDatabase():Promise; @@ -144,6 +146,8 @@ export function GetSimulatorConfig():Promise; export function GetSimulatorStatus():Promise; +export function GetSourceDevices():Promise>; + export function GetSourceRules():Promise>; export function GetSourceTemplates():Promise>; diff --git a/frontend/wailsjs/go/main/App.js b/frontend/wailsjs/go/main/App.js index 2dd14bb..471ed41 100644 --- a/frontend/wailsjs/go/main/App.js +++ b/frontend/wailsjs/go/main/App.js @@ -50,6 +50,10 @@ export function ClearNotifyLog() { return window['go']['main']['App']['ClearNotifyLog'](); } +export function ClearSourceDevices() { + return window['go']['main']['App']['ClearSourceDevices'](); +} + export function CloseBackup() { return window['go']['main']['App']['CloseBackup'](); } @@ -278,6 +282,10 @@ export function GetSimulatorStatus() { return window['go']['main']['App']['GetSimulatorStatus'](); } +export function GetSourceDevices() { + return window['go']['main']['App']['GetSourceDevices'](); +} + export function GetSourceRules() { return window['go']['main']['App']['GetSourceRules'](); } diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts index 8503a88..6f76f36 100644 --- a/frontend/wailsjs/go/models.ts +++ b/frontend/wailsjs/go/models.ts @@ -1408,6 +1408,66 @@ export namespace models { } } + export class SourceDevice { + address: string; + hostname: string; + hostnames: string[]; + appName: string; + protocol: string; + // Go type: time + firstSeen: any; + // Go type: time + lastSeen: any; + messages: number; + perMinute: number; + nonStandard: number; + noPriority: number; + noHostname: number; + ruled: number; + readBy?: string; + lastRaw: string; + + static createFrom(source: any = {}) { + return new SourceDevice(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.address = source["address"]; + this.hostname = source["hostname"]; + this.hostnames = source["hostnames"]; + this.appName = source["appName"]; + this.protocol = source["protocol"]; + this.firstSeen = this.convertValues(source["firstSeen"], null); + this.lastSeen = this.convertValues(source["lastSeen"], null); + this.messages = source["messages"]; + this.perMinute = source["perMinute"]; + this.nonStandard = source["nonStandard"]; + this.noPriority = source["noPriority"]; + this.noHostname = source["noHostname"]; + this.ruled = source["ruled"]; + this.readBy = source["readBy"]; + this.lastRaw = source["lastRaw"]; + } + + convertValues(a: any, classs: any, asMap: boolean = false): any { + if (!a) { + return a; + } + if (a.slice && a.map) { + return (a as any[]).map(elem => this.convertValues(elem, classs)); + } else if ("object" === typeof a) { + if (asMap) { + for (const key of Object.keys(a)) { + a[key] = new classs(a[key]); + } + return a; + } + return new classs(a); + } + return a; + } + } export class SourceSample { diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 63b5ff5..9d3e5ff 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -39,6 +39,8 @@ type Engine struct { logStore *storage.LogStore // reader reads what the server receives: the source rules' reading. reader *sources.Reader + // inventory is the devices it has read messages from since Open. + inventory *sources.Inventory // rereading reads stored messages again with it (reread.go). rereading rereading simulator *simulator.Simulator @@ -126,6 +128,8 @@ func (e *Engine) Open(emitter event.EventEmitter) error { // source rule the way its rule says (package sources). e.reader = sources.New() e.applySourceRules() + e.inventory = sources.NewInventory() + e.reader.SetInventory(e.inventory) e.server.SetReader(e.reader.Read) e.simulator = simulator.New(emitter) diff --git a/internal/engine/sources.go b/internal/engine/sources.go index 38e1d01..c94f8c9 100644 --- a/internal/engine/sources.go +++ b/internal/engine/sources.go @@ -8,6 +8,7 @@ import ( "fmt" "log/slog" "strings" + "time" "SyslogStudio/internal/models" "SyslogStudio/internal/sources" @@ -26,6 +27,22 @@ func (e *Engine) applySourceRules() { } } +// GetSourceDevices returns the devices messages have arrived from since the +// engine started, the last heard from first. +func (e *Engine) GetSourceDevices() []models.SourceDevice { + if e.inventory == nil { + return []models.SourceDevice{} + } + return e.inventory.Devices(time.Now()) +} + +// ClearSourceDevices forgets them: counting starts again from now. +func (e *Engine) ClearSourceDevices() { + if e.inventory != nil { + e.inventory.Clear() + } +} + // GetSourceRules returns the rules, in the order they are tried. func (e *Engine) GetSourceRules() []models.SourceRule { rules := e.configStore.LoadSourceRules() diff --git a/internal/engine/sources_test.go b/internal/engine/sources_test.go index 8948f37..99bf905 100644 --- a/internal/engine/sources_test.go +++ b/internal/engine/sources_test.go @@ -2,6 +2,7 @@ package engine import ( "testing" + "time" "SyslogStudio/internal/event" "SyslogStudio/internal/models" @@ -71,3 +72,22 @@ func TestSourceRules_SavedOrderedAppliedAndBundled(t *testing.T) { t.Fatalf("after deleting: %q", got) } } + +// What the server reads is counted, by address; reading again is not. +func TestSourceDevices(t *testing.T) { + e, _ := openEngine(t, nil) + if d := e.GetSourceDevices(); d == nil || len(d) != 0 { + t.Fatalf("before any message: %v", d) + } + e.reader.Read([]byte("<14>Oct 27 09:14:02 dhcp: lease assigned"), "10.0.0.5", "UDP") + e.reader.Read([]byte("no priority at all"), "10.0.0.5", "UDP") + e.reader.ReadAt([]byte("<14>Oct 27 09:14:02 dhcp: lease assigned"), "10.0.0.6", "UDP", time.Now()) + devs := e.GetSourceDevices() + if len(devs) != 1 || devs[0].Address != "10.0.0.5" || devs[0].Messages != 2 || devs[0].NoPriority != 1 { + t.Fatalf("devices: %+v", devs) + } + e.ClearSourceDevices() + if len(e.GetSourceDevices()) != 0 { + t.Fatal("not cleared") + } +} diff --git a/internal/models/sources.go b/internal/models/sources.go index 64346e1..f5dd98e 100644 --- a/internal/models/sources.go +++ b/internal/models/sources.go @@ -4,6 +4,7 @@ import ( "fmt" "path" "strings" + "time" ) // SourceRule says how the messages from a device, or a group of devices, are @@ -120,6 +121,39 @@ func (r SourceRule) AsTemplate() SourceRule { return r } +// SourceDevice is one address messages arrive from, as the engine has seen +// it since it started: what it sends, how much, and how it is read — so that +// a device that bends the RFCs, or sends no hostname, shows before anyone +// wonders why its messages look wrong. +type SourceDevice struct { + Address string `json:"address"` + // Hostname and AppName are those of its last message, as read; Hostnames + // the different ones read lately, the last first — several behind one + // address is a relay. + Hostname string `json:"hostname"` + Hostnames []string `json:"hostnames"` + AppName string `json:"appName"` + Protocol string `json:"protocol"` + FirstSeen time.Time `json:"firstSeen"` + LastSeen time.Time `json:"lastSeen"` + Messages int64 `json:"messages"` + // PerMinute is how many arrived in the last minute. + PerMinute int64 `json:"perMinute"` + // NonStandard is how many had a header that is not the RFCs' (no + // priority included), NoPriority how many had no priority at all, and + // NoHostname how many were left without a hostname once read. + NonStandard int64 `json:"nonStandard"` + NoPriority int64 `json:"noPriority"` + NoHostname int64 `json:"noHostname"` + // Ruled is how many a source rule read; ReadBy the rule that read the + // last one, "" if none did. + Ruled int64 `json:"ruled"` + ReadBy string `json:"readBy,omitempty"` + // LastRaw is its last message as it arrived, cut short: what to try a + // rule on when the live view holds none of its messages. + LastRaw string `json:"lastRaw"` +} + // RereadOptions says which stored messages to read again with the source // rules. Only messages received from a device are: an imported line was read // by its file's format. diff --git a/internal/remote/backend.go b/internal/remote/backend.go index 8d60901..7fc95a8 100644 --- a/internal/remote/backend.go +++ b/internal/remote/backend.go @@ -128,6 +128,9 @@ type Backend interface { DeleteSourceRule(id string) ReorderSourceRules(ids []string) error TrySourceRule(rule models.SourceRule, samples []models.SourceSample) ([]models.SourceTrial, error) + // The devices messages arrive from, since the engine started. + GetSourceDevices() []models.SourceDevice + ClearSourceDevices() // Stored messages read again with the rules; history:reread follows it. RereadHistory(opts models.RereadOptions) error GetHistoryReread() models.HistoryReread diff --git a/internal/remote/client_gen.go b/internal/remote/client_gen.go index 9a5e95d..1f2771f 100644 --- a/internal/remote/client_gen.go +++ b/internal/remote/client_gen.go @@ -445,6 +445,16 @@ func (c *Client) TrySourceRule(rule models.SourceRule, samples []models.SourceSa return r0, err } +func (c *Client) GetSourceDevices() []models.SourceDevice { + var r0 []models.SourceDevice + c.logLost("GetSourceDevices", c.call("GetSourceDevices", []any{}, &r0)) + return r0 +} + +func (c *Client) ClearSourceDevices() { + c.logLost("ClearSourceDevices", c.call("ClearSourceDevices", []any{})) +} + func (c *Client) RereadHistory(opts models.RereadOptions) error { err := c.call("RereadHistory", []any{opts}) return err diff --git a/internal/remote/switch_gen.go b/internal/remote/switch_gen.go index aaa40cb..77d896c 100644 --- a/internal/remote/switch_gen.go +++ b/internal/remote/switch_gen.go @@ -328,6 +328,14 @@ func (s *Switch) TrySourceRule(rule models.SourceRule, samples []models.SourceSa return s.Current().TrySourceRule(rule, samples) } +func (s *Switch) GetSourceDevices() []models.SourceDevice { + return s.Current().GetSourceDevices() +} + +func (s *Switch) ClearSourceDevices() { + s.Current().ClearSourceDevices() +} + func (s *Switch) RereadHistory(opts models.RereadOptions) error { return s.Current().RereadHistory(opts) } diff --git a/internal/sources/inventory.go b/internal/sources/inventory.go new file mode 100644 index 0000000..f1b6cd4 --- /dev/null +++ b/internal/sources/inventory.go @@ -0,0 +1,142 @@ +package sources + +import ( + "slices" + "sync" + "time" + "unicode/utf8" + + "SyslogStudio/internal/models" + "SyslogStudio/internal/syslog" +) + +// Inventory is the devices messages arrive from since the engine started +// (models.SourceDevice). Bounded: past maxDevices addresses, a new one is not +// kept — a scan from a whole network would otherwise be remembered forever. +type Inventory struct { + mu sync.Mutex + devices map[string]*device +} + +const ( + maxDevices = 5000 + maxHostnames = 8 + maxRaw = 400 + // rateSlot is the width of the slots the last minute is counted in. + rateSlot = 10 * time.Second + rateSlots = 6 +) + +type device struct { + models.SourceDevice + // counts[i] is how many arrived in the slot slots[i], a count of + // rateSlot since the epoch. + counts [rateSlots]int64 + slots [rateSlots]int64 +} + +// NewInventory makes an empty inventory. +func NewInventory() *Inventory { + return &Inventory{devices: map[string]*device{}} +} + +// note counts a message as it was read: rd is how its header was. +func (inv *Inventory) note(msg models.SyslogMessage, rd syslog.Reading) { + at := msg.ReceivedAt + inv.mu.Lock() + defer inv.mu.Unlock() + d := inv.devices[msg.SourceIP] + if d == nil { + if len(inv.devices) >= maxDevices { + return + } + d = &device{SourceDevice: models.SourceDevice{Address: msg.SourceIP, FirstSeen: at}} + inv.devices[msg.SourceIP] = d + } + d.Messages++ + d.LastSeen = at + d.Protocol = msg.Protocol + d.Hostname = msg.Hostname + d.AppName = msg.AppName + if msg.Hostname == "" { + d.NoHostname++ + } else { + // The last first, each once. + d.Hostnames = slices.DeleteFunc(d.Hostnames, func(h string) bool { return h == msg.Hostname }) + d.Hostnames = append([]string{msg.Hostname}, d.Hostnames...) + if len(d.Hostnames) > maxHostnames { + d.Hostnames = d.Hostnames[:maxHostnames] + } + } + if !rd.HasPRI { + d.NoPriority++ + } + if !rd.Standard { + d.NonStandard++ + } + d.ReadBy = msg.ReadBy + if msg.ReadBy != "" { + d.Ruled++ + } + d.LastRaw = cut(msg.RawMessage, maxRaw) + + slot := at.UnixNano() / int64(rateSlot) + i := int(slot % rateSlots) + if d.slots[i] != slot { + d.slots[i], d.counts[i] = slot, 0 + } + d.counts[i]++ +} + +// Devices returns them, the last heard from first. +func (inv *Inventory) Devices(now time.Time) []models.SourceDevice { + current := now.UnixNano() / int64(rateSlot) + inv.mu.Lock() + out := make([]models.SourceDevice, 0, len(inv.devices)) + for _, d := range inv.devices { + dev := d.SourceDevice + dev.Hostnames = slices.Clone(d.Hostnames) + for i := range d.slots { + if d.slots[i] > current-rateSlots && d.slots[i] <= current { + dev.PerMinute += d.counts[i] + } + } + out = append(out, dev) + } + inv.mu.Unlock() + slices.SortFunc(out, func(a, b models.SourceDevice) int { + if c := b.LastSeen.Compare(a.LastSeen); c != 0 { + return c + } + return compareText(a.Address, b.Address) + }) + return out +} + +// Clear forgets every device: counting starts again from now. +func (inv *Inventory) Clear() { + inv.mu.Lock() + defer inv.mu.Unlock() + inv.devices = map[string]*device{} +} + +func compareText(a, b string) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + } + return 0 +} + +// cut shortens s to at most n bytes, on a character boundary. +func cut(s string, n int) string { + if len(s) <= n { + return s + } + for n > 0 && !utf8.RuneStart(s[n]) { + n-- + } + return s[:n] + "…" +} diff --git a/internal/sources/inventory_test.go b/internal/sources/inventory_test.go new file mode 100644 index 0000000..e8b360e --- /dev/null +++ b/internal/sources/inventory_test.go @@ -0,0 +1,88 @@ +package sources + +import ( + "fmt" + "strings" + "testing" + "time" + + "SyslogStudio/internal/models" + "SyslogStudio/internal/syslog" +) + +// What each device sends is counted as it is read: how many, how lately, +// with which header, which hostnames, which rule. +func TestInventory(t *testing.T) { + r := reader(t, models.SourceRule{Name: "camera", Enabled: true, Match: bySource("10.0.0.17"), NoHostname: true}) + inv := NewInventory() + r.SetInventory(inv) + + r.Read([]byte("<14>1 2026-10-27T09:14:02Z web-01 nginx - - - hello"), "10.0.0.5", "UDP") + r.Read([]byte("<14>Oct 27 09:14:03 relay-a sshd: hi"), "10.0.0.9", "UDP") + r.Read([]byte("<14>Oct 27 09:14:04 relay-b sshd: hi"), "10.0.0.9", "UDP") + r.Read([]byte("<14>Oct 27 09:14:05 relay-a sshd: again"), "10.0.0.9", "UDP") + r.Read([]byte("2026-10-27 09:14:02 ERROR motion detection failed"), "10.0.0.17", "UDP") + r.Read([]byte(`<189>date=2026-10-27 time=09:14:02 devname="FGT60F" level="warning" msg="x"`), "10.0.0.1", "UDP") + // Read again and tried: not received, not counted. + r.ReadAt([]byte("<14>Oct 27 09:14:03 relay-c sshd: hi"), "10.0.0.9", "UDP", time.Now()) + Try(models.SourceRule{Name: "t", Enabled: true, Match: bySource("10.0.0.9")}, []models.SourceSample{{Raw: "x", SourceIP: "10.0.0.9", Protocol: "UDP"}}) + + devs := inv.Devices(time.Now()) + by := map[string]models.SourceDevice{} + for _, d := range devs { + by[d.Address] = d + } + if len(devs) != 4 || devs[0].Address != "10.0.0.1" { + t.Fatalf("devices, the last heard from first: %+v", devs) + } + if d := by["10.0.0.5"]; d.Messages != 1 || d.NonStandard != 0 || d.NoPriority != 0 || d.Hostname != "web-01" || d.AppName != "nginx" { + t.Errorf("a standard sender: %+v", d) + } + if d := by["10.0.0.9"]; d.Messages != 3 || d.PerMinute != 3 || strings.Join(d.Hostnames, ",") != "relay-a,relay-b" || d.LastRaw != "<14>Oct 27 09:14:05 relay-a sshd: again" { + t.Errorf("behind a relay: %+v", d) + } + if d := by["10.0.0.17"]; d.NoPriority != 1 || d.NonStandard != 1 || d.Ruled != 1 || d.ReadBy != "camera" || d.Hostname != "10.0.0.17" { + t.Errorf("a camera its rule reads: %+v", d) + } + if d := by["10.0.0.1"]; d.NonStandard != 1 || d.NoPriority != 0 || d.NoHostname != 0 { + t.Errorf("key=value after a priority: %+v", d) + } + + // A minute later, nothing arrived in the last one. + if d := inv.Devices(time.Now().Add(2 * time.Minute)); d[0].PerMinute != 0 { + t.Errorf("per minute, two minutes on: %d", d[0].PerMinute) + } + + inv.Clear() + if len(inv.Devices(time.Now())) != 0 { + t.Error("not cleared") + } +} + +// Past maxDevices addresses, a new one is not kept; those kept still count. +func TestInventory_Bounded(t *testing.T) { + inv := NewInventory() + at := time.Now() + for i := 0; i < maxDevices+10; i++ { + inv.note(models.SyslogMessage{SourceIP: fmt.Sprintf("10.%d.%d.%d", i>>16&255, i>>8&255, i&255), ReceivedAt: at}, syslog.Reading{HasPRI: true, Standard: true}) + } + inv.note(models.SyslogMessage{SourceIP: "10.0.0.0", ReceivedAt: at}, syslog.Reading{}) + devs := inv.Devices(at) + if len(devs) != maxDevices { + t.Fatalf("%d devices kept", len(devs)) + } + for _, d := range devs { + if d.Address == "10.0.0.0" && (d.Messages != 2 || d.NoPriority != 1) { + t.Errorf("a device kept stopped counting: %+v", d) + } + } +} + +func TestCut(t *testing.T) { + if got := cut("abcdé", 5); got != "abcd…" { + t.Errorf("cut in the middle of é: %q", got) + } + if got := cut("abc", 5); got != "abc" { + t.Errorf("short: %q", got) + } +} diff --git a/internal/sources/sources.go b/internal/sources/sources.go index 4b03e78..f50693c 100644 --- a/internal/sources/sources.go +++ b/internal/sources/sources.go @@ -31,8 +31,14 @@ import ( type Reader struct { wire *importer.WireReader rules atomic.Pointer[[]*rule] + // inventory, when set, counts what Read reads: messages received, not + // those read again or tried. + inventory atomic.Pointer[Inventory] } +// SetInventory has what is read from now on counted in inv. +func (r *Reader) SetInventory(inv *Inventory) { r.inventory.Store(inv) } + // New makes a reader with no rules. func New() *Reader { return &Reader{wire: importer.NewWireReader()} @@ -115,25 +121,35 @@ func (r *Reader) SetRules(rules []models.SourceRule) error { // Read reads one message received from sourceIP over protocol. func (r *Reader) Read(raw []byte, sourceIP, protocol string) models.SyslogMessage { - return r.ReadAt(raw, sourceIP, protocol, time.Now()) + msg, rd := r.read(raw, sourceIP, protocol, time.Now()) + if inv := r.inventory.Load(); inv != nil { + inv.note(msg, rd) + } + return msg } // ReadAt is Read for a message that arrived at a given time: a stored one, // read again with the rules as they are now. func (r *Reader) ReadAt(raw []byte, sourceIP, protocol string, at time.Time) models.SyslogMessage { + msg, _ := r.read(raw, sourceIP, protocol, at) + return msg +} + +// read reads a message, and says how its header was as it arrived. +func (r *Reader) read(raw []byte, sourceIP, protocol string, at time.Time) (models.SyslogMessage, syslog.Reading) { msg, rd := syslog.ReadAt(raw, sourceIP, protocol, at) pri := msg.Severity r.wire.Reread(&msg, rd) rules := r.rules.Load() if rules == nil { - return msg + return msg, rd } for _, c := range *rules { if c.matches(msg) { - return r.apply(c, raw, msg, rd, pri) + return r.apply(c, raw, msg, rd, pri), rd } } - return msg + return msg, rd } func (c *rule) matches(msg models.SyslogMessage) bool {