feat(sources): templates of your own, rules shared as files, "sends no hostname"

README.md

@@ -135,7 +135,7 @@ starts with the computer, before anyone signs in, and that the window then drive
 - **Source rules** for the devices that bend it further: right-click a message, say how that
   device is read (any import format, a regex), fix its hostname, application, clock, severity
   or encoding, and watch its latest messages change before saving — then read the stored
-  history again with it
+  history again with it. Start from a template, save your own, and export them to share
 - **Mutual TLS** where senders must prove who they are
 - **Allowed sources** by IP literal or CIDR (hygiene, not authentication — see the note below)
 - **Per-IP connection cap** so one host cannot occupy every slot

app_sources.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/google/uuid"
+
+	"SyslogStudio/internal/models"
+	"SyslogStudio/internal/sources"
+
+	wailsRuntime "github.com/wailsapp/wails/v2/pkg/runtime"
+)
+
+// --- Source rules: templates, and files to share them ------------------------
+//
+// Window-only: the templates are the same everywhere, and a file is read or
+// written where the person sits. Rules read from a file are saved through the
+// Backend like any other, so they reach the service the window drives.
+
+// GetSourceTemplates returns the rules offered to start from: those that
+// come with SyslogStudio, then those this person saved.
+func (a *App) GetSourceTemplates() []models.SourceTemplate {
+	return append(sources.Templates(), a.prefs.LoadSourceTemplates()...)
+}
+
+// SaveSourceTemplate saves a rule's way of reading as one of this person's
+// templates — a new one, or in place of the one with its ID — and returns it.
+// The devices the rule names are not part of it.
+func (a *App) SaveSourceTemplate(t models.SourceTemplate) (models.SourceTemplate, error) {
+	t.Name = strings.TrimSpace(t.Name)
+	t.Rule = t.Rule.AsTemplate()
+	if err := sources.CheckTemplate(t); err != nil {
+		return models.SourceTemplate{}, err
+	}
+	mine := a.prefs.LoadSourceTemplates()
+	replaced := false
+	for i := range mine {
+		if t.ID != "" && mine[i].ID == t.ID {
+			mine[i], replaced = t, true
+		}
+	}
+	if !replaced {
+		t.ID = "mine-" + uuid.NewString()
+		mine = append(mine, t)
+	}
+	a.prefs.SaveSourceTemplates(mine)
+	return t, nil
+}
+
+// DeleteSourceTemplate removes one of this person's templates. The rules
+// started from it keep what they took.
+func (a *App) DeleteSourceTemplate(id string) {
+	mine := a.prefs.LoadSourceTemplates()
+	kept := mine[:0]
+	for _, t := range mine {
+		if t.ID != id {
+			kept = append(kept, t)
+		}
+	}
+	a.prefs.SaveSourceTemplates(kept)
+}
+
+// ExportSourceRules asks where, then writes the rules there, with this
+// person's templates. "" when the person cancelled.
+func (a *App) ExportSourceRules(rules []models.SourceRule) (string, error) {
+	data, err := sources.EncodeFile(rules, a.prefs.LoadSourceTemplates())
+	if err != nil {
+		return "", err
+	}
+	path, err := wailsRuntime.SaveFileDialog(a.ctx, wailsRuntime.SaveDialogOptions{
+		Title:           "Export source rules",
+		DefaultFilename: "syslogstudio-source-rules.json",
+		Filters:         []wailsRuntime.FileFilter{{DisplayName: "Source rules (*.json)", Pattern: "*.json"}},
+	})
+	if err != nil || path == "" {
+		return "", err
+	}
+	if err := os.WriteFile(path, data, 0o644); err != nil {
+		return "", fmt.Errorf("failed to write the rules: %w", err)
+	}
+	return path, nil
+}
+
+// ReadSourceRulesFile asks for a file of exported rules and reads it, every
+// rule and template checked as it would be saved. The templates are added to
+// this person's; the rules are returned, for the window to save — through the
+// Backend, so that they reach the engine it drives. Nothing when the person
+// cancelled.
+func (a *App) ReadSourceRulesFile() (sources.RuleFile, error) {
+	path, err := wailsRuntime.OpenFileDialog(a.ctx, wailsRuntime.OpenDialogOptions{
+		Title:   "Import source rules",
+		Filters: []wailsRuntime.FileFilter{{DisplayName: "Source rules (*.json)", Pattern: "*.json"}},
+	})
+	if err != nil || path == "" {
+		return sources.RuleFile{}, err
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return sources.RuleFile{}, err
+	}
+	defer f.Close()
+	data, err := io.ReadAll(io.LimitReader(f, sources.MaxFileSize+1))
+	if err != nil {
+		return sources.RuleFile{}, err
+	}
+	file, err := sources.DecodeFile(data)
+	if err != nil {
+		return sources.RuleFile{}, err
+	}
+	for i, t := range file.Templates {
+		saved, err := a.SaveSourceTemplate(t)
+		if err != nil {
+			return sources.RuleFile{}, err
+		}
+		file.Templates[i] = saved
+	}
+	return file, nil
+}

app_sources_test.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"testing"
+
+	"SyslogStudio/internal/models"
+	"SyslogStudio/internal/sources"
+	"SyslogStudio/internal/storage"
+)
+
+// A person's templates: saved without the devices a rule names, offered after
+// those that come with SyslogStudio, replaced by ID, deleted.
+func TestSourceTemplates_Mine(t *testing.T) {
+	a := newAppIn(storage.NewConfigStoreAt(t.TempDir()))
+	builtIn := len(sources.Templates())
+
+	rule := models.SourceRule{ID: "source-1", Name: "Lobby cameras", Enabled: true,
+		Match:      models.SourceMatch{Sources: []string{"10.10.7.0/24"}},
+		Format:     models.ImportFormat{Mode: models.ImportCustom, Pattern: `^(?P<time>\S+ \S+) \[(?P<level>\w+)\] (?P<msg>.*)$`},
+		NoHostname: true}
+	if _, err := a.SaveSourceTemplate(models.SourceTemplate{Name: "  ", Rule: rule}); err == nil {
+		t.Error("a template with no name was saved")
+	}
+	bad := rule
+	bad.Format.Pattern = "^(no groups)$"
+	if _, err := a.SaveSourceTemplate(models.SourceTemplate{Name: "broken", Rule: bad}); err == nil {
+		t.Error("a template whose pattern names no field was saved")
+	}
+
+	saved, err := a.SaveSourceTemplate(models.SourceTemplate{Name: " NVR export ", Rule: rule})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if saved.ID == "" || saved.Name != "NVR export" || saved.Rule.Match.Sources != nil || saved.Rule.ID != "" ||
+		saved.Rule.Name != "" || saved.Rule.Format.Pattern != rule.Format.Pattern || !saved.Rule.NoHostname {
+		t.Fatalf("saved as %+v", saved)
+	}
+	all := a.GetSourceTemplates()
+	if len(all) != builtIn+1 || all[builtIn].ID != saved.ID {
+		t.Fatalf("offered: %d templates, the last %+v", len(all), all[len(all)-1])
+	}
+
+	saved.Name = "NVR export, v2"
+	if _, err := a.SaveSourceTemplate(saved); err != nil {
+		t.Fatal(err)
+	}
+	if all := a.GetSourceTemplates(); len(all) != builtIn+1 || all[builtIn].Name != "NVR export, v2" {
+		t.Fatalf("replaced: %+v", all[builtIn:])
+	}
+
+	a.DeleteSourceTemplate(saved.ID)
+	if len(a.GetSourceTemplates()) != builtIn {
+		t.Fatal("not deleted")
+	}
+}

docs/USER_GUIDE.md

@@ -182,7 +182,7 @@ expression with named groups (`time`, `level`, `host`, `app`, `msg`), and the ot
 
 | Setting | What it does |
 |---|---|
-| Hostname | As sent, the address the message came from, or a name you give |
+| Hostname | *As sent*. *It sends none*: the word after an RFC 3164 timestamp is not a hostname but the start of the tag or text, and the hostname is the address it came from. *The address it came from*, replacing whatever was sent. Or a name you give |
 | Application | Replaces the one sent |
 | Severity | *Automatic*: the priority, unless the message declares a level in a field. *Always the priority*. *From the message text*: a level word anywhere in the text wins |
 | Encoding | UTF-8, Windows-1252, ISO-8859-1 or ISO-8859-15, for a device that garbles accents |
@@ -196,6 +196,33 @@ that read a message.
 Rules belong to the engine the window drives: while it drives the system service, they are the
 service's. Installing the service copies the window's rules to it.
 
+#### Templates, and sharing rules
+
+**Start from** in the rule editor fills the format and the corrections from a template, keeping the
+rule's name and addresses. Those that come with SyslogStudio each do one thing the reading as
+received cannot guess:
+
+| Template | What it sets |
+|---|---|
+| Sends no hostname | *It sends none* — routers, switches and appliances that leave it out |
+| Camera or IoT device | *It sends none*, and the severity from the message text |
+| Level in the text | The severity from the message text, for a device with one priority for everything |
+| JSON behind a syslog header | The JSON format, for a container or application logging JSON through syslog |
+| Garbled accents (Windows-1252) | The Windows-1252 encoding |
+| Clock not to be trusted | The arrival time |
+| A format of its own | A regular expression with named groups, to adapt |
+
+**Save as a template…** keeps a rule's way of reading — its format, your regular expression
+included, and its corrections, not its addresses — under a name of your own. Your templates are
+offered in the rule editor and in a file import's **Format** (*From one of your templates*), where
+a format written for a file can be saved as a template too. They stay in this window's settings
+whichever engine it drives, and are listed, to delete, under **Your templates** in Settings >
+Sources.
+
+**Export…** writes the rules and your templates to a JSON file; **Import…** reads one. Imported
+rules are added after yours and **turned off**: the addresses they name are another network's until
+someone has checked them. A file with a rule that would not be saved is refused whole.
+
 #### Messages already stored
 
 Below the rules, **Messages already stored** reads stored messages again from their raw text, with

docs/documentation.html

@@ -161,7 +161,11 @@ <h3>Messages that bend the standard</h3>
         severity, the encoding. Its latest messages are read again at every change, before and
         after side by side, and the rule applies as messages arrive — before the live view, the
         history, the alerts and the routing. Rules are kept in order in
-        <strong>Settings › Sources</strong>, where the messages already stored can be read
+        <strong>Settings › Sources</strong>, exported to a file and imported elsewhere. A rule can
+        start from a template — a device that sends no hostname, a camera, JSON behind a syslog
+        header, garbled accents, a clock nobody set — and its way of reading, a regular
+        expression of your own included, can be saved as a template for the next rule or a file
+        import. The messages already stored can be read
         again from their raw text with the rules as they are now — from every device or from
         one rule's addresses, over the last day, week, month or all of it.
       </p>

frontend/screenshots/bridge/App.js

@@ -33,7 +33,7 @@ import {
   IMPORT_PREVIEW,
   SIMULATOR_CONFIG, SIMULATOR_STATUS,
   NETWORK_INTERFACES, CERT_INFO,
-  BACKGROUND_SETTINGS, SYSTEM_SERVICE, SERVICE_LINK, SOURCE_RULES,
+  BACKGROUND_SETTINGS, SYSTEM_SERVICE, SERVICE_LINK, SOURCE_RULES, SOURCE_TEMPLATES,
   BACKUP_CONFIG, BACKUP_STATUS, BACKUP_FILES, BACKUP_LISTING, RECOVERY_KEY,
 } from '../fixtures.js';
 
@@ -96,6 +96,7 @@ let recoveryKey = clone(RECOVERY_KEY);
 let updateConfig = { checkOnStartup: true, channel: 'stable', skippedVersion: '' };
 let closeAction = 'ask';
 let sourceRules = clone(SOURCE_RULES);
+let sourceTemplates = clone(SOURCE_TEMPLATES);
 
 /**
  * What a rule makes of a message, as far as a page can say without the
@@ -362,6 +363,15 @@ const HANDLERS = {
     return Promise.resolve(null);
   },
   TrySourceRule: (rule, samples) => Promise.resolve(trySource(rule, samples || [])),
+  GetSourceTemplates: () => Promise.resolve(clone(sourceTemplates)),
+  SaveSourceTemplate: (t) => {
+    const saved = { ...t, id: t.id || `mine-${Date.now()}` };
+    sourceTemplates = [...sourceTemplates.filter((x) => x.id !== saved.id), saved];
+    return Promise.resolve(clone(saved));
+  },
+  DeleteSourceTemplate: (id) => { sourceTemplates = sourceTemplates.filter((t) => t.id !== id); return Promise.resolve(null); },
+  ExportSourceRules: () => Promise.resolve('C:\\Users\\operator\\Documents\\syslogstudio-source-rules.json'),
+  ReadSourceRulesFile: () => Promise.resolve({ rules: [], templates: [] }),
   // What reading the stored messages again did last: a page cannot do it.
   GetHistoryReread: () => Promise.resolve({ running: false, total: 1284409, read: 1284409, changed: 3216 }),
   RereadHistory: () => Promise.resolve(null),

frontend/screenshots/fixtures.js

@@ -656,6 +656,21 @@ export const BACKUP_LISTING = { files: BACKUP_FILES, problems: [] };
  * sends no hostname. Read the way the engine reads them, as far as a page can
  * pretend to (bridge/App.js, TrySourceRule).
  */
+/** The templates that come with SyslogStudio (sources.Templates), and one saved. */
+export const SOURCE_TEMPLATES = [
+  { id: 'no-hostname', rule: { format: { mode: 'auto' }, noHostname: true } },
+  { id: 'camera', rule: { format: { mode: 'auto' }, noHostname: true, levelFrom: 'message' } },
+  { id: 'level-in-text', rule: { format: { mode: 'auto' }, levelFrom: 'message' } },
+  { id: 'json', rule: { format: { mode: 'json' } } },
+  { id: 'windows-1252', rule: { format: { mode: 'auto' }, encoding: 'windows-1252' } },
+  { id: 'bad-clock', rule: { format: { mode: 'auto' }, arrivalTime: true } },
+  { id: 'own-format', rule: { format: { mode: 'custom', pattern: '^(?P<time>\\S+ \\S+) (?P<level>\\w+) (?P<msg>.*)$' } } },
+  {
+    id: 'mine-lobby', name: 'Lobby cameras (NVR export)',
+    rule: { format: { mode: 'custom', pattern: '^(?P<time>\\S+ \\S+) \\[(?P<level>\\w+)\\] (?P<msg>.*)$' }, noHostname: true },
+  },
+];
+
 export const SOURCE_RULES = [
   {
     id: 'source-forti', name: 'Edge firewall', enabled: true,

frontend/src/components/ImportDialog.svelte

@@ -27,6 +27,9 @@
     import { pendingImportPath, frozen } from '../lib/stores';
     import { entryFor } from '../lib/formats';
     import FormatFields from './FormatFields.svelte';
+    import SaveTemplate from './SaveTemplate.svelte';
+    import { emptyRule, getSourceTemplates, isOwnTemplate, templatesVersion } from '../lib/sources';
+    import type { SourceTemplate } from '../lib/sources';
 
     // The counts come back keyed by LABEL, while the colours are keyed by
     // severity number. Inverting the label table once is what keeps the two
@@ -44,6 +47,22 @@
         adopted = false;
     }
 
+    // The formats this person saved as templates, from a rule or from here,
+    // and that say more than "automatic".
+    let ownFormats: SourceTemplate[] = [];
+    let ownChosen = '';
+    $: if ($templatesVersion >= 0) getSourceTemplates()
+        .then(t => (ownFormats = t.filter(x => isOwnTemplate(x) && x.rule.format?.mode && x.rule.format.mode !== 'auto')))
+        .catch(() => {});
+
+    function useOwnFormat() {
+        const t = ownFormats.find(x => x.id === ownChosen);
+        if (!t) return;
+        format = { ...t.rule.format, joinContinuations: format.joinContinuations, skipUnmatched: format.skipUnmatched };
+        pickedByHand();
+        refresh();
+    }
+
     let path = '';
     let preview: ImportPreview | null = null;
     let persist = false;
@@ -310,8 +329,22 @@
                     </button>
 
                     {#if showFormat}
+                        {#if ownFormats.length}
+                            <label class="own-format">
+                                <span>{$_('sources.fromTemplate')}</span>
+                                <select bind:value={ownChosen} on:change={useOwnFormat}>
+                                    <option value="">{$_('sources.tplChoose')}</option>
+                                    {#each ownFormats as t (t.id)}<option value={t.id}>{t.name}</option>{/each}
+                                </select>
+                            </label>
+                        {/if}
                         <FormatFields bind:format fileOptions {formatError}
                                       on:pick={pickedByHand} on:change={refresh} />
+                        {#if format.mode !== 'auto'}
+                            <div class="own-format">
+                                <SaveTemplate make={() => ({ ...emptyRule(), format: { ...format, joinContinuations: false, skipUnmatched: false } })} />
+                            </div>
+                        {/if}
                     {/if}
 
                     <label class="check">
@@ -358,6 +391,9 @@
         padding: 16px; display: flex; flex-direction: column; gap: 10px;
         overflow-y: auto;
     }
+    /* Longer than the dialog, it scrolls: nothing in it is squeezed to fit,
+       the file's name and its sample least of all. */
+    .modal-body > :global(*) { flex-shrink: 0; }
     .lead { margin: 0; font-size: 12px; color: var(--text-secondary); line-height: 1.5; }
     .note { margin: 0; font-size: 10px; color: var(--text-secondary); line-height: 1.5; }
     .verdict { margin: 0; font-size: 12px; color: var(--text-primary); line-height: 1.5; }
@@ -421,4 +457,9 @@
     .btn:disabled { opacity: 0.5; cursor: default; }
     .btn.primary { background: var(--accent, #3b82f6); border-color: var(--accent, #3b82f6); color: #fff; }
     .btn.primary:hover:not(:disabled) { background: var(--accent-hover, #2563eb); }
+    .own-format { display: flex; align-items: center; gap: 8px; margin: 6px 0; font-size: 11px; color: var(--text-secondary); }
+    .own-format select {
+        background: var(--bg-primary); color: var(--text-primary);
+        border: 1px solid var(--border-color); border-radius: 4px; padding: 4px 6px; font-size: 12px;
+    }
 </style>