diff --git a/.github/workflows/plugin-ci-workflow.yml b/.github/workflows/plugin-ci-workflow.yml index b235b82d..2652abc9 100644 --- a/.github/workflows/plugin-ci-workflow.yml +++ b/.github/workflows/plugin-ci-workflow.yml @@ -274,6 +274,7 @@ jobs: php tests/regression/threshold_recovery_database_test.php php tests/regression/quality_must_fix_database_test.php php tests/regression/incoming_preservation_database_test.php + php tests/regression/partition_rebuild_database_test.php php tests/regression/review_fixes_database_test.php php tests/regression/review_fixes_paths_database_test.php working-directory: ${{ github.workspace }}/cacti/plugins/syslog diff --git a/CHANGELOG.md b/CHANGELOG.md index 598df61b..18246b42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ --- develop --- +* bug: Keep log column resizing within the results area on both log tabs, preserve message ellipsis, and use an accessible Font Awesome export button +* feature: Standardize both schema and partition CLI help with Cacti-style version banners and options, separate maintenance warnings, and replace the unreleased millisecond converter with --convert-ms +* feature: Add a data-preserving partition rebuild CLI with optional --convert-ms, verified 10,000-row copies, interrupted-copy recovery, and database-size, extra-disk-space and several-hour maintenance warnings +* bug: Trim leading and trailing parentheses and square brackets from incoming program names before reference lookup and rule processing +* bug: Load grouped message details only on expansion, 50 at a time, preserving the displayed query's date range and filters instead of rendering every occurrence into hidden rows +* bug: Place Cancel before an active submit-type Save button in Save Search As, support Enter through a separate dialog form and the existing callback, and show a Font Awesome refresh glyph that spins only while results callbacks are running +* bug: Remove message-row and lightbulb hover tooltips, keep details in the click-open flyout, and fix the recommendation column at 20 pixels with a centered header * bug: Remove recommendation callbacks on message hover, show a theme-aware lightbulb before messages with matching recommendations, and include the CSRF token in the details flyout's fallback POST --- 5.0 --- @@ -52,7 +59,7 @@ * issue#440: Avoid repeating successful schema operations and seed the default dashboard after schema setup, preventing duplicate table creation during installation and upgrade -* feature: Add an install-time timestamp precision choice for the logtime column (issue #325). New MariaDB installs default to millisecond TIMESTAMP(3), which preserves sub-second ordering and still RANGE partition-prunes; MySQL installs are fixed to second precision because MySQL cannot prune a fractional partitioned column, and an explicit millisecond request on MySQL is rejected. Upgrades never migrate the schema automatically; switch an existing MariaDB install with the destructive cli/updatedb_convert_ms.php utility during a maintenance window. README documents forwarder configuration (rsyslog, Cisco, Juniper, Arista, syslog-ng) for delivering sub-second timestamps +* feature: Add an install-time timestamp precision choice for the logtime column (issue #325). New MariaDB installs default to millisecond TIMESTAMP(3), which preserves sub-second ordering and still RANGE partition-prunes; MySQL installs are fixed to second precision because MySQL cannot prune a fractional partitioned column, and an explicit millisecond request on MySQL is rejected. Upgrades never migrate the schema automatically; switch an existing MariaDB install with cli/upgrade_partitioning.php --convert-ms during a maintenance window. README documents forwarder configuration (rsyslog, Cisco, Juniper, Arista, syslog-ng) for delivering sub-second timestamps * feature: Add a manifest.json that drives upgrade-time "housecleaning": when genuinely upgrading from an older version, syslog_prune_files() (run from the shared syslog_check_upgrade() path so every version-aware entry point - the plugin upgrade hook, the re-enable path and cli/syslog_schema.php - cleans up) removes files and directories that older layouts shipped but that have since moved or been deleted (for example the old top-level database.php/functions.php now living under includes/), so a stale module can no longer redeclare a function and fatal the plugin on load after an upgrade. User-data paths (config.php, config_local.php) and VCS metadata (including .git/.md descendants nested under a tombstoned directory) are never touched, tombstones reached through a symlinked parent or that resolve outside the plugin are refused, a fresh install or same-version repair leaves the dev-only tests/ tree intact, and a CI check (tests/bin/validate-manifest.php) keeps the manifest's expected-file list from drifting * bug: Stop partition recovery from wedging with MySQL error 1493 ('VALUES LESS THAN value must be strictly increasing') when an existing partition boundary already covers the day being created. Both syslog_partition_check() and syslog_partition_create() decided a daily partition was missing purely by name, so a table carried over from an older TO_DAYS schema, or left inconsistent by an interrupted 4.2-to-4.11 upgrade, would repeatedly emit a REORGANIZE of dMaxValue that MySQL rejected, leaving writes parked in the dMaxValue safety partition. Both now compare the target boundary against the highest existing concrete boundary, so already-covered days no longer count as missing, no longer consume the bounded per-run recovery budget, and are treated as a successful no-op, while genuine gaps are still created (issue #434) * bug: Stop preloading every distinct program into the query-builder choice list; on a busy collector the unbounded SELECT over syslog_programs built a very large choice array that could exhaust the PHP memory limit and fatal (HTTP 503) when opening the Alert, Removal and Dashboard editors, while the Report editor (which does not use the filter builder) kept working. The program and program_id fields already resolve through the bounded AJAX suggestion endpoint, so only the small facility and priority enumerations are preloaded now diff --git a/README.md b/README.md index 7a09cb2c..a1ab9d82 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,10 @@ Traps and Informs and convert them into Syslog messages on your log server. These syslog messages can then be consumed by the syslog plugin. So, this tool is quite handy. +Incoming program names have leading and trailing parentheses and square brackets +removed before reference lookup and rule processing. Interior punctuation is +preserved. This normalization does not rename programs in existing archived logs. + For log events that continue to be generated frequently on a device, such as smartd's feature to notify every 15 minutes of an impending drive failure, can be quieted using syslog's 'Re-Alert' setting. @@ -142,25 +146,90 @@ MySQL installs always use second precision. automatically, even if the configured precision changes. This avoids the long, blocking table rewrite that an in-place precision change would require. * **Existing production systems** – switch precision only with the CLI utility - below, during a maintenance window, after accepting that stored messages are - destroyed. + below, during a maintenance window. The unified utility preserves messages + by default. ### Converting an existing MariaDB install ```console -php plugins/syslog/cli/updatedb_convert_ms.php +php plugins/syslog/cli/upgrade_partitioning.php --convert-ms +``` + +The `--convert-ms` option is **MariaDB only**. It rebuilds the stored/removed +message tables and converts incoming messages to `TIMESTAMP(3)`, preserving +data in batches of at most **10,000 rows**. Already-converted incoming tables +are left alone. The replication outbox (`syslog_replication_output`) is widened +in place; undelivered rows are never discarded. Sequence high-water marks and +replication identities are preserved, including when sequence numbers repeat +at different timestamps. Existing second-resolution messages gain `.000`; +conversion cannot recover fractional timestamps that were never recorded. + +The default is preservation. The explicit +`--discard-data` option deletes messages in rebuilt +tables. Rules and unrelated settings are not changed. The precision setting +is recorded only after the complete pipeline has converted. + +### Rebuilding partition layouts + +```console +php plugins/syslog/cli/upgrade_partitioning.php ``` -The utility is **MariaDB only** and **destructive**: it truncates the -`syslog`, `syslog_incoming`, and `syslog_removed` tables (deleting all stored -messages) and changes their `logtime` column to `TIMESTAMP(3)`. The replication -outbox (`syslog_replication_output`) is widened in place so any undelivered rows -are preserved, and the sequence high-water marks of the truncated tables are kept -so remote replication identities are never reused. Alert, removal, and report -rules and all settings are preserved. The conversion is all-or-nothing and -re-runnable: the stored precision is only recorded once every target column is -`TIMESTAMP(3)`. It prompts for confirmation; pass `--confirm` to skip the prompt -in automation, and `--help` for usage. Take a database backup first. +Without `--convert-ms`, this rebuilds `syslog` and `syslog_removed` using the +configured storage engine (`syslog_install_engine`), message retention +(`syslog_retention`) and future-partition horizon +(`syslog_partition_ahead_days`), retaining their existing timestamp precision. +It preserves existing columns and indexes, adds `logtime` to a legacy +sequence-only primary key when necessary, and creates UTC daily +`UNIX_TIMESTAMP(logtime)` partitions with a final `dMaxValue` safety partition. +It does not offer weekly or monthly partitioning, since Syslog's maintenance +expects daily partitions. + +Legacy `DATETIME` columns are converted to `TIMESTAMP` using the connection's +original session time zone; verify that this matches the zone used to write +those historical wall-clock values. Existing `TIMESTAMP` rows are copied in UTC +to preserve distinct events across daylight-saving clock changes. + +Every existing message is copied, including older-than-retention messages; +those older rows fit in the earliest RANGE partition. Normal retention pruning +resumes when processing restarts. Indefinite retention creates daily partitions +back to the oldest valid timestamp, and future-dated messages extend the planned +horizon. Plans exceeding the server's 8192-partition limit are rejected rather +than dropping history. Custom triggers, foreign keys, or unique indexes that +exclude `logtime` require a manual migration. + +`cli/syslog_schema.php --upgrade` and `--repair` repair and upgrade schema and +create missing tables; they do **not** rebuild an existing partition layout or +change its timestamp precision. Ordinary retention/horizon changes normally +need no rebuild: poller maintenance creates missing daily partitions, +reorganizes `dMaxValue` safely, and removes expired partitions. Use the explicit +rebuild utility for legacy layouts or engine changes, not routine daily rotation. + +**Maintenance requirements for both commands:** + +* Back up the database, stop Syslog processing, partition maintenance, + replication and **all external database writers**, and buffer new logs + upstream. The utility does not stop these services for you. Keep them stopped + until the rebuild or recovery finishes. Live log tables are temporarily absent + or incomplete; avoid using log views during the rebuild. DDL waits at most + 30 seconds for metadata locks before reporting failure; let active queries + finish before retrying. +* Before confirmation, the utility reports the **estimated size of the current + Syslog database, including data and indexes**. Have **at least that much + additional free disk space on the database server** during transfer, plus + headroom for temporary files, indexes and binary logs. Database metadata is + an estimate; the utility cannot verify free space on a remote database server. +* Depending on table size and storage speed, copying and verification can take + **several hours**. `--confirm` skips the prompt but not these warnings. +* DDL is **not transactional**. Each original is renamed to `_rebuild`; + a replacement is created and populated using bounded primary-key ranges. + Originals are dropped automatically only after row counts and column values + match and sequence high-water marks have been restored. +* A failure leaves originals in the live or `_rebuild` tables. Keep writers + stopped, fix the reported error, then retry with `--preserve-data` and the + same `--convert-ms` option, if used. Existing copied rows are checked rather + than overwritten; mismatches stop recovery without dropping the original. + Do not change settings or manually delete `_rebuild` tables during recovery. ### Delivering sub-second timestamps from your log forwarders @@ -588,11 +657,17 @@ evaluates the rules, and records fired alerts in `syslog_logs`. ## Recommendations Recommendations explain what a message means and what to do about it. A lightbulb -in the narrow column immediately before Message marks matching messages on the -System Logs and Alert Logs tabs, including grouped messages. Availability is -checked in batches when the page is rendered, not through hover callbacks. -Hovering shows the full message already present in the row's title without any -network requests. Click the message or row to read recommendations in the flyout. +in the fixed 20-pixel column immediately before Message marks matching messages +on the System Logs and Alert Logs tabs, including grouped messages. Availability +is checked in batches when the page is rendered, not through hover callbacks. +Message rows and lightbulbs have no hover tooltips or hover-triggered requests. +Click the message or row to read the full message and recommendations in the flyout. + +Grouped Messages initially renders summaries only. Expand a group to load 50 +matching occurrences at a time; Previous and Next replace that detail page. +Expansion preserves the displayed search filters and date range, including the +original bounds of relative time presets. It does not load every duplicate into +hidden rows. Refresh the results if an older grouped view has expired. Clicking a row on the System Logs or Alert Logs tab opens a **Message details** flyout on the right. The flyout fetches the record from the server and shows @@ -747,9 +822,14 @@ master process for its workers. `cli/syslog_schema.php` takes `--audit`, `--repair`, or `--upgrade`, as described under Installation. -`cli/updatedb_convert_ms.php` converts an existing MariaDB install to -millisecond (`TIMESTAMP(3)`) timestamps. It is destructive (it truncates the -message tables) and takes `--confirm` and `--help`; see +`cli/upgrade_partitioning.php` rebuilds daily partitions to match the configured +engine, retention and future horizon, preserving messages by default. +`--convert-ms` also converts the message pipeline to millisecond timestamps on +MariaDB. It reports database size and warns about additional disk space and +potential several-hour duration before confirmation. + +Both utilities provide Cacti-style `--help` and `--version` output. +The partition utility also takes `--confirm`; see [Millisecond Timestamps](#millisecond-timestamps). ## Development and Testing diff --git a/cli/syslog_schema.php b/cli/syslog_schema.php index 180bc1f5..5b72e80c 100644 --- a/cli/syslog_schema.php +++ b/cli/syslog_schema.php @@ -54,6 +54,11 @@ $upgrade = true; break; + case '--version': + case '-V': + case '-v': + syslog_schema_display_version(); + exit(0); case '--help': case '-H': case '-h': @@ -173,15 +178,29 @@ function syslog_schema_report_audit_results(bool $audit = true): bool { return true; } +/** + * Display the Syslog schema utility version. + */ +function syslog_schema_display_version(): void { + $version = plugin_syslog_version(); + print 'Syslog Schema Utility, Version ' . trim($version['version']) . ', ' . COPYRIGHT_YEARS . PHP_EOL; +} + /** * Display the Syslog schema utility usage and options. * * @return void */ function syslog_schema_display_help(): void { - print 'Usage: php syslog_schema.php --audit|--repair|--upgrade' . PHP_EOL; - print ' --audit Report missing tables without changing the schema.' . PHP_EOL; - print ' --repair Repair missing tables and rerun schema migrations.' . PHP_EOL; - print ' --upgrade Upgrade the schema, including missing-table repair.' . PHP_EOL; - print 'Repair and upgrade preserve existing data and preferences.' . PHP_EOL; + syslog_schema_display_version(); + + print PHP_EOL . 'usage: syslog_schema.php --audit | --repair | --upgrade' . PHP_EOL . PHP_EOL; + print 'Cacti utility for auditing and repairing the Syslog database schema.' . PHP_EOL; + print 'Repair and upgrade preserve existing data and preferences.' . PHP_EOL . PHP_EOL; + print 'Options:' . PHP_EOL; + print ' --audit - Report missing tables without changing the schema.' . PHP_EOL; + print ' --repair - Repair missing tables and rerun schema migrations.' . PHP_EOL; + print ' --upgrade - Upgrade the schema, including missing-table repair.' . PHP_EOL; + print ' --version - Display version information.' . PHP_EOL; + print ' --help - Display this help.' . PHP_EOL . PHP_EOL; } diff --git a/cli/updatedb_convert_ms.php b/cli/updatedb_convert_ms.php deleted file mode 100644 index 2b690039..00000000 --- a/cli/updatedb_convert_ms.php +++ /dev/null @@ -1,225 +0,0 @@ - $next) { - if ($next > 1) { - $ok = syslog_db_execute("ALTER TABLE `$syslogdb_default`.`$table` AUTO_INCREMENT = $next") && $ok; - } - } - - // Record success only when every target column is actually TIMESTAMP(3). - foreach ($all_tables as $table) { - if (syslog_db_table_exists($table, false) && syslog_detect_logtime_precision($table) !== 'ms') { - $ok = false; - } - } - - if (!$ok) { - fwrite(STDERR, 'ERROR: Conversion did not fully complete; the logtime precision setting was NOT changed. Re-run this utility after resolving the database error (check the Cacti log).' . PHP_EOL); - - return false; - } - - set_config_option('syslog_install_logtime_precision', 'ms'); - - cacti_log('SYSLOG: Converted message pipeline tables to millisecond (TIMESTAMP(3)) logtime via cli/updatedb_convert_ms.php', false, 'SYSLOG'); - - print 'Conversion complete; Syslog now stores millisecond timestamps.' . PHP_EOL; - print 'Ensure your log forwarders emit sub-second timestamps (see README.md).' . PHP_EOL; - - return true; -} - -/** - * Display the millisecond conversion utility usage and options. - * - * @return void - */ -function updatedb_convert_ms_display_help(): void { - print 'Usage: php updatedb_convert_ms.php [--confirm]' . PHP_EOL; - print ' --confirm Skip the interactive confirmation prompt.' . PHP_EOL; - print ' --help Display this help.' . PHP_EOL; - print PHP_EOL; - print 'Converts the Syslog message pipeline to millisecond TIMESTAMP(3) logtime.' . PHP_EOL; - print 'MariaDB only. TRUNCATES syslog, syslog_incoming and syslog_removed' . PHP_EOL; - print '(deleting stored messages) and widens syslog_replication_output in place;' . PHP_EOL; - print 'rules and settings are preserved.' . PHP_EOL; -} - diff --git a/cli/upgrade_partitioning.php b/cli/upgrade_partitioning.php new file mode 100644 index 00000000..2a513ae4 --- /dev/null +++ b/cli/upgrade_partitioning.php @@ -0,0 +1,584 @@ + 'InnoDB', 'aria' => 'Aria']; + $days = read_config_option('syslog_retention'); + $ahead = syslog_partition_ahead_days(); + + if (!isset($engines[strtolower($engine)]) || (strtolower($engine) === 'aria' && !$mariadb) + || !ctype_digit((string) $days) || (int) $days > 8184) { + fwrite(STDERR, 'ERROR: Unsupported storage engine or retention setting; no changes were made.' . PHP_EOL); + + return false; + } + + $engine = $engines[strtolower($engine)]; + $lock = hash('sha256', $syslogdb_default . '.syslog_upgrade_partitioning'); + + if ((int) syslog_db_fetch_cell_prepared('SELECT GET_LOCK(?, 0)', [$lock]) !== 1) { + fwrite(STDERR, 'ERROR: Cannot acquire the rebuild lock; another rebuild may be running.' . PHP_EOL); + + return false; + } + + try { + $tables = []; + + foreach (['syslog', 'syslog_removed', 'syslog_incoming'] as $table) { + $backup_exists = syslog_db_table_exists($table . '_rebuild', false); + + if ($backup_exists && (!$preserve_data || ($table === 'syslog_incoming' && !$convert_ms))) { + fwrite(STDERR, "ERROR: {$table}_rebuild exists from an unfinished rebuild. Retry with --preserve-data and the original --convert-ms option, if used. No data was discarded." . PHP_EOL); + + return false; + } + + if ($backup_exists || (syslog_db_table_exists($table, false) + && ($table !== 'syslog_incoming' || ($convert_ms && syslog_detect_logtime_precision($table) !== 'ms')))) { + $tables[] = $table; + } + } + + $outbox = $convert_ms && syslog_db_table_exists('syslog_replication_output', false) + && syslog_detect_logtime_precision('syslog_replication_output') !== 'ms'; + + if (!$tables && !$outbox) { + print 'No tables require rebuilding.' . PHP_EOL; + + return true; + } + + $bytes = syslog_db_fetch_cell_prepared('SELECT SUM(DATA_LENGTH + INDEX_LENGTH) + FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?', [$syslogdb_default]); + + if (!is_numeric($bytes)) { + fwrite(STDERR, 'ERROR: Cannot determine the current database size; no tables were changed.' . PHP_EOL); + + return false; + } + + $size = number_format((float) $bytes / 1073741824, 2) . ' GiB (' . number_format((float) $bytes, 0) . ' bytes)'; + print '================================ WARNING ================================' . PHP_EOL; + print "Database $syslogdb_default size (estimated data + indexes): $size" . PHP_EOL; + print "Have AT LEAST another $size free on the DATABASE SERVER during transfer." . PHP_EOL; + print 'Allow additional headroom for temporary files, indexes and binary logs.' . PHP_EOL; + print 'Depending on the size of the Syslog tables and storage speed, this can' . PHP_EOL; + print 'take SEVERAL HOURS. Database metadata is an estimate, not a disk-space check.' . PHP_EOL; + print "Rebuilding daily partitions: engine=$engine, retention=$days days, ahead=$ahead days." . PHP_EOL; + print 'Tables: ' . implode(', ', $tables) . ($outbox ? '; replication outbox widened in place' : '') . PHP_EOL; + print $preserve_data + ? 'Messages are preserved in 10,000-row copies; originals are dropped only after verification.' . PHP_EOL + : 'DESTRUCTIVE: stored messages in the listed rebuild tables will be DELETED.' . PHP_EOL; + print $convert_ms ? 'Converting the complete message pipeline to TIMESTAMP(3).' . PHP_EOL + : 'Existing timestamp precision is preserved.' . PHP_EOL; + print 'Take a database backup. STOP all processing, partition maintenance,' . PHP_EOL; + print 'replication and external database writers; buffer new logs upstream.' . PHP_EOL; + print 'This utility does NOT pause them. Keep them stopped until completion or recovery.' . PHP_EOL; + print 'Live log tables are temporarily absent or incomplete during rebuilding.' . PHP_EOL; + print '========================================================================' . PHP_EOL; + + if (!$confirm) { + print 'Type "yes" to proceed, anything else to abort: '; + + if (strtolower(trim((string) fgets(STDIN))) !== 'yes') { + print 'Aborted; no tables were changed.' . PHP_EOL; + + return false; + } + } + + $zone = syslog_db_fetch_cell('SELECT @@SESSION.time_zone'); + $mode = syslog_db_fetch_cell('SELECT @@SESSION.sql_mode'); + $lock_wait = syslog_db_fetch_cell('SELECT @@SESSION.lock_wait_timeout'); + + if (!is_string($zone) || !is_string($mode) || !ctype_digit((string) $lock_wait)) { + fwrite(STDERR, 'ERROR: Cannot read the database session settings; no tables were changed.' . PHP_EOL); + + return false; + } + + $lock_wait = (int) $lock_wait; + $copy_mode = implode(',', array_unique(array_merge( + array_diff(explode(',', $mode), ['NO_ZERO_DATE', 'NO_ZERO_IN_DATE', '']), + ['NO_AUTO_VALUE_ON_ZERO'] + ))); + + try { + if (!syslog_db_execute("SET SESSION time_zone = '+00:00'") + || !syslog_db_execute_prepared('SET SESSION sql_mode = ?', [$copy_mode]) + || !syslog_db_execute('SET SESSION lock_wait_timeout = 30')) { + fwrite(STDERR, 'ERROR: Cannot configure the copy session; no tables were changed.' . PHP_EOL); + + return false; + } + + foreach ($tables as $table) { + if (!syslog_upart_rebuild($table, $convert_ms, $preserve_data, $engine, (int) $days, $ahead, $zone)) { + return false; + } + } + + if ($outbox) { + $column = syslog_logtime_sql_type('ms'); + + if (!syslog_db_execute_prepared('SET SESSION time_zone = ?', [$zone]) + || !syslog_db_execute("ALTER TABLE `$syslogdb_default`.`syslog_replication_output` MODIFY COLUMN logtime $column")) { + fwrite(STDERR, 'ERROR: Cannot widen the replication outbox; the precision setting was not changed.' . PHP_EOL); + + return false; + } + } + + if ($convert_ms) { + foreach (['syslog', 'syslog_removed', 'syslog_incoming', 'syslog_replication_output'] as $table) { + if (syslog_db_table_exists($table, false) && syslog_detect_logtime_precision($table) !== 'ms') { + fwrite(STDERR, "ERROR: $table did not convert; the precision setting was not changed." . PHP_EOL); + + return false; + } + } + + set_config_option('syslog_install_logtime_precision', 'ms'); + } + } finally { + $restored_zone = syslog_db_execute_prepared('SET SESSION time_zone = ?', [$zone]); + $restored_mode = syslog_db_execute_prepared('SET SESSION sql_mode = ?', [$mode]); + $restored_wait = syslog_db_execute("SET SESSION lock_wait_timeout = $lock_wait"); + + if (!$restored_zone || !$restored_mode || !$restored_wait) { + cacti_log('SYSLOG ERROR: Could not restore the rebuild session settings', false, 'SYSLOG'); + } + } + + cacti_log('SYSLOG: Completed cli/upgrade_partitioning.php' . ($convert_ms ? ' with --convert-ms' : ''), false, 'SYSLOG'); + print 'Rebuild complete.' . ($convert_ms ? ' Syslog now stores millisecond timestamps; configure forwarders accordingly.' : '') . PHP_EOL; + + return true; + } finally { + syslog_db_fetch_cell_prepared('SELECT RELEASE_LOCK(?)', [$lock]); + } +} + +/** + * Rebuild one message table without dropping its original until data agrees. + * A retained original is authoritative on retry; existing copied rows are + * checked, not overwritten, so a failed copy never discards either version. + */ +function syslog_upart_rebuild(string $table, bool $convert_ms, bool $preserve_data, string $engine, int $days, int $ahead, ?string $local_zone = null): bool { + global $syslogdb_default; + + $backup = $table . '_rebuild'; + $source = syslog_db_table_exists($backup, false) ? $backup : $table; + $original = "`$syslogdb_default`.`$source`"; + $target = "`$syslogdb_default`.`$table`"; + $saved = "`$syslogdb_default`.`$backup`"; + + $fail = static function (string $reason) use ($table, $backup): bool { + fwrite(STDERR, "ERROR: $table: $reason. Originals are retained in $table or $backup; keep writers stopped and retry with --preserve-data and the same --convert-ms option, if used. The precision setting was not changed." . PHP_EOL); + + return false; + }; + + $columns = syslog_db_fetch_assoc_prepared('SELECT COLUMN_NAME, EXTRA, DATA_TYPE + FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? + ORDER BY ORDINAL_POSITION', [$syslogdb_default, $source]); + $keys = syslog_db_fetch_assoc_prepared("SELECT COLUMN_NAME FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND INDEX_NAME = 'PRIMARY' + ORDER BY SEQ_IN_INDEX", [$syslogdb_default, $source]); + $next = syslog_db_fetch_cell_prepared('SELECT AUTO_INCREMENT + FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?', [$syslogdb_default, $source]); + $triggers = syslog_db_fetch_cell_prepared('SELECT COUNT(*) FROM information_schema.TRIGGERS + WHERE TRIGGER_SCHEMA = ? AND EVENT_OBJECT_TABLE IN (?, ?)', [$syslogdb_default, $table, $backup]); + $foreign_keys = syslog_db_fetch_cell_prepared('SELECT COUNT(*) FROM information_schema.KEY_COLUMN_USAGE + WHERE REFERENCED_TABLE_NAME IS NOT NULL AND + ((TABLE_SCHEMA = ? AND TABLE_NAME IN (?, ?)) + OR (REFERENCED_TABLE_SCHEMA = ? AND REFERENCED_TABLE_NAME IN (?, ?)))', + [$syslogdb_default, $table, $backup, $syslogdb_default, $table, $backup]); + + if (!is_array($columns) || !$columns || !is_array($keys) || !in_array(array_column($keys, 'COLUMN_NAME'), [['seq'], ['seq', 'logtime']], true) + || !ctype_digit((string) $next) || !is_numeric($triggers) || !is_numeric($foreign_keys)) { + return $fail('Cannot read the table metadata or the primary key is unsupported'); + } + + if ((int) $triggers !== 0 || (int) $foreign_keys !== 0) { + return $fail('Custom triggers or foreign keys require a manual migration'); + } + + $logtime = array_values(array_filter($columns, static fn($column) => $column['COLUMN_NAME'] === 'logtime')); + + if (!$logtime || !in_array($logtime[0]['DATA_TYPE'], ['timestamp', 'datetime'], true)) { + return $fail('Unsupported original logtime type'); + } + + // Legacy DATETIME holds wall-clock time; TIMESTAMP must use UTC to avoid + // duplicate cursor values during a daylight-saving fall-back hour. + $zone = $logtime[0]['DATA_TYPE'] === 'timestamp' ? '+00:00' + : ($local_zone ?? syslog_db_fetch_cell('SELECT @@SESSION.time_zone')); + + if (!is_string($zone) || !syslog_db_execute_prepared('SET SESSION time_zone = ?', [$zone])) { + return $fail('Cannot configure the timestamp copy time zone'); + } + + $precision = $convert_ms ? 'ms' : syslog_detect_logtime_precision($source); + $partitioned = $table !== 'syslog_incoming'; + $partition_sql = null; + + if (!in_array($precision, ['sec', 'ms'], true)) { + return $fail('Cannot determine the original timestamp precision'); + } + + if ($partitioned) { + $unique_keys = syslog_db_fetch_assoc_prepared("SELECT INDEX_NAME, + GROUP_CONCAT(COLUMN_NAME ORDER BY SEQ_IN_INDEX) AS columns_list + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND NON_UNIQUE = 0 GROUP BY INDEX_NAME", + [$syslogdb_default, $source]); + $range = syslog_db_fetch_row("SELECT UNIX_TIMESTAMP(MIN(logtime)) AS first_time, + UNIX_TIMESTAMP(MAX(logtime)) AS last_time FROM $original WHERE logtime > '1970-01-01 00:00:00'"); + + if (!is_array($unique_keys) || !is_array($range) || !array_key_exists('first_time', $range) || !array_key_exists('last_time', $range)) { + return $fail('Cannot read the unique indexes or timestamp range'); + } + + foreach ($unique_keys as $key) { + if ($key['INDEX_NAME'] !== 'PRIMARY' && !in_array('logtime', explode(',', $key['columns_list']), true)) { + return $fail('A custom unique index excludes logtime; partitioning requires a manual migration'); + } + } + + $partition_sql = syslog_upart_partition_sql($days, $ahead, + $range['first_time'] === null ? null : (int) $range['first_time'], + $range['last_time'] === null ? null : (int) $range['last_time']); + + if ($partition_sql === null) { + return $fail('The requested history and future horizon exceed the 8192-partition limit; reduce retention or archive outlying timestamps first'); + } + } + + if ($source === $table) { + if (!syslog_db_execute("RENAME TABLE $target TO $saved")) { + return $fail('Cannot rename the original table'); + } + } + + if (!syslog_db_table_exists($table, false) + && !syslog_db_execute("CREATE TABLE $target LIKE $saved")) { + return $fail('Cannot create the replacement table'); + } + + $count = syslog_db_fetch_cell("SELECT COUNT(*) FROM $target"); + $parts = syslog_db_fetch_assoc_prepared('SELECT PARTITION_NAME, PARTITION_DESCRIPTION, PARTITION_EXPRESSION + FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? + ORDER BY PARTITION_ORDINAL_POSITION', [$syslogdb_default, $table]); + + if (!is_numeric($count) || !is_array($parts) || !$parts) { + return $fail('Cannot read the replacement row count or partition metadata'); + } + + if ((int) $count === 0) { + $column = syslog_logtime_sql_type($precision); + $steps = []; + + if ($partitioned && $parts[0]['PARTITION_NAME'] !== null) { + $steps[] = "ALTER TABLE $target REMOVE PARTITIONING"; + } + + $steps[] = "ALTER TABLE $target MODIFY COLUMN logtime $column"; + + if ($partitioned) { + $row_format = $engine === 'Aria' ? 'PAGE' : 'DYNAMIC'; + $steps[] = "ALTER TABLE $target ENGINE=$engine ROW_FORMAT=$row_format"; + + if (array_column($keys, 'COLUMN_NAME') === ['seq']) { + $steps[] = "ALTER TABLE $target DROP PRIMARY KEY, ADD PRIMARY KEY (seq, logtime)"; + } + + $steps[] = "ALTER TABLE $target $partition_sql"; + } + + foreach ($steps as $sql) { + if (!syslog_db_execute($sql)) { + return $fail('Cannot configure the replacement schema (check the Cacti log)'); + } + } + } elseif (!$preserve_data || syslog_detect_logtime_precision($table) !== $precision) { + return $fail('The nonempty replacement does not match the requested conversion; retry with the original options'); + } elseif ($partitioned) { + $actual_engine = syslog_db_fetch_cell_prepared('SELECT ENGINE FROM information_schema.TABLES + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?', [$syslogdb_default, $table]); + $previous = null; + + if ($actual_engine !== $engine || count($parts) < 2) { + return $fail('The partially copied replacement has a different engine or partition layout'); + } + + foreach ($parts as $index => $part) { + $expression = strtolower(str_replace(['`', ' '], '', (string) $part['PARTITION_EXPRESSION'])); + $boundary = (string) $part['PARTITION_DESCRIPTION']; + + if ($expression !== 'unix_timestamp(logtime)') { + return $fail('The partially copied replacement does not use UTC epoch partitioning'); + } + + if ($index === count($parts) - 1) { + if ($part['PARTITION_NAME'] !== 'dMaxValue' || $boundary !== 'MAXVALUE') { + return $fail('The partially copied replacement has no final dMaxValue partition'); + } + } elseif (!ctype_digit($boundary) || (int) $boundary % 86400 !== 0 + || $part['PARTITION_NAME'] !== 'd' . gmdate('Ymd', (int) $boundary - 86400) + || ($previous !== null && (int) $boundary !== $previous + 86400)) { + return $fail('The partially copied replacement does not have contiguous UTC daily partitions'); + } + + $previous = (int) $boundary; + } + } + + $names = []; + $values = []; + $equal = []; + + foreach ($columns as $column) { + $name = '`' . str_replace('`', '``', $column['COLUMN_NAME']) . '`'; + $binary = $column['COLUMN_NAME'] === 'logtime' ? '' : 'BINARY '; + $equal[] = "({$binary}copied.$name <=> {$binary}source.$name)"; + + if (preg_match('/(?:VIRTUAL|STORED|PERSISTENT) GENERATED/i', $column['EXTRA']) !== 1) { + $names[] = $name; + $values[] = "source.$name"; + } + } + + $join = 'copied.seq = source.seq AND copied.logtime = source.logtime'; + $composite = count($keys) === 2; + $order = $composite ? 'source.seq, source.logtime' : 'source.seq'; + $cursor = null; + $copied = 0; + + while ($preserve_data) { + $where = '1 = 1'; + $params = []; + + if ($cursor !== null) { + $where = $composite ? '(source.seq > ? OR (source.seq = ? AND source.logtime > ?))' : 'source.seq > ?'; + $params = $composite ? [$cursor['seq'], $cursor['seq'], $cursor['logtime']] : [$cursor['seq']]; + } + + $rows = syslog_db_fetch_assoc_prepared("SELECT source.seq, source.logtime FROM $saved AS source + WHERE $where ORDER BY $order LIMIT 10000", $params); + + if (!is_array($rows)) { + return $fail('Cannot read the next copy batch'); + } + + if (!$rows) { + break; + } + + $last = $rows[count($rows) - 1]; + $upper = $composite ? '(source.seq < ? OR (source.seq = ? AND source.logtime <= ?))' : 'source.seq <= ?'; + $params = array_merge($params, $composite ? [$last['seq'], $last['seq'], $last['logtime']] : [$last['seq']]); + $names_sql = implode(', ', $names); + $values_sql = implode(', ', $values); + + if (!syslog_db_execute_prepared("INSERT INTO $target ($names_sql) + SELECT $values_sql FROM $saved AS source + LEFT JOIN $target AS copied ON $join + WHERE $where AND $upper + AND copied.seq IS NULL ORDER BY $order", $params)) { + return $fail('Cannot copy the next batch'); + } + + $cursor = $last; + $copied += count($rows); + print "$table: checked $copied source rows" . PHP_EOL; + + if (count($rows) < 10000) { + break; + } + } + + $old_count = $preserve_data ? syslog_db_fetch_cell("SELECT COUNT(*) FROM $saved") : 0; + $new_count = syslog_db_fetch_cell("SELECT COUNT(*) FROM $target"); + $equal_sql = implode(' AND ', $equal); + $different = $preserve_data ? syslog_db_fetch_cell("SELECT EXISTS (SELECT 1 FROM $saved AS source + LEFT JOIN $target AS copied ON $join + WHERE copied.seq IS NULL OR NOT ($equal_sql))") : 0; + + if (!is_numeric($old_count) || !is_numeric($new_count) || (string) $old_count !== (string) $new_count + || !is_numeric($different) || (int) $different !== 0 + || syslog_detect_logtime_precision($table) !== $precision) { + return $fail('Copied rows, values or timestamp precision do not match'); + } + + if (!syslog_db_execute("ALTER TABLE $target AUTO_INCREMENT = $next") + || !syslog_db_execute("DROP TABLE $saved")) { + return $fail('Cannot restore the sequence high-water mark or drop the verified original'); + } + + print "$table: verified $new_count rows; removed $backup" . PHP_EOL; + + return true; +} + +/** + * Plan UTC daily partitions without deleting out-of-retention rows: the first + * RANGE partition also accepts older timestamps. Indefinite retention starts + * at the oldest valid timestamp; future data gets concrete daily partitions. + */ +function syslog_upart_partition_sql(int $days, int $ahead, ?int $first, ?int $last, ?int $now = null): ?string { + $today = intdiv($now ?? time(), 86400) * 86400; + $start = $days > 0 ? $today - ($days - 1) * 86400 : min($today, intdiv($first ?? $today, 86400) * 86400); + $end = max($today + $ahead * 86400, intdiv($last ?? $today, 86400) * 86400); + + if (($end - $start) / 86400 + 2 > 8192) { + return null; + } + + $parts = []; + + for ($day = $start; $day <= $end; $day += 86400) { + $parts[] = 'PARTITION d' . gmdate('Ymd', $day) . ' VALUES LESS THAN (' . ($day + 86400) . ')'; + } + + $parts[] = 'PARTITION dMaxValue VALUES LESS THAN MAXVALUE'; + + return 'PARTITION BY RANGE (UNIX_TIMESTAMP(logtime)) (' . implode(', ', $parts) . ')'; +} + +function syslog_upart_display_version(): void { + $version = plugin_syslog_version(); + print 'Syslog Partition Upgrade Utility, Version ' . trim($version['version']) . ', ' . COPYRIGHT_YEARS . PHP_EOL; +} + +function syslog_upart_display_help(): void { + syslog_upart_display_version(); + + print PHP_EOL . 'usage: upgrade_partitioning.php [--convert-ms] [--preserve-data | --discard-data] [--confirm]' . PHP_EOL . PHP_EOL; + print 'Rebuilds syslog and syslog_removed with UTC daily partitions matching the' . PHP_EOL; + print 'configured storage engine, message retention and future-partition horizon.' . PHP_EOL . PHP_EOL; + print 'Existing precision is preserved unless --convert-ms is selected.' . PHP_EOL . PHP_EOL; + print 'With preservation, originals are dropped only after row counts and values are verified.' . PHP_EOL . PHP_EOL; + print 'Options:' . PHP_EOL; + print ' --convert-ms - Also convert pipeline timestamps to TIMESTAMP(3); MariaDB only.' . PHP_EOL; + print ' --preserve-data - Preserve existing messages (the default), copying 10,000 rows at a time.' . PHP_EOL; + print ' --discard-data - Delete messages in rebuilt tables; never discard the outbox.' . PHP_EOL; + print ' --confirm - Skip confirmation, not the size and maintenance warnings.' . PHP_EOL; + print ' --version - Display version information.' . PHP_EOL; + print ' --help - Display this help.' . PHP_EOL . PHP_EOL; + print 'Maintenance:' . PHP_EOL; + print 'Take a database backup and stop ALL processing and database writers first.' . PHP_EOL . PHP_EOL; + print 'The current database size is reported before changes. Have at least that' . PHP_EOL; + print 'much ADDITIONAL free disk space on the database server, plus temporary' . PHP_EOL; + print 'file and binary-log headroom. Large transfers may take several hours.' . PHP_EOL . PHP_EOL; + print 'On failure keep writers stopped and retry with --preserve-data and the' . PHP_EOL; + print 'same --convert-ms option, if used; originals remain in *_rebuild tables.' . PHP_EOL . PHP_EOL; +} diff --git a/config.php.dist b/config.php.dist index 5306dfe1..486d3205 100644 --- a/config.php.dist +++ b/config.php.dist @@ -80,8 +80,8 @@ $syslog_replication_main_db_ssl_ca = $database_ssl_ca; * 'ms' - millisecond TIMESTAMP(3), preserves sub-second ordering (MariaDB only, default) * 'sec' - second granularity TIMESTAMP * Millisecond precision is only supported on MariaDB; a 'ms' request on MySQL is - * rejected. On a production system this cannot be changed in place later without - * destroying the stored messages (see cli/updatedb_convert_ms.php). */ + * rejected. To convert an existing MariaDB install while preserving messages, + * run cli/upgrade_partitioning.php --convert-ms during a maintenance window. */ //$syslog_install_options['logtime_precision'] = 'ms'; /* field in the incoming table */ diff --git a/config_local.php.dist b/config_local.php.dist index a6f3eab1..4dead7b2 100644 --- a/config_local.php.dist +++ b/config_local.php.dist @@ -92,8 +92,8 @@ $syslog_install_options['id'] = 'syslog'; * 'ms' - millisecond TIMESTAMP(3), preserves sub-second ordering (MariaDB only, default) * 'sec' - second granularity TIMESTAMP * Millisecond precision is only supported on MariaDB; a 'ms' request on MySQL is - * rejected. On a production system this cannot be changed in place later without - * destroying the stored messages (see cli/updatedb_convert_ms.php). */ + * rejected. To convert an existing MariaDB install while preserving messages, + * run cli/upgrade_partitioning.php --convert-ms during a maintenance window. */ //$syslog_install_options['logtime_precision'] = 'ms'; /** diff --git a/css/search.css b/css/search.css index 2b45ee1b..27d84078 100644 --- a/css/search.css +++ b/css/search.css @@ -623,6 +623,8 @@ resized, and cells clip instead of spilling into their neighbors. */ #syslog_workspace table.syslogFixedColumns { table-layout: fixed; } #syslog_workspace table.syslogFixedColumns td { overflow: hidden; text-overflow: ellipsis; } +#syslog_workspace table.syslogFixedColumns .sortinfo { overflow: hidden; } +#syslog_workspace table.syslogFixedColumns .textSubHeaderDark { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } /* Severity row tints: a translucent hue per priority mixed over the live theme surface, so the same rules read correctly on both light and dark themes. */ #syslog_workspace tr.syslogRow { --syslog-sev: transparent; color: var(--search-text); background: color-mix(in srgb, var(--syslog-sev) 16%, var(--search-card)); } @@ -649,8 +651,10 @@ #syslog_workspace tr:has(.syslogMessageOpen) td:first-child { font-variant-numeric: tabular-nums; } #syslog_workspace td.syslogMessage { width: 100%; max-width: 0; } #syslog_workspace th:has(.syslogRecommendationHeading), -#syslog_workspace td.syslogRecommendation { width: 1%; min-width: 20px; text-align: center; } +#syslog_workspace tr td.syslogRecommendation { box-sizing: border-box; width: 20px; min-width: 20px; max-width: 20px; padding-left: 0; padding-right: 0; text-align: center; } #syslog_workspace .syslogRecommendationIndicator { color: var(--search-accent, currentColor); } +#syslog_workspace .syslog-group-toggle { padding: 0; margin-right: 5px; border: 0; background: transparent; color: inherit; cursor: pointer; } +#syslog_workspace .syslogGroupPage button { margin-left: 8px; } .syslogMessageOpen { display: block; width: 100%; min-width: 0; padding: 0; border: 0; background: transparent; color: inherit; text-align: left; font: 13px/1.35 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Arial, sans-serif; letter-spacing: 0.01em; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; cursor: pointer; } /* Trim "All Text": the whole message wraps inside its column. */ .syslogMessageOpen.syslogMessageWrap { white-space: pre-wrap; overflow-wrap: anywhere; text-overflow: clip; } @@ -962,9 +966,6 @@ outline-offset: -1px; } -/* Hover text comes from the row title; recommendation cards live in the flyout. */ -.ui-tooltip.syslogRecommendationTip { max-width: 520px; padding: 0; overflow: hidden; } -.syslogMessageTip { padding: 8px 10px; font-size: 12px; line-height: 1.4; white-space: pre-wrap; overflow-wrap: anywhere; } .syslogRecSection + .syslogRecSection { margin-top: 6px; } .syslogRecLabel { font-weight: 700; opacity: .75; text-transform: uppercase; font-size: 10px; letter-spacing: .04em; } .syslogRecText { white-space: pre-line; overflow-wrap: anywhere; } diff --git a/includes/functions.php b/includes/functions.php index 2413561d..61dc9f2a 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -356,13 +356,16 @@ function syslog_search_time_presets(): array { * * @param string $operator 'last' or 'during'. * @param string $value A preset value of that operator. + * @param int|null $now Freeze relative ranges at this Unix timestamp. * * @return string The condition on syslog.logtime. * * @throws InvalidArgumentException When the preset is unknown. */ -function syslog_search_time_sql(string $operator, string $value): string { +function syslog_search_time_sql(string $operator, string $value, ?int $now = null): string { $presets = syslog_search_time_presets(); + $clock = $now === null ? 'NOW()' : 'FROM_UNIXTIME(' . $now . ')'; + $today = $now === null ? 'CURDATE()' : 'DATE(' . $clock . ')'; if (!isset($presets[$operator][$value])) { throw new InvalidArgumentException('Invalid date preset.'); @@ -375,14 +378,14 @@ function syslog_search_time_sql(string $operator, string $value): string { $interval = (int) $value . ' SECOND'; } - return '(syslog.logtime BETWEEN DATE_SUB(NOW(), INTERVAL ' . $interval . ') AND NOW())'; + return '(syslog.logtime BETWEEN DATE_SUB(' . $clock . ', INTERVAL ' . $interval . ') AND ' . $clock . ')'; } // Days counted back from today to the configured first day of the week (0 = Sunday). $first = function_exists('read_user_setting') ? (int) read_user_setting('first_weekdayid', 1) : 1; - $week = 'DATE_SUB(CURDATE(), INTERVAL (WEEKDAY(CURDATE()) + ' . (8 - max(0, min(6, $first))) . ') % 7 DAY)'; - $month = "CAST(DATE_FORMAT(CURDATE(), '%Y-%m-01') AS DATE)"; - $year = 'MAKEDATE(YEAR(CURDATE()), 1)'; + $week = 'DATE_SUB(' . $today . ', INTERVAL (WEEKDAY(' . $today . ') + ' . (8 - max(0, min(6, $first))) . ') % 7 DAY)'; + $month = "CAST(DATE_FORMAT($today, '%Y-%m-01') AS DATE)"; + $year = 'MAKEDATE(YEAR(' . $today . '), 1)'; if ($value === 'day_shift') { $minutes = function (string $setting, string $default): int { @@ -393,12 +396,12 @@ function syslog_search_time_sql(string $operator, string $value): string { }; $range = [ - 'CURDATE() + INTERVAL ' . $minutes('day_shift_start', '07:00') . ' MINUTE', - 'CURDATE() + INTERVAL ' . $minutes('day_shift_end', '18:00') . ' MINUTE' + $today . ' + INTERVAL ' . $minutes('day_shift_start', '07:00') . ' MINUTE', + $today . ' + INTERVAL ' . $minutes('day_shift_end', '18:00') . ' MINUTE' ]; } else { [$when, $unit] = explode('_', $value); - $start = ['day' => 'CURDATE()', 'week' => $week, 'month' => $month, 'year' => $year][$unit]; + $start = ['day' => $today, 'week' => $week, 'month' => $month, 'year' => $year][$unit]; $step = ' INTERVAL 1 ' . strtoupper($unit); $range = $when === 'this' ? [$start, $start . ' +' . $step] : [$start . ' -' . $step, $start]; } @@ -555,12 +558,13 @@ function syslog_parse_logical_search(string $input): ?array { * * @param array|null $tree The parsed search tree. * @param string $column The column name ('message' or 'logmsg'). + * @param int|null $now Freeze relative ranges at this Unix timestamp. * * @return string The SQL WHERE clause fragment. * * @throws InvalidArgumentException When the tree or column is invalid. */ -function syslog_logical_search_sql(?array $tree, string $column): string { +function syslog_logical_search_sql(?array $tree, string $column, ?int $now = null): string { if (!in_array($column, ['message', 'logmsg'], true)) { throw new InvalidArgumentException('Invalid message column.'); } @@ -582,7 +586,7 @@ function syslog_logical_search_sql(?array $tree, string $column): string { } if ($operator === 'last' || $operator === 'during') { - return syslog_search_time_sql($operator, (string) $value); + return syslog_search_time_sql($operator, (string) $value, $now); } $target = $field === 'message' ? $column : 'syslog.' . $field; @@ -608,10 +612,10 @@ function syslog_logical_search_sql(?array $tree, string $column): string { } if ($tree[0] == 'NOT') { - return '(NOT ' . syslog_logical_search_sql($tree[1], $column) . ')'; + return '(NOT ' . syslog_logical_search_sql($tree[1], $column, $now) . ')'; } - return '(' . syslog_logical_search_sql($tree[1], $column) . ' ' . $tree[0] . ' ' . syslog_logical_search_sql($tree[2], $column) . ')'; + return '(' . syslog_logical_search_sql($tree[1], $column, $now) . ' ' . $tree[0] . ' ' . syslog_logical_search_sql($tree[2], $column, $now) . ')'; } /** @@ -4551,18 +4555,18 @@ function syslog_priority_class($priority): string { } /** - * Prints the opening row tag with the priority CSS class and message tooltip - * for each row of the syslog table as it is displayed. It supports both the - * legacy as well as the new approach to controlling these colors. + * Prints the opening row tag with the priority CSS class for each row of + * the syslog table as it is displayed. It supports both the legacy as well + * as the new approach to controlling these colors. * * @param mixed $priority The syslog priority level of the row. - * @param mixed $message The syslog message for the row tooltip. + * @param mixed $message The syslog message (unused, kept for compatibility). * * @return string Always an empty string, kept for legacy row composition. */ function syslog_row_color($priority, $message): string { $priority_class = syslog_priority_class($priority); - print ""; + print ""; return ''; } @@ -6594,7 +6598,7 @@ function syslog_get_alert_sql(&$alert, $max_seq, $processing_boundary = true, $c /** * syslog_preprocess_incoming_records - Generate a max_seq to allow moving of * records to done table and mark incoming records with the max_seq and - * then if syslog is configured to strip domains, perform that first. + * then normalize program names and strip configured domains before processing. * * @return array{max_seq: int, incoming: int} The maximum sequence id, allowing * syslog messages that come in randomly to be differentiated @@ -6630,6 +6634,12 @@ function syslog_preprocess_incoming_records(): array { // strip domains if we have requested to do so syslog_strip_incoming_domains($max_seq); + if (!syslog_strip_incoming_programs($max_seq)) { + cacti_log('ERROR: Unable to normalize incoming program names; this batch will be retried.', false, 'SYSLOG'); + + return ['max_seq' => 0, 'incoming' => 0]; + } + api_plugin_hook('plugin_syslog_before_processing'); return ['max_seq' => (int) $max_seq, 'incoming' => (int) $syslog_incoming]; @@ -6638,6 +6648,56 @@ function syslog_preprocess_incoming_records(): array { return ['max_seq' => 0, 'incoming' => 0]; } +/** + * Strip boundary parentheses and square brackets before program reference lookup. + * Batch distinct names without requiring REGEXP_REPLACE or loading every message. + * + * @param int $max_seq The incoming batch's maximum sequence ID. + * + * @return bool Whether every matching program name was normalized. + */ +function syslog_strip_incoming_programs(int $max_seq): bool { + global $syslogdb_default; + + do { + $programs = syslog_db_fetch_assoc_prepared("SELECT DISTINCT program + FROM `$syslogdb_default`.`syslog_incoming` + WHERE `status` = 1 AND `seq` <= ? + AND (HEX(SUBSTR(program, 1, 1)) IN (HEX('('), HEX(')'), HEX('['), HEX(']')) + OR HEX(SUBSTR(program, -1, 1)) IN (HEX('('), HEX(')'), HEX('['), HEX(']'))) + LIMIT 250", [$max_seq]); + + if ($programs === false) { + return false; + } + + if (!$programs) { + return true; + } + + $cases = []; + $params = []; + $names = []; + + foreach ($programs as $program) { + $cases[] = 'WHEN ? THEN ?'; + $params[] = $program['program']; + $params[] = trim($program['program'], '()[]'); + $names[] = $program['program']; + } + + if (!syslog_db_execute_prepared("UPDATE `$syslogdb_default`.`syslog_incoming` + SET program = CASE program " . implode(' ', $cases) . " ELSE program END + WHERE `status` = 1 AND `seq` <= ? + AND program IN (" . implode(', ', array_fill(0, count($names), '?')) . ')', + array_merge($params, [$max_seq], $names))) { + return false; + } + } while (count($programs) === 250); + + return true; +} + /** * syslog_strip_incoming_domains - If syslog is setup to strip DNS domain name suffixes do that * prior to processing the records. @@ -8502,27 +8562,146 @@ function alert_replace_variables($alert, $results, $hostname = '') { } /** - * Fetch a group's messages within the active search predicates. + * Preserve a bounded set of rendered group filters for subsequent detail requests. + * + * @param array $messages The displayed group summaries. + * @param string $sql_where The compiled search and date-range predicates. + * @param int $trim The displayed message trim setting. + * @param array $recommendations Recommendation availability keyed by summary row. + * + * @return string The opaque, session-owned filter context. + */ +function syslog_group_context(array $messages, string $sql_where, int $trim, array $recommendations): string { + $groups = []; + + foreach ($messages as $key => $message) { + if (($message['occurrence_count'] ?? 0) > 1) { + $groups[$key] = [ + 'source' => $message['mtype'], + 'id' => $message['latest_seq'], + 'logtime' => $message['logtime'], + 'available' => $recommendations[$key] + ]; + } + } + + $token = bin2hex(random_bytes(16)); + $_SESSION['syslog_group_contexts'][$token] = ['where' => $sql_where, 'trim' => $trim, 'groups' => $groups]; + $_SESSION['syslog_group_contexts'] = array_slice($_SESSION['syslog_group_contexts'], -5, null, true); + + return $token; +} + +/** + * Fetch one bounded page of a group's messages within the active search predicates. * * @param array $group The grouped message row. * @param string $sql_where The compiled message search clause. * @param string $breach_sql The compiled threshold-breach expression. + * @param int $offset The zero-based detail-page offset. * * @return array|false The matching rows, or false on a database error. */ -function syslog_group_detail_messages(array $group, string $sql_where, string $breach_sql): array|false { +function syslog_group_detail_messages(array $group, string $sql_where, string $breach_sql, int $offset = 0): array|false { global $syslogdb_default; - $seq_array = array_map('intval', explode(',', $group['seq_list'])); - $group_where = $sql_where . ($sql_where === '' ? 'WHERE ' : ' AND ') . - 'syslog.seq IN (' . implode(',', array_fill(0, cacti_sizeof($seq_array), '?')) . ')'; - - return syslog_db_fetch_assoc_prepared("SELECT `syslog`.*, $breach_sql AS threshold_breached, `syslog_programs`.`program` + $group_where = $sql_where === '' ? 'WHERE ' : 'WHERE (' . substr($sql_where, 6) . ') AND '; + $digest = syslog_message_digest_ready(); + $group_where .= 'syslog.host_id = ? AND syslog.program_id = ? AND syslog.facility_id = ? AND syslog.priority_id = ? AND ' . + ($digest ? "IF(syslog.message_digest = '', MD5(syslog.message), syslog.message_digest) = MD5(?)" : 'syslog.message = ?'); + $params = [$group['host_id'], $group['program_id'], $group['facility_id'], $group['priority_id'], $group['message']]; + + return syslog_db_fetch_assoc_prepared("SELECT syslog.seq, syslog.logtime, syslog.message, $breach_sql AS threshold_breached, + (SELECT host FROM `$syslogdb_default`.`syslog_hosts` AS sh WHERE sh.host_id = syslog.host_id) AS host, + syslog_programs.program, + (SELECT facility FROM `$syslogdb_default`.`syslog_facilities` AS sf WHERE sf.facility_id = syslog.facility_id) AS facility, + (SELECT priority FROM `$syslogdb_default`.`syslog_priorities` AS sp WHERE sp.priority_id = syslog.priority_id) AS priority FROM `$syslogdb_default`.`" . (($group['mtype'] === 'main') ? 'syslog' : 'syslog_removed') . "` AS syslog - LEFT JOIN `$syslogdb_default`.`syslog_programs` - ON syslog.program_id = syslog_programs.program_id + LEFT JOIN `$syslogdb_default`.`syslog_programs` AS syslog_programs ON syslog.program_id = syslog_programs.program_id $group_where - ORDER BY syslog.logtime DESC", $seq_array); + ORDER BY syslog.logtime DESC, syslog.seq DESC + LIMIT 51 OFFSET " . $offset, $params); +} + +/** + * Resolve a detail page exclusively from its server-owned filter context. + * + * @param string $token The rendered view's context token. + * @param string $group The rendered summary index. + * @param string $offset The zero-based page offset. + * + * @return array The bounded page or an explicit error. + */ +function syslog_group_details_page(string $token, string $group, string $offset = '0'): array { + global $syslogdb_default; + + $context = $_SESSION['syslog_group_contexts'][$token] ?? null; + + if (!$context || !isset($context['groups'][$group]) || !ctype_digit($offset) + || filter_var($offset, FILTER_VALIDATE_INT, ['options' => ['min_range' => 0, 'max_range' => PHP_INT_MAX - 51]]) === false) { + return ['error' => __('Grouped results are unavailable or expired. Refresh the results and try again.', 'syslog')]; + } + + $anchor = $context['groups'][$group]; + $table = $anchor['source'] === 'main' ? 'syslog' : 'syslog_removed'; + $where = $context['where'] === '' ? 'WHERE ' : 'WHERE (' . substr($context['where'], 6) . ') AND '; + $record = syslog_db_fetch_row_prepared("SELECT syslog.host_id, syslog.program_id, syslog.facility_id, syslog.priority_id, syslog.message + FROM `$syslogdb_default`.`$table` AS syslog + $where syslog.seq = ? AND syslog.logtime = ? + LIMIT 1", [$anchor['id'], $anchor['logtime']]); + + if ($record === false) { + return ['error' => __('Group messages could not be loaded.', 'syslog')]; + } + + if (!$record) { + return ['error' => __('Grouped results are unavailable or expired. Refresh the results and try again.', 'syslog')]; + } + + $record['mtype'] = $anchor['source']; + $messages = syslog_group_detail_messages($record, $context['where'], syslog_message_breach_sql(), (int) $offset); + + if ($messages === false) { + return ['error' => __('Group messages could not be loaded.', 'syslog')]; + } + + return [ + 'rows' => array_slice($messages, 0, 50), + 'more' => cacti_sizeof($messages) > 50, + 'source' => $anchor['source'], + 'available' => $anchor['available'], + 'trim' => $context['trim'] + ]; +} + +/** + * Render a bounded page of group members using the normal message controls. + * + * @param array $messages The detail-page records. + * @param string $source The stored or removed table name. + * @param int $trim The message trim setting. + * @param bool $available Whether the group has a recommendation. + * + * @return string The escaped detail row HTML. + */ +function syslog_group_detail_rows(array $messages, string $source, int $trim, bool $available): string { + ob_start(); + + foreach ($messages as $message) { + print ""; + form_selectable_cell(html_escape($message['logtime']), $message['seq'], '', 'left'); + print "'; + print "'; + form_selectable_cell(syslog_recommendation_indicator($available), $message['seq'], '20px', 'center syslogRecommendation'); + form_selectable_cell(syslog_message_button($message['message'], $message['host'] ?? '', $message['program'] ?? '', $message['facility'] ?? '', $message['priority'] ?? '', $message['logtime'], $message['seq'], $source, $trim), $message['seq'], '', 'left syslogMessage'); + form_selectable_cell(syslog_metadata_label($message['facility'] ?? '', 'facility'), $message['seq'], '', 'left'); + print "'; + form_selectable_cell(!empty($message['threshold_breached']) ? __('Yes', 'syslog') : __('No', 'syslog'), $message['seq'], '1%', 'center'); + form_selectable_cell('', $message['seq'], '', 'right'); + print ''; + } + + return ob_get_clean(); } /** diff --git a/includes/installer.php b/includes/installer.php index 689db107..112fa1c9 100644 --- a/includes/installer.php +++ b/includes/installer.php @@ -66,7 +66,7 @@ function syslog_install_advisor(int $syslog_exists, array $submitted_options = [ 'logtime_precision' => [ 'method' => 'drop_array', 'friendly_name' => __('Timestamp Precision', 'syslog'), - 'description' => __('Millisecond precision stores sub-second syslog timestamps in a TIMESTAMP(3) column and is only available on MariaDB. Second precision uses a standard TIMESTAMP column. On a production system this can only be changed later by running the cli/updatedb_convert_ms.php conversion utility, which truncates the stored messages.', 'syslog'), + 'description' => __('Millisecond precision stores sub-second syslog timestamps in a TIMESTAMP(3) column and is only available on MariaDB. Second precision uses a standard TIMESTAMP column. On a production system, use cli/upgrade_partitioning.php --convert-ms during a maintenance window to convert existing messages without discarding them.', 'syslog'), 'value' => isset($submitted_options['logtime_precision']) ? $submitted_options['logtime_precision'] : 'ms', 'array' => [ 'ms' => __('Millisecond - TIMESTAMP(3) (MariaDB only)', 'syslog'), diff --git a/includes/recommendations.php b/includes/recommendations.php index 29d3a00e..113f6ddd 100644 --- a/includes/recommendations.php +++ b/includes/recommendations.php @@ -608,7 +608,7 @@ function syslog_recommendation_indicator(bool $available): string { $label = __esc('Recommendation available - click the message for details', 'syslog'); - return ''; } diff --git a/includes/schema.php b/includes/schema.php index da732a52..6003f7d6 100644 --- a/includes/schema.php +++ b/includes/schema.php @@ -529,7 +529,7 @@ function syslog_detect_logtime_precision(string $table): string { * * The existing syslog table is authoritative so an upgrade never silently * migrates the schema, even if the configured precision changed (that switch is - * a deliberate, data destroying CLI operation). For a fresh install the + * an explicit CLI operation, optionally preserving data). For a fresh install the * requested value (or stored setting) is used, defaulting to 'ms', and is * clamped to 'sec' on MySQL where fractional partitioning cannot prune. * diff --git a/js/functions.js b/js/functions.js index b866ec90..30b2f34d 100644 --- a/js/functions.js +++ b/js/functions.js @@ -105,6 +105,25 @@ function initSyslogCompactSearch() { }).filter(Boolean); status.textContent = summaries.join(' · '); $('#refresh_results').off('click').on('click', refreshResults); + var pending = $(document).data('syslogFilterRequests') || new Set(); + $(document).data('syslogFilterRequests', pending); + function showRefreshActivity() { + $('#refresh_results i').toggleClass('fa-spin', pending.size > 0); + } + $(document).off('ajaxSend.syslogRefresh ajaxComplete.syslogRefresh') + .on('ajaxSend.syslogRefresh', function(event, xhr, settings) { + var url = new URL(settings.url, window.location.href); + var data = new URLSearchParams(typeof settings.data === 'string' ? settings.data : ''); + var action = url.searchParams.get('action') || data.get('action'); + if (url.origin === window.location.origin && /\/syslog\.php$/.test(url.pathname) && (!action || action === 'view')) { + pending.add(xhr); + showRefreshActivity(); + } + }).on('ajaxComplete.syslogRefresh', function(event, xhr) { + pending.delete(xhr); + showRefreshActivity(); + }); + showRefreshActivity(); var summary = document.getElementById('syslog_search_summary'); summary.textContent = document.getElementById('rfilter').value || builder.dataset.message; // Collapse the filter by default; only a saved 'false' preference keeps it open. @@ -284,7 +303,6 @@ function initSyslogWorkspace() { var row = button.closest('tr'); row.classList.add('syslogRowOpen'); var data = JSON.parse(button.dataset.message); - row.title = data.message || ''; // Show facility and severity through badges, leaving message backgrounds neutral. [ ['facility', 'syslogFacility'], @@ -300,23 +318,17 @@ function initSyslogWorkspace() { } }); }); - function open() { - if ($(row).tooltip('instance')) $(row).tooltip('close'); - deselect(); - active = button; - row.classList.add('syslogSelected'); - button.setAttribute('aria-expanded', 'true'); - show(data); - } - // The message button keeps keyboard access; the rest of the row opens the pane too. - button.addEventListener('click', function(event) { - event.stopPropagation(); - open(); - }); - row.addEventListener('click', function(event) { - if (event.target.closest('a, button, input, select, textarea, summary, .syslog-group-toggle')) return; - open(); - }); + }); + // Delegation also covers group members loaded after the initial render. + $(workspace).off('click.syslogMessage').on('click.syslogMessage', 'tr.syslogRow', function(event) { + var button = this.querySelector('.syslogMessageOpen'); + if (!button || event.target.closest('a, button:not(.syslogMessageOpen), input, select, textarea, summary')) return; + event.stopPropagation(); + deselect(); + active = button; + this.classList.add('syslogSelected'); + button.setAttribute('aria-expanded', 'true'); + show(JSON.parse(button.dataset.message)); }); /** @@ -395,41 +407,86 @@ var SYSLOG_MESSAGE_SORT = '.sortinfo[sort-column="message"], .sortinfo[sort-colu * switch to the fixed layout, which honors header widths. The message column * then absorbs any change. * + * Cacti's resize callback pins preceding columns, including the message, and + * does not bound the combined widths. Keep that column flexible and limit + * each drag to the space available inside the results area instead. + * * Releasing a drag over the header text also fires a click, which Cacti's * sortable headers treat as a sort, so that one click is swallowed. */ function syslogEnableColumnResize(workspace) { + var fitTables = []; workspace.querySelectorAll('tr.tableHeader').forEach(function(header) { var table = header.closest('table'); if (!table || table.classList.contains('syslogFixedColumns')) return; var cells = Array.from(header.children); - var widths = cells.map(function(cell) { return Math.ceil(cell.getBoundingClientRect().width); }); + var message = cells.find(function(cell) { return cell.querySelector(SYSLOG_MESSAGE_SORT); }); + if (!message) return; + var main = table.closest('.syslogResultsMain'); + var indicator = cells.find(function(cell) { return cell.querySelector('.syslogRecommendationHeading'); }); + var minimumMessage = 80; + var widths = cells.map(function(cell) { return $(cell).outerWidth(); }); + var chrome = Math.max(0, table.getBoundingClientRect().width - widths.reduce(function(sum, width) { return sum + width; }, 0)); cells.forEach(function(cell, index) { - var message = cell.querySelector(SYSLOG_MESSAGE_SORT); - if (message) cell.style.removeProperty('width'); - else if (!cell.style.width) cell.style.width = widths[index] + 'px'; + if (cell === message) cell.style.removeProperty('width'); + else $(cell).outerWidth(cell === indicator ? 20 : widths[index]); }); table.classList.add('syslogFixedColumns'); - var resized = 0; - var keep = null; - // Cacti's resize hands the width change to the next column, which can - // squeeze it to nothing; here the message column absorbs it instead. - // The core callback runs after these handlers, so restore on the next frame. - $(header).on('resizestart', 'th', function() { - resized = Date.now(); - var next = this.nextElementSibling; - keep = next && !next.querySelector(SYSLOG_MESSAGE_SORT) ? {cell: next, width: next.style.width} : null; - }).on('resize', 'th', function() { - resized = Date.now(); - if (keep) { - var restore = keep; - window.requestAnimationFrame(function() { restore.cell.style.width = restore.width; }); + function fit() { + message.style.removeProperty('width'); + if (indicator) $(indicator).outerWidth(20); + var fixed = cells.filter(function(cell) { return cell !== message && cell !== indicator; }); + var available = Math.max(main.clientWidth, parseFloat(getComputedStyle(table).minWidth) || 0) - chrome; + var budget = available - minimumMessage - (indicator ? 20 : 0); + var total = fixed.reduce(function(sum, cell) { return sum + $(cell).outerWidth(); }, 0); + if (total > budget) { + var spare = total - fixed.length * 20; + fixed.forEach(function(cell) { + $(cell).outerWidth(20 + Math.max(0, $(cell).outerWidth() - 20) * Math.max(0, budget - fixed.length * 20) / spare); + }); } - }).on('resizestop', 'th', function() { - // Restore before the core stop callback saves the column widths. + } + fit(); + fitTables.push(fit); + + var resized = 0; + $(header).on('resizestart', 'th', function(event, ui) { resized = Date.now(); - if (keep) keep.cell.style.width = keep.width; + fit(); + var active = this; + var initial = cells.map(function(cell) { return $(cell).outerWidth(); }); + var index = cells.indexOf(active); + var messageWidth = $(message).outerWidth(); + var padding = $(active).outerWidth() - $(active).width(); + var donor = message.nextElementSibling; + var donorWidth = donor ? $(donor).outerWidth() : 0; + var originalWidth = ui.originalSize.width; + // Replace only this log header's core callbacks; retain core width persistence. + $(active).resizable('option', { + minWidth: (active === message ? minimumMessage : 20) - padding, + maxWidth: active === message ? originalWidth + donorWidth - 20 : + (active === indicator ? 20 : initial[index] + messageWidth - minimumMessage) - padding, + resize: function(event, ui) { + resized = Date.now(); + cells.forEach(function(cell, index) { + if (cell !== message) $(cell).outerWidth(initial[index]); + }); + if (active === message) { + if (donor) $(donor).outerWidth(donorWidth - (ui.size.width - originalWidth)); + } else { + $(active).width(ui.size.width); + } + message.style.removeProperty('width'); + active.style.height = 'auto'; + }, + stop: function() { + resized = Date.now(); + fit(); + saveTableWidths(false); + message.style.removeProperty('width'); + } + }); }); header.addEventListener('click', function(event) { if (Date.now() - resized < 500) { @@ -438,10 +495,20 @@ function syslogEnableColumnResize(workspace) { } }, true); }); + $(window).off('resize.syslogColumns').on('resize.syslogColumns', function() { + if (!workspace.isConnected) { + $(window).off('resize.syslogColumns'); + return; + } + fitTables.forEach(function(fit) { fit(); }); + }); + // Core may restore persisted widths after the page's inline initialization. + window.requestAnimationFrame(function() { fitTables.forEach(function(fit) { fit(); }); }); } -function initSyslogValueFilters() { - document.querySelectorAll('.syslogValueFilter').forEach(function(button) { +function initSyslogValueFilters(root) { + var buttons = Array.from((root || document).querySelectorAll('.syslogValueFilter')); + buttons.forEach(function(button) { button.addEventListener('click', function(event) { event.stopPropagation(); syslogAddFilterCondition(button.dataset.filterField, button.dataset.filterValue); @@ -451,7 +518,7 @@ function initSyslogValueFilters() { // The search builder initializes on document ready, so mark after it. $(function() { var builder = document.getElementById('syslog_search_builder'); - document.querySelectorAll('.syslogValueFilter').forEach(function(button) { + buttons.forEach(function(button) { var active = builder && syslogFilterInScope(builder.searchRows, button.dataset.filterField, button.dataset.filterValue); button.classList.toggle('syslogValueFilterActive', !!active); button.setAttribute('aria-pressed', active ? 'true' : 'false'); @@ -1173,7 +1240,17 @@ function savedSearchPrompt(name, accept) { // Saving the default search again keeps it the default unless unticked. var selected = document.getElementById('saved_search').selectedOptions[0]; makeDefault.checked = !!(name && selected && selected.dataset.default === '1'); - $('#syslog_saved_prompt').dialog({ + $('#syslog_saved_prompt').off('submit.syslogSavedPrompt').on('submit.syslogSavedPrompt', function(event) { + event.preventDefault(); + event.stopPropagation(); + var value = input.value.trim(); + if (!value) { + input.focus(); + return; + } + $(this).dialog('close'); + accept(value, makeDefault.checked); + }).dialog({ modal: false, appendTo: 'body', width: 420, @@ -1184,16 +1261,11 @@ function savedSearchPrompt(name, accept) { applySyslogTheme(this); }, buttons: [ - {text: text.save, click: function() { - var value = input.value.trim(); - if (!value) { - input.focus(); - return; - } - $(this).dialog('close'); - accept(value, makeDefault.checked); - }}, - {text: text.cancel, click: function() { $(this).dialog('close'); }} + {text: text.cancel, click: function() { $(this).dialog('close'); }}, + {text: text.save, type: 'submit', form: 'syslog_saved_prompt', class: 'ui-state-active', click: function(event) { + event.preventDefault(); + this.requestSubmit(); + }} ] }); } @@ -1379,34 +1451,12 @@ function syslogRecommendationCards($parent, recommendations, labels, clipped) { } /** - * Show the locally available title as text, without requesting the server. - * - * @returns {jQuery} The full message or indicator tooltip, never parsed as HTML. - */ -function syslogRowTooltip() { - return $('
').text($(this).attr('title') || ''); -} - -/** - * Initialize syslog message tooltips and group expand/collapse functionality + * Initialize group expand/collapse and tooltips for non-message buttons. * Call this after the syslog message table is rendered or updated */ function initSyslogMessagesDisplay() { $(function() { applySyslogTheme(document.getElementById('syslog_workspace')); - // Initialize tooltips for syslog rows - $('.syslogRow').tooltip({ - track: true, - show: { - effect: 'fade', - duration: 250, - delay: 125 - }, - position: { my: 'left+15 center', at: 'right center' }, - classes: { 'ui-tooltip': 'syslogRecommendationTip' }, - content: syslogRowTooltip - }); - // Initialize tooltips for buttons $('button').not('.syslogRow button').tooltip({ closed: true @@ -1420,19 +1470,60 @@ function initSyslogMessagesDisplay() { $('.syslog-group-toggle').off('click').on('click', function(e) { e.preventDefault(); e.stopPropagation(); + var toggle = this; + var page = $(toggle).data('syslogGroupPage'); + + if (!page) { + var parent = toggle.closest('tr'); + var control = $('').insertAfter(parent); + var cell = $('').attr('colspan', parent.cells.length).appendTo(control); + page = { + control: control, rows: $(), offset: 0, loaded: false, busy: false, + status: $('').appendTo(cell), + previous: $('').text(toggle.dataset.previous).appendTo(cell), + next: $('').text(toggle.dataset.next).appendTo(cell) + }; + $(toggle).data('syslogGroupPage', page); + page.previous.on('click', function() { load(page.offset - 50); }); + page.next.on('click', function() { load(page.offset + 50); }); + } - var seq = $(this).data('seq'); - var detailRows = $('.syslog-detail-' + seq); - var icon = $(this); - - if (detailRows.is(':visible')) { - // Collapse - detailRows.hide(); - icon.removeClass('fa-chevron-up').addClass('fa-chevron-down'); - } else { - // Expand - detailRows.show(); - icon.removeClass('fa-chevron-down').addClass('fa-chevron-up'); + var expanded = toggle.getAttribute('aria-expanded') !== 'true'; + toggle.setAttribute('aria-expanded', String(expanded)); + $(toggle).find('i').toggleClass('fa-chevron-up', expanded).toggleClass('fa-chevron-down', !expanded); + page.control.toggle(expanded); + page.rows.toggle(expanded); + if (expanded && !page.loaded && !page.busy) load(0); + + function load(offset) { + if (page.busy) return; + page.busy = true; + page.status.text(toggle.dataset.loading); + page.previous.prop('disabled', true); + page.next.prop('disabled', true); + $.getJSON('syslog.php', {action: 'group_details', context: toggle.dataset.context, group: toggle.dataset.group, offset: offset}) + .done(function(result) { + if (!document.documentElement.contains(toggle)) return; + if (!result || typeof result.html !== 'string' || typeof result.more !== 'boolean') { + page.status.text(result && result.error || toggle.dataset.error); + return; + } + var body = $('').html(result.html); + initSyslogValueFilters(body[0]); + page.rows.remove(); + page.rows = body.children('tr').insertAfter(page.control); + page.rows.toggle(toggle.getAttribute('aria-expanded') === 'true'); + page.offset = offset; + page.loaded = true; + page.more = result.more; + page.status.text(page.rows.length ? (offset + 1) + ' - ' + (offset + page.rows.length) : toggle.dataset.empty); + }).fail(function() { + page.status.text(toggle.dataset.error); + }).always(function() { + page.busy = false; + page.previous.prop('disabled', !page.loaded || page.offset === 0); + page.next.prop('disabled', !page.loaded || !page.more); + }); } }); }); diff --git a/locales/po/cacti.pot b/locales/po/cacti.pot index bd969351..bff09f32 100644 --- a/locales/po/cacti.pot +++ b/locales/po/cacti.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: Cacti \n" "Report-Msgid-Bugs-To: developers@cacti.net\n" -"POT-Creation-Date: 2026-11-19 15:22+0000\n" +"POT-Creation-Date: 2026-11-19 16:30+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: Cacti Developers >\n" "Language-Team: Cacti Developers \n" @@ -495,6 +495,26 @@ msgstr "" msgid "Report previews search the most recent %d stored messages; the report itself covers its whole time window. Testing does not send email." msgstr "" +#: includes/functions.php +msgid "Grouped results are unavailable or expired. Refresh the results and try again." +msgstr "" + +#: includes/functions.php syslog.php +msgid "Group messages could not be loaded." +msgstr "" + +#: includes/functions.php syslog.php syslog_alerts.php syslog_dashboards.php +#: syslog_device_rules.php syslog_recommendations.php syslog_removal.php +#: syslog_reports.php +msgid "No" +msgstr "" + +#: includes/functions.php syslog.php syslog_alerts.php syslog_dashboards.php +#: syslog_device_rules.php syslog_recommendations.php syslog_removal.php +#: syslog_reports.php +msgid "Yes" +msgstr "" + #: includes/functions.php msgid "All Text" msgstr "" @@ -558,7 +578,7 @@ msgid "Timestamp Precision" msgstr "" #: includes/installer.php -msgid "Millisecond precision stores sub-second syslog timestamps in a TIMESTAMP(3) column and is only available on MariaDB. Second precision uses a standard TIMESTAMP column. On a production system this can only be changed later by running the cli/updatedb_convert_ms.php conversion utility, which truncates the stored messages." +msgid "Millisecond precision stores sub-second syslog timestamps in a TIMESTAMP(3) column and is only available on MariaDB. Second precision uses a standard TIMESTAMP column. On a production system, use cli/upgrade_partitioning.php --convert-ms during a maintenance window to convert existing messages without discarding them." msgstr "" #: includes/installer.php @@ -1737,11 +1757,11 @@ msgstr "" msgid "Recovery requires no matching messages until the displayed time. Every match restarts the quiet period." msgstr "" -#: lib/syslog_thresholds.php +#: lib/syslog_thresholds.php syslog.php msgid "Previous" msgstr "" -#: lib/syslog_thresholds.php +#: lib/syslog_thresholds.php syslog.php msgid "Next" msgstr "" @@ -2495,14 +2515,16 @@ msgstr "" msgid "Empty message" msgstr "" -#: syslog.php syslog_alerts.php syslog_dashboards.php syslog_device_rules.php -#: syslog_recommendations.php syslog_removal.php syslog_reports.php -msgid "No" +#: syslog.php +msgid "Toggle group messages" msgstr "" -#: syslog.php syslog_alerts.php syslog_dashboards.php syslog_device_rules.php -#: syslog_recommendations.php syslog_removal.php syslog_reports.php -msgid "Yes" +#: syslog.php +msgid "Loading group messages..." +msgstr "" + +#: syslog.php +msgid "No matching messages." msgstr "" #: syslog.php diff --git a/syslog.php b/syslog.php index 413602ec..c4aeb2ed 100644 --- a/syslog.php +++ b/syslog.php @@ -141,6 +141,23 @@ exit; } +if (get_request_var('action') === 'group_details') { + header('Content-Type: application/json; charset=UTF-8'); + $page = syslog_group_details_page( + (string) get_filter_request_var('context', FILTER_VALIDATE_REGEXP, ['options' => ['regexp' => '/^[a-f0-9]{32}$/D']]), + (string) get_filter_request_var('group', FILTER_VALIDATE_INT, ['options' => ['min_range' => 0]]), + isset_request_var('offset') ? (string) get_filter_request_var('offset', FILTER_VALIDATE_INT, ['options' => ['min_range' => 0]]) : '0' + ); + + if (!isset($page['error'])) { + $page['html'] = syslog_group_detail_rows($page['rows'], $page['source'], $page['trim'], $page['available']); + unset($page['rows'], $page['source'], $page['trim'], $page['available']); + } + + print json_encode($page, JSON_INVALID_UTF8_SUBSTITUTE); + exit; +} + $title = __('Syslog Viewer', 'syslog'); // set the default tab @@ -2021,7 +2038,8 @@ function get_syslog_messages(string &$sql_where, $rows, string $tab): array { if (get_request_var('search_mode') === 'logical') { $predicate = !empty($GLOBALS['syslog_search_error']) ? '(1 = 0)' : - syslog_logical_search_sql($GLOBALS['syslog_search_tree'] ?? null, $tab == 'syslog' ? 'message' : 'logmsg'); + syslog_logical_search_sql($GLOBALS['syslog_search_tree'] ?? null, $tab == 'syslog' ? 'message' : 'logmsg', + $tab === 'syslog' && get_request_var('grouping') == '1' ? time() : null); if ($predicate !== '') { $sql_where .= ($sql_where == '' ? 'WHERE ' : ' AND ') . $predicate; @@ -2132,13 +2150,16 @@ function get_syslog_messages(string &$sql_where, $rows, string $tab): array { ? "GROUP BY syslog.host_id, IF(syslog.message_digest = '', MD5(syslog.message), syslog.message_digest), syslog.program_id, syslog.facility_id, syslog.priority_id" : 'GROUP BY syslog.host_id, syslog.message, syslog.program_id, syslog.facility_id, syslog.priority_id'; + // A fixed-width timestamp/sequence pair selects the latest ID without collecting all member IDs. + $latest_seq_sql = "CAST(RIGHT(MAX(CONCAT(DATE_FORMAT(syslog.logtime, '%Y%m%d%H%i%s%f'), LPAD(syslog.seq, 20, '0'))), 20) AS UNSIGNED)"; + if (get_request_var('removal') == '-1') { $query_sql = "SELECT syslog.host_id, syslog.message, syslog.program_id, syslog.facility_id, syslog.priority_id, syslog_programs.program, 'main' AS mtype, COUNT(*) AS occurrence_count, MIN(syslog.logtime) AS first_logtime, MAX(syslog.logtime) AS logtime, MIN(syslog.seq) AS seq, MAX($breach_sql) AS threshold_breached, - GROUP_CONCAT(syslog.seq ORDER BY syslog.logtime DESC SEPARATOR ',') AS seq_list + $latest_seq_sql AS latest_seq FROM `$syslogdb_default`.`syslog` LEFT JOIN `$syslogdb_default`.`syslog_programs` ON syslog.program_id=syslog_programs.program_id @@ -2155,7 +2176,7 @@ function get_syslog_messages(string &$sql_where, $rows, string $tab): array { 'main' AS mtype, COUNT(*) AS occurrence_count, MIN(syslog.logtime) AS first_logtime, MAX(syslog.logtime) AS logtime, MIN(syslog.seq) AS seq, MAX($breach_sql) AS threshold_breached, - GROUP_CONCAT(syslog.seq ORDER BY syslog.logtime DESC SEPARATOR ',') AS seq_list + $latest_seq_sql AS latest_seq FROM `$syslogdb_default`.`syslog` AS syslog LEFT JOIN `$syslogdb_default`.`syslog_programs` ON syslog.program_id=syslog_programs.program_id @@ -2167,7 +2188,7 @@ function get_syslog_messages(string &$sql_where, $rows, string $tab): array { 'remove' AS mtype, COUNT(*) AS occurrence_count, MIN(syslog.logtime) AS first_logtime, MAX(syslog.logtime) AS logtime, MIN(syslog.seq) AS seq, MAX($breach_sql) AS threshold_breached, - GROUP_CONCAT(syslog.seq ORDER BY syslog.logtime DESC SEPARATOR ',') AS seq_list + $latest_seq_sql AS latest_seq FROM `$syslogdb_default`.`syslog_removed` AS syslog LEFT JOIN `$syslogdb_default`.`syslog_programs` ON syslog.program_id=syslog_programs.program_id @@ -2183,7 +2204,7 @@ function get_syslog_messages(string &$sql_where, $rows, string $tab): array { 'remove' AS mtype, COUNT(*) AS occurrence_count, MIN(syslog.logtime) AS first_logtime, MAX(syslog.logtime) AS logtime, MIN(syslog.seq) AS seq, MAX($breach_sql) AS threshold_breached, - GROUP_CONCAT(syslog.seq ORDER BY syslog.logtime DESC SEPARATOR ',') AS seq_list + $latest_seq_sql AS latest_seq FROM `$syslogdb_default`.`syslog_removed` AS syslog LEFT JOIN `$syslogdb_default`.`syslog_programs` AS syslog_programs ON syslog.program_id = syslog_programs.program_id @@ -2497,26 +2518,14 @@ function syslog_filter(string $sql_where, string $tab): void { data-exclude=''> -
' style='display:none'> -
-
- - -
-
- - -
-
-
" . syslog_value_filter_button($message['host'] ?? '', 'host') . '" . syslog_value_filter_button($message['program'] ?? '', 'program') . '" . syslog_value_filter_button($message['priority'] ?? '', 'priority') . '
@@ -2529,6 +2538,18 @@ function syslog_filter(string $sql_where, string $tab): void {
- - + +
'> +
' style='display:none'> +
+
+ + +
+
+ + +
+
+
'center', 'tip' => __('Recommendations', 'syslog') ]; + $group_token = $grouping_enabled ? syslog_group_context($syslog_messages, $sql_where, $trim, $recommendations) : ''; if ($tab == 'syslog') { // Check if grouping is enabled for display @@ -2739,7 +2761,13 @@ function syslog_messages(string $tab = 'syslog'): void { // Display grouped or individual messages if ($grouping_enabled && isset($sm['occurrence_count']) && $sm['occurrence_count'] > 1) { // Grouped message display with expand/collapse - $expand_icon = ""; + $expand_icon = ""; form_selectable_cell($expand_icon . $sm['logtime'], $sm['seq'], '', 'left'); } else { form_selectable_cell($sm['logtime'], $sm['seq'], '', 'left'); @@ -2747,9 +2775,9 @@ function syslog_messages(string $tab = 'syslog'): void { print "" . syslog_value_filter_button($hosts[$sm['host_id']] ?? __('Unknown', 'syslog'), 'host') . ''; print "" . syslog_value_filter_button($sm['program'], 'program') . ''; - print "" . syslog_recommendation_indicator($recommendations[$row]) . ''; + form_selectable_cell(syslog_recommendation_indicator($recommendations[$row]), $sm['seq'], '20px', 'center syslogRecommendation'); // Group summaries show the latest timestamp; use its matching sequence ID. - $rule_id = $grouping_enabled && !empty($sm['seq_list']) ? explode(',', $sm['seq_list'])[0] : $sm[$syslog_incoming_config['id']]; + $rule_id = $grouping_enabled ? $sm['latest_seq'] : $sm[$syslog_incoming_config['id']]; form_selectable_cell(syslog_message_button($sm['message'], $hosts[$sm['host_id']] ?? '', $sm['program'], $facilities[$sm['facility_id']] ?? '', $priorities[$sm['priority_id']] ?? '', $sm['logtime'], $rule_id, $sm['mtype'], $trim), $sm['seq'], '', 'left syslogMessage'); form_selectable_cell(syslog_metadata_label(isset($facilities[$sm['facility_id']]) ? $facilities[$sm['facility_id']] : __('Unknown', 'syslog'), 'facility'), $sm['seq'], '', 'left'); form_selectable_cell(syslog_value_filter_button($priorities[$sm['priority_id']] ?? __('Unknown', 'syslog'), 'priority'), $sm['seq'], '', 'left'); @@ -2763,32 +2791,6 @@ function syslog_messages(string $tab = 'syslog'): void { form_end_row(); - // If grouping is enabled and there are multiple occurrences, add hidden detail rows - if ($grouping_enabled && isset($sm['occurrence_count']) && $sm['occurrence_count'] > 1 && isset($sm['seq_list'])) { - $detail_messages = syslog_group_detail_messages($sm, $sql_where, $breach_sql); - - if (cacti_sizeof($detail_messages)) { - foreach ($detail_messages as $dm) { - print ""; - - print "" . html_escape($dm['logtime']) . ''; - print "" . syslog_value_filter_button($hosts[$dm['host_id']] ?? __('Unknown', 'syslog'), 'host') . ''; - print "" . syslog_value_filter_button($dm['program'], 'program') . ''; - print "" . syslog_recommendation_indicator($recommendations[$row]) . ''; - print "" . syslog_message_button($dm['message'], $hosts[$dm['host_id']] ?? '', $dm['program'], $facilities[$dm['facility_id']] ?? '', $priorities[$dm['priority_id']] ?? '', $dm['logtime'], $dm[$syslog_incoming_config['id']], $sm['mtype'], $trim) . ''; - print "" . syslog_metadata_label(isset($facilities[$dm['facility_id']]) ? $facilities[$dm['facility_id']] : __('Unknown', 'syslog'), 'facility') . ''; - print "" . syslog_value_filter_button($priorities[$dm['priority_id']] ?? __('Unknown', 'syslog'), 'priority') . ''; - - print "" . (!empty($dm['threshold_breached']) ? __('Yes', 'syslog') : __('No', 'syslog')) . ''; - - if ($grouping_enabled) { - print ""; - } - - print ''; - } - } - } } } else { print "" . __('No Syslog Messages', 'syslog') . ''; @@ -2840,7 +2842,7 @@ function syslog_messages(string $tab = 'syslog'): void { print "" . syslog_value_filter_button($log['host'], 'host') . ''; form_selectable_cell(isset($severities[$log['severity']]) ? $severities[$log['severity']] : __('Unknown', 'syslog'), $log['seq'], '', 'left'); form_selectable_cell(filter_value($log['name'] != '' ? $log['name'] : __('Alert Removed', 'syslog'), get_request_var('rfilter'), $config['url_path'] . 'plugins/syslog/syslog.php?id=' . $log['seq'] . '&tab=current'), $log['seq'], '', 'left'); - print "" . syslog_recommendation_indicator($recommendations[$row]) . ''; + form_selectable_cell(syslog_recommendation_indicator($recommendations[$row]), $log['seq'], '20px', 'center syslogRecommendation'); form_selectable_cell(syslog_message_button($log['logmsg'], $log['host'], $log['program'] ?? '', $log['facility'], $log['priority'], $log['logtime'], $log['seq'], 'alert', $trim), $log['seq'], '', 'syslogMessage left'); form_selectable_cell($log['count'], $log['seq'], '', 'right'); diff --git a/tests/Fixtures/compact_workspace.php b/tests/Fixtures/compact_workspace.php index 7114b38e..7252f0d1 100644 --- a/tests/Fixtures/compact_workspace.php +++ b/tests/Fixtures/compact_workspace.php @@ -25,15 +25,15 @@ function read_user_setting($name, $default = false, $force = false) { return $de $saved_active = 7; $saved_admin = true; $saved_share = true; $username = 'tester'; $item_rows = [50=>'50',100=>'100'];$trimvals = [0=>'Fit column',75=>'75 Chars',1024=>'All Text'];$page_refresh_interval = [0=>'Paused',300=>'5 Minutes']; $source = file_get_contents(dirname(__DIR__, 2) . '/syslog.php'); -$start = strpos($source, "
', $start)+7; +$start = strpos($source, "', strpos($source, "

System Logs   |   Alert Logs

'; eval('?>'.substr($source,$start,$end-$start)); echo '
'; $priorities = ['emerg', 'alert', 'crit', 'err', 'warning', 'notice', 'info', 'debug']; $message = 'nf_conntrack: table full, dropping packet '; $long_message = $message . ' ' . str_repeat('width preview ', 4); -foreach(range(1,17) as $i) echo ''; +foreach(range(1,17) as $i) echo ''; echo '
TimeDeviceProgramMessageFacilitySeverity
2026-09-13 00:12:4210.0.0.5kernel'.syslog_recommendation_indicator($i === 1).''.syslog_message_button($i === 1 ? $long_message : $message,'10.0.0.5','kernel','kern',$priorities[$i % 8],'2026-09-13 00:12:42', 100 + $i, $i === 17 ? 'removed' : 'main').'kern'.$priorities[$i % 8].'
2026-09-13 00:12:4210.0.0.5kernel'.syslog_recommendation_indicator($i === 1).''.syslog_message_button($i === 1 ? $long_message : $message,'10.0.0.5','kernel','kern',$priorities[$i % 8],'2026-09-13 00:12:42', 100 + $i, $i === 17 ? 'removed' : 'main').'kern'.$priorities[$i % 8].'
'; $start = strpos($source,"
', $start)+strlen(''); eval('?>'.substr($source,$start,$end-$start)); -echo ''; +echo ''; diff --git a/tests/Fixtures/threshold_messages.php b/tests/Fixtures/threshold_messages.php index 35144682..14e73422 100644 --- a/tests/Fixtures/threshold_messages.php +++ b/tests/Fixtures/threshold_messages.php @@ -6,14 +6,22 @@ function form_selectable_cell($value, $id, $width = '', $class = '') { print '' . $value . ''; } function form_end_row() { print ''; } $source = file_get_contents(dirname(__DIR__, 2) . '/syslog.php'); -$recommendation_header = ['display' => 'Recommendations', 'align' => 'center']; +// Run excerpts of syslog.php in this global scope from a temporary file. +function fixture_excerpt(string $code): string { + $file = tempnam(sys_get_temp_dir(), 'syslog-fixture'); + file_put_contents($file, $code); + register_shutdown_function('unlink', $file); + + return $file; +} +$recommendation_header = ['display' => '', 'align' => 'center']; $recommendations = [true, false]; $start = strpos($source, "\t\t\$display_text = [", strpos($source, 'function syslog_messages(')); $end = strpos($source, "\n\t\t// Add count column", $start); -eval(substr($source, $start, $end - $start)); +require fixture_excerpt(' 'router']; $facilities = [1 => 'user']; $priorities = [3 => 'err']; @@ -22,14 +30,54 @@ function form_end_row() { print ''; } $syslog_messages = [$base + ['threshold_breached' => 1], array_replace($base, ['seq' => 102, 'threshold_breached' => null])]; $grouping_enabled = false; $trim = 0; -print ''; -foreach ($display_text as $header) { print ''; } +function fixture_log_headers($headers) { + foreach ($headers as $key => $header) { + $label = $header['display'] ?? $header[0]; + print ''; + } +} +print '
' . html_escape($header['display'] ?? $header[0]) . ''; + print str_starts_with($key, 'nosort') ? $label : '
' . html_escape($label) . '
'; + print '
'; +fixture_log_headers($display_text); print ''; -eval($rows); -print '
'; +require $rows; +print '
'; $grouping_enabled = true; $sql_where = ''; -$syslog_messages = [$base + ['threshold_breached' => 1, 'occurrence_count' => 2, 'seq_list' => '101,102']]; -test_override('syslog_db_fetch_assoc_prepared', static fn() => [$base + ['threshold_breached' => 1], array_replace($base, ['seq' => 102, 'threshold_breached' => null])]); -eval($rows); -print '
'; +$syslog_messages = [$base + ['threshold_breached' => 1, 'occurrence_count' => 1000000, 'latest_seq' => 102]]; +$group_token = syslog_group_context($syslog_messages, 'WHERE logtime BETWEEN \'2026-10-15 00:00:00\' AND \'2026-10-15 23:59:59\'', 0, [true]); +test_override('syslog_db_fetch_assoc_prepared', static function () { throw new RuntimeException('Group members must not be queried while rendering summaries'); }); +require $rows; +print ''; +$start = strpos($source, "\t\t\$display_text = [", $end); +$end = strpos($source, "\n\t\t\$nav =", $start); +require fixture_excerpt(''; +fixture_log_headers($display_text); +print ''; +$syslog_messages = [[ + 'seq' => 301, 'logtime' => $base['logtime'], 'host' => 'router.example.test', + 'severity' => 2, 'name' => str_repeat('LongAlertName', 12), 'count' => 12345, + 'facility' => 'local7', 'priority' => 'warning', 'logmsg' => str_repeat('Alert message with details ', 40) +]]; +$severities = [2 => 'Critical']; +$config['url_path'] = '/'; +$start = strpos($source, "\t\tif (cacti_sizeof(\$syslog_messages)) {", $end); +$end = strpos($source, "\n\t\thtml_end_box(false);", $start); +$alert_rows = fixture_excerpt('
'; +$details = []; +foreach (range(201, 252) as $seq) { + $details[] = ['seq' => $seq, 'logtime' => $base['logtime'], 'message' => $base['message'], 'host' => 'router', 'program' => 'app', 'facility' => 'user', 'priority' => 'err', 'threshold_breached' => $seq % 2]; +} +print ''; +print ''; +$start = strpos($source, "'); +require fixture_excerpt(substr($source, $start, $end - $start)); +print ''; diff --git a/tests/Unit/IncomingPreservationTest.php b/tests/Unit/IncomingPreservationTest.php index 467aec4d..1c77cf58 100644 --- a/tests/Unit/IncomingPreservationTest.php +++ b/tests/Unit/IncomingPreservationTest.php @@ -72,6 +72,95 @@ expect(syslog_preprocess_incoming_records())->toBe(['max_seq' => 42, 'incoming' => 3]); }); +it('strips program boundary delimiters only from the active incoming batch', function () { + syslog_load_plugin_source('includes/functions.php'); + $GLOBALS['syslogdb_default'] = 'main'; + $db = new PDO('sqlite::memory:', null, null, [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $db->exec('CREATE TABLE syslog_incoming (seq INTEGER PRIMARY KEY, status INTEGER, program TEXT)'); + $insert = $db->prepare('INSERT INTO syslog_incoming VALUES (?, ?, ?)'); + $programs = [ + ['(o-bridge)', 'o-bridge'], + ['(sd-pam)', 'sd-pam'], + ['(systemd)', 'systemd'], + ['gdm-launch-environment]', 'gdm-launch-environment'], + ['[sshd]', 'sshd'], + ['[(systemd)]', 'systemd'], + ['](leading', 'leading'], + ['trailing([', 'trailing'], + ['worker[7](helper)-main', 'worker[7](helper)-main'], + ['plain-program', 'plain-program'], + ['([])', ''], + [null, null] + ]; + foreach ($programs as $index => [$program]) { + $insert->execute([$index + 1, 1, $program]); + } + $insert->execute([100, 0, '(pending)']); + $insert->execute([101, 1, '[next-batch]']); + $insert->execute([102, 2, '(processed)']); + test_override('syslog_db_fetch_assoc_prepared', static function ($sql, $params) use ($db) { + $query = $db->prepare($sql); + $query->execute($params); + + return $query->fetchAll(PDO::FETCH_ASSOC); + }); + test_override('syslog_db_execute_prepared', static function ($sql, $params) use ($db) { + $query = $db->prepare($sql); + + return $query->execute($params); + }); + + expect(syslog_strip_incoming_programs(100))->toBeTrue() + ->and($db->query('SELECT program FROM syslog_incoming ORDER BY seq')->fetchAll(PDO::FETCH_COLUMN)) + ->toBe([...array_column($programs, 1), '(pending)', '[next-batch]', '(processed)']); +}); + +it('bounds program normalization updates and continues beyond the first batch', function () { + syslog_load_plugin_source('includes/functions.php'); + $GLOBALS['syslogdb_default'] = 'main'; + $db = new PDO('sqlite::memory:', null, null, [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); + $db->exec('CREATE TABLE syslog_incoming (seq INTEGER PRIMARY KEY, status INTEGER, program TEXT)'); + $insert = $db->prepare('INSERT INTO syslog_incoming VALUES (?, 1, ?)'); + foreach (range(1, 251) as $seq) { + $insert->execute([$seq, '(worker-' . $seq . ')']); + } + $updates = []; + test_override('syslog_db_fetch_assoc_prepared', static function ($sql, $params) use ($db) { + $query = $db->prepare($sql); + $query->execute($params); + + return $query->fetchAll(PDO::FETCH_ASSOC); + }); + test_override('syslog_db_execute_prepared', static function ($sql, $params) use ($db, &$updates) { + $updates[] = count($params); + $query = $db->prepare($sql); + + return $query->execute($params); + }); + + expect(syslog_strip_incoming_programs(251))->toBeTrue() + ->and($updates)->toBe([751, 4]) + ->and($db->query('SELECT program FROM syslog_incoming ORDER BY seq')->fetchAll(PDO::FETCH_COLUMN)) + ->toBe(array_map(static fn($seq) => 'worker-' . $seq, range(1, 251))); +}); + +it('leaves the incoming batch for retry when program normalization fails', function ($failure) { + syslog_load_plugin_source('includes/functions.php'); + $GLOBALS['syslogdb_default'] = 'syslog'; + $logs = []; + $hooks = []; + test_override('syslog_db_fetch_cell', static fn() => 42); + test_override('syslog_db_fetch_cell_prepared', static fn() => 3); + test_override('syslog_db_fetch_assoc_prepared', static fn() => $failure === 'select' ? false : [['program' => '(systemd)']]); + test_override('syslog_db_execute_prepared', static fn($sql) => $failure !== 'update' || !str_contains($sql, 'CASE program')); + test_override('cacti_log', static function ($message) use (&$logs) { $logs[] = $message; }); + test_override('api_plugin_hook', static function ($hook) use (&$hooks) { $hooks[] = $hook; }); + + expect(syslog_preprocess_incoming_records())->toBe(['max_seq' => 0, 'incoming' => 0]) + ->and($logs)->toBe(['ERROR: Unable to normalize incoming program names; this batch will be retried.']) + ->and($hooks)->toBe([]); +})->with(['select', 'update']); + it('rolls back failed local transfers without reporting archived rows', function ($failure) { syslog_load_plugin_source('includes/functions.php'); $GLOBALS['config']['poller_id'] = 1; diff --git a/tests/Unit/MessageDigestGroupingTest.php b/tests/Unit/MessageDigestGroupingTest.php index 91c0f62d..8e408903 100644 --- a/tests/Unit/MessageDigestGroupingTest.php +++ b/tests/Unit/MessageDigestGroupingTest.php @@ -55,10 +55,16 @@ $db = new PDO('sqlite::memory:', null, null, [PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION]); $db->exec('CREATE TABLE syslog_programs (program_id INTEGER, program TEXT)'); $db->exec("INSERT INTO syslog_programs VALUES (7, 'kernel')"); - $db->exec('CREATE TABLE syslog (seq INTEGER, logtime TEXT, program_id INTEGER, host_id INTEGER, message TEXT, PRIMARY KEY (seq, logtime))'); - $db->exec('CREATE TABLE syslog_removed (seq INTEGER, logtime TEXT, program_id INTEGER, host_id INTEGER, message TEXT, PRIMARY KEY (seq, logtime))'); - $insert_main = $db->prepare("INSERT INTO syslog VALUES (?, ?, 7, ?, 'stored link down')"); - $insert_removed = $db->prepare("INSERT INTO syslog_removed VALUES (?, ?, 7, ?, 'removed link down')"); + $db->exec('CREATE TABLE syslog_hosts (host_id INTEGER, host TEXT)'); + $db->exec("INSERT INTO syslog_hosts VALUES (1, 'router')"); + $db->exec('CREATE TABLE syslog_facilities (facility_id INTEGER, facility TEXT)'); + $db->exec("INSERT INTO syslog_facilities VALUES (1, 'user')"); + $db->exec('CREATE TABLE syslog_priorities (priority_id INTEGER, priority TEXT)'); + $db->exec("INSERT INTO syslog_priorities VALUES (3, 'err')"); + $db->exec('CREATE TABLE syslog (seq INTEGER, logtime TEXT, program_id INTEGER, host_id INTEGER, message TEXT, facility_id INTEGER, priority_id INTEGER, PRIMARY KEY (seq, logtime))'); + $db->exec('CREATE TABLE syslog_removed (seq INTEGER, logtime TEXT, program_id INTEGER, host_id INTEGER, message TEXT, facility_id INTEGER, priority_id INTEGER, PRIMARY KEY (seq, logtime))'); + $insert_main = $db->prepare('INSERT INTO syslog VALUES (?, ?, 7, ?, ?, 1, 3)'); + $insert_removed = $db->prepare('INSERT INTO syslog_removed VALUES (?, ?, 7, ?, ?, 1, 3)'); foreach ([ [101, '2026-01-01 12:00:00', 1], @@ -67,20 +73,27 @@ [102, '2026-01-02 13:00:00', 2], [103, '2026-01-02 14:00:00', 1] ] as $row) { - $insert_main->execute($row); - $insert_removed->execute($row); + $insert_main->execute([...$row, $row[0] === 103 ? 'unrelated message' : 'stored link down']); + $insert_removed->execute([...$row, $row[0] === 103 ? 'unrelated message' : 'removed link down']); } $fetch = static function ($sql, $params = []) use ($db) { + $sql = str_replace(syslog_message_breach_sql(), '0', $sql); $query = $db->prepare($sql); $query->execute($params); return $query->fetchAll(PDO::FETCH_ASSOC); }; test_override('syslog_db_fetch_assoc_prepared', $fetch); + test_override('syslog_db_fetch_row_prepared', static function ($sql, $params) use ($db) { + $query = $db->prepare($sql); + $query->execute($params); + + return $query->fetch(PDO::FETCH_ASSOC) ?: []; + }); $GLOBALS['syslogdb_default'] = 'main'; - $sm = ['mtype' => $type, 'seq_list' => '101,102']; + $sm = ['mtype' => $type, 'host_id' => 1, 'program_id' => 7, 'facility_id' => 1, 'priority_id' => 3, 'message' => $message]; $sql_where = 'WHERE ' . syslog_logical_search_sql(syslog_parse_logical_search( 'logtime >= "2026-01-02 00:00:00" AND logtime <= "2026-01-02 23:59:59" AND host_id = "1"' ), 'message'); @@ -95,9 +108,92 @@ $detail_messages = syslog_group_detail_messages($sm, '', '0'); - expect($detail_messages)->toHaveCount(4) - ->and(array_column($detail_messages, 'seq'))->not->toContain(103); + expect($detail_messages)->toHaveCount(3) + ->and(array_column($detail_messages, 'seq'))->not->toContain(103) + ->and(array_column($detail_messages, 'seq'))->not->toContain(102); + + foreach (range(200, 259) as $seq) { + $insert_main->execute([$seq, '2026-01-02 12:30:00', 1, 'stored link down']); + $insert_removed->execute([$seq, '2026-01-02 12:30:00', 1, 'removed link down']); + } + + $token = syslog_group_context([ + $sm + ['latest_seq' => 101, 'logtime' => '2026-01-02 12:00:00', 'occurrence_count' => 1000000] + ], $sql_where, 75, [true]); + $first = syslog_group_details_page($token, '0'); + $second = syslog_group_details_page($token, '0', '50'); + + expect($first['rows'])->toHaveCount(50) + ->and($first['more'])->toBeTrue() + ->and($second['rows'])->toHaveCount(11) + ->and($second['more'])->toBeFalse() + ->and(array_intersect(array_column($first['rows'], 'seq'), array_column($second['rows'], 'seq')))->toBe([]) + ->and(array_column(array_merge($first['rows'], $second['rows']), 'logtime'))->each->toStartWith('2026-01-02') + ->and($second['rows'][10]['message'])->toBe($message) + ->and($second['rows'][10]['seq'])->toBe(101); })->with([ 'stored messages' => ['main', 'stored link down'], 'removed messages' => ['remove', 'removed link down'] ]); + +it('rejects expired groups and invalid offsets before issuing a detail query', function () { + syslog_load_plugin_source('includes/functions.php'); + test_override('syslog_db_fetch_row_prepared', static function () { + throw new RuntimeException('Unexpected detail query'); + }); + $token = syslog_group_context([[ + 'mtype' => 'main', 'latest_seq' => 1, 'logtime' => '2026-01-02 12:00:00', 'occurrence_count' => 1000000 + ]], 'WHERE syslog.host_id = 1', 0, [false]); + + expect(syslog_group_details_page('missing', '0'))->toHaveKey('error') + ->and(syslog_group_details_page($token, '1'))->toHaveKey('error') + ->and(syslog_group_details_page($token, '0', '-1'))->toHaveKey('error') + ->and(syslog_group_details_page($token, '0', '999999999999999999999999'))->toHaveKey('error'); +}); + +it('reports detail query failures and anchors that left the active results', function () { + syslog_load_plugin_source('includes/functions.php'); + $GLOBALS['syslogdb_default'] = 'main'; + $token = syslog_group_context([[ + 'mtype' => 'main', 'latest_seq' => 1, 'logtime' => '2026-01-02 12:00:00', 'occurrence_count' => 2 + ]], '', 0, [false]); + $anchor = ['host_id' => 1, 'program_id' => 7, 'facility_id' => 1, 'priority_id' => 3, 'message' => 'link down']; + $failure = __('Group messages could not be loaded.', 'syslog'); + + test_override('syslog_db_fetch_row_prepared', static fn() => false); + expect(syslog_group_details_page($token, '0'))->toBe(['error' => $failure]); + + test_override('syslog_db_fetch_row_prepared', static fn() => []); + expect(syslog_group_details_page($token, '0')['error'])->toContain('unavailable or expired'); + + test_override('syslog_db_fetch_row_prepared', static fn() => $anchor); + test_override('syslog_db_fetch_assoc_prepared', static fn() => false); + expect(syslog_group_details_page($token, '0'))->toBe(['error' => $failure]); +}); + +it('renders detail pages as escaped rows using the standard message controls', function () { + syslog_load_plugin_source('includes/functions.php'); + syslog_load_plugin_source('includes/recommendations.php'); + + $html = syslog_group_detail_rows([ + ['seq' => 5, 'logtime' => '2026-01-02 12:00:00', 'message' => 'link down', 'host' => 'router', + 'program' => 'kernel', 'facility' => 'user', 'priority' => 'err', 'threshold_breached' => 1], + ['seq' => 6, 'logtime' => '2026-01-02 12:01:00', 'message' => 'link up'] + ], 'main', 30, true); + + expect(substr_count($html, 'syslogGroupMember'))->toBe(2) + ->and($html)->toContain('<b>link down</b>')->not->toContain('link down') + ->and($html)->toContain('syslogRecommendation')->toContain('syslogMessageOpen') + ->and($html)->toContain('>' . __('Yes', 'syslog') . '<')->toContain('>' . __('No', 'syslog') . '<'); +}); + +it('freezes relative group ranges without rewriting literal message filters', function () { + syslog_load_plugin_source('includes/functions.php'); + test_override('db_qstr', static fn($value) => "'" . str_replace("'", "''", $value) . "'"); + $tree = syslog_parse_logical_search('logtime last "86400" AND NOT (logtime during "previous_day" OR message contains "NOW()")'); + $sql = syslog_logical_search_sql($tree, 'message', 1763568000); + + expect($sql)->toContain('FROM_UNIXTIME(1763568000)', "'NOW()'") + ->not->toContain('DATE_SUB(NOW()') + ->not->toContain('CURDATE()'); +}); diff --git a/tests/Unit/PartitionRebuildTest.php b/tests/Unit/PartitionRebuildTest.php new file mode 100644 index 00000000..598e048b --- /dev/null +++ b/tests/Unit/PartitionRebuildTest.php @@ -0,0 +1,219 @@ + '11.8-MariaDB', + 'tables' => ['syslog'], + 'bytes' => '10737418240', + 'lock' => 1, + 'queries' => [], + 'rebuilds' => [], + 'settings' => [], + 'executions' => [], + 'precision' => 'sec' + ]; + test_override('read_config_option', static fn($key) => [ + 'syslog_install_engine' => 'InnoDB', 'syslog_retention' => '30', + 'syslog_partition_ahead_days' => '3' + ][$key] ?? ''); + test_override('syslog_db_table_exists', static fn($table) => in_array($table, $state->tables, true)); + test_override('syslog_db_fetch_row', static fn() => ['Value' => $state->version]); + test_override('set_config_option', static function ($key, $value) use ($state) { + $state->settings[$key] = $value; + }); + test_override('syslog_db_execute_prepared', static function ($sql, $params) use ($state) { + $state->executions[] = [$sql, $params]; + + return true; + }); + test_override('rebuild_mariadb', static fn() => stripos($state->version, 'mariadb') !== false); + test_override('syslog_db_fetch_cell', static fn($sql) => match (true) { + str_contains($sql, 'time_zone') => '+05:30', + str_contains($sql, 'lock_wait_timeout') => '86400', + default => 'STRICT_TRANS_TABLES,NO_ZERO_DATE' + }); + test_override('syslog_db_fetch_cell_prepared', static function ($sql, $params) use ($state) { + $state->queries[] = [$sql, $params]; + + if (str_contains($sql, 'GET_LOCK')) { + return $state->lock; + } + + if (str_contains($sql, 'COLUMN_TYPE')) { + return $state->precision === 'ms' ? 'timestamp(3)' : 'timestamp'; + } + + return $state->bytes; + }); + test_override('syslog_upart_rebuild', static function (...$args) use ($state) { + $state->rebuilds[] = $args; + $state->precision = $args[1] ? 'ms' : $state->precision; + + return true; + }); + + return $state; +} + +it('reports database size, equal extra space and several-hour duration even without a prompt', function () { + $state = partition_rebuild_setup(); + $run = partition_rebuild_runner(); + ob_start(); + $ok = $run(true, false); + $output = ob_get_clean(); + + expect($ok)->toBeTrue() + ->and($output)->toContain('10.00 GiB (10,737,418,240 bytes)') + ->toContain('AT LEAST another 10.00 GiB (10,737,418,240 bytes)') + ->toContain('DATABASE SERVER')->toContain('SEVERAL HOURS') + ->toContain('does NOT pause them')->toContain('10,000-row copies') + ->not->toContain('Type "yes"') + ->and($state->rebuilds)->toBe([['syslog', false, true, 'InnoDB', 30, 3, '+05:30']]); + expect($state->settings)->toBe([]); + expect(array_column($state->queries, 0))->toContain('SELECT RELEASE_LOCK(?)'); +}); + +it('refuses to rebuild when the database size is unreadable', function () { + $state = partition_rebuild_setup(); + $state->bytes = false; + $run = partition_rebuild_runner(); + + expect($run(true, false))->toBeFalse() + ->and($state->rebuilds)->toBe([]); + expect(array_column($state->queries, 0))->toContain('SELECT RELEASE_LOCK(?)'); +}); + +it('prevents a destructive retry from deleting retained originals', function () { + $state = partition_rebuild_setup(); + $state->tables[] = 'syslog_rebuild'; + $run = partition_rebuild_runner(); + + expect($run(true, true, false))->toBeFalse() + ->and($state->rebuilds)->toBe([]); +}); + +it('rejects millisecond conversion on MySQL before any rebuild', function () { + $state = partition_rebuild_setup(); + $state->version = '8.4.0'; + $run = partition_rebuild_runner(); + + expect($run(true, true))->toBeFalse() + ->and($state->rebuilds)->toBe([]); +}); + +it('leaves the precision setting unchanged and restores the session after a failed copy', function () { + $state = partition_rebuild_setup(); + test_override('syslog_upart_rebuild', static fn() => false); + $run = partition_rebuild_runner(); + ob_start(); + $ok = $run(true, true); + ob_end_clean(); + + expect($ok)->toBeFalse(); + expect($state->settings)->toBe([]) + ->and($state->executions)->toContain(['SET SESSION time_zone = ?', ['+05:30']]) + ->toContain(['SET SESSION sql_mode = ?', ['STRICT_TRANS_TABLES,NO_ZERO_DATE']]); +}); + +it('plans UTC daily boundaries for retention and outlying future records', function () { + $plan = syslog_test_extract_function('cli/upgrade_partitioning.php', 'syslog_upart_partition_sql'); + $today = gmmktime(0, 0, 0, 11, 19, 2026); + $sql = $plan(2, 1, $today - 400 * 86400, $today + 3 * 86400, $today + 43200); + + expect($sql)->toContain('PARTITION d20261118 VALUES LESS THAN (' . $today . ')') + ->toContain('PARTITION d20261122 VALUES LESS THAN (' . ($today + 4 * 86400) . ')') + ->toEndWith('PARTITION dMaxValue VALUES LESS THAN MAXVALUE)') + ->and(substr_count($sql, 'PARTITION d'))->toBe(6); +}); + +it('covers historical days for indefinite retention and refuses excessive partition counts', function () { + $plan = syslog_test_extract_function('cli/upgrade_partitioning.php', 'syslog_upart_partition_sql'); + $today = gmmktime(0, 0, 0, 11, 19, 2026); + + expect($plan(0, 1, $today - 2 * 86400, null, $today)) + ->toContain('PARTITION d20261117 VALUES LESS THAN (' . ($today - 86400) . ')') + ->and($plan(0, 1, $today - 8190 * 86400, null, $today))->toBeNull(); +}); + +it('prints readable CLI help and rejects conflicting preservation arguments', function () { + $base = sys_get_temp_dir() . '/syslog-rebuild-cli-' . bin2hex(random_bytes(6)); + mkdir($base . '/plugins/syslog/cli', 0700, true); + mkdir($base . '/plugins/syslog/includes', 0700); + mkdir($base . '/include', 0700); + $paths = [ + 'include/cli_check.php' => ' plugin_test_read_source('cli/upgrade_partitioning.php'), + 'plugins/syslog/setup.php' => ' "5.0"]; }', + 'plugins/syslog/includes/schema.php' => ' $content) { + file_put_contents($base . '/' . $path, $content); + } + + foreach ([ + ['upgrade_partitioning.php', ['--help'], 0, [ + 'Preserve existing messages (the default)', + 'future-partition horizon.' . PHP_EOL . PHP_EOL . 'Existing precision', + '--convert-ms is selected.' . PHP_EOL . PHP_EOL . 'With preservation', + 'Syslog Partition Upgrade Utility, Version 5.0, Copyright (C)', + PHP_EOL . PHP_EOL . 'usage: upgrade_partitioning.php', + 'values are verified.' . PHP_EOL . PHP_EOL . 'Options:', + 'several hours.' . PHP_EOL . PHP_EOL . 'On failure', + 'writers first.' . PHP_EOL . PHP_EOL . 'The current database size' + ]], + ['upgrade_partitioning.php', ['--version'], 0, 'Syslog Partition Upgrade Utility, Version 5.0, Copyright (C)'], + ['upgrade_partitioning.php', ['-V'], 0, 'Syslog Partition Upgrade Utility, Version 5.0, Copyright (C)'], + ['upgrade_partitioning.php', ['-v'], 0, 'Syslog Partition Upgrade Utility, Version 5.0, Copyright (C)'], + ['upgrade_partitioning.php', ['--discard-data', '--preserve-data'], 1, 'cannot be combined'] + ] as [$entry, $args, $expected_status, $expected_output]) { + $process = proc_open(array_merge([PHP_BINARY, '-d', 'xdebug.mode=off', $base . '/plugins/syslog/cli/' . $entry], $args), + [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], $pipes); + expect(is_resource($process))->toBeTrue(); + fclose($pipes[0]); + $output = stream_get_contents($pipes[1]) . stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + expect(proc_close($process))->toBe($expected_status); + foreach ((array) $expected_output as $text) { + expect($output)->toContain($text); + } + } + } finally { + foreach (array_keys($paths) as $path) { + if (is_file($base . '/' . $path)) { + unlink($base . '/' . $path); + } + } + + foreach (['plugins/syslog/cli', 'plugins/syslog/includes', 'plugins/syslog', 'plugins', 'include', ''] as $directory) { + rmdir($base . '/' . $directory); + } + } +}); diff --git a/tests/Unit/RecommendationsTest.php b/tests/Unit/RecommendationsTest.php index 9dba3c9a..fc334844 100644 --- a/tests/Unit/RecommendationsTest.php +++ b/tests/Unit/RecommendationsTest.php @@ -550,7 +550,7 @@ function recommendation_sample(array $override = []): array { ->and($flag)->toBe([SYSLOG_RECOMMENDATIONS_SEED_FLAG => '1']); }); -it('keeps the escaped full message in the title without enabling hover lookups', function () { +it('renders message rows without hover titles or recommendation lookups', function () { syslog_load_plugin_source('includes/functions.php'); $queries = []; @@ -564,8 +564,8 @@ function recommendation_sample(array $override = []): array { syslog_row_color('3', ''); $html = ob_get_clean(); - expect($html)->not->toContain('data-recommendations') - ->toContain("title='<script>alert("message")</script>'") + expect($html)->not->toContain('title=') + ->not->toContain('data-recommendations') ->and($queries)->toBe([]); }); @@ -661,5 +661,7 @@ function recommendation_sample(array $override = []): array { expect(syslog_recommendation_indicator(false))->toBe('') ->and($html)->toContain('fa-lightbulb', 'role="img"', 'aria-label="Recommendation available - click the message for details"') - ->not->toContain('onclick', 'data-action'); + ->not->toContain('title=') + ->not->toContain('onclick') + ->not->toContain('data-action'); }); diff --git a/tests/bin/patch-coverage.php b/tests/bin/patch-coverage.php index aea51da4..f74cf545 100644 --- a/tests/bin/patch-coverage.php +++ b/tests/bin/patch-coverage.php @@ -251,7 +251,7 @@ function changed_lines($base_ref) { // CLI bootstrap (includes cli_check.php and runs at the top level) requires // Cacti, so it cannot be loaded into the isolated unit process; like the // other CLI entry points it is intentionally absent from Clover. - 'cli/updatedb_convert_ms.php', + 'cli/upgrade_partitioning.php', // Setup modules are loaded by Cacti hook paths rather than directly by the // isolated Pest bootstrap. Their behavior is covered by targeted tests, but // they are intentionally absent from Clover's source list. diff --git a/tests/bootstrap-unit.php b/tests/bootstrap-unit.php index 5387e713..f96dd91f 100644 --- a/tests/bootstrap-unit.php +++ b/tests/bootstrap-unit.php @@ -495,6 +495,14 @@ function form_checkbox_cell($title, $id, $disabled = false) { } } +if (!function_exists('form_selectable_cell')) { + function form_selectable_cell($contents, $id, $width = '', $style_or_class = '', $title = '') { + if (test_call_override('form_selectable_cell', [$contents, $id, $width, $style_or_class, $title], true)) { + print "$contents"; + } + } +} + if (!function_exists('form_end')) { function form_end($ajax = true) { return test_call_override('form_end', [$ajax], null); diff --git a/tests/browser/compact_workspace.cjs b/tests/browser/compact_workspace.cjs index 2e4e7247..f3b5a133 100644 --- a/tests/browser/compact_workspace.cjs +++ b/tests/browser/compact_workspace.cjs @@ -7,6 +7,10 @@ const cacti = process.env.CACTI_ROOT || path.resolve(root, '../cacti'); const longMessage = 'nf_conntrack: table full, dropping packet ' + 'width preview '.repeat(4); const shortMessage = 'nf_conntrack: table full, dropping packet '; const dispatched = []; +const savedSearchPosts = []; +const heldFilters = []; +let holdFilters = false; +let filterReceived; const fixture = execFileSync('php', [path.join(root, 'tests/Fixtures/compact_workspace.php')], {encoding: 'utf8'}); (async()=>{ const browser=await chromium.launch({headless:true,executablePath:process.env.CHROMIUM_PATH,args:['--no-sandbox']}); @@ -25,7 +29,17 @@ const fixture = execFileSync('php', [path.join(root, 'tests/Fixtures/compact_wor if (url.pathname === '/') return r.fulfill({body: url.searchParams.has('error') ? fixture.replace("id='logical_search_error' role='alert'>", "id='logical_search_error' role='alert'>Invalid search") : fixture, contentType: 'text/html; charset=utf-8'}); if (url.pathname === '/syslog.php') { if (r.request().method() === 'POST') assert.equal(new URLSearchParams(r.request().postData()).get('__csrf_magic'), 'fixture', 'Fallback requests carry the CSRF token without relying on the global XHR shim'); + if (holdFilters && url.searchParams.get('header') === 'false') { + heldFilters.push(r); + if (filterReceived) { filterReceived(); filterReceived = null; } + return; + } dispatched.push(r.request().method() + ' ' + (url.searchParams.get('action') || new URLSearchParams(r.request().postData() || '').get('action')) + ' ' + (url.searchParams.get('source') || '') + ':' + (url.searchParams.get('id') || '')); + const data = new URLSearchParams(r.request().postData() || ''); + if (data.get('action') === 'saved_search_save') { + savedSearchPosts.push(Object.fromEntries(data)); + return r.fulfill({body: JSON.stringify({id: 9}), contentType: 'application/json'}); + } if (url.searchParams.get('action') !== 'message_details') return r.fulfill({body: JSON.stringify({found: false, recommendations: []}), contentType: 'application/json'}); const id = url.searchParams.get('id'); const query = 'id=' + id + '&action=newedit&type=0&date=2026-09-13+00%3A12%3A42'; @@ -39,7 +53,22 @@ const fixture = execFileSync('php', [path.join(root, 'tests/Fixtures/compact_wor }); await page.goto('http://fixture/'); await page.waitForTimeout(250); assert.deepEqual(errors,[]); + assert.equal(await page.locator('#refresh_results').getAttribute('aria-label'), 'Refresh Results'); + assert.equal(await page.locator('#refresh_results .fa.fa-sync-alt').count(), 1, 'Refresh uses the Font Awesome glyph'); + assert.equal(await page.locator('#refresh_results .fa-spin').count(), 0, 'Refresh is not spinning while idle'); + assert.equal(await page.locator('#export').getAttribute('aria-label'), 'Export Records to CSV'); + assert.equal(await page.locator('#export .fa.fa-file-export[aria-hidden="true"]').count(), 1, 'Export uses an accessible Font Awesome glyph'); + assert.equal((await page.locator('#export').textContent()).trim(), '', 'Export is glyph-only'); + await page.locator('#export').click(); + assert.equal(await page.evaluate(() => posts.at(-1).export), 'true', 'The glyph keeps the existing export callback'); const messageButton = page.locator('.syslogMessageOpen').first(); + const indicatorColumns = page.locator('th:has(.syslogRecommendationHeading), td.syslogRecommendation'); + async function checkIndicatorWidths() { + const widths = await indicatorColumns.evaluateAll(cells => cells.map(cell => cell.getBoundingClientRect().width)); + assert(widths.length > 1 && widths.every(width => width === 20), 'The lightbulb header and cells stay exactly 20px wide'); + assert.equal(await page.locator('th:has(.syslogRecommendationHeading)').evaluate(cell => getComputedStyle(cell).textAlign), 'center', 'The lightbulb header is centered'); + } + await checkIndicatorWidths(); assert.equal(await messageButton.textContent(), longMessage, 'Preview retains text beyond 100 characters'); const widePreview = await messageButton.evaluate(button => { const range = document.createRange(); @@ -58,6 +87,10 @@ const fixture = execFileSync('php', [path.join(root, 'tests/Fixtures/compact_wor assert.equal(await page.locator('#saved_saveas').isVisible(), true); await page.locator('#saved_saveas').click(); assert.equal(await page.locator('#syslog_saved_prompt').isVisible(), true); + assert.equal(await page.locator('#syslog_saved_prompt_name').evaluate(input => input.form.id), 'syslog_saved_prompt', 'The name belongs to the dialog form, not the main search form'); + await page.locator('#syslog_saved_prompt_name').press('Enter'); + assert.equal(await page.locator('#syslog_saved_prompt').isVisible(), true, 'An empty name keeps the dialog open'); + assert.equal(savedSearchPosts.length, 0, 'An empty name does not save'); const nameLabel = await page.locator('label[for="syslog_saved_prompt_name"]').boundingBox(); const nameInput = await page.locator('#syslog_saved_prompt_name').boundingBox(); const defaultLabel = await page.locator('label[for="syslog_saved_prompt_default"]').boundingBox(); @@ -69,11 +102,27 @@ const fixture = execFileSync('php', [path.join(root, 'tests/Fixtures/compact_wor assert(defaultSwitch.width > 30 && defaultSwitch.height > 15, 'Default toggle has a visible switch track'); await page.locator('label[for="syslog_saved_prompt_default"]').click(); assert.equal(await page.locator('#syslog_saved_prompt_default').isChecked(), true, 'Clicking Default toggles the switch'); + const beforeKeyboardSave = await page.evaluate(() => posts.length); + await page.locator('#syslog_saved_prompt_name').fill(' Keyboard search '); + await page.locator('#syslog_saved_prompt_name').press('Enter'); + await page.waitForFunction(() => posts.at(-1)?.saved === 9); + assert.equal(savedSearchPosts.length, 1, 'Enter issues exactly one save callback POST'); + assert.equal(savedSearchPosts[0].name, 'Keyboard search', 'Keyboard save trims the name'); + assert.equal(savedSearchPosts[0].default, 'on', 'Keyboard save preserves the Default switch'); + assert.equal(savedSearchPosts[0].__csrf_magic, 'fixture', 'Keyboard save includes the CSRF token'); + assert.equal(await page.evaluate(() => posts.length), beforeKeyboardSave + 1, 'Keyboard save applies only the saved search, not the main search submit action'); + assert.equal(await page.locator('#syslog_saved_prompt').isVisible(), false, 'Keyboard save closes the dialog without navigation'); await page.evaluate(() => savedSearchPrompt('New search', (name, isDefault) => { window.savedPromptResult = {name, isDefault}; })); assert.equal(await page.locator('#syslog_saved_prompt_default').isChecked(), false, 'A new search is not default unless selected'); await page.locator('label[for="syslog_saved_prompt_default"]').click(); - await page.locator('.ui-dialog:has(#syslog_saved_prompt) .ui-dialog-buttonpane button').first().click(); + const saveSearchButtons = page.locator('.ui-dialog:has(#syslog_saved_prompt) .ui-dialog-buttonpane button'); + assert.deepEqual(await saveSearchButtons.allTextContents(), ['Cancel', 'Save'], 'Save Search As places Cancel before Save'); + assert.equal(await saveSearchButtons.filter({hasText: /^Cancel$/}).getAttribute('type'), 'button', 'Cancel remains a non-submit button'); + assert.equal(await saveSearchButtons.filter({hasText: /^Save$/}).getAttribute('type'), 'submit', 'Save is a submit button'); + assert.equal(await saveSearchButtons.filter({hasText: /^Save$/}).evaluate(button => button.classList.contains('ui-state-active')), true, 'Save has the active action styling'); + await saveSearchButtons.filter({hasText: /^Save$/}).click(); assert.deepEqual(await page.evaluate(() => savedPromptResult), {name: 'New search', isDefault: true}, 'Saving preserves the toggle value'); + assert.equal(savedSearchPosts.length, 1, 'Reopening replaces the submit callback instead of accumulating handlers'); await page.evaluate(() => { document.querySelector('#saved_search option:checked').dataset.default = '1'; savedSearchPrompt('Existing default', () => {}); }); assert.equal(await page.locator('#syslog_saved_prompt_default').isChecked(), true, 'Editing a default keeps the toggle selected'); await page.keyboard.press('Escape'); @@ -98,22 +147,14 @@ const fixture = execFileSync('php', [path.join(root, 'tests/Fixtures/compact_wor assert.equal(await page.locator('.syslogRow').first().locator('.syslogRecommendationIndicator').count(), 1, 'Matching messages advertise recommendations before a click'); assert.equal(await page.locator('.syslogRow').nth(1).locator('.syslogRecommendation').textContent(), '', 'Unmatched messages leave the indicator column blank'); assert.equal(await page.locator('.syslogRecommendationIndicator').first().evaluate(icon => icon.parentElement.nextElementSibling.classList.contains('syslogMessage')), true, 'The indicator immediately precedes the message'); + assert.equal(await page.locator('.syslogRow[title], .syslogRow [title]').count(), 0, 'Message rows and their contents have no native hover titles'); const beforeHover = dispatched.length; - await messageButton.hover(); - const messageTooltip = page.locator('.ui-tooltip:visible .syslogMessageTip'); - await messageTooltip.waitFor(); - assert.equal(await messageTooltip.textContent(), longMessage, 'Hover shows the full untrimmed message'); - assert.equal(await page.locator('.ui-tooltip:visible script').count(), 0, 'Hover treats message markup as literal text'); - await page.mouse.move(0, 0); - await messageTooltip.waitFor({state: 'hidden'}); - await messageButton.hover(); - await messageTooltip.waitFor(); - assert.equal(await messageTooltip.textContent(), longMessage, 'Repeated hovers preserve the full message'); - await page.mouse.move(0, 0); - await messageTooltip.waitFor({state: 'hidden'}); - await page.locator('.syslogRecommendationIndicator').first().hover(); - await page.locator('.ui-tooltip:visible').filter({hasText: 'Recommendation available - click the message for details'}).waitFor(); - assert.equal(dispatched.length, beforeHover, 'Neither message nor lightbulb hovers issue any requests'); + for (const target of [messageButton, ...await page.locator('.syslogRow').first().locator('td').all(), page.locator('.syslogRecommendationIndicator').first(), messageButton]) { + await target.hover(); + await page.waitForTimeout(400); + assert.equal(await page.locator('.ui-tooltip:visible').count(), 0, 'Hovering messages, metadata, or the lightbulb never opens a tooltip'); + } + assert.equal(dispatched.length, beforeHover, 'Message-row and lightbulb hovers issue no requests'); await page.mouse.move(0, 0); await page.waitForFunction(() => $('.ui-tooltip:visible').length === 0); await page.locator('.syslogMessageOpen').first().click(); @@ -173,12 +214,30 @@ const fixture = execFileSync('php', [path.join(root, 'tests/Fixtures/compact_wor await page.mouse.move(0, 0); await page.waitForFunction(() => $('.ui-tooltip:visible').length === 0); await page.setViewportSize({width:390,height:844});if (process.env.TEST_ARTIFACT_DIR) await page.screenshot({path:path.join(process.env.TEST_ARTIFACT_DIR, 'syslog-workspace-mobile.png'),fullPage:true}); + await checkIndicatorWidths(); assert.equal(await messageButton.textContent(), longMessage, 'Narrow layouts retain the complete preview text'); const narrowPreview = await messageButton.evaluate(button => ({width: button.clientWidth, content: button.scrollWidth, ellipsis: getComputedStyle(button).textOverflow})); assert(narrowPreview.content > narrowPreview.width, 'Long previews overflow their own button on narrow screens'); assert.equal(narrowPreview.ellipsis, 'ellipsis', 'The browser controls visual truncation'); const overflow=await page.evaluate(()=>document.documentElement.scrollWidth>innerWidth);assert.equal(overflow,false,'Page must not overflow horizontally'); await page.locator('#refresh_results').click();assert.equal(await page.evaluate(()=>Object.hasOwn(posts.at(-1),'rfilter')),false); + holdFilters = true; + await page.evaluate(() => { window.mockPostSyslog = postSyslog; postSyslog = realPostSyslog; }); + const refreshReceived = new Promise(resolve => { filterReceived = resolve; }); + await page.locator('#refresh_results').click(); + await refreshReceived; + assert.equal(await page.locator('#refresh_results .fa-spin').count(), 1, 'Clicking Refresh spins its glyph while the callback is pending'); + const searchReceived = new Promise(resolve => { filterReceived = resolve; }); + await page.evaluate(() => applyFilter()); + await searchReceived; + await heldFilters[0].fulfill({body: '', contentType: 'text/html'}); + await page.waitForFunction(() => $.active === 1); + assert.equal(await page.locator('#refresh_results .fa-spin').count(), 1, 'The glyph keeps spinning while another filter callback remains pending'); + await heldFilters[1].fulfill({status: 500, body: 'Fixture filter failure'}); + await page.waitForFunction(() => $.active === 0); + assert.equal(await page.locator('#refresh_results .fa-spin').count(), 0, 'The glyph stops after the final callback, including failures'); + await page.evaluate(() => { postSyslog = mockPostSyslog; }); + holdFilters = false; // A saved expanded preference reopens the panel on reload; an active search error forces it open even when collapse is stored. await page.setViewportSize({width:1600,height:950}); await page.evaluate(()=>localStorage.setItem('syslog.search.collapsed','false')); diff --git a/tests/browser/threshold_messages.cjs b/tests/browser/threshold_messages.cjs index f8134db9..a4b82ca5 100644 --- a/tests/browser/threshold_messages.cjs +++ b/tests/browser/threshold_messages.cjs @@ -2,12 +2,128 @@ const {chromium} = require(process.env.PLAYWRIGHT_MODULE || 'playwright'); const {execFileSync} = require('node:child_process'); const path = require('node:path'); const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const root = path.resolve(__dirname, '../..'); +const cacti = process.env.CACTI_ROOT || path.resolve(root, '../cacti'); +const layout = fs.readFileSync(path.join(cacti, 'include/layout.js'), 'utf8'); +const sizing = layout.slice(layout.indexOf('function saveTableWidths('), layout.indexOf('function appendHeaderSuppression(')); +assert(sizing.includes('function applyTableSizing()'), 'Exercise the actual Cacti column-resize callbacks'); +const storage = `var Storages = {localStorage: { + isSet: key => localStorage.getItem(key) !== null, + get: key => JSON.parse(localStorage.getItem(key)), + set: (key, value) => localStorage.setItem(key, JSON.stringify(value)), + remove: key => localStorage.removeItem(key) +}};`; const html = execFileSync('php', [path.join(__dirname, '../Fixtures/threshold_messages.php')], {encoding: 'utf8'}); (async () => { const browser = await chromium.launch({headless: true, executablePath: process.env.CHROMIUM_PATH, args: ['--no-sandbox']}); try { const page = await browser.newPage({viewport: {width: 1600, height: 900}}); - await page.setContent(html); + const errors = []; + const requests = []; + let pages; + let failNext = false; + page.on('pageerror', error => errors.push(error.message)); + const assets = { + '/main.css': path.join(cacti, 'include/themes/classic/main.css'), + '/jquery-ui.css': path.join(cacti, 'include/themes/classic/jquery-ui.css'), + '/jquery.js': path.join(cacti, 'include/js/jquery.js'), + '/jquery-ui.js': path.join(cacti, 'include/js/jquery-ui.js'), + '/search.css': path.join(root, 'css/search.css'), + '/functions.js': path.join(root, 'js/functions.js') + }; + await page.route('https://fixture/**', async route => { + const url = new URL(route.request().url()); + if (url.pathname === '/') return route.fulfill({body: html, contentType: 'text/html'}); + if (url.pathname === '/table-sizing.js') return route.fulfill({body: storage + sizing, contentType: 'text/javascript'}); + if (url.pathname === '/syslog.php') { + requests.push(url); + if (url.searchParams.get('action') === 'group_details') { + if (failNext) { failNext = false; return route.fulfill({status: 500, body: 'Fixture query failure'}); } + const offset = url.searchParams.get('offset'); + assert(['0', '50'].includes(offset), 'Only one bounded page is requested'); + assert.match(url.searchParams.get('context'), /^[a-f0-9]{32}$/); + assert.equal(url.searchParams.get('group'), '0'); + return route.fulfill({contentType: 'application/json', body: JSON.stringify({html: pages[offset], more: offset === '0'})}); + } + return route.fulfill({contentType: 'application/json', body: JSON.stringify({found: true, record: {id: '201', source: 'main', received: '2026-10-15 12:00:00', message: 'Example message'}, recommendations: []})}); + } + return assets[url.pathname] ? route.fulfill({path: assets[url.pathname], contentType: url.pathname.endsWith('.js') ? 'text/javascript' : 'text/css'}) : route.fulfill({status: 404, body: ''}); + }); + await page.goto('https://fixture/'); + await page.waitForTimeout(100); + const checkFit = async id => { + const sizes = await page.locator(id).evaluate(table => { + const main = table.closest('.syslogResultsMain'); + const message = table.querySelector('.syslogMessageOpen'); + return { + table: table.getBoundingClientRect().width, available: main.clientWidth, + scroll: main.scrollWidth, message: message.clientWidth, text: message.scrollWidth, + ellipsis: getComputedStyle(message).textOverflow, + headers: [...table.querySelectorAll('th')].map(cell => cell.getBoundingClientRect().width) + }; + }); + assert(sizes.table <= sizes.available + 1, `${id} must fit the results area: ${JSON.stringify(sizes)}`); + assert(sizes.scroll <= sizes.available + 1, `${id} must not create a desktop horizontal scrollbar`); + assert(sizes.message >= 50, `${id} keeps a usable message preview`); + return sizes; + }; + const drag = async (id, column, delta) => { + const handle = page.locator(`${id} th`).nth(column).locator('.ui-resizable-e'); + await handle.scrollIntoViewIfNeeded(); + const box = await handle.boundingBox(); + await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2); + await page.mouse.down(); + await page.mouse.move(box.x + box.width / 2 + delta, box.y + box.height / 2, {steps: 12}); + await checkFit(id); + await page.mouse.up(); + await page.waitForTimeout(60); + }; + for (const [id, messageIndex] of [['#individual', 4], ['#alerts', 5]]) { + await checkFit(id); + await drag(id, 0, -100); + const before = await checkFit(id); + await drag(id, 1, 50); + const after = await checkFit(id); + assert(after.headers[1] > before.headers[1] + 30, `${id} metadata columns really resize`); + assert(after.headers[messageIndex] < before.headers[messageIndex] - 30, `${id} the message absorbs metadata resizing`); + await drag(id, messageIndex + 1, 70); + await checkFit(id); + await drag(id, 0, 2500); + await checkFit(id); + await drag(id, 0, -2500); + await checkFit(id); + const beforeMessage = await checkFit(id); + await drag(id, messageIndex, -50); + const smallerMessage = await checkFit(id); + assert(smallerMessage.headers[messageIndex] < beforeMessage.headers[messageIndex] - 30, `${id} the message itself can shrink`); + await drag(id, messageIndex, 50); + const largerMessage = await checkFit(id); + assert(largerMessage.headers[messageIndex] > smallerMessage.headers[messageIndex] + 30, `${id} the message itself can expand`); + } + const preview = await checkFit('#alerts'); + assert(preview.text > preview.message, 'Alert Logs retains the full long preview'); + assert.equal(preview.ellipsis, 'ellipsis', 'Alert Logs visually truncates using ellipsis'); + const wrapped = await page.locator('#alerts .syslogMessageWrap').evaluate(button => ({ + width: button.clientWidth, scroll: button.scrollWidth, height: button.clientHeight, + whitespace: getComputedStyle(button).whiteSpace, ellipsis: getComputedStyle(button).textOverflow + })); + assert.equal(wrapped.whitespace, 'pre-wrap', 'Alert Logs All Text continues to wrap'); + assert.equal(wrapped.ellipsis, 'clip'); + assert(wrapped.height > 30 && wrapped.scroll <= wrapped.width + 1, 'Wrapped alert messages stay inside their column'); + assert.notEqual(await page.evaluate(() => localStorage.getItem('alerts')), null, 'Column drags retain Cacti width persistence'); + await page.evaluate(() => localStorage.setItem('alerts', JSON.stringify([innerWidth, 1800, 1000, 1000, 1000, 20, 1800, 1000, 1000]))); + await page.reload(); + await page.waitForTimeout(100); + await checkFit('#individual'); + await checkFit('#alerts'); + assert.equal(await page.locator('#alerts th:has(.syslogRecommendationHeading)').evaluate(cell => cell.getBoundingClientRect().width), 20, 'Saved oversized widths do not stretch the indicator'); + await page.setViewportSize({width: 1050, height: 900}); + await page.waitForTimeout(60); + await checkFit('#individual'); + await checkFit('#alerts'); + await page.setViewportSize({width: 1600, height: 900}); + pages = await page.locator('template').evaluateAll(templates => Object.fromEntries(templates.map(template => [template.id.slice('group-page-'.length), template.innerHTML]))); assert.equal(await page.locator('#individual th').last().textContent(), 'Breach'); assert.equal(await page.locator('#individual th').last().getAttribute('title'), 'Threshold breached'); assert.ok((await page.locator('#individual th').last().boundingBox()).width < 90, 'Breach column stays compact'); @@ -18,14 +134,38 @@ const html = execFileSync('php', [path.join(__dirname, '../Fixtures/threshold_me assert.equal(await rows.nth(0).locator('td').count(), 8); assert.equal(await rows.nth(0).locator('td').nth(3).locator('.syslogRecommendationIndicator').count(), 1); assert.equal(await rows.nth(1).locator('td').nth(3).textContent(), ''); + assert.equal(await rows.nth(0).locator('td').nth(3).evaluate(cell => cell.style.width), '20px', 'Individual recommendation cells declare their fixed width'); const grouped = page.locator('#grouped tr'); - assert.equal(await grouped.count(), 3); + assert.equal(await grouped.count(), 1, 'Even a million-occurrence group initially renders only its summary'); assert.equal(await grouped.nth(0).locator('td').nth(7).textContent(), 'Yes'); - assert.equal(await grouped.nth(1).locator('td').nth(7).textContent(), 'Yes'); - assert.equal(await grouped.nth(2).locator('td').nth(7).textContent(), 'No'); assert.equal(await grouped.nth(0).locator('td').count(), 9); - assert.equal(await grouped.nth(1).locator('td').count(), 9); - assert.equal(await grouped.locator('td:nth-child(4) .syslogRecommendationIndicator').count(), 3, 'Grouped summaries and their identical messages share the indicator'); - console.log('Threshold message column passed: Yes/No, grouped summary, and individual details'); + assert.equal(requests.length, 0, 'Rendering a collapsed group makes no detail requests'); + const toggle = page.locator('.syslog-group-toggle'); + await toggle.click(); + await page.waitForFunction(() => document.querySelectorAll('#grouped .syslogGroupMember').length === 50); + assert.equal(requests.length, 1); + assert.equal(await page.locator('#grouped .syslogGroupMember').first().locator('td').count(), 9); + assert.equal(await page.locator('#grouped .syslogGroupMember').first().locator('td').nth(7).textContent(), 'Yes'); + assert.equal(await page.locator('#grouped .syslogGroupMember').nth(1).locator('td').nth(7).textContent(), 'No'); + await toggle.click(); + assert.equal(await page.locator('#grouped .syslogGroupMember:visible').count(), 0); + await toggle.click(); + assert.equal(requests.length, 1, 'Reopening reuses the already-loaded bounded page'); + const next = page.locator('.syslogGroupPage button').filter({hasText: /^Next$/}); + failNext = true; + await next.click(); + await page.waitForFunction(() => document.querySelector('.syslogGroupPage [role="status"]').textContent === 'Group messages could not be loaded.'); + assert.equal(await page.locator('#grouped .syslogGroupMember').count(), 50, 'A failed page request preserves the current members and reports an error'); + await next.click(); + await page.waitForFunction(() => document.querySelectorAll('#grouped .syslogGroupMember').length === 2); + assert.equal(await next.isDisabled(), true); + await page.locator('.syslogGroupPage button').filter({hasText: /^Previous$/}).click(); + await page.waitForFunction(() => document.querySelectorAll('#grouped .syslogGroupMember').length === 50); + assert.equal(await page.locator('#grouped td.syslogRecommendation').evaluateAll(cells => cells.every(cell => cell.style.width === '20px')), true); + await page.locator('#grouped .syslogGroupMember .syslogMessageOpen').first().click(); + await page.waitForFunction(() => !document.getElementById('syslog_message_details').hidden); + assert.equal(await page.locator('#syslog_details_raw').textContent(), 'Example message', 'Dynamically loaded members still open the details flyout'); + assert.deepEqual(errors, []); + console.log('Log browser checks passed: bounded groups, constrained resizing, saved widths, alert ellipsis/wrapping, and flyout'); } finally { await browser.close(); } })().catch(error => { console.error(error); process.exit(1); }); diff --git a/tests/regression/cli_schema_upgrade_test.php b/tests/regression/cli_schema_upgrade_test.php index 6ab018dd..78f25aad 100644 --- a/tests/regression/cli_schema_upgrade_test.php +++ b/tests/regression/cli_schema_upgrade_test.php @@ -99,10 +99,11 @@ function syslog_config_safe() { throw new RuntimeException('CLI bootstrap ran th mkdir($fixture . '/plugins/syslog/cli', 0700, true); mkdir($fixture . '/plugins/syslog/includes', 0700, true); copy($root . '/cli/syslog_schema.php', $fixture . '/plugins/syslog/cli/syslog_schema.php'); -file_put_contents($fixture . '/include/cli_check.php', ' getenv("SYSLOG_TEST_REMOTE") ? 2 : 1];'); +file_put_contents($fixture . '/include/cli_check.php', ' getenv("SYSLOG_TEST_REMOTE") ? 2 : 1];'); file_put_contents($fixture . '/plugins/syslog/setup.php', <<<'STUB' '5.0']; } function syslog_determine_config() { define('SYSLOG_CONFIG', __FILE__); } function syslog_connect() { return true; } function read_config_option($name) { return $name === 'syslog_install_days' ? '0' : ''; } @@ -164,7 +165,14 @@ function run_cli($fixture, $arguments = ['--repair']) { check(run_cli($fixture, [])[0] === 1, 'CLI ran without an explicit action'); foreach (['--help', '-h', '-H'] as $flag) { [$status, $output] = run_cli($fixture, [$flag]); - check($status === 0 && strpos($output, 'Usage: php syslog_schema.php') !== false, 'CLI help failed: ' . $flag); + check($status === 0 && strpos($output, 'usage: syslog_schema.php') !== false, 'CLI help failed: ' . $flag); + check(strpos($output, 'Syslog Schema Utility, Version 5.0, Copyright (C)') === 0, 'CLI help is missing the version/copyright banner'); + check(strpos($output, PHP_EOL . PHP_EOL . 'Options:' . PHP_EOL . ' --audit - ') !== false, 'CLI help must separate its description and aligned options'); + } + foreach (['--version', '-v', '-V'] as $flag) { + [$status, $output] = run_cli($fixture, [$flag]); + check($status === 0 && strpos($output, 'Syslog Schema Utility, Version 5.0, Copyright (C)') === 0, 'CLI version failed: ' . $flag); + check(strpos($output, 'usage:') === false, 'Version must not print help'); } check(run_cli($fixture, ['--truncate'])[0] === 1, 'CLI accepted destructive option'); putenv('SYSLOG_TEST_MISSING=syslog_dashboards'); diff --git a/tests/regression/partition_rebuild_database_test.php b/tests/regression/partition_rebuild_database_test.php new file mode 100644 index 00000000..d115a2cc --- /dev/null +++ b/tests/regression/partition_rebuild_database_test.php @@ -0,0 +1,215 @@ + PDO::ERRMODE_EXCEPTION, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => false +]); +$syslogdb_default = 'syslog_rebuild_test'; +$options = ['syslog_install_engine' => 'InnoDB', 'syslog_retention' => '2', 'syslog_partition_ahead_days' => '2']; +$copy_attempts = 0; +$fail_copy_at = 0; +$fail_prefix = ''; +$batch_sizes = []; +$checks = 0; + +function rebuild_check(bool $ok, string $message): void { + $GLOBALS['checks']++; + + if (!$ok) { + throw new RuntimeException($message); + } +} + +function syslog_connect() { return true; } +function read_config_option($name) { return $GLOBALS['options'][$name] ?? ''; } +function set_config_option($name, $value) { $GLOBALS['options'][$name] = $value; } +function cacti_log(...$args) {} +function syslog_db_execute_prepared($sql, $params = [], ...$args) { + if ($GLOBALS['fail_prefix'] !== '' && str_starts_with($sql, $GLOBALS['fail_prefix'])) { + return false; + } + + $copy = str_starts_with($sql, 'INSERT INTO'); + + if ($copy && ++$GLOBALS['copy_attempts'] === $GLOBALS['fail_copy_at']) { + return false; + } + + $query = $GLOBALS['pdo']->prepare($sql); + $ok = $query->execute($params); + + if ($copy) { + $GLOBALS['batch_sizes'][] = $query->rowCount(); + } + + return $ok; +} +function syslog_db_execute($sql, ...$args) { return syslog_db_execute_prepared($sql); } +function syslog_db_fetch_assoc_prepared($sql, $params = [], ...$args) { + $query = $GLOBALS['pdo']->prepare($sql); + $query->execute($params); + + return $query->fetchAll(); +} +function syslog_db_fetch_row($sql) { + $rows = syslog_db_fetch_assoc_prepared($sql); + + return $rows[0] ?? []; +} +function syslog_db_fetch_cell_prepared($sql, $params = [], ...$args) { + $query = $GLOBALS['pdo']->prepare($sql); + $query->execute($params); + + return $query->fetchColumn(); +} +function syslog_db_fetch_cell($sql) { return syslog_db_fetch_cell_prepared($sql); } +function syslog_db_table_exists($table, ...$args) { + return (bool) syslog_db_fetch_cell_prepared('SELECT COUNT(*) FROM information_schema.TABLES + WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?', [$GLOBALS['syslogdb_default'], $table]); +} +function syslog_partition_ahead_days() { return (int) read_config_option('syslog_partition_ahead_days'); } + +require dirname(__DIR__, 2) . '/includes/schema.php'; +$source = file_get_contents(dirname(__DIR__, 2) . '/cli/upgrade_partitioning.php'); +$offset = strpos($source, "\nfunction syslog_upart_run("); + +if ($offset === false) { + throw new RuntimeException('Cannot find rebuild functions'); +} + +$fixture = tempnam(sys_get_temp_dir(), 'syslog-rebuild-'); + +try { + file_put_contents($fixture, "exec('DROP DATABASE IF EXISTS `syslog_rebuild_test`'); + $pdo->exec('CREATE DATABASE `syslog_rebuild_test`'); + $pdo->exec('USE `syslog_rebuild_test`'); + $pdo->exec("SET SESSION time_zone = '+00:00'"); + $pdo->exec("SET SESSION sql_mode = 'STRICT_TRANS_TABLES,NO_AUTO_VALUE_ON_ZERO'"); + $pdo->exec("CREATE TABLE syslog ( + seq BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, logtime DATETIME NOT NULL, + message TEXT COLLATE utf8mb4_general_ci NOT NULL, nullable_value VARCHAR(40), + replication_source_event_id BIGINT UNSIGNED, + PRIMARY KEY(seq,logtime), KEY logtime(logtime) + ) ENGINE=InnoDB PARTITION BY RANGE(TO_DAYS(logtime)) + (PARTITION legacy VALUES LESS THAN MAXVALUE)"); + $pdo->exec("INSERT INTO syslog (seq,logtime,message,replication_source_event_id) + SELECT seq,TIMESTAMPADD(SECOND,seq,UTC_DATE()),CONCAT('Event ',seq),seq FROM seq_1_to_10005"); + $pdo->exec("INSERT INTO syslog VALUES + (0,'0000-00-00 00:00:00','zero',NULL,0), + (10000,TIMESTAMPADD(SECOND,10001,UTC_DATE()),'repeated seq','',10000), + (10006,UTC_DATE() - INTERVAL 400 DAY,'outside retention',NULL,10006), + (18446744073709551600,UTC_DATE(),UNHEX('410042'),NULL,18446744073709551600)"); + $pdo->exec('ALTER TABLE syslog AUTO_INCREMENT=18446744073709551610'); + $total = syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog'); + $fail_copy_at = 2; + rebuild_check(!rebuild_table('syslog'), 'The injected second-batch failure must stop the rebuild'); + rebuild_check(syslog_db_table_exists('syslog_rebuild'), 'A failed copy must retain the original'); + rebuild_check((int) syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog') === 10000, 'The first copy is exactly 10000 rows'); + rebuild_check((int) syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog_rebuild') === (int) $total, 'The original is intact'); + $fail_copy_at = 0; + $pdo->exec("UPDATE syslog SET message='event 1' WHERE seq=1"); + rebuild_check(!rebuild_table('syslog'), 'Case-only differences must fail binary value verification'); + rebuild_check(syslog_db_table_exists('syslog_rebuild'), 'A value mismatch must retain the original even with matching counts'); + $pdo->exec('UPDATE syslog AS copied JOIN syslog_rebuild AS source + ON copied.seq=source.seq AND copied.logtime=source.logtime SET copied.message=source.message'); + rebuild_check(rebuild_table('syslog'), 'A repaired copy resumes successfully'); + rebuild_check(!syslog_db_table_exists('syslog_rebuild'), 'Verified originals are removed automatically'); + rebuild_check((int) syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog') === (int) $total, 'Every row survives retries'); + rebuild_check((int) syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog WHERE seq=10000') === 2, 'The composite cursor preserves repeated sequence numbers across batches'); + rebuild_check(syslog_db_fetch_cell('SELECT HEX(message) FROM syslog WHERE seq=18446744073709551600') === '410042', 'Binary message contents survive'); + rebuild_check((int) syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog WHERE seq=0') === 1, 'Zero sequence values must not be regenerated'); + rebuild_check((string) syslog_db_fetch_cell("SELECT AUTO_INCREMENT FROM information_schema.TABLES + WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='syslog'") === '18446744073709551610', 'Unsigned high-water marks must not overflow PHP integers'); + + $pdo->exec('CREATE TABLE syslog_removed (seq BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, logtime TIMESTAMP NOT NULL, message TEXT) ENGINE=Aria'); + rebuild_check(rebuild_table('syslog_removed', false, 'Aria'), 'An empty legacy table can be repartitioned'); + rebuild_check(syslog_detect_logtime_precision('syslog_removed') === 'sec', 'A plain rebuild must preserve second precision'); + rebuild_check(syslog_db_fetch_cell("SELECT ENGINE FROM information_schema.TABLES + WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='syslog_removed'") === 'Aria', 'Configured Aria engine is supported'); + $pdo->exec('ALTER TABLE syslog_removed REMOVE PARTITIONING'); + $pdo->exec('ALTER TABLE syslog_removed MODIFY logtime DATETIME NOT NULL'); + $pdo->exec("SET SESSION time_zone = '+05:30'"); + $pdo->exec("INSERT INTO syslog_removed VALUES(8,CONCAT(UTC_DATE(),' 12:00:00'),'removed')"); + $removed_epoch = syslog_db_fetch_cell('SELECT UNIX_TIMESTAMP(logtime) FROM syslog_removed'); + $fail_prefix = "CREATE TABLE `$syslogdb_default`.`syslog_removed`"; + rebuild_check(!rebuild_table('syslog_removed', false, 'Aria'), 'A create failure after rename must be recoverable'); + rebuild_check(!syslog_db_table_exists('syslog_removed') && syslog_db_table_exists('syslog_removed_rebuild'), 'The renamed original survives a create failure'); + $fail_prefix = ''; + rebuild_check(rebuild_table('syslog_removed', false, 'Aria'), 'A missing replacement can be recreated on retry'); + rebuild_check((int) syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog_removed') === 1, 'Aria data survives recovery'); + rebuild_check(syslog_db_fetch_cell('SELECT UNIX_TIMESTAMP(logtime) FROM syslog_removed') === $removed_epoch, 'Legacy DATETIME history preserves its original non-UTC interpretation'); + + $pdo->exec('CREATE TABLE syslog_incoming (seq BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, logtime TIMESTAMP NOT NULL, status INT, message TEXT) ENGINE=InnoDB'); + $pdo->exec("INSERT INTO syslog_incoming VALUES(30,UTC_DATE(),1,'queued'),(40,UTC_DATE(),0,'new')"); + $pdo->exec("INSERT INTO syslog_incoming SELECT seq+100,UTC_DATE(),0,CONCAT('queued ',seq) FROM seq_1_to_10001"); + $incoming_count = syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog_incoming'); + $pdo->exec("SET SESSION time_zone = '+05:30'"); + $pdo->exec('CREATE TABLE syslog_replication_output (seq BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, logtime DATETIME NOT NULL, message TEXT) ENGINE=InnoDB'); + $pdo->exec("INSERT INTO syslog_replication_output VALUES(4,CONCAT(UTC_DATE(),' 12:00:00'),'undelivered')"); + $outbox_epoch = syslog_db_fetch_cell('SELECT UNIX_TIMESTAMP(logtime) FROM syslog_replication_output'); + $options['syslog_install_logtime_precision'] = 'sec'; + $fail_prefix = "ALTER TABLE `$syslogdb_default`.`syslog_replication_output`"; + ob_start(); + rebuild_check(!syslog_upart_run(true, true), 'Outbox conversion failure must fail the overall operation'); + $output = ob_get_clean(); + rebuild_check(str_contains($output, 'size (estimated data + indexes):') && str_contains($output, 'AT LEAST another') + && str_contains($output, 'SEVERAL HOURS'), 'Size, extra-space and duration warnings are mandatory with --confirm'); + rebuild_check($options['syslog_install_logtime_precision'] === 'sec', 'Partial conversion must not update the precision setting'); + rebuild_check(syslog_db_fetch_cell('SELECT @@SESSION.time_zone') === '+05:30', 'The original non-UTC session zone is restored after failure'); + rebuild_check((int) syslog_db_fetch_cell("SELECT IS_FREE_LOCK('" . hash('sha256', $syslogdb_default . '.syslog_upgrade_partitioning') . "')") === 1, 'The named lock is released after failure'); + $fail_prefix = ''; + ob_start(); + rebuild_check(syslog_upart_run(true, true), 'The complete pipeline can finish after the outbox failure is fixed'); + ob_end_clean(); + rebuild_check($options['syslog_install_logtime_precision'] === 'ms', 'Only complete conversion records millisecond precision'); + rebuild_check(syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog_incoming') === $incoming_count, 'All queued incoming messages survive multiple sequence-only batches'); + rebuild_check((int) syslog_db_fetch_cell('SELECT status FROM syslog_incoming WHERE seq=30') === 1, 'Incoming processing state survives'); + rebuild_check((int) syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog_replication_output') === 1, 'Undelivered replication messages survive'); + rebuild_check((string) syslog_db_fetch_cell('SELECT UNIX_TIMESTAMP(logtime) FROM syslog_replication_output') === $outbox_epoch . '.000', 'A legacy DATETIME outbox uses its original session time zone'); + rebuild_check((int) syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog') === (int) $total, 'Previously converted history survives subsequent pipeline retries'); + rebuild_check((int) syslog_db_fetch_cell("SELECT COUNT(*) FROM information_schema.PARTITIONS + WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('syslog','syslog_removed') + AND REPLACE(PARTITION_EXPRESSION,'`','')='unix_timestamp(logtime)'") > 2, 'Stored and removed tables use epoch partitions'); + ob_start(); + rebuild_check(syslog_upart_rebuild('syslog_removed', false, false, 'InnoDB', 2, 2), 'Explicit discard mode completes'); + ob_end_clean(); + rebuild_check((int) syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog_removed') === 0, 'Only explicit discard mode empties a rebuilt table'); + rebuild_check(!syslog_db_table_exists('syslog_removed_rebuild'), 'Explicit discard removes the original after verification'); + rebuild_check((int) syslog_db_fetch_cell('SELECT COUNT(*) FROM syslog_replication_output') === 1, 'Discard mode does not delete the outbox'); + rebuild_check(max($batch_sizes) <= 10000, 'No stored or incoming INSERT SELECT may copy more than 10000 rows'); + + print "$checks partition rebuild database checks passed.\n"; +} finally { + $pdo->exec('DROP DATABASE `syslog_rebuild_test`'); +}