From 9e4b26bc8f7f7dcb9f5da645676672b6594f04b4 Mon Sep 17 00:00:00 2001 From: Thomas Vincent Date: Wed, 18 Nov 2026 00:34:59 -0800 Subject: [PATCH 1/4] ci: check source-derived manual statements against Cacti develop The Galera table list drifted from cacti.sql because nothing compared them. Add a checker that compares the manual with a Cacti develop checkout: the Galera conversions against the MEMORY tables in cacti.sql, the PHP floor against composer.json and CACTI_PHP_VERSION_MINIMUM, the extension list against composer.json, and the configuration examples against include/config.php.dist. Run it on pull requests, on develop and weekly so Cacti changes that make the manual stale are reported. The first run found data_source_stats_hourly_cache in the Galera list. The 1.3.0 upgrade converts it to InnoDB, and new installs create it as InnoDB, so it no longer needs converting. Signed-off-by: Thomas Vincent --- .github/workflows/code-claims.yml | 47 ++++++++ General-Installing-Instructions.md | 1 - bin/check_code_claims.py | 142 ++++++++++++++++++++++++ bin/tests/test_code_claims.py | 168 +++++++++++++++++++++++++++++ 4 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/code-claims.yml create mode 100644 bin/check_code_claims.py create mode 100644 bin/tests/test_code_claims.py diff --git a/.github/workflows/code-claims.yml b/.github/workflows/code-claims.yml new file mode 100644 index 00000000..3454e02f --- /dev/null +++ b/.github/workflows/code-claims.yml @@ -0,0 +1,47 @@ +name: Source-derived statements +on: + pull_request: + merge_group: + types: [checks_requested] + push: + branches: [develop] + schedule: + # Catch Cacti source changes that make the manual stale between edits. + - cron: '17 6 * * 1' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: docs-code-claims-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + claims: + name: Check statements against Cacti develop + runs-on: ubuntu-24.04 + timeout-minutes: 5 + env: + PYTHONDONTWRITEBYTECODE: '1' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Check out the Cacti source this manual documents + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + repository: Cacti/cacti + ref: develop + path: cacti-source + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: '3.12' + - name: Test the checker + run: | + python3 -m pip install --quiet coverage==7.13.2 + python3 -m coverage run --branch --include=bin/check_code_claims.py \ + -m unittest discover -s bin/tests -p test_code_claims.py + python3 -m coverage report --fail-under=100 + - name: Compare the manual with the Cacti source + run: | + echo "Cacti develop $(git -C cacti-source rev-parse --short=12 HEAD)" + python3 bin/check_code_claims.py --cacti cacti-source diff --git a/General-Installing-Instructions.md b/General-Installing-Instructions.md index 626c725a..a9567baf 100644 --- a/General-Installing-Instructions.md +++ b/General-Installing-Instructions.md @@ -277,7 +277,6 @@ InnoDB engine: MariaDB [(none)]> use cacti; MariaDB [cacti]> ALTER TABLE `automation_ips` ENGINE=InnoDB; MariaDB [cacti]> ALTER TABLE `automation_processes` ENGINE=InnoDB; -MariaDB [cacti]> ALTER TABLE `data_source_stats_hourly_cache` ENGINE=InnoDB; MariaDB [cacti]> ALTER TABLE `data_source_stats_hourly_last` ENGINE=InnoDB; MariaDB [cacti]> ALTER TABLE `poller_output` ENGINE=InnoDB; MariaDB [cacti]> ALTER TABLE `poller_output_boost_local_data_ids` ENGINE=InnoDB; diff --git a/bin/check_code_claims.py b/bin/check_code_claims.py new file mode 100644 index 00000000..79fa439b --- /dev/null +++ b/bin/check_code_claims.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Check manual statements that are derived from the Cacti source tree. + +Copyright (c) 2004-2026 The Cacti Group. GPL-2.0-or-later. + +Usage: bin/check_code_claims.py --cacti PATH [--docs PATH] + +PATH is a Cacti checkout of the branch this manual documents. Each check +compares one documented list or value with the file it was copied from, so a +change on either side fails until the other is updated. +""" +import argparse +import json +from pathlib import Path +import re +import sys + +ROOT = Path(__file__).resolve().parent.parent + +def section(text, heading): + """Return the body of the Markdown section that starts with heading.""" + match = re.search(r'^(#+) ' + re.escape(heading) + r'\s*$', text, re.M) + if not match: + raise ValueError('Missing section: ' + heading) + level = len(match.group(1)) + end = re.compile(r'^#{1,%d} ' % level, re.M).search(text, match.end()) + return text[match.end():end.start() if end else len(text)] + + +def fence_after(text, marker, language): + """Return the first fenced block of language that follows marker.""" + start = text.find(marker) + if start < 0: + raise ValueError('Missing marker: ' + marker) + match = re.compile(r'^```' + language + r'\n(.*?)^```', re.M | re.S).search(text, start) + if not match: + raise ValueError(f'Missing {language} block after: {marker}') + return match.group(1) + + +def memory_tables(schema): + tables, current = set(), None + for line in schema.splitlines(): + create = re.match(r'CREATE TABLE `(\w+)`', line) + if create: + current = create.group(1) + elif current and re.search(r'ENGINE=MEMORY', line): + tables.add(current) + return tables + + +def check_galera(docs, cacti): + text = (docs / 'General-Installing-Instructions.md').read_text() + block = fence_after(text, 'Galera clustering:', 'sql') + documented = set(re.findall(r'ALTER TABLE `(\w+)` ENGINE=InnoDB;', block)) + expected = memory_tables((cacti / 'cacti.sql').read_text()) + problems = [f'Galera list is missing MEMORY table {t}' for t in sorted(expected - documented)] + problems += [f'Galera list converts {t}, which cacti.sql does not create as MEMORY' + for t in sorted(documented - expected)] + return problems + + +def composer(cacti): + return json.loads((cacti / 'composer.json').read_text())['require'] + + +def check_php_floor(docs, cacti): + text = (docs / 'Requirements.md').read_text() + row = re.search(r'^\| PHP \| PHP (\d+\.\d+) or later.*\(`\^(\d+\.\d+)` in Composer\) \|$', text, re.M) + if not row: + return ['Requirements.md has no PHP version row'] + constraint = re.fullmatch(r'\^(\d+\.\d+)', composer(cacti)['php']) + constant = re.search(r"define\('CACTI_PHP_VERSION_MINIMUM', '(\d+\.\d+)\.\d+'\)", + (cacti / 'include/global_constants.php').read_text()) + actual = {constraint.group(1) if constraint else composer(cacti)['php'], + constant.group(1) if constant else 'undefined'} + if len(actual) != 1: + return ['composer.json and CACTI_PHP_VERSION_MINIMUM disagree: ' + ', '.join(sorted(actual))] + floor = actual.pop() + return [f'Requirements.md states PHP {v}; the source requires {floor}' + for v in sorted(set(row.groups())) if v != floor] + + +def check_extensions(docs, cacti): + body = section((docs / 'Requirements.md').read_text(), 'PHP extensions') + listed = {name.lower() for name in re.findall(r'`(\w+)`', body)} + required = {name[4:] for name in composer(cacti) if name.startswith('ext-')} + return [f'Requirements.md does not list required extension {name}' + for name in sorted(required) if name.lower() not in listed] + + +def literal_kind(value): + value = value.strip() + if re.fullmatch(r'-?\d+', value): + return 'integer' + if value in ('true', 'false'): + return 'boolean' + if re.fullmatch(r"'[^']*'|\"[^\"]*\"", value): + return 'string' + return 'expression' + + +def assignments(text, commented=False): + prefix = r'#?' if commented else '' + return {name: literal_kind(value) for name, value in + re.findall(r'^' + prefix + r'\$(\w+)\s*=\s*(.+?);', text, re.M)} + + +def check_config_examples(docs, cacti): + dist = (cacti / 'include/config.php.dist').read_text() + active, all_names = assignments(dist), assignments(dist, commented=True) + problems = [] + install = (docs / 'General-Installing-Instructions.md').read_text() + for name, kind in assignments(fence_after(install, 'Edit the copied file', 'php')).items(): + if name not in active: + problems.append(f'General-Installing-Instructions.md sets ${name}, which config.php.dist does not define') + elif kind != active[name]: + problems.append(f'General-Installing-Instructions.md shows ${name} as {kind}; config.php.dist uses {active[name]}') + remote = (docs / 'How-To-Setup-Remote-Pollers.md').read_text() + for name in sorted(set(re.findall(r'\$(rdatabase_\w+)', remote))): + if name not in all_names: + problems.append(f'How-To-Setup-Remote-Pollers.md uses ${name}, which config.php.dist does not define') + return problems + + +CHECKS = (check_galera, check_php_floor, check_extensions, check_config_examples) + + +def main(argv=None): + parser = argparse.ArgumentParser(description='Check manual statements derived from the Cacti source.') + parser.add_argument('--cacti', type=Path, required=True, help='Cacti source checkout') + parser.add_argument('--docs', type=Path, default=ROOT, help='documentation checkout') + args = parser.parse_args(argv) + problems = [problem for check in CHECKS for problem in check(args.docs, args.cacti)] + for problem in problems: + print('error: ' + problem, file=sys.stderr) + print(f'{len(CHECKS)} source-derived checks, {len(problems)} problems') + return 1 if problems else 0 + + +if __name__ == '__main__': # pragma: no cover + sys.exit(main()) diff --git a/bin/tests/test_code_claims.py b/bin/tests/test_code_claims.py new file mode 100644 index 00000000..e31a0481 --- /dev/null +++ b/bin/tests/test_code_claims.py @@ -0,0 +1,168 @@ +"""Tests for bin/check_code_claims.py using small source and manual fixtures.""" +import contextlib +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import check_code_claims as claims + +SCHEMA = """CREATE TABLE `poller_output` ( + `local_data_id` int(10) unsigned NOT NULL +) ENGINE=MEMORY; +CREATE TABLE `processes` ( + `pid` int(10) unsigned NOT NULL +) ENGINE=MEMORY; +CREATE TABLE `host` ( + `id` int(10) unsigned NOT NULL +) ENGINE=InnoDB; +""" + +INSTALL = """# Installing + +Galera clustering: convert these tables. + +```sql +MariaDB [cacti]> ALTER TABLE `poller_output` ENGINE=InnoDB; +MariaDB [cacti]> ALTER TABLE `processes` ENGINE=InnoDB; +``` + +Edit the copied file to match the account created above: + +```php +$database_hostname = 'localhost'; +$database_port = 3306; +``` +""" + +REQUIREMENTS = """# Requirements + +| Component | Requirement | +| --- | --- | +| PHP | PHP 8.3 or later in the PHP 8 series (`^8.3` in Composer) | + +### PHP extensions + +- `gd`, `PDO`, and `posix`. + +### Composer and source deployments + +Run `composer install`. +""" + +REMOTE = """# Remote pollers + +```php +$rdatabase_hostname = 'main'; +$rdatabase_ssl_capath = ''; +``` +""" + +CONFIG = """ ALTER TABLE `processes` ENGINE=InnoDB;\n", + "MariaDB [cacti]> ALTER TABLE `host` ENGINE=InnoDB;\n") + status, errors = self.run_checks() + self.assertEqual(status, 1) + self.assertIn('missing MEMORY table processes', errors) + self.assertIn('converts host', errors) + + def test_php_floor_must_match_composer_and_constant(self): + self.composer({'php': '^8.4', 'ext-gd': '*', 'ext-PDO': '*', 'ext-posix': '*'}) + self.assertIn('disagree: 8.3, 8.4', self.run_checks()[1]) + self.write(self.cacti / 'include/global_constants.php', + "=8.3', 'ext-gd': '*', 'ext-PDO': '*', 'ext-posix': '*'}) + self.assertIn('disagree: >=8.3, undefined', self.run_checks()[1]) + + def test_required_extensions_must_be_listed(self): + self.composer({'php': '^8.3', 'ext-gd': '*', 'ext-PDO': '*', 'ext-posix': '*', 'ext-intl': '*'}) + self.assertIn('required extension intl', self.run_checks()[1]) + + def test_config_example_names_and_types_follow_dist(self): + self.edit('General-Installing-Instructions.md', '$database_port = 3306;', + "$database_port = '3306';\n$database_typo = 'x';") + errors = self.run_checks()[1] + self.assertIn('$database_port as string; config.php.dist uses integer', errors) + self.assertIn('sets $database_typo', errors) + + def test_remote_poller_variables_must_exist(self): + self.edit('How-To-Setup-Remote-Pollers.md', '$rdatabase_ssl_capath', '$rdatabase_ssl_path') + self.assertIn('uses $rdatabase_ssl_path', self.run_checks()[1]) + + def test_literal_kinds(self): + self.assertEqual([claims.literal_kind(v) for v in ('-5', 'true', '"x"', 'CONSTANT')], + ['integer', 'boolean', 'string', 'expression']) + + def test_missing_structure_is_reported(self): + with self.assertRaisesRegex(ValueError, 'Missing section'): + claims.section('# Title\n', 'PHP extensions') + with self.assertRaisesRegex(ValueError, 'Missing marker'): + claims.fence_after('text', 'Galera clustering:', 'sql') + with self.assertRaisesRegex(ValueError, 'Missing sql block'): + claims.fence_after('Galera clustering: none', 'Galera clustering:', 'sql') + + def test_last_section_runs_to_end_of_file(self): + self.assertEqual(claims.section('## A\nbody\n', 'A'), '\nbody\n') + + +if __name__ == '__main__': + unittest.main() From 1739c4f04f942fa83130487b9c8e7db4102c3e1f Mon Sep 17 00:00:00 2001 From: Thomas Vincent Date: Wed, 18 Nov 2026 00:35:00 -0800 Subject: [PATCH 2/4] build: add a container build that matches CI Host builds embed the host's fonts, so a macOS build produces Arial and Andale Mono PDFs while CI embeds DejaVu. The documented macOS workaround also fails: env and uv run do not pass DYLD_FALLBACK_LIBRARY_PATH to Python, so WeasyPrint cannot load Homebrew's Pango. Add a script that runs the locked build in Ubuntu 24.04 with the CI font and library packages and a checksum-verified uv, and document it in place of the macOS workaround. Signed-off-by: Thomas Vincent --- bin/build_manuals_in_container.sh | 53 +++++++++++++++++++++++++++++++ resources/build/Other-Manuals.md | 20 ++++++++---- 2 files changed, 67 insertions(+), 6 deletions(-) create mode 100755 bin/build_manuals_in_container.sh diff --git a/bin/build_manuals_in_container.sh b/bin/build_manuals_in_container.sh new file mode 100755 index 00000000..fc31261c --- /dev/null +++ b/bin/build_manuals_in_container.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Build the PDF and text manual editions in the same environment as CI. +# +# Copyright (c) 2004-2026 The Cacti Group. GPL-2.0-or-later. +# +# Host builds embed whatever fonts the host provides (Arial on macOS, for +# example), so their PDFs differ from the published editions. This script runs +# the CI build in Ubuntu 24.04 with the CI font and library packages and the +# locked Python toolchain, then returns the outputs to the caller's account. +# +# Usage: bin/build_manuals_in_container.sh [build_pdf_docs.py arguments] +# Default arguments: --only manual --companions +set -euo pipefail + +root=$(cd "$(dirname "$0")/.." && pwd) +image='ubuntu:24.04@sha256:73b999393d07f2bf65a40373778a44dc9a6d54ad7718d29247d8ff9413c4d6a8' +uv_version='0.12.23' +# A worktree keeps its Git metadata outside the checkout; mount it read-only at +# the same path so the builder can record the source revision. +git_dir=$(git -C "$root" rev-parse --absolute-git-dir) +git_common=$(cd "$(git -C "$root" rev-parse --git-common-dir)" && pwd) +if [ "$#" -eq 0 ]; then + set -- --only manual --companions +fi + +docker run --rm -i \ + -v "$root:/src" -w /src \ + -v "$git_dir:$git_dir:ro" -v "$git_common:$git_common:ro" \ + -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" -e UV_VERSION="$uv_version" \ + "$image" bash -s -- "$@" <<'CONTAINER' +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y -qq --no-install-recommends ca-certificates curl git \ + libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz-subset0 \ + fonts-dejavu-core fonts-dejavu-extra poppler-utils >/dev/null + +case "$(uname -m)" in + x86_64) arch=x86_64; sum=9167d72b3319674b6303c4cbe071854bba13ebdf3d76b1a7cbdc175471fb66d6 ;; + aarch64) arch=aarch64; sum=6524bd338177ed50d035d39354e12545e993bbeba2ecbddf0480c5b3a81d313f ;; + *) echo "Unsupported architecture: $(uname -m)" >&2; exit 1 ;; +esac +archive="uv-$arch-unknown-linux-gnu.tar.gz" +curl -fsSL -o "/tmp/$archive" "https://github.com/astral-sh/uv/releases/download/$UV_VERSION/$archive" +echo "$sum /tmp/$archive" | sha256sum -c --quiet +tar -xzf "/tmp/$archive" -C /usr/local/bin --strip-components=1 + +git config --global --add safe.directory '*' +status=0 +uv run --locked --python 3.12.12 --script bin/build_pdf_docs.py "$@" || status=$? +chown -R "$HOST_UID:$HOST_GID" resource manual-artifacts 2>/dev/null || true +exit "$status" +CONTAINER diff --git a/resources/build/Other-Manuals.md b/resources/build/Other-Manuals.md index 7e9832d6..cbfe34ff 100644 --- a/resources/build/Other-Manuals.md +++ b/resources/build/Other-Manuals.md @@ -10,7 +10,8 @@ Use Python 3.12 and uv through `mise`. The existing PDF builder's committed script lock pins MarkdownIt, WeasyPrint, Mammoth, and Pillow. WeasyPrint also needs native Pango libraries and fonts with German, Turkish, and Cyrillic glyphs. CI installs DejaVu fonts. Native libraries and fonts affect pagination; -checksums identify built artifacts, not cross-platform reproducible bytes. +checksums identify built artifacts, not cross-platform reproducible bytes. The +container build below uses the CI fonts and libraries. The main PDF and text editions contain every main chapter selected by `bin/check_navigation.py`. German and Turkish editions add localized entries @@ -35,11 +36,18 @@ mise exec python@3.12 uv@0.12.23 -- uv run --locked --python python \ --script bin/build_pdf_docs.py --only manual --companions ``` -On macOS with Homebrew, set `DYLD_FALLBACK_LIBRARY_PATH` to the Homebrew `lib` -directory after selecting runtimes if WeasyPrint cannot locate Pango. For -example, use `mise exec python@3.12 uv@0.12.23 -- env` followed by -`DYLD_FALLBACK_LIBRARY_PATH=/opt/homebrew/lib uv run` and the remaining build -arguments on an Apple Silicon Homebrew installation. +To build in the same environment as CI, with its fonts, native libraries and +locked toolchain, run the container build. It needs Docker and accepts the same +arguments: + +```sh +bin/build_manuals_in_container.sh --only manual --companions +``` + +Host builds use the host's fonts, so their pagination and checksums differ +from the published editions. On macOS, `env` and `uv run` do not pass +`DYLD_FALLBACK_LIBRARY_PATH` to Python, so WeasyPrint cannot find Homebrew's +Pango through the command above. Use the container build there. The command above preserves the existing ReportIt edition. The default is the Cacti manual; rebuilding From 2d1d6bcbedde0175bd8e2a5795607ccc2aa2e6d8 Mon Sep 17 00:00:00 2001 From: Thomas Vincent Date: Wed, 18 Nov 2026 00:35:01 -0800 Subject: [PATCH 3/4] docs: deny unprotected paths in the Apache example Cacti ships .htaccess files for bin, cli, contrib, log, mibs, rra, scripts, tests and the private cache directories, but none for service/, include/config.php, include/config_local.php or PHP files under include/vendor. The nginx guides deny those paths; the Apache example relied on .htaccess alone. Deny them explicitly, matching the nginx rules. Signed-off-by: Thomas Vincent --- Install-Under-Enterprise-Linux-LAMP.md | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/Install-Under-Enterprise-Linux-LAMP.md b/Install-Under-Enterprise-Linux-LAMP.md index 0620500c..32400595 100644 --- a/Install-Under-Enterprise-Linux-LAMP.md +++ b/Install-Under-Enterprise-Linux-LAMP.md @@ -82,13 +82,32 @@ executed by FPM rather than served as text. This example exposes only Require all granted DirectoryIndex index.php + + # Cacti ships no .htaccess for these paths. + + Require all denied + + + + Require all denied + + + + + Require all denied + + ``` Install `mod_ssl` if not already present. `AllowOverride All` lets Cacti's -shipped `.htaccess` restrictions protect private paths; if your policy disables -overrides, reproduce those restrictions in the server configuration. Do not -grant general access to private directories as a troubleshooting step. +shipped `.htaccess` files deny `bin`, `cli`, `contrib`, `log`, `mibs`, `rra`, +`scripts`, `tests` and the private `cache` subdirectories. Cacti ships no +`.htaccess` for `service`, the `include/config.php` files or PHP files under +`include/vendor`, so the example denies them explicitly, as the nginx guides +do. If your policy disables overrides, reproduce the shipped restrictions in +the server configuration. Do not grant general access to private directories +as a troubleshooting step. ```console dnf install -y mod_ssl From 69a4eb9f5d1e91a973a13f6f9c7dc15c815c8ec3 Mon Sep 17 00:00:00 2001 From: Thomas Vincent Date: Wed, 18 Nov 2026 00:41:39 -0800 Subject: [PATCH 4/4] build: address static-analysis findings in the new tooling Use [[ for the argument test, and install coverage from a wheel only so pip cannot run a source distribution's build scripts. Signed-off-by: Thomas Vincent --- .github/workflows/code-claims.yml | 2 +- bin/build_manuals_in_container.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code-claims.yml b/.github/workflows/code-claims.yml index 3454e02f..f6e3ac52 100644 --- a/.github/workflows/code-claims.yml +++ b/.github/workflows/code-claims.yml @@ -37,7 +37,7 @@ jobs: python-version: '3.12' - name: Test the checker run: | - python3 -m pip install --quiet coverage==7.13.2 + python3 -m pip install --quiet --only-binary :all: coverage==7.13.2 python3 -m coverage run --branch --include=bin/check_code_claims.py \ -m unittest discover -s bin/tests -p test_code_claims.py python3 -m coverage report --fail-under=100 diff --git a/bin/build_manuals_in_container.sh b/bin/build_manuals_in_container.sh index fc31261c..29544441 100755 --- a/bin/build_manuals_in_container.sh +++ b/bin/build_manuals_in_container.sh @@ -19,7 +19,7 @@ uv_version='0.12.23' # the same path so the builder can record the source revision. git_dir=$(git -C "$root" rev-parse --absolute-git-dir) git_common=$(cd "$(git -C "$root" rev-parse --git-common-dir)" && pwd) -if [ "$#" -eq 0 ]; then +if [[ "$#" -eq 0 ]]; then set -- --only manual --companions fi