diff --git a/collection/stages/roles/openstack_test/defaults/main.yml b/collection/stages/roles/openstack_test/defaults/main.yml index 2ec13f5d..5e5386cf 100644 --- a/collection/stages/roles/openstack_test/defaults/main.yml +++ b/collection/stages/roles/openstack_test/defaults/main.yml @@ -14,3 +14,26 @@ openstack_test_suite: openstack-test/all openstack_test_ote_binary_relpath: bin/openstack-test-tests-ext # Run filtered OTE tests one-by-one (required for [lb][Serial] under Octavia limits). openstack_test_ote_run_serial: false +# Serial coda: after the main filtered batch suite, run selected tests +# serially in fixed order. Starts with the bogus Machine leak cluster +# (bz_2073398 cleanup, then Machine trio), then known suite-load flakes +# that pass alone but time out in the randomized batch (egressIP FIP +# failover AAP wait, prometheus PVC resize pod-ready wait), then topology +# pair ordered so the AZ check runs while enable_topology is still true +# before the mutator ([Serial] set enable_topology=false) can race it in +# batch. Keeps the bulk suite in batch to limit wall time. No-ops when +# the matchers are absent (e.g. lb_tests allowlists). Requires OTE +# filtered run path. +openstack_test_machines_leak_cluster_serial: true +# Ordered substrings matched against list_of_tests_to_run.txt (one phrase +# each; written to a matchers file so shell never splits them). For the +# MachineSet replica matcher, ControlPlane variants are skipped. +openstack_test_machines_leak_cluster_matchers: + - "Bugfix bz_2073398: [Serial] MachineSet scale-in does not leak OpenStack ports" + - "Machine are in phase Running" + - "MachineSet replica number corresponds to the number of Machines" + - "Machine ProviderSpec is correctly applied to OpenStack instances" + - "egressIP attached to a floating IP should be kept after EgressIP node failover" + - "should follow PVC specs during resizing for prometheus" + - "enables topology aware scheduling when compute and volume AZs are identical" + - "should allow the manual setting of enable_topology to false" diff --git a/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py new file mode 100644 index 00000000..ccf137f6 --- /dev/null +++ b/collection/stages/roles/openstack_test/files/split_machines_leak_cluster.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Split serial-coda tests out of an OTE test list. + +Reads the filtered list_of_tests_to_run.txt and an ordered matchers file +(one substring per line), writes: + - leak_cluster_serial.txt: ordered matches for serial coda + (Machine leak cluster, suite-load flakes, topology pair) + - list_of_tests_to_run.txt: remaining tests for the batch suite + +Matchers are plain substrings (one full phrase per line). The MachineSet +replica matcher skips any line that also contains "ControlPlane". + +When any matcher hits, every matcher must resolve to exactly one test; +missing or non-unique matches fail hard. Zero hits keep the allowlist +no-op (empty serial file, batch unchanged). +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +def load_matchers(path: str) -> list[str]: + with open(path, "r", encoding="utf-8") as f: + return [ln.strip() for ln in f if ln.strip() and not ln.strip().startswith("#")] + + +def pick(remaining: list[str], matcher: str) -> list[str]: + """Return all candidate lines for matcher (ControlPlane skipped for MachineSet replica).""" + candidates: list[str] = [] + for ln in remaining: + if not ln.strip(): + continue + if matcher not in ln: + continue + # Avoid ControlPlane MachineSet when matching worker MachineSet replica. + if "MachineSet replica number corresponds to the number of Machines" in matcher: + if "ControlPlane" in ln: + continue + candidates.append(ln) + return candidates + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("tests_to_run_path") + parser.add_argument("leak_cluster_path") + parser.add_argument("matchers_file") + args = parser.parse_args() + + matchers = load_matchers(args.matchers_file) + if not matchers: + print("leak_cluster=0 batch_remaining=unchanged (empty matchers file)") + Path(args.leak_cluster_path).write_text("", encoding="utf-8") + return 0 + + with open(args.tests_to_run_path, "r", encoding="utf-8") as f: + lines = [ln.rstrip("\n") for ln in f] + + remaining = list(lines) + resolved: list[tuple[str, list[str]]] = [] + any_hit = False + + for matcher in matchers: + candidates = pick(remaining, matcher) + if candidates: + any_hit = True + resolved.append((matcher, candidates)) + + if not any_hit: + # Allowlist no-op: none of the coda tests are in this filtered list. + print("leak_cluster=0 batch_remaining=unchanged (no matchers hit)") + Path(args.leak_cluster_path).write_text("", encoding="utf-8") + return 0 + + selected: list[str] = [] + for matcher, candidates in resolved: + if len(candidates) == 0: + print( + f"error: matcher matched no tests: {matcher!r}", + file=sys.stderr, + ) + return 1 + if len(candidates) > 1: + print( + f"error: matcher matched {len(candidates)} tests (expected 1): {matcher!r}", + file=sys.stderr, + ) + for c in candidates: + print(f" candidate: {c}", file=sys.stderr) + return 1 + selected.append(candidates[0]) + remaining = [ln for ln in remaining if ln != candidates[0]] + + with open(args.leak_cluster_path, "w", encoding="utf-8") as f: + for ln in selected: + f.write(ln + "\n") + + with open(args.tests_to_run_path, "w", encoding="utf-8") as f: + for ln in remaining: + f.write(ln + "\n") + + print( + f"leak_cluster={len(selected)} batch_remaining={len([x for x in remaining if x.strip()])}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml new file mode 100644 index 00000000..4af26764 --- /dev/null +++ b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml @@ -0,0 +1,101 @@ +--- +# Serial coda: serially run the ordered coda list after the main batch suite +# (Machine leak cluster, egressIP / prometheus, then topology AZ check +# before enable_topology=false mutator). +# Expects openstack_test_leak_cluster_path to already list the ordered tests. +- name: Stat machines leak-cluster serial list + ansible.builtin.stat: + path: "{{ openstack_test_leak_cluster_path }}" + register: openstack_test_leak_cluster_stat + +- name: Count machines leak-cluster serial tests + ansible.builtin.command: + argv: + - bash + - -c + - "grep -cve '^[[:space:]]*$' '{{ openstack_test_leak_cluster_path }}' || true" + register: openstack_test_leak_cluster_count + changed_when: false + when: openstack_test_leak_cluster_stat.stat.exists | default(false) + +- name: Skip machines leak-cluster serial coda when empty + ansible.builtin.debug: + msg: >- + No machines leak-cluster tests matched the filtered list; skipping serial coda. + when: > + (not (openstack_test_leak_cluster_stat.stat.exists | default(false))) + or ((openstack_test_leak_cluster_count.stdout | default('0') | trim | int) == 0) + +- name: Run machines leak-cluster tests serially and merge into suite log/junit + when: + - openstack_test_leak_cluster_stat.stat.exists | default(false) + - (openstack_test_leak_cluster_count.stdout | default('0') | trim | int) > 0 + block: + - name: Execute machines leak-cluster run-test loop + ansible.builtin.shell: | + set -o pipefail + suite_exit=0 + py_exit=0 + resolve="{{ openstack_test_ote_resolve_script }}" + leak_log="{{ openstack_test_results_dir }}/machines_leak_cluster.log" + leak_list="{{ openstack_test_leak_cluster_path }}" + # Pre-count from the list file (do not increment in-loop; avoids + # false UNSTABLE when resolvable!=expected after a successful coda). + expected="$(grep -cve '^[[:space:]]*$' "$leak_list" || true)" + : > "$leak_log" + while IFS= read -r tname || [ -n "${tname:-}" ]; do + [ -z "${tname// }" ] && continue + tmp=$(mktemp) + rc=0 + {{ openstack_test_executable }} run-test --output=json -c 1 "$tname" \ + > "$tmp" 2>&1 || rc=$? + cat "$tmp" >> "$leak_log" + cat "$tmp" >> {{ openstack_test_log_path }} + if [ "$rc" -ne 0 ]; then + failed=$(python3 "$resolve" count "$tmp" failed || echo 1) + skipped=$(python3 "$resolve" count "$tmp" skipped || echo 0) + passed=$(python3 "$resolve" count "$tmp" passed || echo 0) + unknown=$(python3 "$resolve" count "$tmp" unknown || echo 0) + if [ "$failed" -gt 0 ] || [ "$unknown" -gt 0 ]; then + suite_exit=1 + elif [ $((failed + skipped + passed + unknown)) -eq 0 ]; then + if grep -q 'FAIL!' "$tmp" 2>/dev/null; then + suite_exit=1 + elif ! grep -qE 'SUCCESS!|SKIP' "$tmp" 2>/dev/null; then + suite_exit=1 + fi + fi + fi + rm -f "$tmp" + done < "$leak_list" + python3 "$resolve" junit \ + "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$? + failed_count=$(python3 "$resolve" count "$leak_log" failed || echo 0) + skipped_count=$(python3 "$resolve" count "$leak_log" skipped || echo 0) + passed_count=$(python3 "$resolve" count "$leak_log" passed || echo 0) + unknown_count=$(python3 "$resolve" count "$leak_log" unknown || echo 0) + resolvable=$(( ${passed_count:-0} + ${skipped_count:-0} + ${failed_count:-0} )) + echo "machines_leak_cluster counts: passed=${passed_count} failed=${failed_count} skipped=${skipped_count} unknown=${unknown_count} expected=${expected} resolvable=${resolvable}" + if [ "$suite_exit" -ne 0 ] || [ "$py_exit" -ne 0 ] \ + || [ "${failed_count:-0}" -gt 0 ] || [ "${unknown_count:-0}" -gt 0 ] \ + || [ "$resolvable" -ne "$expected" ]; then + exit 1 + fi + exit 0 + environment: + OS_CLOUD: "{{ user_cloud }}" + KUBECONFIG: "{{ kubeconfig }}" + RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}" + SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}" + changed_when: true + register: openstack_test_leak_cluster_run + + rescue: + - name: Mark openstack-test UNSTABLE after machines leak-cluster serial failure + ansible.builtin.include_role: + name: tools_stage_results + tasks_from: mark_test_stage_unstable.yml + vars: + unstable_msg: >- + openstack_test serial coda failed (Machine leak cluster, egressIP / + prometheus resize, and/or topology enable_topology pair). diff --git a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml index 179da042..70d5bdb8 100644 --- a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml +++ b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml @@ -141,6 +141,53 @@ ansible.builtin.set_fact: openstack_test_filtering: "{{ (openstack_tests_allowlist_file | length > 0) or (openstack_tests_blocklist_file | length > 0) }}" +# Serial coda: pull matchers (Machine leak cluster + suite-load flakes) +# out of the batch list for an ordered serial run after the batch. +- name: Set machines leak-cluster serial list path + ansible.builtin.set_fact: + openstack_test_leak_cluster_path: >- + {{ openstack_test_results_dir }}/machines_leak_cluster_serial.txt + openstack_test_leak_cluster_matchers_path: >- + {{ openstack_test_results_dir }}/machines_leak_cluster_matchers.txt + when: + - openstack_test_use_ote | bool + - openstack_test_machines_leak_cluster_serial | bool + - openstack_test_filtering | bool + +# Write matchers one-per-line so multi-word phrases are never shell-split. +- name: Write machines leak-cluster matchers file + ansible.builtin.copy: + dest: "{{ openstack_test_leak_cluster_matchers_path }}" + mode: "0644" + content: "{{ openstack_test_machines_leak_cluster_matchers | map('trim') | join('\n') }}\n" + when: + - openstack_test_use_ote | bool + - openstack_test_machines_leak_cluster_serial | bool + - openstack_test_filtering | bool + +- name: Split machines leak-cluster tests from batch list for serial coda + ansible.builtin.command: + argv: + - python3 + - "{{ role_path }}/files/split_machines_leak_cluster.py" + - "{{ tests_to_run_path }}" + - "{{ openstack_test_leak_cluster_path }}" + - "{{ openstack_test_leak_cluster_matchers_path }}" + register: openstack_test_leak_cluster_split + changed_when: true + when: + - openstack_test_use_ote | bool + - openstack_test_machines_leak_cluster_serial | bool + - openstack_test_filtering | bool + +- name: Print machines leak-cluster split summary + ansible.builtin.debug: + msg: "{{ openstack_test_leak_cluster_split.stdout | default('leak cluster split skipped') }}" + when: + - openstack_test_use_ote | bool + - openstack_test_machines_leak_cluster_serial | bool + - openstack_test_filtering | bool + # OTE run-suite often marks every outer JSON result as failed when stderr has # klog lines (Deserializaion Error: invalid character 'I'...). True outcomes # live in nested STDOUT JSON / ginkgo SUCCESS!|FAIL! lines — see @@ -385,3 +432,12 @@ name: tools_must-gather vars: must_gather_suffix: "openstack-tests" + +# Serial coda after the main batch (or filtered serial) suite. +- name: Run serial-coda tests after main suite + ansible.builtin.include_tasks: run_machines_leak_cluster_serial.yml + when: + - openstack_test_use_ote | bool + - openstack_test_machines_leak_cluster_serial | bool + - openstack_test_filtering | bool + - openstack_test_leak_cluster_path is defined