diff --git a/collection/stages/roles/lb_tests/tasks/main.yml b/collection/stages/roles/lb_tests/tasks/main.yml index ba1b687b..f5ee0813 100644 --- a/collection/stages/roles/lb_tests/tasks/main.yml +++ b/collection/stages/roles/lb_tests/tasks/main.yml @@ -99,6 +99,10 @@ openstack_tests_allowlist_file: "{{ lb_ovn_allowlist_resolved }}" openstack_tests_blocklist_file: "{{ lb_ingress_blocklist_resolved }}" openstack_test_ote_run_serial: true + # LB allowlists can include egressIP (a coda matcher) without the Machine + # leak cluster; serial-coda uniqueness then fails. Coda belongs on the + # main openstack_test stage only. + openstack_test_machines_leak_cluster_serial: false openstack_testsuite_name: openstack_tests_lb_ovn openstack_reset_result_dir: no # As we want to keep the logs generated in the previous step diff --git a/collection/stages/roles/lb_tests/tasks/run_amphora_openstack_test.yml b/collection/stages/roles/lb_tests/tasks/run_amphora_openstack_test.yml index 32b75d3f..28cf868f 100644 --- a/collection/stages/roles/lb_tests/tasks/run_amphora_openstack_test.yml +++ b/collection/stages/roles/lb_tests/tasks/run_amphora_openstack_test.yml @@ -8,5 +8,7 @@ openstack_tests_allowlist_file: "{{ lb_amphora_openstack_test_allowlist }}" openstack_tests_blocklist_file: "{{ lb_amphora_openstack_test_blocklist }}" openstack_test_ote_run_serial: true + # Same as OVN LB: do not apply Machine serial coda on LB allowlists. + openstack_test_machines_leak_cluster_serial: false openstack_testsuite_name: "{{ lb_amphora_openstack_testsuite_name }}" openstack_reset_result_dir: "{{ lb_amphora_openstack_reset_result_dir | default(false) }}" diff --git a/collection/stages/roles/openstack_test/defaults/main.yml b/collection/stages/roles/openstack_test/defaults/main.yml index 5e5386cf..dd8fb39d 100644 --- a/collection/stages/roles/openstack_test/defaults/main.yml +++ b/collection/stages/roles/openstack_test/defaults/main.yml @@ -25,6 +25,11 @@ openstack_test_ote_run_serial: false # the matchers are absent (e.g. lb_tests allowlists). Requires OTE # filtered run path. openstack_test_machines_leak_cluster_serial: true +# Dedicated Zuul/artifact folder for the serial coda (same pattern as +# lb_ovn / lb_amphora_*): own log, junit, list, and test_results copy. +# Main openstack-test-results stays batch-only after the split. +openstack_test_serial_coda_results_dir: "{{ artifacts_dir }}/openstack_test_serial_coda" +openstack_test_serial_coda_testsuite_name: openstack_tests_serial_coda # Ordered substrings matched against list_of_tests_to_run.txt (one phrase # each; written to a matchers file so shell never splits them). For the # MachineSet replica matcher, ControlPlane variants are skipped. diff --git a/collection/stages/roles/openstack_test/tasks/main.yml b/collection/stages/roles/openstack_test/tasks/main.yml index bfb0418c..23a183b6 100644 --- a/collection/stages/roles/openstack_test/tasks/main.yml +++ b/collection/stages/roles/openstack_test/tasks/main.yml @@ -38,6 +38,15 @@ - name: Include Openstack-Test tasks ansible.builtin.include_tasks: run_openstack_test.yml + always: + # Always publish batch results when junit exists, even if run_openstack_test + # (or serial coda) failed after the suite wrote artifacts. + - name: Stat main openstack-test junit before post + ansible.builtin.stat: + path: "{{ openstack_test_results_dir }}/junit_e2e_openstack.xml" + register: openstack_test_main_junit_stat + failed_when: false + - name: Post openshift-test ansible.builtin.include_role: name: tools_openshift_tests @@ -47,8 +56,8 @@ key_for_filtering_results: "openstack" test_name: "{{ openstack_test_name }}" results_dir: "{{ openstack_test_results_dir }}" + when: openstack_test_main_junit_stat.stat.exists | default(false) - always: - name: Restore hypervisor's cifmw-dnsmasq service ansible.builtin.import_role: name: tools_cifmw_dnsmasq diff --git a/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml index 4af26764..defc84da 100644 --- a/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml +++ b/collection/stages/roles/openstack_test/tasks/run_machines_leak_cluster_serial.yml @@ -2,7 +2,9 @@ # Serial coda: serially run the ordered coda list after the main batch suite # (Machine leak cluster, egressIP / prometheus, then topology AZ check # before enable_topology=false mutator). -# Expects openstack_test_leak_cluster_path to already list the ordered tests. +# Artifacts go to openstack_test_serial_coda/ (separate from batch +# openstack-test-results/), same layout as lb_ovn / lb_amphora_*. +# Expects openstack_test_leak_cluster_path and coda log/junit paths set. - name: Stat machines leak-cluster serial list ansible.builtin.stat: path: "{{ openstack_test_leak_cluster_path }}" @@ -26,7 +28,7 @@ (not (openstack_test_leak_cluster_stat.stat.exists | default(false))) or ((openstack_test_leak_cluster_count.stdout | default('0') | trim | int) == 0) -- name: Run machines leak-cluster tests serially and merge into suite log/junit +- name: Run machines leak-cluster tests serially into dedicated results dir when: - openstack_test_leak_cluster_stat.stat.exists | default(false) - (openstack_test_leak_cluster_count.stdout | default('0') | trim | int) > 0 @@ -34,10 +36,10 @@ - name: Execute machines leak-cluster run-test loop ansible.builtin.shell: | set -o pipefail - suite_exit=0 py_exit=0 resolve="{{ openstack_test_ote_resolve_script }}" - leak_log="{{ openstack_test_results_dir }}/machines_leak_cluster.log" + leak_log="{{ openstack_test_serial_coda_log_path }}" + leak_junit="{{ openstack_test_serial_coda_junit_path }}" leak_list="{{ openstack_test_leak_cluster_path }}" # Pre-count from the list file (do not increment in-loop; avoids # false UNSTABLE when resolvable!=expected after a successful coda). @@ -46,37 +48,21 @@ while IFS= read -r tname || [ -n "${tname:-}" ]; do [ -z "${tname// }" ] && continue tmp=$(mktemp) - rc=0 + # OTE often exits non-zero on pass/skip; ignore per-test rc — final + # resolver counts are the source of truth (same as LB serial / batch). {{ openstack_test_executable }} run-test --output=json -c 1 "$tname" \ - > "$tmp" 2>&1 || rc=$? + > "$tmp" 2>&1 || true cat "$tmp" >> "$leak_log" - cat "$tmp" >> {{ openstack_test_log_path }} - if [ "$rc" -ne 0 ]; then - failed=$(python3 "$resolve" count "$tmp" failed || echo 1) - skipped=$(python3 "$resolve" count "$tmp" skipped || echo 0) - passed=$(python3 "$resolve" count "$tmp" passed || echo 0) - unknown=$(python3 "$resolve" count "$tmp" unknown || echo 0) - if [ "$failed" -gt 0 ] || [ "$unknown" -gt 0 ]; then - suite_exit=1 - elif [ $((failed + skipped + passed + unknown)) -eq 0 ]; then - if grep -q 'FAIL!' "$tmp" 2>/dev/null; then - suite_exit=1 - elif ! grep -qE 'SUCCESS!|SKIP' "$tmp" 2>/dev/null; then - suite_exit=1 - fi - fi - fi rm -f "$tmp" done < "$leak_list" - python3 "$resolve" junit \ - "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$? + python3 "$resolve" junit "$leak_log" "$leak_junit" || py_exit=$? failed_count=$(python3 "$resolve" count "$leak_log" failed || echo 0) skipped_count=$(python3 "$resolve" count "$leak_log" skipped || echo 0) passed_count=$(python3 "$resolve" count "$leak_log" passed || echo 0) unknown_count=$(python3 "$resolve" count "$leak_log" unknown || echo 0) resolvable=$(( ${passed_count:-0} + ${skipped_count:-0} + ${failed_count:-0} )) echo "machines_leak_cluster counts: passed=${passed_count} failed=${failed_count} skipped=${skipped_count} unknown=${unknown_count} expected=${expected} resolvable=${resolvable}" - if [ "$suite_exit" -ne 0 ] || [ "$py_exit" -ne 0 ] \ + if [ "$py_exit" -ne 0 ] \ || [ "${failed_count:-0}" -gt 0 ] || [ "${unknown_count:-0}" -gt 0 ] \ || [ "$resolvable" -ne "$expected" ]; then exit 1 @@ -91,6 +77,61 @@ register: openstack_test_leak_cluster_run rescue: + - name: Resolve machines leak-cluster serial failure counts + ansible.builtin.command: + argv: + - python3 + - "{{ openstack_test_ote_resolve_script }}" + - count + - "{{ openstack_test_serial_coda_log_path }}" + - "{{ item }}" + loop: + - failed + - passed + - skipped + - unknown + register: openstack_test_leak_cluster_rescue_counts + changed_when: false + failed_when: false + + - name: Normalize machines leak-cluster rescue counts + ansible.builtin.set_fact: + openstack_test_leak_failed: >- + {{ (openstack_test_leak_cluster_rescue_counts.results + | selectattr('item', 'equalto', 'failed') + | map(attribute='stdout') | first | default('0')) | trim }} + openstack_test_leak_passed: >- + {{ (openstack_test_leak_cluster_rescue_counts.results + | selectattr('item', 'equalto', 'passed') + | map(attribute='stdout') | first | default('0')) | trim }} + openstack_test_leak_skipped: >- + {{ (openstack_test_leak_cluster_rescue_counts.results + | selectattr('item', 'equalto', 'skipped') + | map(attribute='stdout') | first | default('0')) | trim }} + openstack_test_leak_unknown: >- + {{ (openstack_test_leak_cluster_rescue_counts.results + | selectattr('item', 'equalto', 'unknown') + | map(attribute='stdout') | first | default('0')) | trim }} + openstack_test_leak_expected: >- + {{ (openstack_test_leak_cluster_count.stdout | default('0')) | trim }} + + - name: Compute machines leak-cluster resolvable count + ansible.builtin.set_fact: + openstack_test_leak_resolvable: >- + {{ ( + (openstack_test_leak_passed | int) + + (openstack_test_leak_skipped | int) + + (openstack_test_leak_failed | int) + ) }} + + - name: Print machines leak-cluster rescue counts + ansible.builtin.debug: + msg: >- + machines_leak_cluster rescue counts: passed={{ openstack_test_leak_passed }}, + failed={{ openstack_test_leak_failed }}, skipped={{ openstack_test_leak_skipped }}, + unknown={{ openstack_test_leak_unknown }}, expected={{ openstack_test_leak_expected }}, + resolvable={{ openstack_test_leak_resolvable }} + - name: Mark openstack-test UNSTABLE after machines leak-cluster serial failure ansible.builtin.include_role: name: tools_stage_results @@ -99,3 +140,35 @@ unstable_msg: >- openstack_test serial coda failed (Machine leak cluster, egressIP / prometheus resize, and/or topology enable_topology pair). + when: > + ((openstack_test_leak_failed | default('0')) | int > 0) + or ((openstack_test_leak_unknown | default('0')) | int > 0) + or ( + (openstack_test_leak_resolvable | default('0')) | int + != (openstack_test_leak_expected | default('0')) | int + ) + + always: + # Publish coda results like lb_ovn when junit exists. No ignore_errors: + # publish failure must fail the stage so results are not silently missing. + - name: Stat serial-coda junit before post + ansible.builtin.stat: + path: "{{ openstack_test_serial_coda_junit_path }}" + register: openstack_test_serial_coda_junit_stat + failed_when: false + when: + - openstack_test_serial_coda_junit_path is defined + - openstack_test_leak_cluster_stat.stat.exists | default(false) + - (openstack_test_leak_cluster_count.stdout | default('0') | trim | int) > 0 + + - name: Post serial-coda openshift-test results + ansible.builtin.include_role: + name: tools_openshift_tests + tasks_from: post_openshift_tests.yml + vars: + testsuite_name: "{{ openstack_test_serial_coda_testsuite_name }}" + key_for_filtering_results: "openstack" + test_name: "{{ openstack_test_name }}" + results_dir: "{{ openstack_test_serial_coda_results_dir }}" + when: + - openstack_test_serial_coda_junit_stat.stat.exists | default(false) diff --git a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml index 70d5bdb8..5ea818d1 100644 --- a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml +++ b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml @@ -143,12 +143,27 @@ # Serial coda: pull matchers (Machine leak cluster + suite-load flakes) # out of the batch list for an ordered serial run after the batch. +# Artifacts land in openstack_test_serial_coda/ (LB-style separate folder). +- name: Ensure serial coda results directory exists + ansible.builtin.file: + path: "{{ openstack_test_serial_coda_results_dir }}" + state: directory + mode: "0755" + when: + - openstack_test_use_ote | bool + - openstack_test_machines_leak_cluster_serial | bool + - openstack_test_filtering | bool + - name: Set machines leak-cluster serial list path ansible.builtin.set_fact: openstack_test_leak_cluster_path: >- - {{ openstack_test_results_dir }}/machines_leak_cluster_serial.txt + {{ openstack_test_serial_coda_results_dir }}/list_of_tests_to_run.txt openstack_test_leak_cluster_matchers_path: >- - {{ openstack_test_results_dir }}/machines_leak_cluster_matchers.txt + {{ openstack_test_serial_coda_results_dir }}/machines_leak_cluster_matchers.txt + openstack_test_serial_coda_log_path: >- + {{ openstack_test_serial_coda_results_dir }}/{{ openstack_test_name }}.log + openstack_test_serial_coda_junit_path: >- + {{ openstack_test_serial_coda_results_dir }}/junit_e2e_openstack.xml when: - openstack_test_use_ote | bool - openstack_test_machines_leak_cluster_serial | bool @@ -235,7 +250,6 @@ ansible.builtin.shell: | set -o pipefail : > {{ openstack_test_log_path }} - suite_exit=0 py_exit=0 expected=0 resolve="{{ openstack_test_ote_resolve_script }}" @@ -243,33 +257,18 @@ [[ -z "${test// }" ]] && continue expected=$((expected + 1)) tmp=$(mktemp) - rc=0 - {{ openstack_test_executable }} run-test --output=json "$test" \ - > "$tmp" 2>&1 || rc=$? + # OTE often exits non-zero on pass/skip (Deserializaion Error on + # leading I... klog). Ignore per-test rc; end-of-suite resolver is + # the source of truth (same as stdin-batch path). + {{ openstack_test_executable }} run-test --output=json -c 1 "$test" \ + > "$tmp" 2>&1 || true cat "$tmp" >> {{ openstack_test_log_path }} - # Skip-only (and pass) often exit non-zero from run-test; ignore that - # only when the resolver confirms skipped/passed. Failed/unknown keep - # the suite failed. Empty resolve on this chunk alone is deferred to - # the end-of-suite resolvable==expected check (OTE often exits - # non-zero before JSON is parseable as a standalone result). - if [ "$rc" -ne 0 ]; then - failed=$(python3 "$resolve" count "$tmp" failed || echo 1) - skipped=$(python3 "$resolve" count "$tmp" skipped || echo 0) - passed=$(python3 "$resolve" count "$tmp" passed || echo 0) - unknown=$(python3 "$resolve" count "$tmp" unknown || echo 0) - if [ "$failed" -gt 0 ] || [ "$unknown" -gt 0 ]; then - suite_exit=1 - elif [ $((failed + skipped + passed + unknown)) -eq 0 ]; then - if grep -q 'FAIL!' "$tmp" 2>/dev/null; then - suite_exit=1 - elif ! grep -qE 'SUCCESS!|SKIP' "$tmp" 2>/dev/null; then - # No resolvable outcome and no ginkgo success/skip signal. - suite_exit=1 - fi - fi - fi rm -f "$tmp" done < {{ tests_to_run_path }} + if [ "$expected" -eq 0 ]; then + echo "ote serial: empty filtered test list (expected=0); failing" + exit 1 + fi python3 "$resolve" junit \ "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$? failed_count=$(python3 "$resolve" count \ @@ -283,7 +282,8 @@ # Require every requested test to resolve as pass/skip/fail (no unknown, # no missing entries). Exit 0 with empty/unresolvable output must fail. resolvable=$(( ${passed_count:-0} + ${skipped_count:-0} + ${failed_count:-0} )) - if [ "$suite_exit" -ne 0 ] || [ "$py_exit" -ne 0 ] \ + echo "ote serial counts: passed=${passed_count:-0} failed=${failed_count:-0} skipped=${skipped_count:-0} unknown=${unknown_count:-0} expected=$expected resolvable=$resolvable" + if [ "$py_exit" -ne 0 ] \ || [ "${failed_count:-0}" -gt 0 ] || [ "${unknown_count:-0}" -gt 0 ] \ || [ "$resolvable" -ne "$expected" ]; then exit 1 @@ -305,11 +305,37 @@ set -o pipefail run_exit=0 py_exit=0 + resolve="{{ openstack_test_ote_resolve_script }}" cat {{ tests_to_run_path }} | {{ openstack_test_executable }} run-test \ > {{ openstack_test_log_path }} || run_exit=$? - python3 "{{ openstack_test_ote_resolve_script }}" junit \ + # OTE run-test often exits non-zero on pass/skip; ignore that when + # the resolver confirms outcomes (same as serial path). + python3 "$resolve" junit \ "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$? - if [ "$run_exit" -ne 0 ] || [ "$py_exit" -ne 0 ]; then + failed_count=$(python3 "$resolve" count \ + "{{ openstack_test_log_path }}" failed || echo 0) + skipped_count=$(python3 "$resolve" count \ + "{{ openstack_test_log_path }}" skipped || echo 0) + passed_count=$(python3 "$resolve" count \ + "{{ openstack_test_log_path }}" passed || echo 0) + unknown_count=$(python3 "$resolve" count \ + "{{ openstack_test_log_path }}" unknown || echo 0) + expected=$(grep -cve '^[[:space:]]*$' {{ tests_to_run_path }} || true) + # Empty filtered list: run-test exits non-zero with no names, and all + # counts stay 0 so resolvable==expected would otherwise pass. + if [ "${expected:-0}" -eq 0 ]; then + echo "ote batch: empty filtered test list (expected=0); failing" + exit 1 + fi + # Require every requested test to resolve as pass/skip/fail (no unknown, + # no missing entries). Exit 0 with empty/unresolvable output must fail. + resolvable=$(( ${passed_count:-0} + ${skipped_count:-0} + ${failed_count:-0} )) + if [ "$run_exit" -ne 0 ] && [ "${failed_count:-0}" -eq 0 ]; then + echo "ote batch run_exit=$run_exit ignored; resolved passed=${passed_count:-0} failed=0 skipped=${skipped_count:-0} unknown=${unknown_count:-0} expected=$expected" + fi + if [ "$py_exit" -ne 0 ] \ + || [ "${failed_count:-0}" -gt 0 ] || [ "${unknown_count:-0}" -gt 0 ] \ + || [ "$resolvable" -ne "$expected" ]; then exit 1 fi exit 0 @@ -340,17 +366,9 @@ when: not (openstack_test_use_ote | bool) rescue: - - name: Mark the openshift tests as UNSTABLE - ansible.builtin.include_role: - name: tools_stage_results - tasks_from: mark_test_stage_unstable.yml - vars: - unstable_msg: >- - The openshift test Suite failed. - - # Use distinct registers for OTE vs legacy. Skipped tasks still overwrite - # a shared register in Ansible, which wiped OTE counts and falsely hit - # "no tests where run!" after a real suite run. + # Resolve counts BEFORE marking UNSTABLE. Shell already ignores false OTE + # non-zero rc; rescue must still UNSTABLE on failed, unknown, or + # resolvable!=expected so incomplete resolve cannot silently pass. - name: Get the number of failed openstack tests (OTE resolved log) ansible.builtin.command: argv: @@ -377,6 +395,45 @@ failed_when: false when: openstack_test_use_ote | bool + - name: Get the number of unknown openstack tests (OTE resolved log) + ansible.builtin.command: + argv: + - python3 + - "{{ openstack_test_ote_resolve_script }}" + - count + - "{{ openstack_test_log_path }}" + - unknown + register: openstack_test_ote_unknown_count + changed_when: false + failed_when: false + when: openstack_test_use_ote | bool + + - name: Get the number of skipped openstack tests (OTE resolved log) + ansible.builtin.command: + argv: + - python3 + - "{{ openstack_test_ote_resolve_script }}" + - count + - "{{ openstack_test_log_path }}" + - skipped + register: openstack_test_ote_skipped_count + changed_when: false + failed_when: false + when: openstack_test_use_ote | bool + + - name: Count expected openstack tests from filtered list + ansible.builtin.command: + argv: + - bash + - -c + - "grep -cve '^[[:space:]]*$' '{{ tests_to_run_path }}' || true" + register: openstack_test_expected_count_cmd + changed_when: false + failed_when: false + when: + - openstack_test_use_ote | bool + - tests_to_run_path is defined + - name: Get the number of failed openstack tests (legacy log) ansible.builtin.shell: > set -o pipefail && @@ -409,12 +466,61 @@ if (openstack_test_use_ote | bool) else openstack_test_legacy_passed_count.stdout | default('0') ) | trim }} + number_of_unknown_tests: >- + {{ ( + openstack_test_ote_unknown_count.stdout | default('0') + if (openstack_test_use_ote | bool) + else '0' + ) | trim }} + number_of_skipped_tests: >- + {{ ( + openstack_test_ote_skipped_count.stdout | default('0') + if (openstack_test_use_ote | bool) + else '0' + ) | trim }} + number_of_expected_tests: >- + {{ ( + openstack_test_expected_count_cmd.stdout | default('0') + if (openstack_test_use_ote | bool) + else '0' + ) | trim }} + + - name: Compute resolvable openstack-test count + ansible.builtin.set_fact: + number_of_resolvable_tests: >- + {{ ( + (number_of_passed_tests | int) + + (number_of_skipped_tests | int) + + (number_of_failed_tests | int) + ) }} - name: Print resolved openstack-test pass/fail counts ansible.builtin.debug: msg: >- openstack-test counts: passed={{ number_of_passed_tests }}, - failed={{ number_of_failed_tests }} + failed={{ number_of_failed_tests }}, + skipped={{ number_of_skipped_tests | default('0') }}, + unknown={{ number_of_unknown_tests | default('0') }}, + expected={{ number_of_expected_tests | default('0') }}, + resolvable={{ number_of_resolvable_tests | default('0') }} + + - name: Mark the openshift tests as UNSTABLE + ansible.builtin.include_role: + name: tools_stage_results + tasks_from: mark_test_stage_unstable.yml + vars: + unstable_msg: >- + The openshift test Suite failed. + when: > + ((number_of_failed_tests | default('0')) | int > 0) + or ((number_of_unknown_tests | default('0')) | int > 0) + or ( + (openstack_test_use_ote | bool) + and ( + (number_of_resolvable_tests | default('0')) | int + != (number_of_expected_tests | default('0')) | int + ) + ) # This fail task is added to detect failures in openstack-test execution - name: Fail the playbook in case there are no failed and passed tests @@ -425,13 +531,13 @@ - (number_of_failed_tests | default("0")) | int == 0 - (number_of_passed_tests | default("0")) | int == 0 - # must-gather is retrieved if openstack-test is successfully executed and there are failing - # tests + # must-gather only when there are real resolved failures - name: Run must-gather ansible.builtin.include_role: name: tools_must-gather vars: must_gather_suffix: "openstack-tests" + when: (number_of_failed_tests | default('0')) | int > 0 # Serial coda after the main batch (or filtered serial) suite. - name: Run serial-coda tests after main suite