From a50cc970b4ae631e1ff9b092bf3f7adefdcdb302 Mon Sep 17 00:00:00 2001 From: Roberto Montero Date: Tue, 27 Oct 2026 16:05:48 +0100 Subject: [PATCH] Share one GitLab after_script for Docker SSI, AWS SSI, and K8s lib injection. The three jobs repeated the same cleanup in bash. One script keeps that cleanup in one place and still finds the system-tests checkout when a tracer launches the pipeline. Co-authored-by: Cursor --- .gitlab/ssi_gitlab-ci.yml | 182 +------------ .../test_the_test/test_gitlab_finalize_job.py | 213 +++++++++++++++ .../ci_orchestrators/gitlab_exporter.py | 21 +- utils/scripts/gitlab_finalize_job.py | 255 ++++++++++++++++++ utils/scripts/gitlab_finalize_job.sh | 20 ++ 5 files changed, 516 insertions(+), 175 deletions(-) create mode 100644 tests/test_the_test/test_gitlab_finalize_job.py create mode 100644 utils/scripts/gitlab_finalize_job.py create mode 100755 utils/scripts/gitlab_finalize_job.sh diff --git a/.gitlab/ssi_gitlab-ci.yml b/.gitlab/ssi_gitlab-ci.yml index 0c66ced2557..623ff1cdfb9 100644 --- a/.gitlab/ssi_gitlab-ci.yml +++ b/.gitlab/ssi_gitlab-ci.yml @@ -26,61 +26,10 @@ ssi_tests: variables: # Force gitlab to keep the exit code as 3 if the job fails with exit code 3 FF_USE_NEW_BASH_EVAL_STRATEGY: true - after_script: | - export DATADOG_SITE="${DATADOG_SITE:-datadoghq.com}" - DATADOG_API_KEY=$(read-system-tests-secret test_optimization dd_api_key) || DATADOG_API_KEY="" - export DATADOG_API_KEY - # Tag the job as part of the official system-tests workflow, first so it's done even if a later command fails. - # Skip the tag if the API key could not be fetched. - if [ -n "$DATADOG_API_KEY" ]; then - datadog-ci tag --level job --tags "ci.framework.name:system-tests" || echo "Failed to tag the job with ci.framework.name:system-tests" - else - echo "Failed to fetch the Datadog API key, skipping the ci.framework.name:system-tests tag" - fi - - SCENARIO_SUFIX=$(echo "$SCENARIO" | tr "[:upper:]" "[:lower:]") - mkdir -p reports/ - if [ "$CI_PROJECT_NAME" = "system-tests" ]; then - cp -R logs_"${SCENARIO_SUFIX}" reports/ - else - cp -R /system-tests/logs_"${SCENARIO_SUFIX}" reports/ - fi - if [ "$CI_COMMIT_BRANCH" = "main" ] && [ "$CI_PROJECT_NAME" = "system-tests" ]; then - export FP_IMPORT_URL=$(read-system-tests-secret fpd fp_import_url) - export FP_API_KEY=$(read-system-tests-secret fpd fp_api_key) - for folder in reports/logs*/ ; do - echo "Checking folder: ${folder}" - for filename in ./${folder}feature_parity.json; do - if [ -e ${filename} ] - then - echo "Processing report: ${filename}" - curl -X POST ${FP_IMPORT_URL} --fail --header "Content-Type: application/json" --header "FP_API_KEY: ${FP_API_KEY}" --data "@${filename}" --include - fi - done - done - fi - - # Push JUnit results to Datadog Test Optimization - export DD_TAGS="test.configuration.ci_environment:${CI_ENVIRONMENT:-prod}" - if compgen -G "reports/logs*/reportJunit.xml" > /dev/null; then - datadog-ci junit upload \ - reports/logs*/reportJunit.xml \ - --service system-tests \ - --env ci \ - --verbose \ - --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']" - else - echo "No reports/logs*/reportJunit.xml found; skipping Test Optimization upload" - fi - - # Render all report logs as collapsible sections + run summary + banner. - if [ "$CI_PROJECT_NAME" = "system-tests" ]; then - SCRIPTS_DIR="./utils/scripts" - else - SCRIPTS_DIR="/system-tests/utils/scripts" - fi - export SCENARIO_SUFIX - bash "${SCRIPTS_DIR}/gitlab_print_logs.sh" + # gitlab_exporter.py points it to the system-tests checkout when another repository runs the job. + SYSTEM_TESTS_DIR: $CI_PROJECT_DIR + after_script: + - bash "${SYSTEM_TESTS_DIR}/utils/scripts/gitlab_finalize_job.sh" --profile docker retry: max: 2 when: @@ -108,6 +57,8 @@ ssi_tests: variables: # Force gitlab to keep the exit code as 3 if the job fails with exit code 3 FF_USE_NEW_BASH_EVAL_STRATEGY: true + # gitlab_exporter.py points it to the system-tests checkout when another repository runs the job. + SYSTEM_TESTS_DIR: $CI_PROJECT_DIR script: - | if [[ -n "${DD_INSTALL_SCRIPT_VERSION}" ]]; then @@ -124,80 +75,8 @@ ssi_tests: # Run the tests and capture stdout+stderr to a file set -o pipefail timeout 3000 ./run.sh "$SCENARIO" --vm-weblog "${WEBLOG}" --vm-env "${ONBOARDING_FILTER_ENV}" --vm-library "${TEST_LIBRARY}" --vm-provider aws --vm-default-vms All --vm-only "${VM}" --report-run-url "${CI_JOB_URL}" --report-environment "${ONBOARDING_FILTER_ENV}" 2>&1 | tee "${CI_PROJECT_DIR}/run_output.log" - after_script: | - # Tag the job as part of the official system-tests workflow, first so it's done even if a later command fails. - # The image does not ship datadog-ci, so the standalone binary is downloaded. - # Skip the tag if the API key could not be fetched. - DATADOG_API_KEY=$(read-system-tests-secret test_optimization dd_api_key) || DATADOG_API_KEY="" - if [ -n "$DATADOG_API_KEY" ]; then - DATADOG_SITE="${DATADOG_SITE:-datadoghq.com}" DATADOG_API_KEY="${DATADOG_API_KEY}" datadog-ci tag --level job --tags "ci.framework.name:system-tests" \ - || echo "Failed to tag the job with ci.framework.name:system-tests" - else - echo "Failed to fetch the Datadog API key, skipping the ci.framework.name:system-tests tag" - fi - - # Only fetch FP params on main in system-tests repo (use default CI creds) - if [ "$CI_COMMIT_BRANCH" = "main" ] && [ "$CI_PROJECT_NAME" = "system-tests" ]; then - export FP_IMPORT_URL=$(read-system-tests-secret fpd fp_import_url) - export FP_API_KEY=$(read-system-tests-secret fpd fp_api_key) - fi - # Re-establish AWS credentials for S3 operations (after_script runs in new shell session) - echo "Setting up AWS credentials for after_script..." - mkdir -p ~/.aws - - # Clean up any existing config and recreate it - rm -f ~/.aws/config - read-system-tests-secret apm-ecosystems-reliability-profile config > ~/.aws/config - export AWS_PROFILE=apm-ecosystems-reliability-ci - - SCENARIO_SUFIX=$(echo "$SCENARIO" | tr "[:upper:]" "[:lower:]") - mkdir -p reports/ - # Upload logs to S3 and delete large files - # When this template is included from downstream repos (dd-trace-*), the - # system-tests checkout lives under a `system-tests/` subdirectory, so - # all relative paths into the system-tests repo need that prefix. - if [ "$CI_PROJECT_NAME" = "system-tests" ]; then - LOGS_FOLDER="logs_${SCENARIO_SUFIX}" - SCRIPTS_DIR="./utils/scripts" - else - LOGS_FOLDER="system-tests/logs_${SCENARIO_SUFIX}" - SCRIPTS_DIR="./system-tests/utils/scripts" - fi - AWS_CONSOLE_URL="" - if [ -d "$LOGS_FOLDER" ]; then - EXECUTION_UNIQUE_IDENTIFIER="${CI_PIPELINE_ID:-datadog-local}" - LAST_DIRECTORY=$(basename "$LOGS_FOLDER") - S3_BUCKET="system-tests-aws-ssi-apm" - S3_DESTINATION="s3://${S3_BUCKET}/${EXECUTION_UNIQUE_IDENTIFIER}/${LAST_DIRECTORY}/${VM}_${WEBLOG}/" - AWS_CONSOLE_URL="https://s3.console.aws.amazon.com/s3/buckets/${S3_BUCKET}?region=us-east-1&prefix=${EXECUTION_UNIQUE_IDENTIFIER}/${LAST_DIRECTORY}/${VM}_${WEBLOG}/" - - aws s3 cp "$LOGS_FOLDER" "$S3_DESTINATION" --recursive --quiet || echo "Failed to upload logs to S3" - - # Delete large files (>50MB) to avoid CI/CD archiving - find "$LOGS_FOLDER" -type f -size +50M -exec sh -c 'echo "Deleting large file ($(du -h "$1" | cut -f1)): $1" && rm "$1"' _ {} \; || echo "Failed to delete some large files" - fi - - # Logs to be archived by gitlab - cp -R "$LOGS_FOLDER" reports/ - - # Feature parity - if [ "$CI_COMMIT_BRANCH" = "main" ] && [ "$CI_PROJECT_NAME" = "system-tests" ]; then - for folder in reports/logs*/ ; do - echo "Checking folder: ${folder}" - for filename in ./${folder}*_feature_parity.json; do - if [ -e ${filename} ] - then - echo "Processing report: ${filename}" - curl -X POST ${FP_IMPORT_URL} --fail --header "Content-Type: application/json" --header "FP_API_KEY: ${FP_API_KEY}" --data "@${filename}" --include - fi - done - done - fi - - # Render all report logs as collapsible sections + run summary + banner. - export SCENARIO_SUFIX AWS_CONSOLE_URL - bash "${SCRIPTS_DIR}/gitlab_print_logs.sh" - + after_script: + - bash "${SYSTEM_TESTS_DIR}/utils/scripts/gitlab_finalize_job.sh" --profile aws retry: max: 2 when: @@ -227,6 +106,8 @@ ssi_tests: variables: # Force gitlab to keep the exit code as 3 if the job fails with exit code 3 FF_USE_NEW_BASH_EVAL_STRATEGY: true + # gitlab_exporter.py points it to the system-tests checkout when another repository runs the job. + SYSTEM_TESTS_DIR: $CI_PROJECT_DIR script: - | # Wrap the build steps (runner image, venv, ECR login, weblog image) in a @@ -254,47 +135,8 @@ ssi_tests: set -o pipefail ./run.sh ${K8S_SCENARIO} --k8s-library ${TEST_LIBRARY} --k8s-weblog ${K8S_WEBLOG} --k8s-weblog-img ${K8S_WEBLOG_IMG}:${TAG_NAME} --k8s-lib-init-img ${K8S_LIB_INIT_IMG} --k8s-injector-img ${K8S_INJECTOR_IMG:-None} --k8s-cluster-img ${K8S_CLUSTER_IMG:-None} --report-run-url $CI_JOB_URL --report-environment ${REPORT_ENVIRONMENT} 2>&1 | tee "${CI_PROJECT_DIR}/run_output.log" - after_script: | - # Tag the job as part of the official system-tests workflow, first so it's done even if a later command fails. - # Skip the tag if the API key could not be fetched. - TAG_DD_API_KEY=$(read-system-tests-secret test_optimization dd_api_key) || TAG_DD_API_KEY="" - if [ -n "$TAG_DD_API_KEY" ]; then - DATADOG_SITE="${DATADOG_SITE:-datadoghq.com}" DATADOG_API_KEY="${TAG_DD_API_KEY}" \ - datadog-ci tag --level job --tags "ci.framework.name:system-tests" || echo "Failed to tag the job with ci.framework.name:system-tests" - else - echo "Failed to fetch the Datadog API key, skipping the ci.framework.name:system-tests tag" - fi - - kind delete clusters --all || true - mkdir -p reports - if [ "$CI_PROJECT_NAME" = "system-tests" ]; then - cp -R logs_*/ reports/ - else - cp -R /system-tests/logs_*/ reports/ - fi - if [ "$CI_COMMIT_BRANCH" = "main" ] && [ "$CI_PROJECT_NAME" = "system-tests" ]; then - export FP_IMPORT_URL=$(read-system-tests-secret fpd fp_import_url) - export FP_API_KEY=$(read-system-tests-secret fpd fp_api_key) - for folder in reports/logs*/ ; do - echo "Checking folder: ${folder}" - for filename in ./${folder}feature_parity.json; do - if [ -e ${filename} ] - then - echo "Processing report: ${filename}" - curl -X POST ${FP_IMPORT_URL} --fail --header "Content-Type: application/json" --header "FP_API_KEY: ${FP_API_KEY}" --data "@${filename}" --include - fi - done - done - fi - - # Render all report logs as collapsible sections + run summary + banner. - if [ "$CI_PROJECT_NAME" = "system-tests" ]; then - SCRIPTS_DIR="./utils/scripts" - else - SCRIPTS_DIR="/system-tests/utils/scripts" - fi - export SCENARIO_SUFIX=$(echo "${K8S_SCENARIO:-logs}" | tr "[:upper:]" "[:lower:]") - bash "${SCRIPTS_DIR}/gitlab_print_logs.sh" + after_script: + - bash "${SYSTEM_TESTS_DIR}/utils/scripts/gitlab_finalize_job.sh" --profile k8s retry: max: 2 when: diff --git a/tests/test_the_test/test_gitlab_finalize_job.py b/tests/test_the_test/test_gitlab_finalize_job.py new file mode 100644 index 00000000000..9a331157867 --- /dev/null +++ b/tests/test_the_test/test_gitlab_finalize_job.py @@ -0,0 +1,213 @@ +"""Tests for utils/scripts/gitlab_finalize_job.py and for the GitLab jobs that call it.""" + +import os +from pathlib import Path +import subprocess + +import yaml + +from utils import pytest, scenarios +from utils.scripts import gitlab_finalize_job as finalizer +from utils.scripts.ci_orchestrators.gitlab_exporter import build_ssi_gitlab_pipeline + +BASE_JOBS_PROFILES = { + ".base_docker_ssi_job": "docker", + ".base_job_onboarding_system_tests": "aws", + ".k8s_lib_injection_base": "k8s", +} + + +@pytest.fixture +def commands(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: + """Record the commands run by the steps instead of running them.""" + recorded: list[list[str]] = [] + + def run(command: list[str], env: dict[str, str] | None = None, cwd: Path | None = None) -> None: # noqa: ARG001 + recorded.append(command) + + monkeypatch.setattr(finalizer, "run", run) + monkeypatch.setattr(finalizer, "read_secret", lambda *path: "/".join(path)) + return recorded + + +def _job(profile: str, tmp_path: Path, **env: str) -> finalizer.Job: + return finalizer.Job( + profile=finalizer.PROFILES[profile], + env=env, + project_dir=tmp_path / "project", + system_tests_dir=tmp_path / "system-tests", + ) + + +def _write(path: Path, content: str = "") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + return path + + +@scenarios.test_the_test +def test_docker_profile_publishes_reports_on_system_tests_main(tmp_path: Path, commands: list[list[str]]): + logs = tmp_path / "system-tests" / "logs_docker_ssi" + _write(logs / "feature_parity.json", "{}") + _write(logs / "reportJunit.xml", "") + job = _job("docker", tmp_path, SCENARIO="DOCKER_SSI", CI_PROJECT_NAME="system-tests", CI_COMMIT_BRANCH="main") + + assert finalizer.finalize(job) == 0 + + reports = tmp_path / "project" / "reports" / "logs_docker_ssi" + assert (reports / "reportJunit.xml").is_file() + assert [command[:3] for command in commands] == [ + ["datadog-ci", "tag", "--level"], + ["curl", "-X", "POST"], + ["datadog-ci", "junit", "upload"], + ["bash", str(tmp_path / "system-tests" / "utils" / "scripts" / "gitlab_print_logs.sh")], + ] + assert f"@{reports / 'feature_parity.json'}" in commands[1] + + +@scenarios.test_the_test +def test_feature_parity_is_only_published_on_system_tests_main(tmp_path: Path, commands: list[list[str]]): + _write(tmp_path / "system-tests" / "logs_docker_ssi" / "feature_parity.json", "{}") + job = _job("docker", tmp_path, SCENARIO="DOCKER_SSI", CI_PROJECT_NAME="dd-trace-java", CI_COMMIT_BRANCH="main") + + assert finalizer.finalize(job) == 0 + assert not [command for command in commands if command[0] == "curl"] + + +@scenarios.test_the_test +def test_aws_profile_uploads_to_s3_and_keeps_large_files_out_of_the_artifacts( + tmp_path: Path, commands: list[list[str]], monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setenv("HOME", str(tmp_path / "home")) + logs = tmp_path / "system-tests" / "logs_simple_installer_auto_injection" + _write(logs / "vm_feature_parity.json", "{}") + _write(logs / "feature_parity.json", "{}") + with (logs / "large.log").open("wb") as large_file: + large_file.truncate(finalizer.MAX_ARTIFACT_FILE_SIZE + 1) + job = _job( + "aws", + tmp_path, + SCENARIO="SIMPLE_INSTALLER_AUTO_INJECTION", + VM="Ubuntu_22_amd64", + WEBLOG="test-app-java", + CI_PIPELINE_ID="42", + CI_PROJECT_NAME="system-tests", + CI_COMMIT_BRANCH="main", + ) + + assert finalizer.finalize(job) == 0 + + prefix = "42/logs_simple_installer_auto_injection/Ubuntu_22_amd64_test-app-java/" + assert commands[1][:5] == ["aws", "s3", "cp", str(logs), f"s3://system-tests-aws-ssi-apm/{prefix}"] + assert job.aws_console_url.endswith(f"prefix={prefix}") + assert (tmp_path / "home" / ".aws" / "config").read_text() == "apm-ecosystems-reliability-profile/config\n" + reports = tmp_path / "project" / "reports" / "logs_simple_installer_auto_injection" + assert not (reports / "large.log").exists() + uploaded = [command[-1] for command in commands if command[0] == "curl"] + assert uploaded == [f"@{reports / 'vm_feature_parity.json'}"] + + +@scenarios.test_the_test +def test_k8s_profile_deletes_clusters_and_collects_every_log_folder(tmp_path: Path, commands: list[list[str]]): + _write(tmp_path / "system-tests" / "logs_k8s_lib_injection" / "tests.log") + _write(tmp_path / "system-tests" / "logs_other" / "tests.log") + job = _job("k8s", tmp_path, K8S_SCENARIO="K8S_LIB_INJECTION", CI_PROJECT_NAME="dd-trace-php") + + assert finalizer.finalize(job) == 0 + + assert commands[1] == ["kind", "delete", "clusters", "--all"] + reports = tmp_path / "project" / "reports" + assert sorted(path.name for path in reports.iterdir()) == ["logs_k8s_lib_injection", "logs_other"] + + +@scenarios.test_the_test +def test_log_folders_with_sockets_are_collected(tmp_path: Path, commands: list[list[str]]): + logs = tmp_path / "system-tests" / "logs_docker_ssi" + _write(logs / "interfaces" / "library" / "00001.json", "{}") + os.mkfifo(logs / "interfaces" / "agent.fifo") + job = _job("docker", tmp_path, SCENARIO="DOCKER_SSI") + + assert finalizer.finalize(job) == 0 + + reports = tmp_path / "project" / "reports" / "logs_docker_ssi" / "interfaces" + assert (reports / "library" / "00001.json").is_file() + assert not (reports / "agent.fifo").exists() + assert commands[-1][0] == "bash" + + +@scenarios.test_the_test +def test_a_failing_step_does_not_stop_the_next_ones( + tmp_path: Path, commands: list[list[str]], monkeypatch: pytest.MonkeyPatch +): + def run(command: list[str], env: dict[str, str] | None = None, cwd: Path | None = None) -> None: # noqa: ARG001 + if command[0] == "datadog-ci": + raise subprocess.CalledProcessError(1, command) + commands.append(command) + + monkeypatch.setattr(finalizer, "run", run) + job = _job("docker", tmp_path, SCENARIO="DOCKER_SSI") + + assert finalizer.finalize(job) == 1 + + assert [command[0] for command in commands] == ["bash"] + + +@scenarios.test_the_test +def test_logs_are_printed_even_without_log_folder(tmp_path: Path, commands: list[list[str]]): + job = _job("docker", tmp_path, SCENARIO="DOCKER_SSI") + + assert finalizer.finalize(job) == 1 + + assert commands[-1][0] == "bash" + assert (tmp_path / "project" / "reports").is_dir() + + +@scenarios.test_the_test +def test_entry_point_runs_the_finalizer(): + result = subprocess.run( + ["bash", "utils/scripts/gitlab_finalize_job.sh", "--help"], check=True, capture_output=True, text=True + ) + assert "--profile {aws,docker,k8s}" in result.stdout + + +@scenarios.test_the_test +def test_base_jobs_call_the_finalizer_from_the_system_tests_checkout(): + pipeline = yaml.safe_load(Path(".gitlab/ssi_gitlab-ci.yml").read_text()) + + for base_job, profile in BASE_JOBS_PROFILES.items(): + assert pipeline[base_job]["variables"]["SYSTEM_TESTS_DIR"] == "$CI_PROJECT_DIR" + assert pipeline[base_job]["after_script"] == [ + f'bash "${{SYSTEM_TESTS_DIR}}/utils/scripts/gitlab_finalize_job.sh" --profile {profile}' + ] + + +@pytest.mark.parametrize( + ("project", "expected_dirs"), + [ + ("system-tests", dict.fromkeys(BASE_JOBS_PROFILES, "$CI_PROJECT_DIR")), + ( + "dd-trace-php", + { + ".base_docker_ssi_job": "/system-tests", + ".base_job_onboarding_system_tests": "$CI_PROJECT_DIR/system-tests", + ".k8s_lib_injection_base": "/system-tests", + }, + ), + ], +) +@scenarios.test_the_test +def test_generated_jobs_point_to_the_checkout_they_run_in( + project: str, expected_dirs: dict[str, str], monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setenv("CI_PROJECT_NAME", project) + for name in ("SYSTEM_TESTS_REF", "K8S_INJECTOR_IMG", "K8S_LIB_INIT_IMG", "K8S_WEBLOG_IMG"): + monkeypatch.delenv(name, raising=False) + matrix_data: dict[str, dict] = { + "aws_ssi_scenario_defs": {"SIMPLE_INSTALLER_AUTO_INJECTION": {"test-app-php": ["Ubuntu_22_amd64"]}}, + "dockerssi_scenario_defs": {"DOCKER_SSI": {"php-app": [{"ubuntu:22.04": ["8.3"], "arch": "linux/amd64"}]}}, + "libinjection_scenario_defs": {"K8S_LIB_INJECTION": ["dd-lib-php-init-test-app"]}, + } + + pipeline = build_ssi_gitlab_pipeline("php", matrix_data, "dev") + + assert {job: pipeline[job]["variables"]["SYSTEM_TESTS_DIR"] for job in expected_dirs} == expected_dirs diff --git a/utils/scripts/ci_orchestrators/gitlab_exporter.py b/utils/scripts/ci_orchestrators/gitlab_exporter.py index 273cf14cef7..c47e73f0534 100644 --- a/utils/scripts/ci_orchestrators/gitlab_exporter.py +++ b/utils/scripts/ci_orchestrators/gitlab_exporter.py @@ -68,6 +68,15 @@ def print_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_enviro def print_ssi_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_environment: str) -> None: + result_pipeline = build_ssi_gitlab_pipeline(language, matrix_data, ci_environment) + pipeline_yml = yaml.dump(result_pipeline, sort_keys=False, default_flow_style=False) + output_file = f"{language}_ssi_gitlab_pipeline.yml" + with open(output_file, "w") as file: + file.write(pipeline_yml) + print("Pipeline file generated: ", output_file) + + +def build_ssi_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_environment: str) -> dict: result_pipeline = {} # type: dict result_pipeline["workflow"] = {"name": f"{language} SSI"} result_pipeline["include"] = [] @@ -99,10 +108,15 @@ def print_ssi_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_en result_pipeline[".base_job_onboarding_system_tests"]["script"].insert( 0, "git clone https://github.com/DataDog/system-tests.git system-tests" ) + result_pipeline[".base_job_onboarding_system_tests"]["variables"]["SYSTEM_TESTS_DIR"] = ( + "$CI_PROJECT_DIR/system-tests" + ) print_aws_gitlab_pipeline(language, matrix_data["aws_ssi_scenario_defs"], ci_environment, result_pipeline) if matrix_data["dockerssi_scenario_defs"]: # Copy the base job for the docker ssi system tests result_pipeline[".base_docker_ssi_job"] = pipeline_data[".base_docker_ssi_job"] + if os.getenv("CI_PROJECT_NAME") != "system-tests": + result_pipeline[".base_docker_ssi_job"]["variables"]["SYSTEM_TESTS_DIR"] = "/system-tests" print_docker_ssi_gitlab_pipeline( language, matrix_data["dockerssi_scenario_defs"], ci_environment, result_pipeline ) @@ -116,14 +130,11 @@ def print_ssi_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_en ) result_pipeline[".k8s_lib_injection_base"]["script"].insert(0, "git pull") result_pipeline[".k8s_lib_injection_base"]["script"].insert(0, "cd /system-tests") + result_pipeline[".k8s_lib_injection_base"]["variables"]["SYSTEM_TESTS_DIR"] = "/system-tests" print_k8s_gitlab_pipeline(language, matrix_data["libinjection_scenario_defs"], ci_environment, result_pipeline) - pipeline_yml = yaml.dump(result_pipeline, sort_keys=False, default_flow_style=False) - output_file = f"{language}_ssi_gitlab_pipeline.yml" - with open(output_file, "w") as file: - file.write(pipeline_yml) - print("Pipeline file generated: ", output_file) + return result_pipeline def get_k8s_scenario_components( diff --git a/utils/scripts/gitlab_finalize_job.py b/utils/scripts/gitlab_finalize_job.py new file mode 100644 index 00000000000..dcacc93b667 --- /dev/null +++ b/utils/scripts/gitlab_finalize_job.py @@ -0,0 +1,255 @@ +"""Finalize a Docker SSI, AWS SSI or K8s lib injection GitLab job. + +Runs in the after_script of those jobs, in system-tests and in the repositories that launch them. +Each profile is an ordered list of steps. A failing step is reported and the next steps still run, +so the job is always tagged and its logs are always collected and printed. + +To add a feature, write a step function and add it to the profiles that need it. + +Uses only the standard library: the system-tests virtualenv does not exist when the job fails before +./build.sh. Generated pipelines of older system-tests commits call this script with --profile, so keep +that option backward compatible. +""" + +# The wrapper falls back to the system python3, which may predate the annotation syntax used here. +from __future__ import annotations + +import argparse +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from functools import cache +import os +from pathlib import Path +import shutil +import subprocess +import sys + +SYSTEM_TESTS_DIR = Path(__file__).resolve().parents[2] + +AWS_LOGS_BUCKET = "system-tests-aws-ssi-apm" +MAX_ARTIFACT_FILE_SIZE = 50 * 1024 * 1024 + + +@dataclass +class Job: + """What the steps know about the job being finalized.""" + + profile: Profile + env: Mapping[str, str] + project_dir: Path + system_tests_dir: Path + aws_console_url: str = "" + + @property + def scenario(self) -> str: + return self.env.get(self.profile.scenario_variable, "logs").lower() + + @property + def reports_dir(self) -> Path: + return self.project_dir / "reports" + + @property + def logs_pattern(self) -> str: + return self.profile.logs_pattern.format(scenario=self.scenario) + + @property + def log_folders(self) -> list[Path]: + return sorted(folder for folder in self.system_tests_dir.glob(self.logs_pattern) if folder.is_dir()) + + @property + def is_system_tests_main(self) -> bool: + return self.env.get("CI_PROJECT_NAME") == "system-tests" and self.env.get("CI_COMMIT_BRANCH") == "main" + + def datadog_env(self) -> dict[str, str]: + return { + "DATADOG_SITE": self.env.get("DATADOG_SITE", "datadoghq.com"), + "DATADOG_API_KEY": read_secret("test_optimization", "dd_api_key"), + } + + +Step = Callable[[Job], None] + + +def run(command: list[str], env: Mapping[str, str] | None = None, cwd: Path | None = None) -> None: + subprocess.run(command, check=True, cwd=cwd, env=None if env is None else {**os.environ, **env}) + + +@cache +def read_secret(*path: str) -> str: + result = subprocess.run(["read-system-tests-secret", *path], check=True, stdout=subprocess.PIPE, text=True) + return result.stdout.strip() + + +def tag_official_workflow(job: Job) -> None: + run(["datadog-ci", "tag", "--level", "job", "--tags", "ci.framework.name:system-tests"], env=job.datadog_env()) + + +def delete_kind_clusters(_job: Job) -> None: + run(["kind", "delete", "clusters", "--all"]) + + +def upload_logs_to_s3(job: Job) -> None: + aws_config = Path.home() / ".aws" / "config" + aws_config.parent.mkdir(parents=True, exist_ok=True) + aws_config.write_text(read_secret("apm-ecosystems-reliability-profile", "config") + "\n") + + for folder in job.log_folders: + pipeline_id = job.env.get("CI_PIPELINE_ID", "datadog-local") + prefix = f"{pipeline_id}/{folder.name}/{job.env['VM']}_{job.env['WEBLOG']}/" + run( + ["aws", "s3", "cp", str(folder), f"s3://{AWS_LOGS_BUCKET}/{prefix}", "--recursive", "--quiet"], + env={"AWS_PROFILE": "apm-ecosystems-reliability-ci"}, + ) + job.aws_console_url = ( + f"https://s3.console.aws.amazon.com/s3/buckets/{AWS_LOGS_BUCKET}?region=us-east-1&prefix={prefix}" + ) + + +def delete_large_log_files(job: Job) -> None: + for folder in job.log_folders: + for file in sorted(folder.rglob("*")): + if file.is_file() and not file.is_symlink() and file.stat().st_size > MAX_ARTIFACT_FILE_SIZE: + print(f"Deleting large file ({file.stat().st_size // (1024 * 1024)}M): {file}") + file.unlink() + + +def _sockets_and_fifos(directory: str, names: list[str]) -> set[str]: + return {name for name in names if Path(directory, name).is_socket() or Path(directory, name).is_fifo()} + + +def collect_logs(job: Job) -> None: + job.reports_dir.mkdir(parents=True, exist_ok=True) + if not job.log_folders: + raise FileNotFoundError(f"No {job.logs_pattern} folder in {job.system_tests_dir}") + for folder in job.log_folders: + shutil.copytree( + folder, job.reports_dir / folder.name, symlinks=True, ignore=_sockets_and_fifos, dirs_exist_ok=True + ) + + +def publish_feature_parity(job: Job) -> None: + if not job.is_system_tests_main: + return + url = read_secret("fpd", "fp_import_url") + api_key = read_secret("fpd", "fp_api_key") + for report in sorted(job.reports_dir.glob(f"logs*/{job.profile.feature_parity_pattern}")): + print(f"Processing report: {report}") + run( + [ + "curl", + "-X", + "POST", + url, + "--fail", + "--include", + "--header", + "Content-Type: application/json", + "--header", + f"FP_API_KEY: {api_key}", + "--data", + f"@{report}", + ] + ) + + +def upload_junit_reports(job: Job) -> None: + reports = sorted(job.reports_dir.glob("logs*/reportJunit.xml")) + if not reports: + print("No reports/logs*/reportJunit.xml found; skipping Test Optimization upload") + return + run( + [ + "datadog-ci", + "junit", + "upload", + *[str(report) for report in reports], + "--service", + "system-tests", + "--env", + "ci", + "--verbose", + "--xpath-tag", + "test.codeowners=/testcase/properties/property[@name='test.codeowners']", + ], + env={ + **job.datadog_env(), + "DD_TAGS": f"test.configuration.ci_environment:{job.env.get('CI_ENVIRONMENT', 'prod')}", + }, + ) + + +def print_logs(job: Job) -> None: + run( + ["bash", str(job.system_tests_dir / "utils" / "scripts" / "gitlab_print_logs.sh")], + env={"SCENARIO_SUFIX": job.scenario, "AWS_CONSOLE_URL": job.aws_console_url}, + cwd=job.project_dir, + ) + + +@dataclass(frozen=True) +class Profile: + scenario_variable: str + logs_pattern: str + feature_parity_pattern: str + steps: tuple[Step, ...] + + +PROFILES: dict[str, Profile] = { + "docker": Profile( + scenario_variable="SCENARIO", + logs_pattern="logs_{scenario}", + feature_parity_pattern="feature_parity.json", + steps=(tag_official_workflow, collect_logs, publish_feature_parity, upload_junit_reports, print_logs), + ), + "aws": Profile( + scenario_variable="SCENARIO", + logs_pattern="logs_{scenario}", + feature_parity_pattern="*_feature_parity.json", + steps=( + tag_official_workflow, + upload_logs_to_s3, + delete_large_log_files, + collect_logs, + publish_feature_parity, + print_logs, + ), + ), + "k8s": Profile( + scenario_variable="K8S_SCENARIO", + logs_pattern="logs_*", + feature_parity_pattern="feature_parity.json", + steps=(tag_official_workflow, delete_kind_clusters, collect_logs, publish_feature_parity, print_logs), + ), +} + + +def finalize(job: Job) -> int: + failed_steps: list[str] = [] + for step in job.profile.steps: + try: + step(job) + except Exception as error: + print(f"❌ Step {step.__name__} failed: {error}", flush=True) + failed_steps.append(step.__name__) + if failed_steps: + print(f"❌ Failed steps: {', '.join(failed_steps)}") + return 1 + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Finalize a Docker SSI, AWS SSI or K8s lib injection GitLab job.") + parser.add_argument("--profile", required=True, choices=sorted(PROFILES)) + args = parser.parse_args() + + job = Job( + profile=PROFILES[args.profile], + env=os.environ, + project_dir=Path(os.environ.get("CI_PROJECT_DIR", Path.cwd())), + system_tests_dir=SYSTEM_TESTS_DIR, + ) + return finalize(job) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/scripts/gitlab_finalize_job.sh b/utils/scripts/gitlab_finalize_job.sh new file mode 100755 index 00000000000..325d3e436fc --- /dev/null +++ b/utils/scripts/gitlab_finalize_job.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# after_script entry point of the Docker SSI, AWS SSI and K8s lib injection GitLab jobs. +# All the logic lives in gitlab_finalize_job.py. +# +# Usage: +# gitlab_finalize_job.sh --profile + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SYSTEM_TESTS_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +# The virtualenv is missing when the job fails before ./build.sh creates it. +PYTHON="${SYSTEM_TESTS_DIR}/venv/bin/python" +if [[ ! -x "${PYTHON}" ]]; then + PYTHON="python3" +fi + +exec "${PYTHON}" -u "${SCRIPT_DIR}/gitlab_finalize_job.py" "$@"