diff --git a/.gitlab/ssi_gitlab-ci.yml b/.gitlab/ssi_gitlab-ci.yml
index 0c66ced2557..623ff1cdfb9 100644
--- a/.gitlab/ssi_gitlab-ci.yml
+++ b/.gitlab/ssi_gitlab-ci.yml
@@ -26,61 +26,10 @@ ssi_tests:
variables:
# Force gitlab to keep the exit code as 3 if the job fails with exit code 3
FF_USE_NEW_BASH_EVAL_STRATEGY: true
- after_script: |
- export DATADOG_SITE="${DATADOG_SITE:-datadoghq.com}"
- DATADOG_API_KEY=$(read-system-tests-secret test_optimization dd_api_key) || DATADOG_API_KEY=""
- export DATADOG_API_KEY
- # Tag the job as part of the official system-tests workflow, first so it's done even if a later command fails.
- # Skip the tag if the API key could not be fetched.
- if [ -n "$DATADOG_API_KEY" ]; then
- datadog-ci tag --level job --tags "ci.framework.name:system-tests" || echo "Failed to tag the job with ci.framework.name:system-tests"
- else
- echo "Failed to fetch the Datadog API key, skipping the ci.framework.name:system-tests tag"
- fi
-
- SCENARIO_SUFIX=$(echo "$SCENARIO" | tr "[:upper:]" "[:lower:]")
- mkdir -p reports/
- if [ "$CI_PROJECT_NAME" = "system-tests" ]; then
- cp -R logs_"${SCENARIO_SUFIX}" reports/
- else
- cp -R /system-tests/logs_"${SCENARIO_SUFIX}" reports/
- fi
- if [ "$CI_COMMIT_BRANCH" = "main" ] && [ "$CI_PROJECT_NAME" = "system-tests" ]; then
- export FP_IMPORT_URL=$(read-system-tests-secret fpd fp_import_url)
- export FP_API_KEY=$(read-system-tests-secret fpd fp_api_key)
- for folder in reports/logs*/ ; do
- echo "Checking folder: ${folder}"
- for filename in ./${folder}feature_parity.json; do
- if [ -e ${filename} ]
- then
- echo "Processing report: ${filename}"
- curl -X POST ${FP_IMPORT_URL} --fail --header "Content-Type: application/json" --header "FP_API_KEY: ${FP_API_KEY}" --data "@${filename}" --include
- fi
- done
- done
- fi
-
- # Push JUnit results to Datadog Test Optimization
- export DD_TAGS="test.configuration.ci_environment:${CI_ENVIRONMENT:-prod}"
- if compgen -G "reports/logs*/reportJunit.xml" > /dev/null; then
- datadog-ci junit upload \
- reports/logs*/reportJunit.xml \
- --service system-tests \
- --env ci \
- --verbose \
- --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']"
- else
- echo "No reports/logs*/reportJunit.xml found; skipping Test Optimization upload"
- fi
-
- # Render all report logs as collapsible sections + run summary + banner.
- if [ "$CI_PROJECT_NAME" = "system-tests" ]; then
- SCRIPTS_DIR="./utils/scripts"
- else
- SCRIPTS_DIR="/system-tests/utils/scripts"
- fi
- export SCENARIO_SUFIX
- bash "${SCRIPTS_DIR}/gitlab_print_logs.sh"
+ # gitlab_exporter.py points it to the system-tests checkout when another repository runs the job.
+ SYSTEM_TESTS_DIR: $CI_PROJECT_DIR
+ after_script:
+ - bash "${SYSTEM_TESTS_DIR}/utils/scripts/gitlab_finalize_job.sh" --profile docker
retry:
max: 2
when:
@@ -108,6 +57,8 @@ ssi_tests:
variables:
# Force gitlab to keep the exit code as 3 if the job fails with exit code 3
FF_USE_NEW_BASH_EVAL_STRATEGY: true
+ # gitlab_exporter.py points it to the system-tests checkout when another repository runs the job.
+ SYSTEM_TESTS_DIR: $CI_PROJECT_DIR
script:
- |
if [[ -n "${DD_INSTALL_SCRIPT_VERSION}" ]]; then
@@ -124,80 +75,8 @@ ssi_tests:
# Run the tests and capture stdout+stderr to a file
set -o pipefail
timeout 3000 ./run.sh "$SCENARIO" --vm-weblog "${WEBLOG}" --vm-env "${ONBOARDING_FILTER_ENV}" --vm-library "${TEST_LIBRARY}" --vm-provider aws --vm-default-vms All --vm-only "${VM}" --report-run-url "${CI_JOB_URL}" --report-environment "${ONBOARDING_FILTER_ENV}" 2>&1 | tee "${CI_PROJECT_DIR}/run_output.log"
- after_script: |
- # Tag the job as part of the official system-tests workflow, first so it's done even if a later command fails.
- # The image does not ship datadog-ci, so the standalone binary is downloaded.
- # Skip the tag if the API key could not be fetched.
- DATADOG_API_KEY=$(read-system-tests-secret test_optimization dd_api_key) || DATADOG_API_KEY=""
- if [ -n "$DATADOG_API_KEY" ]; then
- DATADOG_SITE="${DATADOG_SITE:-datadoghq.com}" DATADOG_API_KEY="${DATADOG_API_KEY}" datadog-ci tag --level job --tags "ci.framework.name:system-tests" \
- || echo "Failed to tag the job with ci.framework.name:system-tests"
- else
- echo "Failed to fetch the Datadog API key, skipping the ci.framework.name:system-tests tag"
- fi
-
- # Only fetch FP params on main in system-tests repo (use default CI creds)
- if [ "$CI_COMMIT_BRANCH" = "main" ] && [ "$CI_PROJECT_NAME" = "system-tests" ]; then
- export FP_IMPORT_URL=$(read-system-tests-secret fpd fp_import_url)
- export FP_API_KEY=$(read-system-tests-secret fpd fp_api_key)
- fi
- # Re-establish AWS credentials for S3 operations (after_script runs in new shell session)
- echo "Setting up AWS credentials for after_script..."
- mkdir -p ~/.aws
-
- # Clean up any existing config and recreate it
- rm -f ~/.aws/config
- read-system-tests-secret apm-ecosystems-reliability-profile config > ~/.aws/config
- export AWS_PROFILE=apm-ecosystems-reliability-ci
-
- SCENARIO_SUFIX=$(echo "$SCENARIO" | tr "[:upper:]" "[:lower:]")
- mkdir -p reports/
- # Upload logs to S3 and delete large files
- # When this template is included from downstream repos (dd-trace-*), the
- # system-tests checkout lives under a `system-tests/` subdirectory, so
- # all relative paths into the system-tests repo need that prefix.
- if [ "$CI_PROJECT_NAME" = "system-tests" ]; then
- LOGS_FOLDER="logs_${SCENARIO_SUFIX}"
- SCRIPTS_DIR="./utils/scripts"
- else
- LOGS_FOLDER="system-tests/logs_${SCENARIO_SUFIX}"
- SCRIPTS_DIR="./system-tests/utils/scripts"
- fi
- AWS_CONSOLE_URL=""
- if [ -d "$LOGS_FOLDER" ]; then
- EXECUTION_UNIQUE_IDENTIFIER="${CI_PIPELINE_ID:-datadog-local}"
- LAST_DIRECTORY=$(basename "$LOGS_FOLDER")
- S3_BUCKET="system-tests-aws-ssi-apm"
- S3_DESTINATION="s3://${S3_BUCKET}/${EXECUTION_UNIQUE_IDENTIFIER}/${LAST_DIRECTORY}/${VM}_${WEBLOG}/"
- AWS_CONSOLE_URL="https://s3.console.aws.amazon.com/s3/buckets/${S3_BUCKET}?region=us-east-1&prefix=${EXECUTION_UNIQUE_IDENTIFIER}/${LAST_DIRECTORY}/${VM}_${WEBLOG}/"
-
- aws s3 cp "$LOGS_FOLDER" "$S3_DESTINATION" --recursive --quiet || echo "Failed to upload logs to S3"
-
- # Delete large files (>50MB) to avoid CI/CD archiving
- find "$LOGS_FOLDER" -type f -size +50M -exec sh -c 'echo "Deleting large file ($(du -h "$1" | cut -f1)): $1" && rm "$1"' _ {} \; || echo "Failed to delete some large files"
- fi
-
- # Logs to be archived by gitlab
- cp -R "$LOGS_FOLDER" reports/
-
- # Feature parity
- if [ "$CI_COMMIT_BRANCH" = "main" ] && [ "$CI_PROJECT_NAME" = "system-tests" ]; then
- for folder in reports/logs*/ ; do
- echo "Checking folder: ${folder}"
- for filename in ./${folder}*_feature_parity.json; do
- if [ -e ${filename} ]
- then
- echo "Processing report: ${filename}"
- curl -X POST ${FP_IMPORT_URL} --fail --header "Content-Type: application/json" --header "FP_API_KEY: ${FP_API_KEY}" --data "@${filename}" --include
- fi
- done
- done
- fi
-
- # Render all report logs as collapsible sections + run summary + banner.
- export SCENARIO_SUFIX AWS_CONSOLE_URL
- bash "${SCRIPTS_DIR}/gitlab_print_logs.sh"
-
+ after_script:
+ - bash "${SYSTEM_TESTS_DIR}/utils/scripts/gitlab_finalize_job.sh" --profile aws
retry:
max: 2
when:
@@ -227,6 +106,8 @@ ssi_tests:
variables:
# Force gitlab to keep the exit code as 3 if the job fails with exit code 3
FF_USE_NEW_BASH_EVAL_STRATEGY: true
+ # gitlab_exporter.py points it to the system-tests checkout when another repository runs the job.
+ SYSTEM_TESTS_DIR: $CI_PROJECT_DIR
script:
- |
# Wrap the build steps (runner image, venv, ECR login, weblog image) in a
@@ -254,47 +135,8 @@ ssi_tests:
set -o pipefail
./run.sh ${K8S_SCENARIO} --k8s-library ${TEST_LIBRARY} --k8s-weblog ${K8S_WEBLOG} --k8s-weblog-img ${K8S_WEBLOG_IMG}:${TAG_NAME} --k8s-lib-init-img ${K8S_LIB_INIT_IMG} --k8s-injector-img ${K8S_INJECTOR_IMG:-None} --k8s-cluster-img ${K8S_CLUSTER_IMG:-None} --report-run-url $CI_JOB_URL --report-environment ${REPORT_ENVIRONMENT} 2>&1 | tee "${CI_PROJECT_DIR}/run_output.log"
- after_script: |
- # Tag the job as part of the official system-tests workflow, first so it's done even if a later command fails.
- # Skip the tag if the API key could not be fetched.
- TAG_DD_API_KEY=$(read-system-tests-secret test_optimization dd_api_key) || TAG_DD_API_KEY=""
- if [ -n "$TAG_DD_API_KEY" ]; then
- DATADOG_SITE="${DATADOG_SITE:-datadoghq.com}" DATADOG_API_KEY="${TAG_DD_API_KEY}" \
- datadog-ci tag --level job --tags "ci.framework.name:system-tests" || echo "Failed to tag the job with ci.framework.name:system-tests"
- else
- echo "Failed to fetch the Datadog API key, skipping the ci.framework.name:system-tests tag"
- fi
-
- kind delete clusters --all || true
- mkdir -p reports
- if [ "$CI_PROJECT_NAME" = "system-tests" ]; then
- cp -R logs_*/ reports/
- else
- cp -R /system-tests/logs_*/ reports/
- fi
- if [ "$CI_COMMIT_BRANCH" = "main" ] && [ "$CI_PROJECT_NAME" = "system-tests" ]; then
- export FP_IMPORT_URL=$(read-system-tests-secret fpd fp_import_url)
- export FP_API_KEY=$(read-system-tests-secret fpd fp_api_key)
- for folder in reports/logs*/ ; do
- echo "Checking folder: ${folder}"
- for filename in ./${folder}feature_parity.json; do
- if [ -e ${filename} ]
- then
- echo "Processing report: ${filename}"
- curl -X POST ${FP_IMPORT_URL} --fail --header "Content-Type: application/json" --header "FP_API_KEY: ${FP_API_KEY}" --data "@${filename}" --include
- fi
- done
- done
- fi
-
- # Render all report logs as collapsible sections + run summary + banner.
- if [ "$CI_PROJECT_NAME" = "system-tests" ]; then
- SCRIPTS_DIR="./utils/scripts"
- else
- SCRIPTS_DIR="/system-tests/utils/scripts"
- fi
- export SCENARIO_SUFIX=$(echo "${K8S_SCENARIO:-logs}" | tr "[:upper:]" "[:lower:]")
- bash "${SCRIPTS_DIR}/gitlab_print_logs.sh"
+ after_script:
+ - bash "${SYSTEM_TESTS_DIR}/utils/scripts/gitlab_finalize_job.sh" --profile k8s
retry:
max: 2
when:
diff --git a/tests/test_the_test/test_gitlab_finalize_job.py b/tests/test_the_test/test_gitlab_finalize_job.py
new file mode 100644
index 00000000000..9a331157867
--- /dev/null
+++ b/tests/test_the_test/test_gitlab_finalize_job.py
@@ -0,0 +1,213 @@
+"""Tests for utils/scripts/gitlab_finalize_job.py and for the GitLab jobs that call it."""
+
+import os
+from pathlib import Path
+import subprocess
+
+import yaml
+
+from utils import pytest, scenarios
+from utils.scripts import gitlab_finalize_job as finalizer
+from utils.scripts.ci_orchestrators.gitlab_exporter import build_ssi_gitlab_pipeline
+
+BASE_JOBS_PROFILES = {
+ ".base_docker_ssi_job": "docker",
+ ".base_job_onboarding_system_tests": "aws",
+ ".k8s_lib_injection_base": "k8s",
+}
+
+
+@pytest.fixture
+def commands(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]:
+ """Record the commands run by the steps instead of running them."""
+ recorded: list[list[str]] = []
+
+ def run(command: list[str], env: dict[str, str] | None = None, cwd: Path | None = None) -> None: # noqa: ARG001
+ recorded.append(command)
+
+ monkeypatch.setattr(finalizer, "run", run)
+ monkeypatch.setattr(finalizer, "read_secret", lambda *path: "/".join(path))
+ return recorded
+
+
+def _job(profile: str, tmp_path: Path, **env: str) -> finalizer.Job:
+ return finalizer.Job(
+ profile=finalizer.PROFILES[profile],
+ env=env,
+ project_dir=tmp_path / "project",
+ system_tests_dir=tmp_path / "system-tests",
+ )
+
+
+def _write(path: Path, content: str = "") -> Path:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(content)
+ return path
+
+
+@scenarios.test_the_test
+def test_docker_profile_publishes_reports_on_system_tests_main(tmp_path: Path, commands: list[list[str]]):
+ logs = tmp_path / "system-tests" / "logs_docker_ssi"
+ _write(logs / "feature_parity.json", "{}")
+ _write(logs / "reportJunit.xml", "")
+ job = _job("docker", tmp_path, SCENARIO="DOCKER_SSI", CI_PROJECT_NAME="system-tests", CI_COMMIT_BRANCH="main")
+
+ assert finalizer.finalize(job) == 0
+
+ reports = tmp_path / "project" / "reports" / "logs_docker_ssi"
+ assert (reports / "reportJunit.xml").is_file()
+ assert [command[:3] for command in commands] == [
+ ["datadog-ci", "tag", "--level"],
+ ["curl", "-X", "POST"],
+ ["datadog-ci", "junit", "upload"],
+ ["bash", str(tmp_path / "system-tests" / "utils" / "scripts" / "gitlab_print_logs.sh")],
+ ]
+ assert f"@{reports / 'feature_parity.json'}" in commands[1]
+
+
+@scenarios.test_the_test
+def test_feature_parity_is_only_published_on_system_tests_main(tmp_path: Path, commands: list[list[str]]):
+ _write(tmp_path / "system-tests" / "logs_docker_ssi" / "feature_parity.json", "{}")
+ job = _job("docker", tmp_path, SCENARIO="DOCKER_SSI", CI_PROJECT_NAME="dd-trace-java", CI_COMMIT_BRANCH="main")
+
+ assert finalizer.finalize(job) == 0
+ assert not [command for command in commands if command[0] == "curl"]
+
+
+@scenarios.test_the_test
+def test_aws_profile_uploads_to_s3_and_keeps_large_files_out_of_the_artifacts(
+ tmp_path: Path, commands: list[list[str]], monkeypatch: pytest.MonkeyPatch
+):
+ monkeypatch.setenv("HOME", str(tmp_path / "home"))
+ logs = tmp_path / "system-tests" / "logs_simple_installer_auto_injection"
+ _write(logs / "vm_feature_parity.json", "{}")
+ _write(logs / "feature_parity.json", "{}")
+ with (logs / "large.log").open("wb") as large_file:
+ large_file.truncate(finalizer.MAX_ARTIFACT_FILE_SIZE + 1)
+ job = _job(
+ "aws",
+ tmp_path,
+ SCENARIO="SIMPLE_INSTALLER_AUTO_INJECTION",
+ VM="Ubuntu_22_amd64",
+ WEBLOG="test-app-java",
+ CI_PIPELINE_ID="42",
+ CI_PROJECT_NAME="system-tests",
+ CI_COMMIT_BRANCH="main",
+ )
+
+ assert finalizer.finalize(job) == 0
+
+ prefix = "42/logs_simple_installer_auto_injection/Ubuntu_22_amd64_test-app-java/"
+ assert commands[1][:5] == ["aws", "s3", "cp", str(logs), f"s3://system-tests-aws-ssi-apm/{prefix}"]
+ assert job.aws_console_url.endswith(f"prefix={prefix}")
+ assert (tmp_path / "home" / ".aws" / "config").read_text() == "apm-ecosystems-reliability-profile/config\n"
+ reports = tmp_path / "project" / "reports" / "logs_simple_installer_auto_injection"
+ assert not (reports / "large.log").exists()
+ uploaded = [command[-1] for command in commands if command[0] == "curl"]
+ assert uploaded == [f"@{reports / 'vm_feature_parity.json'}"]
+
+
+@scenarios.test_the_test
+def test_k8s_profile_deletes_clusters_and_collects_every_log_folder(tmp_path: Path, commands: list[list[str]]):
+ _write(tmp_path / "system-tests" / "logs_k8s_lib_injection" / "tests.log")
+ _write(tmp_path / "system-tests" / "logs_other" / "tests.log")
+ job = _job("k8s", tmp_path, K8S_SCENARIO="K8S_LIB_INJECTION", CI_PROJECT_NAME="dd-trace-php")
+
+ assert finalizer.finalize(job) == 0
+
+ assert commands[1] == ["kind", "delete", "clusters", "--all"]
+ reports = tmp_path / "project" / "reports"
+ assert sorted(path.name for path in reports.iterdir()) == ["logs_k8s_lib_injection", "logs_other"]
+
+
+@scenarios.test_the_test
+def test_log_folders_with_sockets_are_collected(tmp_path: Path, commands: list[list[str]]):
+ logs = tmp_path / "system-tests" / "logs_docker_ssi"
+ _write(logs / "interfaces" / "library" / "00001.json", "{}")
+ os.mkfifo(logs / "interfaces" / "agent.fifo")
+ job = _job("docker", tmp_path, SCENARIO="DOCKER_SSI")
+
+ assert finalizer.finalize(job) == 0
+
+ reports = tmp_path / "project" / "reports" / "logs_docker_ssi" / "interfaces"
+ assert (reports / "library" / "00001.json").is_file()
+ assert not (reports / "agent.fifo").exists()
+ assert commands[-1][0] == "bash"
+
+
+@scenarios.test_the_test
+def test_a_failing_step_does_not_stop_the_next_ones(
+ tmp_path: Path, commands: list[list[str]], monkeypatch: pytest.MonkeyPatch
+):
+ def run(command: list[str], env: dict[str, str] | None = None, cwd: Path | None = None) -> None: # noqa: ARG001
+ if command[0] == "datadog-ci":
+ raise subprocess.CalledProcessError(1, command)
+ commands.append(command)
+
+ monkeypatch.setattr(finalizer, "run", run)
+ job = _job("docker", tmp_path, SCENARIO="DOCKER_SSI")
+
+ assert finalizer.finalize(job) == 1
+
+ assert [command[0] for command in commands] == ["bash"]
+
+
+@scenarios.test_the_test
+def test_logs_are_printed_even_without_log_folder(tmp_path: Path, commands: list[list[str]]):
+ job = _job("docker", tmp_path, SCENARIO="DOCKER_SSI")
+
+ assert finalizer.finalize(job) == 1
+
+ assert commands[-1][0] == "bash"
+ assert (tmp_path / "project" / "reports").is_dir()
+
+
+@scenarios.test_the_test
+def test_entry_point_runs_the_finalizer():
+ result = subprocess.run(
+ ["bash", "utils/scripts/gitlab_finalize_job.sh", "--help"], check=True, capture_output=True, text=True
+ )
+ assert "--profile {aws,docker,k8s}" in result.stdout
+
+
+@scenarios.test_the_test
+def test_base_jobs_call_the_finalizer_from_the_system_tests_checkout():
+ pipeline = yaml.safe_load(Path(".gitlab/ssi_gitlab-ci.yml").read_text())
+
+ for base_job, profile in BASE_JOBS_PROFILES.items():
+ assert pipeline[base_job]["variables"]["SYSTEM_TESTS_DIR"] == "$CI_PROJECT_DIR"
+ assert pipeline[base_job]["after_script"] == [
+ f'bash "${{SYSTEM_TESTS_DIR}}/utils/scripts/gitlab_finalize_job.sh" --profile {profile}'
+ ]
+
+
+@pytest.mark.parametrize(
+ ("project", "expected_dirs"),
+ [
+ ("system-tests", dict.fromkeys(BASE_JOBS_PROFILES, "$CI_PROJECT_DIR")),
+ (
+ "dd-trace-php",
+ {
+ ".base_docker_ssi_job": "/system-tests",
+ ".base_job_onboarding_system_tests": "$CI_PROJECT_DIR/system-tests",
+ ".k8s_lib_injection_base": "/system-tests",
+ },
+ ),
+ ],
+)
+@scenarios.test_the_test
+def test_generated_jobs_point_to_the_checkout_they_run_in(
+ project: str, expected_dirs: dict[str, str], monkeypatch: pytest.MonkeyPatch
+):
+ monkeypatch.setenv("CI_PROJECT_NAME", project)
+ for name in ("SYSTEM_TESTS_REF", "K8S_INJECTOR_IMG", "K8S_LIB_INIT_IMG", "K8S_WEBLOG_IMG"):
+ monkeypatch.delenv(name, raising=False)
+ matrix_data: dict[str, dict] = {
+ "aws_ssi_scenario_defs": {"SIMPLE_INSTALLER_AUTO_INJECTION": {"test-app-php": ["Ubuntu_22_amd64"]}},
+ "dockerssi_scenario_defs": {"DOCKER_SSI": {"php-app": [{"ubuntu:22.04": ["8.3"], "arch": "linux/amd64"}]}},
+ "libinjection_scenario_defs": {"K8S_LIB_INJECTION": ["dd-lib-php-init-test-app"]},
+ }
+
+ pipeline = build_ssi_gitlab_pipeline("php", matrix_data, "dev")
+
+ assert {job: pipeline[job]["variables"]["SYSTEM_TESTS_DIR"] for job in expected_dirs} == expected_dirs
diff --git a/utils/scripts/ci_orchestrators/gitlab_exporter.py b/utils/scripts/ci_orchestrators/gitlab_exporter.py
index 273cf14cef7..c47e73f0534 100644
--- a/utils/scripts/ci_orchestrators/gitlab_exporter.py
+++ b/utils/scripts/ci_orchestrators/gitlab_exporter.py
@@ -68,6 +68,15 @@ def print_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_enviro
def print_ssi_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_environment: str) -> None:
+ result_pipeline = build_ssi_gitlab_pipeline(language, matrix_data, ci_environment)
+ pipeline_yml = yaml.dump(result_pipeline, sort_keys=False, default_flow_style=False)
+ output_file = f"{language}_ssi_gitlab_pipeline.yml"
+ with open(output_file, "w") as file:
+ file.write(pipeline_yml)
+ print("Pipeline file generated: ", output_file)
+
+
+def build_ssi_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_environment: str) -> dict:
result_pipeline = {} # type: dict
result_pipeline["workflow"] = {"name": f"{language} SSI"}
result_pipeline["include"] = []
@@ -99,10 +108,15 @@ def print_ssi_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_en
result_pipeline[".base_job_onboarding_system_tests"]["script"].insert(
0, "git clone https://github.com/DataDog/system-tests.git system-tests"
)
+ result_pipeline[".base_job_onboarding_system_tests"]["variables"]["SYSTEM_TESTS_DIR"] = (
+ "$CI_PROJECT_DIR/system-tests"
+ )
print_aws_gitlab_pipeline(language, matrix_data["aws_ssi_scenario_defs"], ci_environment, result_pipeline)
if matrix_data["dockerssi_scenario_defs"]:
# Copy the base job for the docker ssi system tests
result_pipeline[".base_docker_ssi_job"] = pipeline_data[".base_docker_ssi_job"]
+ if os.getenv("CI_PROJECT_NAME") != "system-tests":
+ result_pipeline[".base_docker_ssi_job"]["variables"]["SYSTEM_TESTS_DIR"] = "/system-tests"
print_docker_ssi_gitlab_pipeline(
language, matrix_data["dockerssi_scenario_defs"], ci_environment, result_pipeline
)
@@ -116,14 +130,11 @@ def print_ssi_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_en
)
result_pipeline[".k8s_lib_injection_base"]["script"].insert(0, "git pull")
result_pipeline[".k8s_lib_injection_base"]["script"].insert(0, "cd /system-tests")
+ result_pipeline[".k8s_lib_injection_base"]["variables"]["SYSTEM_TESTS_DIR"] = "/system-tests"
print_k8s_gitlab_pipeline(language, matrix_data["libinjection_scenario_defs"], ci_environment, result_pipeline)
- pipeline_yml = yaml.dump(result_pipeline, sort_keys=False, default_flow_style=False)
- output_file = f"{language}_ssi_gitlab_pipeline.yml"
- with open(output_file, "w") as file:
- file.write(pipeline_yml)
- print("Pipeline file generated: ", output_file)
+ return result_pipeline
def get_k8s_scenario_components(
diff --git a/utils/scripts/gitlab_finalize_job.py b/utils/scripts/gitlab_finalize_job.py
new file mode 100644
index 00000000000..dcacc93b667
--- /dev/null
+++ b/utils/scripts/gitlab_finalize_job.py
@@ -0,0 +1,255 @@
+"""Finalize a Docker SSI, AWS SSI or K8s lib injection GitLab job.
+
+Runs in the after_script of those jobs, in system-tests and in the repositories that launch them.
+Each profile is an ordered list of steps. A failing step is reported and the next steps still run,
+so the job is always tagged and its logs are always collected and printed.
+
+To add a feature, write a step function and add it to the profiles that need it.
+
+Uses only the standard library: the system-tests virtualenv does not exist when the job fails before
+./build.sh. Generated pipelines of older system-tests commits call this script with --profile, so keep
+that option backward compatible.
+"""
+
+# The wrapper falls back to the system python3, which may predate the annotation syntax used here.
+from __future__ import annotations
+
+import argparse
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from functools import cache
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+SYSTEM_TESTS_DIR = Path(__file__).resolve().parents[2]
+
+AWS_LOGS_BUCKET = "system-tests-aws-ssi-apm"
+MAX_ARTIFACT_FILE_SIZE = 50 * 1024 * 1024
+
+
+@dataclass
+class Job:
+ """What the steps know about the job being finalized."""
+
+ profile: Profile
+ env: Mapping[str, str]
+ project_dir: Path
+ system_tests_dir: Path
+ aws_console_url: str = ""
+
+ @property
+ def scenario(self) -> str:
+ return self.env.get(self.profile.scenario_variable, "logs").lower()
+
+ @property
+ def reports_dir(self) -> Path:
+ return self.project_dir / "reports"
+
+ @property
+ def logs_pattern(self) -> str:
+ return self.profile.logs_pattern.format(scenario=self.scenario)
+
+ @property
+ def log_folders(self) -> list[Path]:
+ return sorted(folder for folder in self.system_tests_dir.glob(self.logs_pattern) if folder.is_dir())
+
+ @property
+ def is_system_tests_main(self) -> bool:
+ return self.env.get("CI_PROJECT_NAME") == "system-tests" and self.env.get("CI_COMMIT_BRANCH") == "main"
+
+ def datadog_env(self) -> dict[str, str]:
+ return {
+ "DATADOG_SITE": self.env.get("DATADOG_SITE", "datadoghq.com"),
+ "DATADOG_API_KEY": read_secret("test_optimization", "dd_api_key"),
+ }
+
+
+Step = Callable[[Job], None]
+
+
+def run(command: list[str], env: Mapping[str, str] | None = None, cwd: Path | None = None) -> None:
+ subprocess.run(command, check=True, cwd=cwd, env=None if env is None else {**os.environ, **env})
+
+
+@cache
+def read_secret(*path: str) -> str:
+ result = subprocess.run(["read-system-tests-secret", *path], check=True, stdout=subprocess.PIPE, text=True)
+ return result.stdout.strip()
+
+
+def tag_official_workflow(job: Job) -> None:
+ run(["datadog-ci", "tag", "--level", "job", "--tags", "ci.framework.name:system-tests"], env=job.datadog_env())
+
+
+def delete_kind_clusters(_job: Job) -> None:
+ run(["kind", "delete", "clusters", "--all"])
+
+
+def upload_logs_to_s3(job: Job) -> None:
+ aws_config = Path.home() / ".aws" / "config"
+ aws_config.parent.mkdir(parents=True, exist_ok=True)
+ aws_config.write_text(read_secret("apm-ecosystems-reliability-profile", "config") + "\n")
+
+ for folder in job.log_folders:
+ pipeline_id = job.env.get("CI_PIPELINE_ID", "datadog-local")
+ prefix = f"{pipeline_id}/{folder.name}/{job.env['VM']}_{job.env['WEBLOG']}/"
+ run(
+ ["aws", "s3", "cp", str(folder), f"s3://{AWS_LOGS_BUCKET}/{prefix}", "--recursive", "--quiet"],
+ env={"AWS_PROFILE": "apm-ecosystems-reliability-ci"},
+ )
+ job.aws_console_url = (
+ f"https://s3.console.aws.amazon.com/s3/buckets/{AWS_LOGS_BUCKET}?region=us-east-1&prefix={prefix}"
+ )
+
+
+def delete_large_log_files(job: Job) -> None:
+ for folder in job.log_folders:
+ for file in sorted(folder.rglob("*")):
+ if file.is_file() and not file.is_symlink() and file.stat().st_size > MAX_ARTIFACT_FILE_SIZE:
+ print(f"Deleting large file ({file.stat().st_size // (1024 * 1024)}M): {file}")
+ file.unlink()
+
+
+def _sockets_and_fifos(directory: str, names: list[str]) -> set[str]:
+ return {name for name in names if Path(directory, name).is_socket() or Path(directory, name).is_fifo()}
+
+
+def collect_logs(job: Job) -> None:
+ job.reports_dir.mkdir(parents=True, exist_ok=True)
+ if not job.log_folders:
+ raise FileNotFoundError(f"No {job.logs_pattern} folder in {job.system_tests_dir}")
+ for folder in job.log_folders:
+ shutil.copytree(
+ folder, job.reports_dir / folder.name, symlinks=True, ignore=_sockets_and_fifos, dirs_exist_ok=True
+ )
+
+
+def publish_feature_parity(job: Job) -> None:
+ if not job.is_system_tests_main:
+ return
+ url = read_secret("fpd", "fp_import_url")
+ api_key = read_secret("fpd", "fp_api_key")
+ for report in sorted(job.reports_dir.glob(f"logs*/{job.profile.feature_parity_pattern}")):
+ print(f"Processing report: {report}")
+ run(
+ [
+ "curl",
+ "-X",
+ "POST",
+ url,
+ "--fail",
+ "--include",
+ "--header",
+ "Content-Type: application/json",
+ "--header",
+ f"FP_API_KEY: {api_key}",
+ "--data",
+ f"@{report}",
+ ]
+ )
+
+
+def upload_junit_reports(job: Job) -> None:
+ reports = sorted(job.reports_dir.glob("logs*/reportJunit.xml"))
+ if not reports:
+ print("No reports/logs*/reportJunit.xml found; skipping Test Optimization upload")
+ return
+ run(
+ [
+ "datadog-ci",
+ "junit",
+ "upload",
+ *[str(report) for report in reports],
+ "--service",
+ "system-tests",
+ "--env",
+ "ci",
+ "--verbose",
+ "--xpath-tag",
+ "test.codeowners=/testcase/properties/property[@name='test.codeowners']",
+ ],
+ env={
+ **job.datadog_env(),
+ "DD_TAGS": f"test.configuration.ci_environment:{job.env.get('CI_ENVIRONMENT', 'prod')}",
+ },
+ )
+
+
+def print_logs(job: Job) -> None:
+ run(
+ ["bash", str(job.system_tests_dir / "utils" / "scripts" / "gitlab_print_logs.sh")],
+ env={"SCENARIO_SUFIX": job.scenario, "AWS_CONSOLE_URL": job.aws_console_url},
+ cwd=job.project_dir,
+ )
+
+
+@dataclass(frozen=True)
+class Profile:
+ scenario_variable: str
+ logs_pattern: str
+ feature_parity_pattern: str
+ steps: tuple[Step, ...]
+
+
+PROFILES: dict[str, Profile] = {
+ "docker": Profile(
+ scenario_variable="SCENARIO",
+ logs_pattern="logs_{scenario}",
+ feature_parity_pattern="feature_parity.json",
+ steps=(tag_official_workflow, collect_logs, publish_feature_parity, upload_junit_reports, print_logs),
+ ),
+ "aws": Profile(
+ scenario_variable="SCENARIO",
+ logs_pattern="logs_{scenario}",
+ feature_parity_pattern="*_feature_parity.json",
+ steps=(
+ tag_official_workflow,
+ upload_logs_to_s3,
+ delete_large_log_files,
+ collect_logs,
+ publish_feature_parity,
+ print_logs,
+ ),
+ ),
+ "k8s": Profile(
+ scenario_variable="K8S_SCENARIO",
+ logs_pattern="logs_*",
+ feature_parity_pattern="feature_parity.json",
+ steps=(tag_official_workflow, delete_kind_clusters, collect_logs, publish_feature_parity, print_logs),
+ ),
+}
+
+
+def finalize(job: Job) -> int:
+ failed_steps: list[str] = []
+ for step in job.profile.steps:
+ try:
+ step(job)
+ except Exception as error:
+ print(f"❌ Step {step.__name__} failed: {error}", flush=True)
+ failed_steps.append(step.__name__)
+ if failed_steps:
+ print(f"❌ Failed steps: {', '.join(failed_steps)}")
+ return 1
+ return 0
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="Finalize a Docker SSI, AWS SSI or K8s lib injection GitLab job.")
+ parser.add_argument("--profile", required=True, choices=sorted(PROFILES))
+ args = parser.parse_args()
+
+ job = Job(
+ profile=PROFILES[args.profile],
+ env=os.environ,
+ project_dir=Path(os.environ.get("CI_PROJECT_DIR", Path.cwd())),
+ system_tests_dir=SYSTEM_TESTS_DIR,
+ )
+ return finalize(job)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/utils/scripts/gitlab_finalize_job.sh b/utils/scripts/gitlab_finalize_job.sh
new file mode 100755
index 00000000000..325d3e436fc
--- /dev/null
+++ b/utils/scripts/gitlab_finalize_job.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# after_script entry point of the Docker SSI, AWS SSI and K8s lib injection GitLab jobs.
+# All the logic lives in gitlab_finalize_job.py.
+#
+# Usage:
+# gitlab_finalize_job.sh --profile
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SYSTEM_TESTS_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+# The virtualenv is missing when the job fails before ./build.sh creates it.
+PYTHON="${SYSTEM_TESTS_DIR}/venv/bin/python"
+if [[ ! -x "${PYTHON}" ]]; then
+ PYTHON="python3"
+fi
+
+exec "${PYTHON}" -u "${SCRIPT_DIR}/gitlab_finalize_job.py" "$@"