diff --git a/.github/workflows/junit5-sdk.yml b/.github/workflows/junit5-sdk.yml new file mode 100644 index 0000000..bdaa300 --- /dev/null +++ b/.github/workflows/junit5-sdk.yml @@ -0,0 +1,77 @@ +name: keploy-junit5 + +on: + pull_request: + paths: + - "keploy-junit5/**" + - ".github/workflows/junit5-sdk.yml" + push: + branches: [main] + paths: + - "keploy-junit5/**" + - ".github/workflows/junit5-sdk.yml" + # keploy-junit5-release.yml runs these rows and the e2e lane on a release + # tag before it publishes. + workflow_call: + +concurrency: + group: keploy-junit5-${{ github.ref }} + cancel-in-progress: true + +jobs: + rows: + name: rows (JDK ${{ matrix.java }}, JUnit ${{ matrix.junit }}) + runs-on: ubuntu-latest + strategy: + # One leg failing must not cancel the others: each leg's result is its + # own. + fail-fast: false + matrix: + # The oldest JUnit this extension supports (README.md) on the oldest + # JDK it supports, the last JUnit 5, and JUnit 6 (JDK 17 or later). + include: + - { java: "8", junit: "5.10.5" } + - { java: "17", junit: "5.14.4" } + - { java: "21", junit: "6.1.3" } + defaults: + run: + working-directory: keploy-junit5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1 + with: + distribution: temurin + java-version: ${{ matrix.java }} + cache: maven + - run: mvn -B -ntp -Djunit.version=${{ matrix.junit }} verify + + e2e: + # e2e/lane.sh: this commit's extension, installed with Maven into the + # lane's own Maven repository and used by a small JUnit 5 suite, run with + # Maven under keploy mock record with its provider up, then keploy mock + # replay with the provider stopped; a test whose call has no recorded + # answer, and one that no longer makes its recorded call, fail with + # keploy's text; a quarantined test is skipped with keploy's reason. + name: e2e (keploy mock record and replay) + runs-on: ubuntu-latest + # The lane takes a few minutes, Maven's downloads included: a run past 15 + # is a hang. + timeout-minutes: 15 + env: + # keploy's enterprise build, which judges each test under + # KEPLOY_PROVEN=1. Pinned, and checked against its digest, so that a + # new keploy release cannot change this job's result unseen. + KEPLOY_VERSION: "3.8.106" + KEPLOY_SHA256: "2c4c9bcf0a703b65827d1cee5ca88fe9d7856844a08cac368b42b83345fef3ce" + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1 + with: + distribution: temurin + java-version: "17" + - name: install keploy ${{ env.KEPLOY_VERSION }} + run: | + curl -fsSL --retry 5 -o "$RUNNER_TEMP/keploy" "https://keploy.io/ent/dl/$KEPLOY_VERSION/enterprise_linux_amd64" + echo "$KEPLOY_SHA256 $RUNNER_TEMP/keploy" | sha256sum -c - + chmod +x "$RUNNER_TEMP/keploy" + - run: KEPLOY_BIN="$RUNNER_TEMP/keploy" bash keploy-junit5/e2e/lane.sh diff --git a/.github/workflows/keploy-junit5-release.yml b/.github/workflows/keploy-junit5-release.yml new file mode 100644 index 0000000..168fdf8 --- /dev/null +++ b/.github/workflows/keploy-junit5-release.yml @@ -0,0 +1,141 @@ +name: keploy-junit5 release + +# Publishes io.keploy:keploy-junit5 to Maven Central from a tag. +# +# To release X.Y.Z: push the tag keploy-junit5-vX.Y.Z on a commit of main. +# The rows on every JDK and JUnit leg and the e2e lane (junit5-sdk.yml) run +# on the tagged commit first; the publish runs only when they all pass. It +# sets keploy-junit5/pom.xml's version to X.Y.Z in this run only (the pom on +# main stays a SNAPSHOT), builds the jar with its sources and javadoc jars, +# signs each, and deploys them to Central, which refuses a version it already +# has. +# +# keploy-junit5 is a Maven project of its own, released with its own +# version: release.yml (v*.*.* tags) builds and deploys only the root pom's +# modules, and never this one. This workflow signs and deploys with the same +# key and the same secret names as release.yml: CENTRAL_USERNAME and +# CENTRAL_PASSWORD (a Central Portal user token for the io.keploy namespace), +# MAVEN_GPG_PRIVATE_KEY (the ASCII-armored release key, id 8541784E4EC36FB8) +# and MAVEN_GPG_PASSPHRASE. A step below fails, naming each one that is not +# set, before anything is built or signed. +# Every action this workflow runs, the rows' included, is pinned by commit: +# the publish job holds the signing key and the Central token. +# +# On a pull request the same release build runs with no signing and no +# deploy, so a javadoc error or a missing jar shows before a tag. +on: + push: + tags: ["keploy-junit5-v*"] + pull_request: + paths: + - "keploy-junit5/**" + - ".github/workflows/keploy-junit5-release.yml" + - ".github/workflows/junit5-sdk.yml" + +permissions: + contents: read + +# Never let two releases of the same tag overlap, and never cancel one +# mid-deploy. +concurrency: + group: keploy-junit5-release-${{ github.ref }} + cancel-in-progress: false + +jobs: + rows: + if: github.event_name == 'push' + uses: ./.github/workflows/junit5-sdk.yml + + publish: + name: build, sign and publish to Maven Central + if: github.event_name == 'push' + needs: rows + runs-on: ubuntu-latest + # central-publishing-maven-plugin waits until Central reports the + # deployment PUBLISHED (waitMaxTime 7200 s), plus the build ahead of it. + timeout-minutes: 150 + env: + # The public key id of keploy's release signing key, as release.yml. + GPG_KEY_ID: 8541784E4EC36FB8 + defaults: + run: + shell: bash # -eo pipefail: a failed command before a pipe fails the step + working-directory: keploy-junit5 + steps: + - name: The tag names a release version + id: ver + # Before the signing key is imported. The tag name is interpolated + # into Maven's arguments below, so only digits and dots pass. + working-directory: . + run: | + version="${GITHUB_REF_NAME#keploy-junit5-v}" + if [[ "$GITHUB_REF_NAME" != keploy-junit5-v* || ! "$version" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then + echo "::error::refusing to publish from '$GITHUB_REF_NAME': the tag must be keploy-junit5-vX.Y.Z, X.Y.Z a release version with no leading zeros" + exit 1 + fi + echo "version=$version" >> "$GITHUB_OUTPUT" + - name: The Central and signing secrets are set + working-directory: . + env: + CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + CENTRAL_PASSWORD: ${{ secrets.CENTRAL_PASSWORD }} + MAVEN_GPG_PRIVATE_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + run: | + missing=() + for s in CENTRAL_USERNAME CENTRAL_PASSWORD MAVEN_GPG_PRIVATE_KEY MAVEN_GPG_PASSPHRASE; do + [[ -n "${!s}" ]] || missing+=("$s") + done + if (( ${#missing[@]} )); then + echo "::error::the secret(s) ${missing[*]} are not set; refusing to publish" + exit 1 + fi + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - name: Set up JDK 17 + uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1 + with: + distribution: temurin + java-version: "17" + # Writes ~/.m2/settings.xml with a `central` server entry read from + # the env vars named here, and imports the signing key. + server-id: central + server-username: CENTRAL_USERNAME + server-password: CENTRAL_PASSWORD + gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Set the release version in the pom + # Pinned, as release.yml pins it: an unpinned plugin would run + # unreviewed code in the job that holds the signing key. + env: + VERSION: ${{ steps.ver.outputs.version }} + run: mvn -B -ntp org.codehaus.mojo:versions-maven-plugin:2.21.0:set -DnewVersion="$VERSION" -DgenerateBackupPoms=false + - name: Build, sign and publish + env: + CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }} + CENTRAL_PASSWORD: ${{ secrets.CENTRAL_PASSWORD }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + run: mvn -B -ntp -P release -DskipTests clean deploy -Dgpg.keyname="$GPG_KEY_ID" + + dry-run: + name: release build, dry run + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + defaults: + run: + shell: bash # -eo pipefail: a failed command before a pipe fails the step + working-directory: keploy-junit5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1 + with: + distribution: temurin + java-version: "17" + - name: Release build, unsigned + run: mvn -B -ntp -P release -DskipTests -Dgpg.skip clean verify + - name: The release carries its jar, sources and javadoc + run: | + v=$(mvn -B -ntp -q help:evaluate -Dexpression=project.version -DforceStdout) + for f in "keploy-junit5-$v.jar" "keploy-junit5-$v-sources.jar" "keploy-junit5-$v-javadoc.jar"; do + test -s "target/$f" || { echo "::error::the release would ship without $f"; exit 1; } + done + unzip -l "target/keploy-junit5-$v.jar" | grep -q 'io/keploy/junit5/' || { echo "::error::keploy-junit5-$v.jar holds no io/keploy/junit5 classes"; exit 1; } diff --git a/keploy-junit5/README.md b/keploy-junit5/README.md new file mode 100644 index 0000000..be02354 --- /dev/null +++ b/keploy-junit5/README.md @@ -0,0 +1,122 @@ +# keploy-junit5 + +A JUnit 5 (Jupiter) extension that joins a test suite to keploy's Universal +Mocker. `keploy mock record` keeps the calls each test's code makes to its +dependencies; `keploy mock replay` answers them from the recording, so the +suite runs with every dependency stopped. keploy judges each test: a call with +no recorded answer, or a recorded call the test no longer makes, fails that +test with keploy's text. + +The extension only tells keploy where each test begins and ends; keploy keeps +every rule. It speaks the same wire as go-sdk's `keploytest`, +`@keploy/playwright`, `@keploy/jest` and `keploy-pytest`. + +## Use + +JUnit 5.10 or later (JUnit 6 included), Java 8 or later. The package is not +published yet: install it from this folder with `mvn install`, then add it to +the test scope: + +```xml + + io.keploy + keploy-junit5 + 0.1.0-SNAPSHOT + test + +``` + +Let JUnit load it for the whole run, in `src/test/resources/junit-platform.properties`: + +```properties +junit.jupiter.extensions.autodetection.enabled=true +``` + +(Gradle: `test { useJUnitPlatform(); systemProperty 'junit.jupiter.extensions.autodetection.enabled', 'true' }`.) + +keploy needs the project's `keploy.yml` to name the recording and CI's test +command; without `mock.command`, keploy refuses the run with +`no keploy project here: set mock.command ...`: + +```yaml +# keploy.yml +mock: + name: shop # the recording: keploy/shop/ + command: mvn test # CI's test command +``` + +Then run the suite under keploy with `KEPLOY_PROVEN=1`, in one test JVM. +`--local` keeps the recording on disk, in `keploy//`: + +```sh +KEPLOY_PROVEN=1 keploy mock record --local -c "mvn test" +KEPLOY_PROVEN=1 keploy mock replay --local -c "mvn test" +``` + +Without keploy (`KEPLOY_MOCK_AGENT` unset or `off`) the extension does +nothing, and the suite runs as plain JUnit. + +## What it does under keploy + +- The test JVM claims keploy's run once, before the first test class's + `@BeforeAll`, naming keploy's project folder (`KEPLOY_MOCK_PROJECT`). +- Each test begins after its class's `@BeforeAll` methods, before every other + extension's BeforeEach callback and its own `@BeforeEach` methods. It ends + after its `@AfterEach` methods and every other extension's AfterEach + callback, with its outcome: passed, failed, or skipped (a failed + assumption). The calls `@BeforeAll` and `@AfterAll` methods make belong to + the test JVM, outside every test. +- A test keploy set aside is aborted (JUnit's skip) with keploy's reason, and + none of its `@BeforeEach`, test or `@AfterEach` methods runs (other + extensions' AfterEach callbacks do, as JUnit runs them after any BeforeEach + callback stops a test). A line on stderr names it. +- A test keploy failed at its end fails with keploy's text; when it had failed + already, keploy's text is added to its failure. +- Its ID is `::`: the class's binary name + (`com.shop.OrdersTest$Refunds` for a `@Nested` class) and JUnit's name of + the method, with a parameterized or repeated test's invocation index: + `com.shop.OrdersTest::price(java.lang.String)[2]`. Each invocation of a + `@ParameterizedClass` (JUnit 5.13 and later) runs each of the class's tests + as a test of its own, with the class's invocation index before the method: + `com.shop.OrdersTest::[2]price(java.lang.String)`. A test run through a + JUnit Platform `@Suite` has the same ID as when it runs directly. + +## Limits + +Each of these fails loudly under keploy, so no Jupiter test runs unjudged: + +- One test JVM at a time: keploy refuses a second claim. Keep Surefire's + `forkCount` at 1 (its default), and Gradle's `maxParallelForks` at 1. +- JUnit's parallel execution (`junit.jupiter.execution.parallel.enabled=true`) + fails every test class under keploy. +- Registering the extension only with `@ExtendWith`, without autodetection, + fails every test class under keploy: some classes could run without it, + and it must wrap every other extension. +- A `@TestFactory`'s dynamic tests run inside one BeforeEach/AfterEach pair, + so keploy cannot judge them one by one: the factory fails under keploy. + Write them as `@ParameterizedTest`. +- A test begun a second time in one JVM fails (a rerun, such as Surefire's + `rerunFailingTestsCount`, or a second `@Suite` that selects the same test + class): keploy names a test by its class and name. +- A JVM cannot change its own environment, so it does not export + `KEPLOY_TEST_PID` to the processes its tests start; one of them that runs + tests under a keploy SDK claims too, and keploy refuses it. + +Only JUnit Jupiter tests are judged. The tests of every other JUnit Platform +engine run unjudged, as without keploy: JUnit 4 tests on the Vintage engine, +and Spock, Kotest, jqwik and Cucumber tests. This extension is a Jupiter +extension, and no other engine calls it; their calls are the test JVM's. + +## Releasing + +keploy-junit5 is released on its own tag, with its own version, apart from +the repository's `v*.*.*` releases of the agent. To release X.Y.Z, push the +tag `keploy-junit5-vX.Y.Z` on a commit of `main`: +`git tag keploy-junit5-vX.Y.Z && git push origin keploy-junit5-vX.Y.Z`. +The tag runs the rows and the e2e lane, then +`.github/workflows/keploy-junit5-release.yml` sets the pom's version to X.Y.Z +for that run only, builds the jar with its sources and javadoc jars, signs +each, and deploys them to Maven Central. It uses the repository secrets that +`release.yml` uses: `CENTRAL_USERNAME` and `CENTRAL_PASSWORD` (a Central +Portal user token for the `io.keploy` namespace), `MAVEN_GPG_PRIVATE_KEY` and +`MAVEN_GPG_PASSPHRASE`; it refuses to start, naming each one that is not set. diff --git a/keploy-junit5/e2e/lane.sh b/keploy-junit5/e2e/lane.sh new file mode 100755 index 0000000..56cdbf2 --- /dev/null +++ b/keploy-junit5/e2e/lane.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# The e2e lane of keploy-junit5: this commit's extension, installed with +# Maven into a Maven repository of the lane's own, used by a small JUnit 5 +# suite (e2e/shop) run with Maven under a real keploy that judges each test. +# The suite's code calls a provider over HTTP; keploy records those calls +# with the provider up, and replays them with the provider stopped. +# +# KEPLOY_BIN=/path/to/keploy bash keploy-junit5/e2e/lane.sh +# +# KEPLOY_BIN is a keploy build that judges each test under KEPLOY_PROVEN=1 +# (keploy's enterprise build; the CI job downloads a pinned release). keploy +# needs root for its eBPF hooks: the lane runs it through sudo unless it is +# root already. java (17 or later: the suite's release, and the provider runs +# as a source file) and mvn must be on PATH. +# +# Checks, each judged by what a user reads (exit codes, Surefire's summary +# and each test's report, the files keploy wrote): +# 1. without keploy the suite passes as plain JUnit, and nothing says +# keploy; +# 2. keploy mock record keeps each test's calls under the test's name; +# 3. keploy mock replay, with the provider stopped, passes every test; +# 4. a test whose call has no recorded answer fails, with keploy's text +# added to its failure; the others pass; +# 5. a test that no longer makes its recorded call fails, by keploy, with +# keploy's text; the others pass; +# 6. a test quarantined in the set is skipped by JUnit with keploy's +# reason, and the replay passes. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PKG="$(cd "$HERE/.." && pwd)" +KEPLOY_BIN="${KEPLOY_BIN:?set KEPLOY_BIN to a keploy binary}" +PORT="${SHOP_PORT:-18751}" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/junit5-e2e.XXXXXX")" +FAILED=0 +PROVIDER_PID="" +export SHOP_PROVIDER="http://127.0.0.1:$PORT" +# The lane's own Maven repository: this commit's extension, and everything +# the suite needs, fetched once (step 0) so that every later run is offline. +export MAVEN_OPTS="${MAVEN_OPTS:+$MAVEN_OPTS }-Dmaven.repo.local=$WORK/m2" + +cleanup() { + [ -z "$PROVIDER_PID" ] || kill "$PROVIDER_PID" 2>/dev/null + wait 2>/dev/null +} +trap cleanup EXIT + +ok() { echo " ok: $*"; } +fail() { echo " FAIL: $*"; FAILED=1; } +# show ID: the run's output, for a failing check. +show() { + local stream + for stream in out err; do + echo " --- $1: std$stream (last 150 lines, keploy's banner left out) ---" + grep -av '38;5;' "$WORK/logs/$1.$stream" | tail -n 150 | sed 's/^/ | /' + done +} + +if [ "$(id -u)" -eq 0 ]; then AS_ROOT=(env); else AS_ROOT=(sudo -E env "PATH=$PATH" "HOME=$HOME"); fi + +# The extension as Maven Central would serve it (its jar, installed), and +# the suite, from a copy of e2e/shop. +mkdir -p "$WORK/logs" "$WORK/pkg" +tar -C "$PKG" --exclude=./e2e --exclude=./target -cf - . | tar -C "$WORK/pkg" -xf - +cp -R "$HERE/shop" "$WORK/shop" +(cd "$WORK/pkg" && mvn -B -ntp -DskipTests install) >"$WORK/logs/install.log" 2>&1 || + { cat "$WORK/logs/install.log"; echo "FAIL: could not build and install the extension"; exit 1; } +JAR=$(ls "$WORK"/m2/io/keploy/keploy-junit5/*/keploy-junit5-*.jar) +cd "$WORK/shop" || exit 1 +echo "installed $(basename "$JAR") into $WORK/m2; $(mvn -v 2>&1 | head -n 1); $(java -version 2>&1 | head -n 1); the suite is at $WORK/shop" + +start_provider() { + : >"$WORK/logs/provider.log" + java "$WORK/shop/Provider.java" "$PORT" >"$WORK/logs/provider.log" 2>&1 & + PROVIDER_PID=$! + until grep -qx ready "$WORK/logs/provider.log"; do + kill -0 "$PROVIDER_PID" 2>/dev/null || { cat "$WORK/logs/provider.log"; echo "FAIL: the provider exited before it listened"; exit 1; } + sleep 0.2 + done +} +stop_provider() { + kill "$PROVIDER_PID" 2>/dev/null + wait "$PROVIDER_PID" 2>/dev/null + PROVIDER_PID="" +} + +# run ID CMD...: one run, stdin closed, stdout and stderr kept apart. Sets EC. +run() { + local id=$1 + shift + echo + echo "== $id: $*" + "$@" "$WORK/logs/$id.out" 2>"$WORK/logs/$id.err" + EC=$? + echo " exit $EC" + # LANE_VERBOSE=1 shows every run's output, not only a failing check's. + [ -z "${LANE_VERBOSE:-}" ] || show "$id" +} +# keploy ID ARGS...: one keploy run under the proven switch, as root. +keploy() { + local id=$1 + shift + run "$id" "${AS_ROOT[@]}" KEPLOY_PROVEN=1 KEPLOY_INVOKER=ci "SHOP_PROVIDER=$SHOP_PROVIDER" \ + "SHOP_CHANGE=${SHOP_CHANGE:-}" "MAVEN_OPTS=$MAVEN_OPTS" ${JAVA_HOME:+"JAVA_HOME=$JAVA_HOME"} "$KEPLOY_BIN" "$@" --disable-tele +} +# report_of ID TEST: Surefire's report of the failed test TEST in a run +# ("[ERROR] TEST -- Time elapsed: ... <<< FAILURE!"): the lines after it, up +# to Maven's next log line. +report_of() { + awk -v h="[ERROR] $2 -- " ' + on && /^\[[A-Z]+\] / { on = 0 } + index($0, h) == 1 && / <<< (FAILURE|ERROR)!$/ { on = 1; next } + on' "$WORK/logs/$1.out" +} +# surefire_summary ID: Surefire's closing count of a run. +surefire_summary() { + grep -aE '^\[[A-Z]+\] Tests run: [0-9]+, Failures: [0-9]+, Errors: [0-9]+, Skipped: [0-9]+$' "$WORK/logs/$1.out" | + tail -n 1 | sed -E 's/^\[[A-Z]+\] //' +} +# has ID TEXT: whether the run's output (Maven's and keploy's) has a line +# that is TEXT. +has() { grep -aqxF -- "$2" "$WORK/logs/$1.out" "$WORK/logs/$1.err"; } + +MVN='mvn -B -ntp -o test' +# keploy's IDs: class::name, the class's binary name and JUnit's name of the +# method in it, with a parameterized test's invocation index. +PRICES='shop.OrdersTest$Books::pricesABook()' +RESERVES='shop.OrdersTest$Books::reservesABook()' +CHARGES='shop.PaymentsTest::chargesACard(int)[1]' +# judge ID WHAT: reports a check from the reasons WHY holds against it. +judge() { + if [ "${#WHY[@]}" -eq 0 ]; then ok "$2"; return; fi + fail "$2" + printf ' - %s\n' "${WHY[@]}" + show "$1" +} + +# 0. Fill the lane's Maven repository: one plain run, online, with the +# provider up. Every later run is offline. +start_provider +run warm mvn -B -ntp test +[ "$EC" -eq 0 ] || { show warm; echo "FAIL: the suite does not pass online without keploy"; exit 1; } + +# 1. Without keploy: plain JUnit, offline. +run inert $MVN +WHY=() +[ "$EC" -eq 0 ] || WHY+=("exit $EC, want 0") +[ "$(surefire_summary inert)" = "Tests run: 3, Failures: 0, Errors: 0, Skipped: 0" ] || WHY+=("Surefire says '$(surefire_summary inert)'") +! cat "$WORK/logs/inert.out" "$WORK/logs/inert.err" | grep -aqi keploy || WHY+=("a line says keploy") +judge inert "without keploy the suite passes as plain JUnit, and nothing says keploy" + +# 2. Record, with the provider up: keploy's own rounds replay it twice more. +keploy record mock record --local -c "$MVN" +stop_provider +WHY=() +[ "$EC" -eq 0 ] || WHY+=("exit $EC, want 0") +has record "keploy: committed keploy/shop and its verdicts.yaml: 3 replayable, 0 quarantined" || + WHY+=("keploy did not commit 3 replayable tests") +for t in "$PRICES" "$RESERVES" "$CHARGES"; do + grep -aqxF -- " - id: $t" keploy/shop/mappings.yaml 2>/dev/null || WHY+=("keploy/shop/mappings.yaml does not name $t") +done +judge record "keploy mock record keeps each test's calls under its class and JUnit's name of it" + +# 3. Replay with the provider stopped. +keploy replay mock replay --local -c "$MVN" +WHY=() +[ "$EC" -eq 0 ] || WHY+=("exit $EC, want 0") +[ "$(surefire_summary replay)" = "Tests run: 3, Failures: 0, Errors: 0, Skipped: 0" ] || WHY+=("Surefire says '$(surefire_summary replay)'") +has replay "keploy: shop: 3 test(s) ran: 3 passed, 0 failed (0 held by keploy), 0 skipped by the test; 0 quarantined (skipped by keploy)" || + WHY+=("keploy did not judge 3 tests passed") +has replay "VERDICT passed" || WHY+=("keploy's verdict is not passed") +judge replay "keploy mock replay with the provider stopped passes every test" + +# 4. A call the recording has no answer for: keploy answers it with an +# error, so the test fails by itself (an error: its IOException), and keploy's +# text is added to that failure; only that test fails. +SHOP_CHANGE=miss keploy replay-miss mock replay --local -c "$MVN" +WHY=() +[ "$EC" -eq 1 ] || WHY+=("exit $EC, want 1") +[ "$(surefire_summary replay-miss)" = "Tests run: 3, Failures: 0, Errors: 1, Skipped: 0" ] || WHY+=("Surefire says '$(surefire_summary replay-miss)'") +# keploy 3.8.106 names this client's destination "[127.0.0.1]:PORT (live +# 127.0.0.1:PORT)": the check takes any form that names the address and port. +report_of replay-miss 'shop.OrdersTest$Books.pricesABook' | + grep -aqE "^[[:space:]]+Suppressed: io\.keploy\.junit5\.KeployException: keploy: GET /price/pen to .*127\.0\.0\.1.*:$PORT.* had no recorded answer \(path differs\)" || + WHY+=("Surefire's report of pricesABook does not carry keploy's miss, added to its failure") +has replay-miss "FAILED_BY_KEPLOY $PRICES (failed): keploy/unmocked_call" || WHY+=("keploy does not hold the miss against pricesABook") +judge replay-miss "a call with no recorded answer fails its own test with keploy's text; the others pass" + +# 5. A recorded call the test no longer makes: keploy fails it at its end. +SHOP_CHANGE=unconsumed keploy replay-unconsumed mock replay --local -c "$MVN" +WHY=() +[ "$EC" -eq 1 ] || WHY+=("exit $EC, want 1") +[ "$(surefire_summary replay-unconsumed)" = "Tests run: 3, Failures: 0, Errors: 1, Skipped: 0" ] || WHY+=("Surefire says '$(surefire_summary replay-unconsumed)'") +# The mock's number follows the order the recording made its calls in, and +# Surefire runs the classes in the order the file system lists them. +report_of replay-unconsumed 'shop.OrdersTest$Books.reservesABook' | + grep -aqE "^keploy: recorded call POST /reserve \(mock-[0-9]+\) was never made\." || + WHY+=("Surefire's report of reservesABook does not carry keploy's unmade call") +has replay-unconsumed "FAILED_BY_KEPLOY $RESERVES (failed): keploy/unconsumed_mock" || WHY+=("keploy does not hold the unmade call against reservesABook") +judge replay-unconsumed "a recorded call the test no longer makes fails that test with keploy's text; the others pass" + +# 6. A test quarantined in the set: JUnit skips it, before its own +# callbacks, with keploy's reason; the replay passes. +keploy quarantine mock quarantine "$CHARGES" --reason 'e2e: set aside by the lane' --local --name shop +WHY=() +[ "$EC" -eq 0 ] || WHY+=("keploy mock quarantine exited $EC") +keploy replay-quarantined mock replay --local -c "$MVN" +[ "$EC" -eq 0 ] || WHY+=("replay exit $EC, want 0") +[ "$(surefire_summary replay-quarantined)" = "Tests run: 3, Failures: 0, Errors: 0, Skipped: 1" ] || WHY+=("Surefire says '$(surefire_summary replay-quarantined)'") +# Surefire's XML report, as CI's test reports read it: the abort's message is +# the skip's text. +grep -aA1 -F '/dev/null | + grep -aqF '&1 | grep -av '&1 | sed 's/^/ | /'; } +judge replay-quarantined "a quarantined test is skipped by JUnit with keploy's reason; the replay passes" + +echo +if [ "$FAILED" -ne 0 ]; then echo "e2e: FAILED (logs in $WORK/logs)"; exit 1; fi +echo "e2e: every check passed" diff --git a/keploy-junit5/e2e/shop/Provider.java b/keploy-junit5/e2e/shop/Provider.java new file mode 100644 index 0000000..9487843 --- /dev/null +++ b/keploy-junit5/e2e/shop/Provider.java @@ -0,0 +1,56 @@ +// The provider the shop calls: up while keploy records, stopped while it +// replays. Prints "ready" once it listens. +// +// java Provider.java PORT +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class Provider { + static final Map PRICES = Map.of("book", 4200, "pen", 150); + + public static void main(String[] args) throws IOException { + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", Integer.parseInt(args[0])), 0); + server.createContext("/", Provider::handle); + server.start(); + System.out.println("ready"); + System.out.flush(); + } + + static void handle(HttpExchange ex) throws IOException { + String method = ex.getRequestMethod(); + String path = ex.getRequestURI().getPath(); + String body = new String(ex.getRequestBody().readAllBytes(), StandardCharsets.UTF_8); + String out = null; + if (method.equals("GET") && path.startsWith("/price/")) { + out = "{\"cents\":" + PRICES.getOrDefault(path.substring("/price/".length()), 0) + "}"; + } else if (method.equals("POST") && path.equals("/reserve")) { + out = "{\"ok\":" + PRICES.containsKey(field(body, "item")) + "}"; + } else if (method.equals("POST") && path.equals("/charge")) { + String cents = field(body, "cents"); + out = "{\"status\":\"" + (cents != null && Integer.parseInt(cents) > 0 ? "captured" : "declined") + "\"}"; + } + byte[] data = (out == null ? "no route" : out).getBytes(StandardCharsets.UTF_8); + ex.getResponseHeaders().set("content-type", out == null ? "text/plain" : "application/json"); + ex.sendResponseHeaders(out == null ? 404 : 200, data.length); + try (OutputStream o = ex.getResponseBody()) { + o.write(data); + } + } + + // The value of a member of a JSON object, a string or an integer: enough + // for the shop's bodies. + static String field(String json, String key) { + Matcher m = Pattern.compile("\"" + key + "\"\\s*:\\s*(?:\"([^\"]*)\"|(-?[0-9]+))").matcher(json); + if (!m.find()) { + return null; + } + return m.group(1) != null ? m.group(1) : m.group(2); + } +} diff --git a/keploy-junit5/e2e/shop/keploy.yml b/keploy-junit5/e2e/shop/keploy.yml new file mode 100644 index 0000000..31d30a5 --- /dev/null +++ b/keploy-junit5/e2e/shop/keploy.yml @@ -0,0 +1,6 @@ +# The keploy project of the e2e suite: the set's name and CI's test command, +# one test JVM (Surefire's default forkCount, 1): keploy judges one test at a +# time. Offline: e2e/lane.sh fills the Maven repository first. +mock: + name: shop + command: mvn -B -ntp -o test diff --git a/keploy-junit5/e2e/shop/pom.xml b/keploy-junit5/e2e/shop/pom.xml new file mode 100644 index 0000000..b70ab1d --- /dev/null +++ b/keploy-junit5/e2e/shop/pom.xml @@ -0,0 +1,62 @@ + + + 4.0.0 + + + example.shop + shop + 0.1.0 + jar + + + UTF-8 + 17 + + + + + + org.junit + junit-bom + 5.14.4 + pom + import + + + + + + + org.junit.jupiter + junit-jupiter + test + + + io.keploy + keploy-junit5 + 0.1.0-SNAPSHOT + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + + + diff --git a/keploy-junit5/e2e/shop/src/main/java/shop/Shop.java b/keploy-junit5/e2e/shop/src/main/java/shop/Shop.java new file mode 100644 index 0000000..3017a7f --- /dev/null +++ b/keploy-junit5/e2e/shop/src/main/java/shop/Shop.java @@ -0,0 +1,74 @@ +package shop; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * The code under test: a shop that prices, reserves and charges through a + * provider over HTTP (the JDK's own client, keep-alive as the JDK sets it). + */ +public final class Shop { + private static final String BASE = base(); + + private Shop() { + } + + private static String base() { + String v = System.getenv("SHOP_PROVIDER"); + return v == null || v.isEmpty() ? "http://127.0.0.1:18751" : v; + } + + public static int price(String item) throws IOException { + return Integer.parseInt(field(call("GET", "/price/" + item, null), "cents")); + } + + public static boolean reserve(String item) throws IOException { + return Boolean.parseBoolean(field(call("POST", "/reserve", "{\"item\":\"" + item + "\"}"), "ok")); + } + + public static String charge(int cents) throws IOException { + return field(call("POST", "/charge", "{\"cents\":" + cents + "}"), "status"); + } + + static String call(String method, String path, String body) throws IOException { + HttpURLConnection c = (HttpURLConnection) new URL(BASE + path).openConnection(); + c.setRequestMethod(method); + if (body != null) { + byte[] data = body.getBytes(StandardCharsets.UTF_8); + c.setDoOutput(true); + c.setRequestProperty("content-type", "application/json"); + c.setFixedLengthStreamingMode(data.length); + try (OutputStream o = c.getOutputStream()) { + o.write(data); + } + } + int status = c.getResponseCode(); + InputStream in = status < 400 ? c.getInputStream() : c.getErrorStream(); + String text = ""; + if (in != null) { + try (InputStream i = in) { + text = new String(i.readAllBytes(), StandardCharsets.UTF_8); + } + } + if (status != 200) { + throw new IOException(method + " " + path + ": HTTP " + status + ": " + text); + } + return text; + } + + // The value of a member of a JSON object: a string's, a number's or a + // boolean's. Enough for the provider's answers. + static String field(String json, String key) throws IOException { + Matcher m = Pattern.compile("\"" + key + "\"\\s*:\\s*(?:\"([^\"]*)\"|([-0-9a-z]+))").matcher(json); + if (!m.find()) { + throw new IOException("no " + key + " in " + json); + } + return m.group(1) != null ? m.group(1) : m.group(2); + } +} diff --git a/keploy-junit5/e2e/shop/src/test/java/shop/OrdersTest.java b/keploy-junit5/e2e/shop/src/test/java/shop/OrdersTest.java new file mode 100644 index 0000000..097d53c --- /dev/null +++ b/keploy-junit5/e2e/shop/src/test/java/shop/OrdersTest.java @@ -0,0 +1,36 @@ +package shop; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +class OrdersTest { + // SHOP_CHANGE: the change a replay of e2e/lane.sh makes to the code under + // test. miss: the price test asks for a pen, which no recorded call + // answers. unconsumed: the reserve test no longer makes its recorded call. + static final String CHANGE = System.getenv().getOrDefault("SHOP_CHANGE", ""); + + // A @Nested class: keploy names its tests by its binary name, + // shop.OrdersTest$Books. + @Nested + class Books { + @Test + void pricesABook() throws Exception { + if (CHANGE.equals("miss")) { + assertEquals(150, Shop.price("pen")); + } else { + assertEquals(4200, Shop.price("book")); + } + } + + @Test + void reservesABook() throws Exception { + if (CHANGE.equals("unconsumed")) { + return; + } + assertTrue(Shop.reserve("book")); + } + } +} diff --git a/keploy-junit5/e2e/shop/src/test/java/shop/PaymentsTest.java b/keploy-junit5/e2e/shop/src/test/java/shop/PaymentsTest.java new file mode 100644 index 0000000..b28fb86 --- /dev/null +++ b/keploy-junit5/e2e/shop/src/test/java/shop/PaymentsTest.java @@ -0,0 +1,16 @@ +package shop; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class PaymentsTest { + // A parameterized test: keploy names each invocation by its index, + // chargesACard(int)[1]. + @ParameterizedTest + @ValueSource(ints = {4200}) + void chargesACard(int cents) throws Exception { + assertEquals("captured", Shop.charge(cents)); + } +} diff --git a/keploy-junit5/e2e/shop/src/test/resources/junit-platform.properties b/keploy-junit5/e2e/shop/src/test/resources/junit-platform.properties new file mode 100644 index 0000000..f05ea8f --- /dev/null +++ b/keploy-junit5/e2e/shop/src/test/resources/junit-platform.properties @@ -0,0 +1,3 @@ +# keploy-junit5 wraps every test of the run, outside every other extension, +# only when JUnit loads it for the whole run. +junit.jupiter.extensions.autodetection.enabled=true diff --git a/keploy-junit5/pom.xml b/keploy-junit5/pom.xml new file mode 100644 index 0000000..23db0dc --- /dev/null +++ b/keploy-junit5/pom.xml @@ -0,0 +1,185 @@ + + + 4.0.0 + + + io.keploy + keploy-junit5 + 0.1.0-SNAPSHOT + jar + + keploy-junit5 + A JUnit 5 extension that tells keploy mock record and replay where each test begins and ends + https://github.com/keploy/java-sdk + + + + The Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + + + + + + keploy + Keploy + hello@keploy.io + Keploy + https://keploy.io + + + + + scm:git:https://github.com/keploy/java-sdk.git + scm:git:ssh://git@github.com/keploy/java-sdk.git + https://github.com/keploy/java-sdk.git + + + + UTF-8 + 1.8 + 1.8 + + 5.10.5 + + + + + + org.junit + junit-bom + ${junit.version} + pom + import + + + + + + + + org.junit.jupiter + junit-jupiter-api + provided + + + org.junit.platform + junit-platform-engine + provided + + + + org.junit.jupiter + junit-jupiter + test + + + org.junit.platform + junit-platform-testkit + test + + + + org.junit.platform + junit-platform-suite + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + + + + + + + release + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + attach-sources + verify + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.4.1 + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + org.sonatype.central + central-publishing-maven-plugin + 0.10.0 + true + + central + true + published + 7200 + + + + + + + diff --git a/keploy-junit5/src/main/java/io/keploy/junit5/AgentClient.java b/keploy-junit5/src/main/java/io/keploy/junit5/AgentClient.java new file mode 100644 index 0000000..378b9bb --- /dev/null +++ b/keploy-junit5/src/main/java/io/keploy/junit5/AgentClient.java @@ -0,0 +1,238 @@ +package io.keploy.junit5; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.net.HttpURLConnection; +import java.net.Proxy; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The client of keploy's agent: scope protocol v1, the same wire go-sdk's + * keploytest, @keploy/playwright, @keploy/jest and keploy-pytest speak (POST + * /agent/instance, /agent/scope/begin, /agent/scope/end). + * + *

Every answer of the agent carries the protocol version as "v"; an answer + * without it, or with another version, is an error, so this extension on an + * old agent (or the reverse) fails loudly instead of running unjudged. Every + * way an answer can fail to be trusted is an error. Nothing is retried, and no + * clock decides anything: there is no timeout, as the agent answers once the + * work it owes is done. + */ +final class AgentClient { + static final int PROTOCOL_VERSION = 1; + + static final String PATH_INSTANCE = "/agent/instance"; + static final String PATH_BEGIN = "/agent/scope/begin"; + static final String PATH_END = "/agent/scope/end"; + + static final String ACTION_SKIP = "skip"; + static final String ACTION_FAIL = "fail"; + + // Bounds what is read of one answer, and how much of one an error quotes. + static final int MAX_ANSWER = 1 << 20; + static final int MAX_QUOTED = 512; + + private final String url; + private final String token; + + AgentClient(String url, String token) { + this.url = url; + this.token = token; + } + + static String quoted(String text) { + String s = text.trim(); + if (s.length() > MAX_QUOTED) { + return Json.quote(s.substring(0, MAX_QUOTED)) + "..."; + } + return Json.quote(s); + } + + /** Sends one call and reads its answer: the answer's object. */ + Map post(String path, Map body) { + byte[] out = Json.object(body).getBytes(StandardCharsets.UTF_8); + int status; + InputStream in; + try { + // No proxy from the environment (the agent is keploy's own, on + // this machine), and no redirect: a redirect is an answer like + // any other non-2xx, and following it would send the call again, + // or to somewhere else. + HttpURLConnection c = (HttpURLConnection) new URL(url + path).openConnection(Proxy.NO_PROXY); + c.setInstanceFollowRedirects(false); + c.setRequestMethod("POST"); + c.setDoOutput(true); + // A fixed length streams the body, and HttpURLConnection never + // sends a streamed request a second time (it may resend a POST + // whose answer it failed to read otherwise). + c.setFixedLengthStreamingMode(out.length); + c.setRequestProperty("content-type", "application/json"); + if (!token.isEmpty()) { + c.setRequestProperty("authorization", "Bearer " + token); + } + try (OutputStream o = c.getOutputStream()) { + o.write(out); + } + status = c.getResponseCode(); + in = status >= 400 ? c.getErrorStream() : c.getInputStream(); + } catch (IOException e) { + throw new KeployException("keploy: cannot reach the keploy agent at " + url + ": " + e, 0, ""); + } + String data; + try { + data = readBounded(in); + } catch (IOException e) { + throw new KeployException("keploy: cannot read the keploy agent's answer to " + path + ": " + e, status, ""); + } + if (status < 200 || status > 299) { + Object e = parseOrNull(data); + if (e instanceof Map && text((Map) e, "message") != null) { + Map m = (Map) e; + String code = text(m, "error"); + String remedy = text(m, "remedy"); + throw new KeployException("keploy: " + text(m, "message") + (remedy == null ? "" : " (" + remedy + ")"), + status, code == null ? "" : code); + } + throw new KeployException("keploy: the keploy agent answered " + status + " to " + path + ": " + + quoted(data), status, ""); + } + Object a; + try { + a = Json.parse(data); + } catch (IllegalArgumentException e) { + throw new KeployException("keploy: the keploy agent's answer to " + path + " cannot be read (" + e.getMessage() + + "): " + quoted(data), status, ""); + } + if (!(a instanceof Map)) { + throw new KeployException("keploy: the keploy agent's answer to " + path + " cannot be read (not an object): " + + quoted(data), status, ""); + } + @SuppressWarnings("unchecked") + Map m = (Map) a; + if (m.get("v") == null) { + throw new KeployException("keploy: the keploy agent answered " + path + " with " + quoted(data) + + ", which names no scope protocol version; this keploy-junit5 needs a keploy agent that speaks scope protocol v" + + PROTOCOL_VERSION + ": upgrade keploy", status, ""); + } + Object v = m.get("v"); + if (!(v instanceof BigDecimal) || ((BigDecimal) v).compareTo(BigDecimal.valueOf(PROTOCOL_VERSION)) != 0) { + throw new KeployException("keploy: the keploy agent answered " + path + " in scope protocol v" + + (v instanceof String ? Json.quote((String) v) : String.valueOf(v)) + " (" + quoted(data) + + "); this keploy-junit5 speaks scope protocol v" + PROTOCOL_VERSION + + ": use a keploy and a keploy-junit5 of the same release", status, ""); + } + return m; + } + + /** Claims keploy's run for this test process. */ + Map claim(String key, String dir) { + Map body = new LinkedHashMap<>(); + body.put("key", key); + body.put("dir", dir); + Map a; + try { + a = post(PATH_INSTANCE, body); + } catch (KeployException e) { + if (e.status == 404 && e.code.isEmpty()) { + // keploy speaks scope protocol v1 only under its proven + // switch. With it off (keploy's default) the agent has no + // claim route and answers the mux's plain 404, as a keploy + // from before the protocol does. The claim is this + // extension's first call, so it is the one that names the + // switch. + throw new KeployException("keploy: this keploy agent answered 404 to POST " + PATH_INSTANCE + + ", so this JUnit process cannot claim keploy's run: keploy speaks keploy-junit5's wire, scope protocol v1," + + " only when it runs with KEPLOY_PROVEN=1 (KEPLOY_PROVEN=1 keploy mock record|replay -c \"mvn test\");" + + " a keploy run without KEPLOY_PROVEN=1, or one from before scope protocol v1, answers 404 there", + 404, ""); + } + throw e; + } + if (text(a, "instance") == null) { + throw new KeployException("keploy: the keploy agent answered the claim of this test process without an instance", 0, ""); + } + return a; + } + + /** Asks the agent to open a test. */ + Map begin(TestRef t) { + Map body = t.wire(); + body.put("canSkip", Boolean.TRUE); + Map a = post(PATH_BEGIN, body); + if (!"ok".equals(a.get("status"))) { + throw new KeployException("keploy: the keploy agent answered the begin of " + t.name + " with status " + + shown(a.get("status")), 0, ""); + } + if (!none(a.get("action")) && !ACTION_SKIP.equals(a.get("action"))) { + throw new KeployException("keploy: the keploy agent answered the begin of " + t.name + " with the unknown action " + + shown(a.get("action")), 0, ""); + } + return a; + } + + /** Tells the agent the test ended, with its outcome. */ + Map end(TestRef t, String outcome) { + Map body = t.wire(); + body.put("outcome", outcome); + Map a = post(PATH_END, body); + if (!"ok".equals(a.get("status"))) { + throw new KeployException("keploy: the keploy agent answered the end of " + t.name + " with status " + + shown(a.get("status")), 0, ""); + } + if (!none(a.get("action")) && !ACTION_FAIL.equals(a.get("action"))) { + throw new KeployException("keploy: the keploy agent answered the end of " + t.name + " with the unknown action " + + shown(a.get("action")), 0, ""); + } + return a; + } + + /** Whether an answer's action is none: absent, null or "" (anything else is an action). */ + static boolean none(Object action) { + return action == null || "".equals(action); + } + + /** A string member of an answer, or null when it is absent, empty or no string. */ + static String text(Map m, String key) { + Object v = m.get(key); + return v instanceof String && !((String) v).isEmpty() ? (String) v : null; + } + + private static String shown(Object v) { + if (v == null) { + return "\"\""; + } + return v instanceof String ? Json.quote((String) v) : String.valueOf(v); + } + + private static Object parseOrNull(String data) { + try { + return Json.parse(data); + } catch (IllegalArgumentException e) { + return null; // not the protocol's error body + } + } + + private static String readBounded(InputStream in) throws IOException { + if (in == null) { + return ""; // an answer with no body + } + try (InputStream r = in) { + ByteArrayOutputStream b = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int n; + while ((n = r.read(buf)) > 0) { + b.write(buf, 0, n); + if (b.size() > MAX_ANSWER) { + throw new IOException("the answer is larger than " + MAX_ANSWER + " bytes"); + } + } + return new String(b.toByteArray(), StandardCharsets.UTF_8); + } + } +} diff --git a/keploy-junit5/src/main/java/io/keploy/junit5/Joining.java b/keploy-junit5/src/main/java/io/keploy/junit5/Joining.java new file mode 100644 index 0000000..11c387c --- /dev/null +++ b/keploy-junit5/src/main/java/io/keploy/junit5/Joining.java @@ -0,0 +1,146 @@ +package io.keploy.junit5; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Map; +import java.util.function.LongPredicate; + +/** + * Whether this process reports to keploy: the same rule go-sdk's keploytest + * follows. keploy mock record and keploy mock replay export KEPLOY_MOCK_AGENT + * (the agent's URL), KEPLOY_MOCK_AGENT_TOKEN and KEPLOY_MOCK_PROJECT (the + * project's directory) into the test command. Unset or "off", the extension + * is inert and every test runs as it would without keploy. + */ +final class Joining { + static final String ENV_AGENT = "KEPLOY_MOCK_AGENT"; + static final String ENV_TOKEN = "KEPLOY_MOCK_AGENT_TOKEN"; + static final String ENV_PROJECT = "KEPLOY_MOCK_PROJECT"; + // The claimer's pid, exported by a test process (of go-sdk, keploy-pytest, + // ...) once it has claimed the run, so the processes its tests start know + // they belong to it and do not claim the run themselves. A JVM cannot + // change its own environment, so this extension exports none: a process + // its tests start that claims too is refused by the agent. + static final String ENV_TEST_PID = "KEPLOY_TEST_PID"; + + static final Joining INERT = new Joining(null, "", "", null); + + /** The agent's URL, without a trailing slash; null when inert or refused. */ + final String url; + final String token; + /** KEPLOY_MOCK_PROJECT, trimmed; "" when keploy names none. */ + final String project; + /** Why this process cannot join keploy's run; null when it can. */ + final String refusal; + + private Joining(String url, String token, String project, String refusal) { + this.url = url; + this.token = token; + this.project = project; + this.refusal = refusal; + } + + boolean inert() { + return url == null && refusal == null; + } + + /** Inert, refused or active, from the environment of process pid. */ + static Joining decide(Map env, long pid, LongPredicate runsInside) { + String v = get(env, ENV_AGENT).trim(); + if (v.isEmpty() || v.equalsIgnoreCase("off")) { + return INERT; + } + if (!httpURL(v)) { + return new Joining(null, "", "", "keploy: " + ENV_AGENT + "=" + Json.quote(v) + + " is neither off nor the http URL of a keploy agent; set it to off to run without keploy"); + } + String tp = get(env, ENV_TEST_PID).trim(); + if (!tp.equals(Long.toString(pid))) { + // Another process joined keploy's run. This one is a member only + // when it runs inside that process; a marker naming any other + // process was inherited from elsewhere, and this process claims + // like any test process (the agent refuses a second live one). + long n = 0; + try { + n = Long.parseLong(tp); + } catch (NumberFormatException e) { + // no marker ("") or no pid + } + if (n > 0 && runsInside.test(n)) { + return INERT; + } + } + return new Joining(v.replaceAll("/+$", ""), get(env, ENV_TOKEN), get(env, ENV_PROJECT).trim(), null); + } + + private static String get(Map env, String key) { + String v = env.get(key); + return v == null ? "" : v; + } + + private static boolean httpURL(String v) { + try { + URI u = new URI(v); + // getHost is null when the authority is no host[:port] (a port + // that is not a number, say). + return ("http".equalsIgnoreCase(u.getScheme()) || "https".equalsIgnoreCase(u.getScheme())) && u.getHost() != null; + } catch (URISyntaxException e) { + return false; + } + } + + /** This process's pid. */ + static long pid() { + String n = ManagementFactory.getRuntimeMXBean().getName(); + return Long.parseLong(n.substring(0, n.indexOf('@'))); + } + + /** + * Whether this process runs inside process pid (pid is an ancestor), read + * from /proc. Without /proc (a system other than Linux) it cannot tell, + * and takes the marker at its word. + */ + static boolean runsInsideProc(long pid) { + if (!Files.exists(Paths.get("/proc/self/stat"))) { + return true; + } + long p = parent("self"); + // Bounded: a walk that cycles (a pid reused while it is read) ends. + for (int k = 0; k < 128; k++) { + if (p == pid) { + return true; + } + p = parent(Long.toString(p)); + if (p < 0) { // past the first process (its parent, 0, has no entry), or gone + return false; + } + } + return false; + } + + /** The parent pid of /proc/{proc}; -1 when it cannot be read. */ + static long parent(String proc) { + try { + return ppid(new String(Files.readAllBytes(Paths.get("/proc", proc, "stat")), StandardCharsets.ISO_8859_1)); + } catch (IOException e) { + return -1; + } + } + + /** The parent pid in a /proc stat line; -1 when it holds none. */ + static long ppid(String stat) { + try { + // The command name, in parentheses, may hold spaces and + // parentheses: the fields after it start at the last ')'. + String[] rest = stat.substring(stat.lastIndexOf(')') + 2).split(" "); + return Long.parseLong(rest[1]); + } catch (RuntimeException e) { + return -1; + } + } +} diff --git a/keploy-junit5/src/main/java/io/keploy/junit5/Json.java b/keploy-junit5/src/main/java/io/keploy/junit5/Json.java new file mode 100644 index 0000000..336e7cc --- /dev/null +++ b/keploy-junit5/src/main/java/io/keploy/junit5/Json.java @@ -0,0 +1,246 @@ +package io.keploy.junit5; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The JSON of keploy's wire, without a library on the test suite's class + * path: a strict reader (RFC 8259; anything else is an error) and a writer of + * flat objects. An object reads as a Map, an array as a List, a number as a + * BigDecimal, a string as a String, true and false as Boolean, null as null. + */ +final class Json { + // A deeper answer is no answer of keploy's: bounded so that one cannot + // overflow the stack. + static final int MAX_DEPTH = 64; + + private final String s; + private int i; + + private Json(String s) { + this.s = s; + } + + /** Reads one JSON text; IllegalArgumentException says why it is none. */ + static Object parse(String s) { + Json j = new Json(s); + Object v = j.value(0); + j.space(); + if (j.i != s.length()) { + throw j.error("text after the value"); + } + return v; + } + + /** Writes a flat object of String, Boolean and Number values, in order. */ + static String object(Map m) { + StringBuilder b = new StringBuilder("{"); + for (Map.Entry e : m.entrySet()) { + if (b.length() > 1) { + b.append(','); + } + b.append(quote(e.getKey())).append(':'); + Object v = e.getValue(); + b.append(v instanceof String ? quote((String) v) : String.valueOf(v)); + } + return b.append('}').toString(); + } + + /** s as a JSON string. */ + static String quote(String s) { + StringBuilder b = new StringBuilder("\""); + for (int k = 0; k < s.length(); k++) { + char c = s.charAt(k); + if (c == '"' || c == '\\') { + b.append('\\').append(c); + } else if (c < 0x20) { + b.append(String.format("\\u%04x", (int) c)); + } else { + b.append(c); + } + } + return b.append('"').toString(); + } + + private IllegalArgumentException error(String what) { + return new IllegalArgumentException(what + " at offset " + i); + } + + private void space() { + while (i < s.length() && " \t\r\n".indexOf(s.charAt(i)) >= 0) { + i++; + } + } + + private Object value(int depth) { + if (depth > MAX_DEPTH) { + throw error("nested deeper than " + MAX_DEPTH); + } + space(); + if (i >= s.length()) { + throw error("no value"); + } + char c = s.charAt(i); + switch (c) { + case '{': + return object(depth); + case '[': + return array(depth); + case '"': + return string(); + case 't': + return word("true", Boolean.TRUE); + case 'f': + return word("false", Boolean.FALSE); + case 'n': + return word("null", null); + default: + return number(); + } + } + + private Object word(String w, Object v) { + if (!s.startsWith(w, i)) { + throw error("not a value"); + } + i += w.length(); + return v; + } + + private Map object(int depth) { + Map m = new LinkedHashMap<>(); + i++; + space(); + if (i < s.length() && s.charAt(i) == '}') { + i++; + return m; + } + while (true) { + space(); + if (i >= s.length() || s.charAt(i) != '"') { + throw error("no member name"); + } + String k = string(); + space(); + expect(':'); + m.put(k, value(depth + 1)); + space(); + if (i < s.length() && s.charAt(i) == ',') { + i++; + continue; + } + expect('}'); + return m; + } + } + + private List array(int depth) { + List l = new ArrayList<>(); + i++; + space(); + if (i < s.length() && s.charAt(i) == ']') { + i++; + return l; + } + while (true) { + l.add(value(depth + 1)); + space(); + if (i < s.length() && s.charAt(i) == ',') { + i++; + continue; + } + expect(']'); + return l; + } + } + + private void expect(char c) { + if (i >= s.length() || s.charAt(i) != c) { + throw error("want '" + c + "'"); + } + i++; + } + + private String string() { + StringBuilder b = new StringBuilder(); + i++; + while (true) { + if (i >= s.length()) { + throw error("unterminated string"); + } + char c = s.charAt(i++); + if (c == '"') { + return b.toString(); + } + if (c < 0x20) { + throw error("control character in a string"); + } + if (c != '\\') { + b.append(c); + continue; + } + if (i >= s.length()) { + throw error("unterminated escape"); + } + char e = s.charAt(i++); + int at = "\"\\/bfnrt".indexOf(e); + if (at >= 0) { + b.append("\"\\/\b\f\n\r\t".charAt(at)); + } else if (e == 'u') { + int u = 0; + for (int k = 0; k < 4; k++) { + // ASCII hex digits only (Character.digit takes any + // script's digits). + int d = i < s.length() ? "0123456789abcdef".indexOf(Character.toLowerCase(s.charAt(i++))) : -1; + if (d < 0) { + throw error("bad \\u escape"); + } + u = u * 16 + d; + } + b.append((char) u); + } else { + throw error("bad escape"); + } + } + } + + private BigDecimal number() { + int start = i; + if (i < s.length() && s.charAt(i) == '-') { + i++; + } + // JSON's grammar: no leading zero, no leading '+', no bare '.'. + if (i < s.length() && s.charAt(i) == '0') { + i++; + } else if (!digits()) { + throw error("not a value"); + } + if (i < s.length() && s.charAt(i) == '.') { + i++; + if (!digits()) { + throw error("no digit after '.'"); + } + } + if (i < s.length() && (s.charAt(i) == 'e' || s.charAt(i) == 'E')) { + i++; + if (i < s.length() && (s.charAt(i) == '+' || s.charAt(i) == '-')) { + i++; + } + if (!digits()) { + throw error("no digit in the exponent"); + } + } + return new BigDecimal(s.substring(start, i)); + } + + private boolean digits() { + int start = i; + while (i < s.length() && s.charAt(i) >= '0' && s.charAt(i) <= '9') { + i++; + } + return i > start; + } +} diff --git a/keploy-junit5/src/main/java/io/keploy/junit5/KeployException.java b/keploy-junit5/src/main/java/io/keploy/junit5/KeployException.java new file mode 100644 index 0000000..3a3ab1a --- /dev/null +++ b/keploy-junit5/src/main/java/io/keploy/junit5/KeployException.java @@ -0,0 +1,27 @@ +package io.keploy.junit5; + +/** + * An answer of keploy's agent this extension cannot trust, the agent's own + * refusal, or keploy's judgment of a test. Its message is keploy's text, and + * it carries no stack trace: the text is the whole report. + */ +public final class KeployException extends RuntimeException { + private static final long serialVersionUID = 1L; + + /** The HTTP status of a non-2xx answer (0 when there was none). */ + final int status; + /** The scope protocol's error code of a refusal ("not_open", ...), or "". */ + final String code; + + KeployException(String message, int status, String code) { + // Suppression stays on: JUnit adds a later failure of the same test + // to the first one as a suppressed exception. + super(message, null, true, false); + this.status = status; + this.code = code; + } + + KeployException(String message) { + this(message, 0, ""); + } +} diff --git a/keploy-junit5/src/main/java/io/keploy/junit5/KeployExtension.java b/keploy-junit5/src/main/java/io/keploy/junit5/KeployExtension.java new file mode 100644 index 0000000..dbb63be --- /dev/null +++ b/keploy-junit5/src/main/java/io/keploy/junit5/KeployExtension.java @@ -0,0 +1,218 @@ +package io.keploy.junit5; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.opentest4j.TestAbortedException; + +/** + * keploy-junit5 joins a JUnit 5 (Jupiter) suite to keploy's Universal Mocker. + * + *

keploy records the calls each test's code makes (keploy mock record) and + * replays them (keploy mock replay), so the suite runs with every dependency + * answered from the recording. keploy judges every recorded test and lists + * the ones it cannot replay, with a reason. This extension is how a test tells + * keploy where it begins and ends, and how keploy skips a test it set aside + * or fails a test whose calls changed. It speaks the same scope protocol (v1) + * as go-sdk's keploytest, @keploy/playwright, @keploy/jest and keploy-pytest; + * it decides nothing itself. + * + *

JUnit loads it for the whole run (junit.jupiter.extensions.autodetection.enabled=true), + * so it wraps every test, outside every other extension. Without keploy + * (KEPLOY_MOCK_AGENT unset or "off") it does nothing. Under keploy, with one + * test JVM (keploy's first release runs one test process at a time, and + * refuses a second): + *

    + *
  • the JVM claims keploy's run before the first test class's + * {@code @BeforeAll} (POST /agent/instance), with keploy's project folder + * (KEPLOY_MOCK_PROJECT); the current folder only when keploy names none;
  • + *
  • each test begins (POST /agent/scope/begin) once its class's + * {@code @BeforeAll} methods ran, before any other extension's BeforeEach + * callback and before its {@code @BeforeEach} methods; a test keploy set + * aside is aborted (JUnit's skip) with keploy's reason, and none of its + * {@code @BeforeEach} or {@code @AfterEach} methods runs;
  • + *
  • the test ends (POST /agent/scope/end) after its {@code @AfterEach} + * methods and every other extension's AfterEach callback, before its class's + * {@code @AfterAll}, with its outcome; when keploy saw a call with no + * recorded answer, or a recorded call the test no longer made, the test fails + * with keploy's text.
  • + *
+ * The calls a {@code @BeforeAll} or {@code @AfterAll} method makes are the + * test process's, outside every test, as JUnit scopes them. Any answer of the + * agent this extension cannot trust fails the test it belongs to; a claim that + * fails fails every test class. A test begun a second time in one JVM (a + * rerun) fails with the reason; none runs unjudged. + */ +public final class KeployExtension implements BeforeAllCallback, BeforeEachCallback, AfterEachCallback { + static final String AUTODETECT = "junit.jupiter.extensions.autodetection.enabled"; + static final String PARALLEL = "junit.jupiter.execution.parallel.enabled"; + + private static final ExtensionContext.Namespace NS = ExtensionContext.Namespace.create(KeployExtension.class); + private static final String OPEN = "open"; + + // The JVM's run: keploy's claim is one per process. + private static Run run; + + static synchronized Run run() { + if (run == null) { + run = new Run(Joining.decide(System.getenv(), Joining.pid(), Joining::runsInsideProc)); + } + return run; + } + + /** Sets the JVM's run (the rows' fake agent). */ + static synchronized void use(Run r) { + run = r; + } + + @Override + public void beforeAll(ExtensionContext ctx) { + run().ready(ctx); + } + + @Override + public void beforeEach(ExtensionContext ctx) { + Run r = run(); + if (!r.ready(ctx)) { + return; + } + TestRef t = TestRef.of(ctx.getRequiredTestClass().getName(), ctx.getUniqueId()); + String skip = r.begin(t); + if (skip != null) { + // JUnit's own skip: no @BeforeEach, test or @AfterEach method + // runs, and the reason is JUnit's abort reason. No end follows. + System.err.println("keploy-junit5: skipped " + t.id() + " (" + skip + ")"); + throw new TestAbortedException(skip); + } + ctx.getStore(NS).put(OPEN, t); + } + + @Override + public void afterEach(ExtensionContext ctx) { + TestRef t = ctx.getStore(NS).remove(OPEN, TestRef.class); + if (t == null) { + return; // not begun: inert, set aside, or its begin failed it + } + String failure = run().end(t, outcome(ctx.getExecutionException().orElse(null))); + if (failure != null) { + throw new KeployException(failure); + } + } + + /** + * A test's outcome as keploy's end takes it, from what it threw through + * its {@code @AfterEach} methods: an abort (a failed assumption) is + * skipped. + */ + static String outcome(Throwable thrown) { + if (thrown == null) { + return "passed"; + } + return thrown instanceof TestAbortedException ? "skipped" : "failed"; + } + + /** + * keploy's reason as the test shows it: once prefixed with "keploy: " (a + * judging reason from the agent already is; a skip reason, + * "<class>/<reason>: <detail>", is not), or fallback when + * the agent gave none, as go-sdk's keploytest shows it. + */ + static String keployText(String reason, String fallback) { + if (reason == null) { + return fallback; + } + return reason.startsWith("keploy: ") ? reason : "keploy: " + reason; + } + + /** One JVM's part in keploy's run. */ + static final class Run { + private final Joining joining; + private boolean claimed; + private AgentClient client; + // Why no test of this JVM can run judged: a claim that failed. + private String error; + // Every test ID begun in this JVM: a second begin of one fails. + private final Set begun = new HashSet<>(); + + Run(Joining joining) { + this.joining = joining; + } + + /** + * Whether this JVM's tests report to keploy: false when it is inert; + * throws when they cannot run judged (a refused KEPLOY_MOCK_AGENT, a + * run JUnit does not let this extension wrap whole, a failed claim). + * The claim is made at the first call, once. + */ + synchronized boolean ready(ExtensionContext ctx) { + if (joining.inert()) { + return false; + } + if (joining.refusal != null) { + throw new KeployException(joining.refusal); + } + // JUnit reads both with Boolean.parseBoolean. + if (!ctx.getConfigurationParameter(AUTODETECT).map(Boolean::parseBoolean).orElse(false)) { + throw new KeployException("keploy: keploy-junit5 wraps every test of the run, outside every other extension," + + " only when JUnit loads it for the whole run: set " + AUTODETECT + "=true (in" + + " src/test/resources/junit-platform.properties, or as a system property) instead of naming it in @ExtendWith"); + } + if (ctx.getConfigurationParameter(PARALLEL).map(Boolean::parseBoolean).orElse(false)) { + throw new KeployException("keploy: keploy runs one test at a time, and this run has JUnit's parallel execution on (" + + PARALLEL + "=true): turn it off for runs under keploy"); + } + if (!claimed) { + claimed = true; + claim(); + } + if (client == null) { + throw new KeployException(error); + } + return true; + } + + private void claim() { + String d = joining.project.isEmpty() ? System.getProperty("user.dir") : joining.project; + AgentClient c = new AgentClient(joining.url, joining.token); + try { + c.claim(".", new File(d).getCanonicalPath()); + } catch (KeployException e) { + error = e.getMessage(); + return; + } catch (IOException e) { + error = "keploy: cannot resolve keploy's project folder " + d + ": " + e; + return; + } + client = c; + } + + /** Begins t: keploy's skip reason when keploy set it aside, else null. */ + synchronized String begin(TestRef t) { + if (!begun.add(t.id())) { + throw new KeployException("keploy: " + t.name + " in " + t.file + " would begin a second time, and keploy names a" + + " test by its class and name: run each test once (no reruns, such as Surefire's rerunFailingTestsCount)"); + } + Map a = client.begin(t); + if (AgentClient.ACTION_SKIP.equals(a.get("action"))) { + return keployText(AgentClient.text(a, "reason"), "keploy: set aside (the agent gave no reason)"); + } + return null; + } + + /** Ends t: keploy's text when keploy failed it, else null. */ + synchronized String end(TestRef t, String outcome) { + Map a = client.end(t, outcome); + if (AgentClient.ACTION_FAIL.equals(a.get("action"))) { + return keployText(AgentClient.text(a, "reason"), "keploy: the keploy agent failed this test and gave no reason"); + } + return null; + } + } +} diff --git a/keploy-junit5/src/main/java/io/keploy/junit5/TestRef.java b/keploy-junit5/src/main/java/io/keploy/junit5/TestRef.java new file mode 100644 index 0000000..2945dfb --- /dev/null +++ b/keploy-junit5/src/main/java/io/keploy/junit5/TestRef.java @@ -0,0 +1,69 @@ +package io.keploy.junit5; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.junit.platform.engine.UniqueId; + +/** + * A test's name on keploy's wire: keploy builds its ID as file::name. file is + * the test's class (its binary name, Outer$Inner for a @Nested class); name + * is JUnit's own name of the method within it, from the test's unique ID: + * {@code orders()}, {@code price(java.lang.String)}, and a template's + * invocation index after it, {@code price(java.lang.String)[2]}. A test of a + * class template (a @ParameterizedClass, JUnit 5.13 and later) has the class's + * invocation index before it, {@code [2]price(java.lang.String)}, one per + * class template it runs in, outermost first: each invocation of the class + * runs each of its tests in a BeforeEach/AfterEach pair of its own. A test run + * through a JUnit Platform @Suite is named as when it runs directly: the + * suite and its engine are not the test. + */ +final class TestRef { + final String file; + final String name; + + TestRef(String file, String name) { + this.file = file; + this.name = name; + } + + String id() { + return file + "::" + name; + } + + Map wire() { + Map m = new LinkedHashMap<>(); + m.put("name", name); + m.put("file", file); + return m; + } + + static TestRef of(String testClass, String uniqueId) { + StringBuilder name = new StringBuilder(); + for (UniqueId.Segment s : UniqueId.parse(uniqueId).getSegments()) { + switch (s.getType()) { + case "engine": + case "suite": + case "class": + case "nested-class": + case "class-template": + case "nested-class-template": + break; + case "method": + case "test-template": + name.append(s.getValue()); + break; + case "test-template-invocation": + case "class-template-invocation": + name.append('[').append(s.getValue().replace("#", "")).append(']'); + break; + default: + throw new KeployException("keploy: keploy-junit5 begins and ends each test at JUnit's BeforeEach and AfterEach" + + " callbacks, and cannot name " + uniqueId + " as one test (a " + s.getType() + "; a @TestFactory's" + + " dynamic tests, for one, all run inside one of them): write it as a @Test or a @ParameterizedTest," + + " or run it without keploy"); + } + } + return new TestRef(testClass, name.toString()); + } +} diff --git a/keploy-junit5/src/main/resources/META-INF/services/org.junit.jupiter.api.extension.Extension b/keploy-junit5/src/main/resources/META-INF/services/org.junit.jupiter.api.extension.Extension new file mode 100644 index 0000000..bf7c4a7 --- /dev/null +++ b/keploy-junit5/src/main/resources/META-INF/services/org.junit.jupiter.api.extension.Extension @@ -0,0 +1 @@ +io.keploy.junit5.KeployExtension diff --git a/keploy-junit5/src/test/java/io/keploy/junit5/AgentClientTest.java b/keploy-junit5/src/test/java/io/keploy/junit5/AgentClientTest.java new file mode 100644 index 0000000..ab7b9c3 --- /dev/null +++ b/keploy-junit5/src/test/java/io/keploy/junit5/AgentClientTest.java @@ -0,0 +1,207 @@ +package io.keploy.junit5; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.net.ServerSocket; +import java.util.Map; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.Executable; + +class AgentClientTest { + static final TestRef T = new TestRef("shop.OrdersTest", "prices()"); + FakeAgent agent; + + @BeforeEach + void start() throws IOException { + agent = new FakeAgent(); + } + + @AfterEach + void stop() { + agent.close(); + } + + void answer(int status, String body) { + agent.answer = (p, b) -> new FakeAgent.Answer(status, body); + } + + String error(Executable call) { + return assertThrows(KeployException.class, call).getMessage(); + } + + @Test + void sendsTheWireAndItsToken() { + AgentClient c = new AgentClient(agent.url(), "tok"); + assertEquals("i-1", c.claim(".", "/p").get("instance")); + assertEquals("Bearer tok", agent.lastAuth); + c.begin(T); + c.end(T, "passed"); + assertEquals("POST /agent/instance {\"key\":\".\",\"dir\":\"/p\"}", agent.calls.get(0)); + assertEquals("POST /agent/scope/begin {\"name\":\"prices()\",\"file\":\"shop.OrdersTest\",\"canSkip\":true}", agent.calls.get(1)); + assertEquals("POST /agent/scope/end {\"name\":\"prices()\",\"file\":\"shop.OrdersTest\",\"outcome\":\"passed\"}", agent.calls.get(2)); + agent.client().begin(T); + assertNull(agent.lastAuth); + } + + @Test + void takesAny2xxAndAVersionThatEqualsOne() { + answer(299, "{\"v\":1.0,\"status\":\"ok\",\"action\":\"\"}"); + assertEquals("ok", agent.client().begin(T).get("status")); + } + + @Test + void takesAnAnswerOfTheLargestSize() { + String head = "{\"v\":1,\"status\":\"ok\",\"pad\":\""; + StringBuilder b = new StringBuilder(head); + while (b.length() < AgentClient.MAX_ANSWER - 2) { + b.append('x'); + } + answer(200, b.append("\"}").toString()); + agent.client().begin(T); + answer(200, b.insert(head.length(), 'x').toString()); + assertEquals("keploy: cannot read the keploy agent's answer to /agent/scope/begin: java.io.IOException: the answer is larger than 1048576 bytes", + error(() -> agent.client().begin(T))); + } + + @Test + void namesTheProtocolsErrorWithItsRemedy() { + answer(409, "{\"error\":\"not_open\",\"message\":\"no test is open\",\"remedy\":\"begin it first\"}"); + KeployException e = assertThrows(KeployException.class, () -> agent.client().end(T, "passed")); + assertEquals("keploy: no test is open (begin it first)", e.getMessage()); + assertEquals(409, e.status); + assertEquals("not_open", e.code); + answer(400, "{\"message\":\"bad name\",\"remedy\":\"\",\"error\":7}"); + e = assertThrows(KeployException.class, () -> agent.client().end(T, "passed")); + assertEquals("keploy: bad name", e.getMessage()); + assertEquals("", e.code); + } + + @Test + void quotesAnyOtherNon2xxAnswer() { + answer(500, " boom "); + assertEquals("keploy: the keploy agent answered 500 to /agent/scope/begin: \"boom\"", error(() -> agent.client().begin(T))); + answer(503, "{\"message\":\"\"}"); + assertEquals("keploy: the keploy agent answered 503 to /agent/scope/begin: \"{\\\"message\\\":\\\"\\\"}\"", error(() -> agent.client().begin(T))); + answer(502, "[\"message\"]"); + assertEquals("keploy: the keploy agent answered 502 to /agent/scope/begin: \"[\\\"message\\\"]\"", error(() -> agent.client().begin(T))); + answer(500, ""); + assertEquals("keploy: the keploy agent answered 500 to /agent/scope/begin: \"\"", error(() -> agent.client().begin(T))); + StringBuilder b = new StringBuilder(); + for (int k = 0; k < AgentClient.MAX_QUOTED; k++) { + b.append('y'); + } + answer(500, b.toString()); + assertEquals("keploy: the keploy agent answered 500 to /agent/scope/begin: \"" + b + "\"", error(() -> agent.client().begin(T))); + answer(500, b + "z"); + assertEquals("keploy: the keploy agent answered 500 to /agent/scope/begin: \"" + b + "\"...", error(() -> agent.client().begin(T))); + } + + @Test + void followsNoRedirect() { + int[] seen = {0}; + agent.answer = (p, b) -> { + seen[0]++; + return seen[0] == 1 ? new FakeAgent.Answer(307, "", agent.url() + "/other") : FakeAgent.ok(p, b); + }; + assertEquals("keploy: the keploy agent answered 307 to /agent/scope/begin: \"\"", error(() -> agent.client().begin(T))); + answer(300, "{\"v\":1,\"status\":\"ok\"}"); + assertTrue(error(() -> agent.client().begin(T)).startsWith("keploy: the keploy agent answered 300 to /agent/scope/begin: ")); + assertEquals(2, agent.calls.size()); + } + + @Test + void refusesAnAnswerItCannotTrust() { + answer(200, "{\"status\":\"ok\"}"); + assertEquals("keploy: the keploy agent answered /agent/scope/begin with \"{\\\"status\\\":\\\"ok\\\"}\", which names no scope protocol" + + " version; this keploy-junit5 needs a keploy agent that speaks scope protocol v1: upgrade keploy", error(() -> agent.client().begin(T))); + answer(200, "{\"v\":null}"); + assertTrue(error(() -> agent.client().begin(T)).contains("which names no scope protocol version")); + answer(200, "{\"v\":2}"); + assertEquals("keploy: the keploy agent answered /agent/scope/begin in scope protocol v2 (\"{\\\"v\\\":2}\"); this keploy-junit5 speaks" + + " scope protocol v1: use a keploy and a keploy-junit5 of the same release", error(() -> agent.client().begin(T))); + answer(200, "{\"v\":\"1\"}"); + assertTrue(error(() -> agent.client().begin(T)).startsWith("keploy: the keploy agent answered /agent/scope/begin in scope protocol v\"1\" (")); + answer(200, "{\"v\":true}"); + assertTrue(error(() -> agent.client().begin(T)).startsWith("keploy: the keploy agent answered /agent/scope/begin in scope protocol vtrue (")); + answer(200, "[1]"); + assertEquals("keploy: the keploy agent's answer to /agent/scope/begin cannot be read (not an object): \"[1]\"", error(() -> agent.client().begin(T))); + answer(200, "{\"v\":1"); + assertEquals("keploy: the keploy agent's answer to /agent/scope/begin cannot be read (want '}' at offset 6): \"{\\\"v\\\":1\"", + error(() -> agent.client().begin(T))); + } + + @Test + void refusesABeginOrEndThatIsNotOk() { + answer(200, "{\"v\":1,\"status\":\"held\"}"); + assertEquals("keploy: the keploy agent answered the begin of prices() with status \"held\"", error(() -> agent.client().begin(T))); + assertEquals("keploy: the keploy agent answered the end of prices() with status \"held\"", error(() -> agent.client().end(T, "passed"))); + answer(200, "{\"v\":1}"); + assertEquals("keploy: the keploy agent answered the begin of prices() with status \"\"", error(() -> agent.client().begin(T))); + answer(200, "{\"v\":1,\"status\":2}"); + assertEquals("keploy: the keploy agent answered the end of prices() with status 2", error(() -> agent.client().end(T, "passed"))); + answer(200, "{\"v\":1,\"status\":\"ok\",\"action\":\"fail\"}"); + assertEquals("keploy: the keploy agent answered the begin of prices() with the unknown action \"fail\"", error(() -> agent.client().begin(T))); + assertEquals("fail", agent.client().end(T, "passed").get("action")); + answer(200, "{\"v\":1,\"status\":\"ok\",\"action\":\"skip\"}"); + assertEquals("skip", agent.client().begin(T).get("action")); + assertEquals("keploy: the keploy agent answered the end of prices() with the unknown action \"skip\"", error(() -> agent.client().end(T, "passed"))); + answer(200, "{\"v\":1,\"status\":\"ok\",\"action\":false}"); + assertEquals("keploy: the keploy agent answered the begin of prices() with the unknown action false", error(() -> agent.client().begin(T))); + answer(200, "{\"v\":1,\"status\":\"ok\",\"action\":null}"); + agent.client().end(T, "passed"); + } + + @Test + void namesKeploysProvenSwitchOnAPlain404Claim() { + answer(404, "404 page not found\n"); + KeployException e = assertThrows(KeployException.class, () -> agent.client().claim(".", "/p")); + assertEquals("keploy: this keploy agent answered 404 to POST /agent/instance, so this JUnit process cannot claim keploy's run:" + + " keploy speaks keploy-junit5's wire, scope protocol v1, only when it runs with KEPLOY_PROVEN=1" + + " (KEPLOY_PROVEN=1 keploy mock record|replay -c \"mvn test\"); a keploy run without KEPLOY_PROVEN=1, or one from before" + + " scope protocol v1, answers 404 there", e.getMessage()); + answer(404, "{\"error\":\"no_run\",\"message\":\"no run\"}"); + assertEquals("keploy: no run", error(() -> agent.client().claim(".", "/p"))); + answer(410, "gone"); + assertEquals("keploy: the keploy agent answered 410 to /agent/instance: \"gone\"", error(() -> agent.client().claim(".", "/p"))); + answer(404, "x"); + assertEquals("keploy: the keploy agent answered 404 to /agent/scope/end: \"x\"", error(() -> agent.client().end(T, "passed"))); + } + + @Test + void refusesAClaimWithoutAnInstance() { + for (String body : new String[] {"{\"v\":1}", "{\"v\":1,\"instance\":\"\"}", "{\"v\":1,\"instance\":true}"}) { + answer(200, body); + assertEquals("keploy: the keploy agent answered the claim of this test process without an instance", + error(() -> agent.client().claim(".", "/p")), body); + } + } + + @Test + void namesAnAgentItCannotReach() throws IOException { + int port; + try (ServerSocket s = new ServerSocket(0)) { + port = s.getLocalPort(); + } + String url = "http://127.0.0.1:" + port; + assertTrue(error(() -> new AgentClient(url, "").begin(T)).startsWith("keploy: cannot reach the keploy agent at " + url + ": java.net.ConnectException")); + } + + @Test + void readsTheReasonOfAnAnswer() { + Map m = new java.util.HashMap<>(); + m.put("a", ""); + m.put("b", 1); + m.put("c", "x"); + assertNull(AgentClient.text(m, "a")); + assertNull(AgentClient.text(m, "b")); + assertNull(AgentClient.text(m, "z")); + assertEquals("x", AgentClient.text(m, "c")); + } +} diff --git a/keploy-junit5/src/test/java/io/keploy/junit5/ExtensionTest.java b/keploy-junit5/src/test/java/io/keploy/junit5/ExtensionTest.java new file mode 100644 index 0000000..d9196ba --- /dev/null +++ b/keploy-junit5/src/test/java/io/keploy/junit5/ExtensionTest.java @@ -0,0 +1,343 @@ +package io.keploy.junit5; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.platform.testkit.engine.Event; +import org.junit.platform.testkit.engine.EventType; +import org.opentest4j.TestAbortedException; + +class ExtensionTest { + static final String F = "io.keploy.junit5.Fixtures$"; + FakeAgent agent; + String project; + + @BeforeEach + void start(@TempDir Path tmp) throws IOException { + agent = new FakeAgent(); + FakeAgent.LOG.clear(); + // keploy's project folder through a symlink: the claim names the folder itself. + Path real = Files.createDirectory(tmp.resolve("real")); + project = real.toFile().getCanonicalPath(); + KeployExtension.use(new KeployExtension.Run(joining(Files.createSymbolicLink(tmp.resolve("link"), real).toString()))); + } + + @AfterEach + void stop() { + KeployExtension.use(null); + agent.close(); + } + + Joining joining(String projectDir) { + return Joining.decide(JoiningTest.env("KEPLOY_MOCK_AGENT", agent.url(), "KEPLOY_MOCK_PROJECT", projectDir), 10, p -> false); + } + + /** Runs one fixture class through JUnit, keploy-junit5 loaded for the whole run, and more parameters. */ + static List run(Class c, String... params) { + EngineTestKit.Builder b = EngineTestKit.engine("junit-jupiter").selectors(DiscoverySelectors.selectClass(c)) + .configurationParameter(KeployExtension.AUTODETECT, "true"); + for (int k = 0; k < params.length; k += 2) { + b.configurationParameter(params[k], params[k + 1]); + } + return b.execute().allEvents().list(); + } + + /** Each test's and failed class's result: its outcome and message, by JUnit's name of it. */ + static Map results(List events) { + Map m = new LinkedHashMap<>(); + for (Event e : events) { + String name = e.getTestDescriptor().getUniqueId().getLastSegment().getValue(); + if (e.getType() == EventType.SKIPPED) { + m.put(name, "SKIPPED"); + } else if (e.getType() == EventType.FINISHED) { + TestExecutionResult r = e.getPayload(TestExecutionResult.class).get(); + if (e.getTestDescriptor().isTest() || r.getStatus() != TestExecutionResult.Status.SUCCESSFUL) { + m.put(name, r.getStatus() + r.getThrowable().map(t -> ": " + t.getMessage() + + (t.getSuppressed().length > 0 ? " + " + t.getSuppressed()[0].getMessage() : "")).orElse("")); + } + } + } + return m; + } + + static Map map(String... kv) { + Map m = new LinkedHashMap<>(); + for (int k = 0; k < kv.length; k += 2) { + m.put(kv[k], kv[k + 1]); + } + return m; + } + + static List log() { + return new ArrayList<>(FakeAgent.LOG); + } + + void answer(String what, String name, String body) { + agent.answer = (p, b) -> p.endsWith(what) && name.equals(b.get("name")) ? new FakeAgent.Answer(200, body) : FakeAgent.ok(p, b); + } + + @Test + void wrapsEachTestOutsideEveryOtherExtensionWithItsOutcome() { + assertEquals(map("assumes()", "ABORTED: Assumption failed: not here", "fails()", "FAILED: boom", "passes()", "SUCCESSFUL"), + results(run(Fixtures.Orders.class))); + List want = new ArrayList<>(Arrays.asList("claim . " + project, "@BeforeAll")); + for (String[] t : new String[][] {{"assumes", "skipped"}, {"fails", "failed"}, {"passes", "passed"}}) { + want.addAll(Arrays.asList("begin " + F + "Orders::" + t[0] + "()", "other.beforeEach", "@BeforeEach " + t[0], t[0], + "@AfterEach " + t[0], "other.afterEach", "end " + F + "Orders::" + t[0] + "() " + t[1])); + } + want.add("@AfterAll"); + assertEquals(want, log()); + } + + @Test + void claimsOncePerJvmFromTheCurrentFolderWhenKeployNamesNone() throws IOException { + KeployExtension.use(new KeployExtension.Run(joining(""))); + run(Fixtures.Payments.class); + run(Fixtures.AfterEachFails.class); + assertEquals(Arrays.asList("claim . " + new File(System.getProperty("user.dir")).getCanonicalPath(), "begin " + F + "Payments::charges()", + "charges", "end " + F + "Payments::charges() passed", "begin " + F + "AfterEachFails::cleans()", "cleans", + "end " + F + "AfterEachFails::cleans() failed"), log()); + } + + @Test + void skipsATestKeploySetAsideBeforeAnyOfItsCode() { + answer("begin", "passes()", "{\"v\":1,\"status\":\"ok\",\"action\":\"skip\",\"reason\":\"quarantined/flaky: set aside by a row\"}"); + assertEquals("ABORTED: keploy: quarantined/flaky: set aside by a row", results(run(Fixtures.Orders.class)).get("passes()")); + List l = log(); + int at = l.indexOf("begin " + F + "Orders::passes()"); + // JUnit runs every extension's AfterEach callback after a BeforeEach + // callback threw; none of the test's own methods runs. + assertEquals(Arrays.asList("begin " + F + "Orders::passes()", "other.afterEach", "@AfterAll"), l.subList(at, l.size())); + FakeAgent.LOG.clear(); + answer("begin", "charges()", "{\"v\":1,\"status\":\"ok\",\"action\":\"skip\"}"); + assertEquals(map("charges()", "ABORTED: keploy: set aside (the agent gave no reason)"), results(run(Fixtures.Payments.class))); + answer("begin", "cleans()", "{\"v\":1,\"status\":\"ok\",\"action\":\"skip\",\"reason\":\"keploy: held\"}"); + assertEquals(map("cleans()", "ABORTED: keploy: held"), results(run(Fixtures.AfterEachFails.class))); + assertEquals(Arrays.asList("begin " + F + "Payments::charges()", "begin " + F + "AfterEachFails::cleans()"), log()); + } + + @Test + void anEmptyActionIsNoAction() { + agent.answer = (p, b) -> "charges()".equals(b.get("name")) && (p.endsWith("begin") || p.endsWith("end")) + ? new FakeAgent.Answer(200, "{\"v\":1,\"status\":\"ok\",\"action\":\"\"}") + : FakeAgent.ok(p, b); + assertEquals(map("charges()", "SUCCESSFUL"), results(run(Fixtures.Payments.class))); + assertEquals(Arrays.asList("claim . " + project, "begin " + F + "Payments::charges()", "charges", "end " + F + "Payments::charges() passed"), + log()); + } + + @Test + void failsATestKeployFailedAtItsEnd() { + agent.answer = (p, b) -> p.endsWith("end") && "passes()".equals(b.get("name")) + ? new FakeAgent.Answer(200, "{\"v\":1,\"status\":\"ok\",\"action\":\"fail\",\"reason\":\"keploy: GET /x had no recorded answer\"}") + : p.endsWith("end") && "fails()".equals(b.get("name")) + ? new FakeAgent.Answer(200, "{\"v\":1,\"status\":\"ok\",\"action\":\"fail\",\"reason\":\"unmocked\"}") + : FakeAgent.ok(p, b); + Map r = results(run(Fixtures.Orders.class)); + assertEquals("FAILED: keploy: GET /x had no recorded answer", r.get("passes()")); + assertEquals("FAILED: boom + keploy: unmocked", r.get("fails()")); + answer("end", "charges()", "{\"v\":1,\"status\":\"ok\",\"action\":\"fail\"}"); + assertEquals(map("charges()", "FAILED: keploy: the keploy agent failed this test and gave no reason"), results(run(Fixtures.Payments.class))); + } + + @Test + void failsTheTestAnAnswerItCannotTrustBelongsTo() { + agent.answer = (p, b) -> p.endsWith("begin") && "passes()".equals(b.get("name")) ? new FakeAgent.Answer(500, "down") + : p.endsWith("end") && "fails()".equals(b.get("name")) ? new FakeAgent.Answer(200, "{\"v\":2}") : FakeAgent.ok(p, b); + Map r = results(run(Fixtures.Orders.class)); + assertEquals("FAILED: keploy: the keploy agent answered 500 to /agent/scope/begin: \"down\"", r.get("passes()")); + assertTrue(r.get("fails()").startsWith("FAILED: boom + keploy: the keploy agent answered /agent/scope/end in scope protocol v2")); + List l = log(); + assertEquals(Arrays.asList("begin " + F + "Orders::passes()", "other.afterEach", "@AfterAll"), + l.subList(l.indexOf("begin " + F + "Orders::passes()"), l.size())); + } + + @Test + void failsEveryClassWhenTheClaimFails() { + agent.answer = (p, b) -> new FakeAgent.Answer(404, "404 page not found"); + assertTrue(results(run(Fixtures.Payments.class)).get(F + "Payments") + .startsWith("FAILED: keploy: this keploy agent answered 404 to POST /agent/instance")); + assertTrue(results(run(Fixtures.Orders.class)).get(F + "Orders").startsWith("FAILED: keploy: this keploy agent answered 404")); + assertEquals(Collections.singletonList("claim . " + project), log()); + } + + @Test + void failsEveryClassWhenKeploysAgentIsRefused() { + KeployExtension.use(new KeployExtension.Run(Joining.decide(JoiningTest.env("KEPLOY_MOCK_AGENT", "nope"), 10, p -> false))); + assertEquals(map(F + "Payments", "FAILED: keploy: KEPLOY_MOCK_AGENT=\"nope\" is neither off nor the http URL of a keploy agent;" + + " set it to off to run without keploy"), results(run(Fixtures.Payments.class))); + assertEquals(Collections.emptyList(), log()); + } + + @Test + void doesNothingWhenInert() { + KeployExtension.use(new KeployExtension.Run(Joining.INERT)); + assertEquals(map("assumes()", "ABORTED: Assumption failed: not here", "fails()", "FAILED: boom", "passes()", "SUCCESSFUL"), + results(run(Fixtures.Orders.class, KeployExtension.PARALLEL, "true"))); + assertEquals(map("extended()", "SUCCESSFUL"), results(EngineTestKit.engine("junit-jupiter") + .selectors(DiscoverySelectors.selectClass(Fixtures.Extended.class)).execute().allEvents().list())); + assertTrue(agent.calls.isEmpty()); + assertTrue(!log().contains("claim")); + } + + @Test + void refusesARunItDoesNotWrapWhole() { + String text = "FAILED: keploy: keploy-junit5 wraps every test of the run, outside every other extension, only when JUnit loads it" + + " for the whole run: set junit.jupiter.extensions.autodetection.enabled=true (in src/test/resources/junit-platform.properties," + + " or as a system property) instead of naming it in @ExtendWith"; + assertEquals(map(F + "Extended", text), results(EngineTestKit.engine("junit-jupiter") + .selectors(DiscoverySelectors.selectClass(Fixtures.Extended.class)).execute().allEvents().list())); + assertEquals(map(F + "Extended", text), results(run(Fixtures.Extended.class, KeployExtension.AUTODETECT, "TRUE "))); + assertEquals(map("extended()", "SUCCESSFUL"), results(run(Fixtures.Extended.class, KeployExtension.AUTODETECT, "True"))); + assertEquals(map(F + "Payments", "FAILED: keploy: keploy runs one test at a time, and this run has JUnit's parallel execution on" + + " (junit.jupiter.execution.parallel.enabled=true): turn it off for runs under keploy"), + results(run(Fixtures.Payments.class, KeployExtension.PARALLEL, "TRUE"))); + assertEquals(map("charges()", "SUCCESSFUL"), results(run(Fixtures.Payments.class, KeployExtension.PARALLEL, "false"))); + } + + @Test + void failsATestBegunASecondTime() { + run(Fixtures.Payments.class); + assertEquals(map("charges()", "FAILED: keploy: charges() in " + F + "Payments would begin a second time, and keploy names a test by its" + + " class and name: run each test once (no reruns, such as Surefire's rerunFailingTestsCount)"), results(run(Fixtures.Payments.class))); + assertEquals(1, log().stream().filter(s -> s.startsWith("begin ")).count()); + } + + @Test + void namesEachTestAsJUnitDoes() { + Map r = results(run(Fixtures.Named.class)); + assertTrue(r.get("dynamic()").startsWith("FAILED: keploy: keploy-junit5 begins and ends each test at JUnit's BeforeEach and AfterEach" + + " callbacks, and cannot name [engine:junit-jupiter]/[class:" + F + "Named]/[test-factory:dynamic()] as one test (a test-factory;")); + assertEquals(Arrays.asList("claim . " + project, + "begin " + F + "Named::price(int)[1]", "end " + F + "Named::price(int)[1] passed", + "begin " + F + "Named::price(int)[2]", "end " + F + "Named::price(int)[2] passed", + "begin " + F + "Named::repeats()[1]", "end " + F + "Named::repeats()[1] passed", + "begin " + F + "Named::repeats()[2]", "end " + F + "Named::repeats()[2] passed", + "begin " + F + "Named$Inner::inner()", "end " + F + "Named$Inner::inner() passed"), log()); + } + + /** Runs suite classes through JUnit's suite engine, keploy-junit5 loaded for the whole run. */ + static List suites(Class... suites) { + EngineTestKit.Builder b = EngineTestKit.engine("junit-platform-suite").configurationParameter(KeployExtension.AUTODETECT, "true"); + for (Class c : suites) { + b.selectors(DiscoverySelectors.selectClass(c)); + } + return b.execute().allEvents().list(); + } + + @Test + void namesATestRunThroughASuiteAsWhenItRunsDirectly() { + Map r = results(suites(Fixtures.NamedSuite.class)); + assertEquals("SUCCESSFUL", r.get("inner()")); + assertTrue(r.get("dynamic()").startsWith("FAILED: keploy: keploy-junit5 begins and ends each test at JUnit's BeforeEach and AfterEach" + + " callbacks, and cannot name [engine:junit-platform-suite]/[suite:" + F + "NamedSuite]/[engine:junit-jupiter]/[class:" + F + + "Named]/[test-factory:dynamic()] as one test (a test-factory;"), r.get("dynamic()")); + assertEquals(Arrays.asList("claim . " + project, + "begin " + F + "Named::price(int)[1]", "end " + F + "Named::price(int)[1] passed", + "begin " + F + "Named::price(int)[2]", "end " + F + "Named::price(int)[2] passed", + "begin " + F + "Named::repeats()[1]", "end " + F + "Named::repeats()[1] passed", + "begin " + F + "Named::repeats()[2]", "end " + F + "Named::repeats()[2] passed", + "begin " + F + "Named$Inner::inner()", "end " + F + "Named$Inner::inner() passed"), log()); + } + + @Test + void failsATestTwoSuitesRunInOneJvm() { + Map r = results(suites(Fixtures.PaymentsSuite.class, Fixtures.PaymentsSuiteAgain.class)); + assertEquals("FAILED: keploy: charges() in " + F + "Payments would begin a second time, and keploy names a test by its" + + " class and name: run each test once (no reruns, such as Surefire's rerunFailingTestsCount)", r.get("charges()")); + assertEquals(Arrays.asList("claim . " + project, "begin " + F + "Payments::charges()", "charges", + "end " + F + "Payments::charges() passed"), log()); + } + + /** + * A @ParameterizedClass (JUnit 5.13 and later) runs each of its tests once per invocation of the class, each in a + * BeforeEach/AfterEach pair of its own: each is a test of its own, named with the class's invocation index. The + * rows compile on JUnit 5.10, which has no class templates, so the fixture is compiled here when JUnit has them. + */ + @Test + void namesEachInvocationOfAParameterizedClassAsATestOfItsOwn(@TempDir Path tmp) throws Exception { + Class annotation; + try { + annotation = Class.forName("org.junit.jupiter.params.ParameterizedClass"); + } catch (ClassNotFoundException e) { + String v = Test.class.getPackage().getImplementationVersion(); + assertTrue(v != null && v.matches("5\\.([0-9]|1[0-2])\\..*"), "JUnit " + v + " has no @ParameterizedClass"); + return; + } + Path src = Files.createDirectories(tmp.resolve("src/classtemplates")).resolve("Prices.java"); + Files.write(src, String.join("\n", + "package classtemplates;", + "import org.junit.jupiter.api.*;", + "import org.junit.jupiter.params.*;", + "import org.junit.jupiter.params.provider.ValueSource;", + "@ParameterizedClass @ValueSource(ints = {1, 2})", + "@TestMethodOrder(MethodOrderer.MethodName.class) @TestClassOrder(ClassOrderer.ClassName.class)", + "public class Prices {", + " @Parameter int n;", + " @Test void price() {}", + " @ParameterizedTest @ValueSource(strings = {\"a\", \"b\"}) void tax(String s) {}", + " @Nested @ParameterizedClass @ValueSource(strings = {\"x\", \"y\"}) class Coupons {", + " @Parameter String c;", + " @Test void coupon() {}", + " }", + " @Nested class Refunds { @Test void refund() {} }", + "}").getBytes(StandardCharsets.UTF_8)); + StringBuilder cp = new StringBuilder(); + for (Class c : new Class[] {Test.class, annotation, Class.forName("org.apiguardian.api.API"), + Class.forName("org.junit.platform.commons.annotation.Testable")}) { + cp.append(new File(c.getProtectionDomain().getCodeSource().getLocation().toURI())).append(File.pathSeparator); + } + Path out = Files.createDirectory(tmp.resolve("out")); + JavaCompiler javac = ToolProvider.getSystemJavaCompiler(); + assertEquals(0, javac.run(null, null, null, "-proc:none", "-cp", cp.toString(), "-d", out.toString(), src.toString())); + try (URLClassLoader l = new URLClassLoader(new URL[] {out.toUri().toURL()}, getClass().getClassLoader())) { + Map r = results(run(l.loadClass("classtemplates.Prices"))); + assertEquals(map("price()", "SUCCESSFUL", "#1", "SUCCESSFUL", "#2", "SUCCESSFUL", "coupon()", "SUCCESSFUL", + "refund()", "SUCCESSFUL"), r); + } + List want = new ArrayList<>(); + want.add("claim . " + project); + for (int n = 1; n <= 2; n++) { + String i = "[" + n + "]"; + for (String t : new String[] {"Prices::" + i + "price()", "Prices::" + i + "tax(java.lang.String)[1]", + "Prices::" + i + "tax(java.lang.String)[2]", "Prices$Coupons::" + i + "[1]coupon()", + "Prices$Coupons::" + i + "[2]coupon()", "Prices$Refunds::" + i + "refund()"}) { + want.add("begin classtemplates." + t); + want.add("end classtemplates." + t + " passed"); + } + } + assertEquals(want, log()); + } + + @Test + void takesAnAbortAsSkipped() { + assertEquals("passed", KeployExtension.outcome(null)); + assertEquals("skipped", KeployExtension.outcome(new TestAbortedException("x"))); + assertEquals("failed", KeployExtension.outcome(new AssertionError("x"))); + assertEquals("failed", KeployExtension.outcome(new KeployException("x"))); + } +} diff --git a/keploy-junit5/src/test/java/io/keploy/junit5/FakeAgent.java b/keploy-junit5/src/test/java/io/keploy/junit5/FakeAgent.java new file mode 100644 index 0000000..f332921 --- /dev/null +++ b/keploy-junit5/src/test/java/io/keploy/junit5/FakeAgent.java @@ -0,0 +1,99 @@ +package io.keploy.junit5; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; + +import com.sun.net.httpserver.HttpServer; + +/** A keploy agent for the rows: it logs each call and answers as told. */ +final class FakeAgent implements AutoCloseable { + /** What the agent saw and the fixtures did, in order. */ + static final List LOG = Collections.synchronizedList(new ArrayList<>()); + + static final class Answer { + final int status; + final String body; + final String location; + + Answer(int status, String body) { + this(status, body, null); + } + + Answer(int status, String body, String location) { + this.status = status; + this.body = body; + this.location = location; + } + } + + /** keploy's answers to a call that went well. */ + static Answer ok(String path, Map body) { + if (path.equals(AgentClient.PATH_INSTANCE)) { + return new Answer(200, "{\"v\":1,\"instance\":\"i-1\",\"timeFile\":\"\"}"); + } + return new Answer(200, "{\"v\":1,\"status\":\"ok\"}"); + } + + final HttpServer server; + volatile BiFunction, Answer> answer = FakeAgent::ok; + volatile String lastAuth; + final List calls = Collections.synchronizedList(new ArrayList<>()); + + FakeAgent() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", x -> { + String body = read(x.getRequestBody()); + lastAuth = x.getRequestHeaders().getFirst("authorization"); + @SuppressWarnings("unchecked") + Map m = (Map) Json.parse(body); + String path = x.getRequestURI().getPath(); + String line = x.getRequestMethod() + " " + path + " " + body; + calls.add(line); + String what = path.substring(path.lastIndexOf('/') + 1); + LOG.add(what.equals("instance") ? "claim " + m.get("key") + " " + m.get("dir") + : what + " " + m.get("file") + "::" + m.get("name") + (m.containsKey("outcome") ? " " + m.get("outcome") : "")); + Answer a = answer.apply(path, m); + byte[] out = a.body.getBytes(StandardCharsets.UTF_8); + if (a.location != null) { + x.getResponseHeaders().set("location", a.location); + } + x.sendResponseHeaders(a.status, out.length == 0 ? -1 : out.length); + try (OutputStream o = x.getResponseBody()) { + o.write(out); + } + }); + server.start(); + } + + String url() { + return "http://127.0.0.1:" + server.getAddress().getPort(); + } + + AgentClient client() { + return new AgentClient(url(), ""); + } + + @Override + public void close() { + server.stop(0); + } + + private static String read(InputStream in) throws IOException { + ByteArrayOutputStream b = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int n; + while ((n = in.read(buf)) > 0) { + b.write(buf, 0, n); + } + return new String(b.toByteArray(), StandardCharsets.UTF_8); + } +} diff --git a/keploy-junit5/src/test/java/io/keploy/junit5/Fixtures.java b/keploy-junit5/src/test/java/io/keploy/junit5/Fixtures.java new file mode 100644 index 0000000..033eb37 --- /dev/null +++ b/keploy-junit5/src/test/java/io/keploy/junit5/Fixtures.java @@ -0,0 +1,161 @@ +package io.keploy.junit5; + +import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.util.Collections; +import java.util.List; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.RepeatedTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.platform.suite.api.SelectClasses; +import org.junit.platform.suite.api.Suite; + +/** The suites the rows run through JUnit (EngineTestKit). Static nested: Surefire runs none of them itself. */ +final class Fixtures { + static void log(String s) { + FakeAgent.LOG.add(s); + } + + static String method(TestInfo i) { + return i.getTestMethod().get().getName(); + } + + /** Another extension, which keploy's must wrap. */ + static final class Other implements BeforeEachCallback, AfterEachCallback { + @Override + public void beforeEach(ExtensionContext ctx) { + log("other.beforeEach"); + } + + @Override + public void afterEach(ExtensionContext ctx) { + log("other.afterEach"); + } + } + + @ExtendWith(Other.class) + @TestMethodOrder(MethodOrderer.MethodName.class) + static class Orders { + @BeforeAll + static void beforeAll() { + log("@BeforeAll"); + } + + @BeforeEach + void beforeEach(TestInfo i) { + log("@BeforeEach " + method(i)); + } + + @Test + void assumes() { + log("assumes"); + assumeTrue(false, "not here"); + } + + @Test + void fails() { + log("fails"); + fail("boom"); + } + + @Test + void passes() { + log("passes"); + } + + @AfterEach + void afterEach(TestInfo i) { + log("@AfterEach " + method(i)); + } + + @AfterAll + static void afterAll() { + log("@AfterAll"); + } + } + + static class Payments { + @Test + void charges() { + log("charges"); + } + } + + static class AfterEachFails { + @Test + void cleans() { + log("cleans"); + } + + @AfterEach + void afterEach() { + throw new IllegalStateException("cleanup broke"); + } + } + + @TestMethodOrder(MethodOrderer.MethodName.class) + static class Named { + @ParameterizedTest + @ValueSource(ints = {1, 2}) + void price(int n) { + } + + @RepeatedTest(2) + void repeats() { + } + + @TestFactory + List dynamic() { + return Collections.singletonList(DynamicTest.dynamicTest("d", () -> log("dynamic ran"))); + } + + @Nested + class Inner { + @Test + void inner() { + } + } + } + + /** Named, run through a JUnit Platform suite. */ + @Suite + @SelectClasses(Named.class) + static class NamedSuite { + } + + /** Two suites that run one class: each test of it runs twice in one JVM. */ + @Suite + @SelectClasses(Payments.class) + static class PaymentsSuite { + } + + @Suite + @SelectClasses(Payments.class) + static class PaymentsSuiteAgain { + } + + @ExtendWith(KeployExtension.class) + static class Extended { + @Test + void extended() { + log("extended"); + } + } +} diff --git a/keploy-junit5/src/test/java/io/keploy/junit5/JoiningTest.java b/keploy-junit5/src/test/java/io/keploy/junit5/JoiningTest.java new file mode 100644 index 0000000..b0f5e84 --- /dev/null +++ b/keploy-junit5/src/test/java/io/keploy/junit5/JoiningTest.java @@ -0,0 +1,108 @@ +package io.keploy.junit5; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.function.LongPredicate; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class JoiningTest { + static final LongPredicate NEVER_ASKED = p -> { + throw new AssertionError("asked whether this process runs inside " + p); + }; + + static Map env(String... kv) { + Map m = new HashMap<>(); + for (int k = 0; k < kv.length; k += 2) { + m.put(kv[k], kv[k + 1]); + } + return m; + } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "off", " OFF "}) + void isInertWithoutAnAgent(String v) { + assertSame(Joining.INERT, Joining.decide(env("KEPLOY_MOCK_AGENT", v), 10, NEVER_ASKED)); + assertTrue(Joining.INERT.inert()); + } + + @Test + void isInertWithNoAgentSet() { + assertSame(Joining.INERT, Joining.decide(env(), 10, NEVER_ASKED)); + } + + @ParameterizedTest + @ValueSource(strings = {"localhost:16789", "ftp://h:1", "http://", "http://h:abc", "http://a b", "mailto:x@h"}) + void refusesAnAgentThatIsNoHttpURL(String v) { + Joining j = Joining.decide(env("KEPLOY_MOCK_AGENT", v), 10, NEVER_ASKED); + assertEquals("keploy: KEPLOY_MOCK_AGENT=" + Json.quote(v) + " is neither off nor the http URL of a keploy agent;" + + " set it to off to run without keploy", j.refusal); + assertNull(j.url); + assertFalse(j.inert()); + } + + @Test + void joinsWithTheAgentsURLTokenAndProject() { + Joining j = Joining.decide(env("KEPLOY_MOCK_AGENT", " HTTP://127.0.0.1:16789// ", "KEPLOY_MOCK_AGENT_TOKEN", " t ", + "KEPLOY_MOCK_PROJECT", " /p "), 10, NEVER_ASKED); + assertEquals("HTTP://127.0.0.1:16789", j.url); + assertEquals(" t ", j.token); + assertEquals("/p", j.project); + assertNull(j.refusal); + assertFalse(j.inert()); + j = Joining.decide(env("KEPLOY_MOCK_AGENT", "https://[::1]:1"), 10, NEVER_ASKED); + assertEquals("https://[::1]:1", j.url); + assertEquals("", j.token); + assertEquals("", j.project); + } + + @Test + void isAMemberOnlyInsideTheProcessTheMarkerNames() { + String agent = "http://127.0.0.1:1"; + // Its own pid: the marker it exported itself. + assertNull(Joining.decide(env("KEPLOY_MOCK_AGENT", agent, "KEPLOY_TEST_PID", " 10 "), 10, NEVER_ASKED).refusal); + assertFalse(Joining.decide(env("KEPLOY_MOCK_AGENT", agent, "KEPLOY_TEST_PID", "10"), 10, NEVER_ASKED).inert()); + for (String tp : new String[] {"abc", "0", "-5", ""}) { + assertFalse(Joining.decide(env("KEPLOY_MOCK_AGENT", agent, "KEPLOY_TEST_PID", tp), 10, NEVER_ASKED).inert(), tp); + } + long[] asked = {0}; + assertSame(Joining.INERT, Joining.decide(env("KEPLOY_MOCK_AGENT", agent, "KEPLOY_TEST_PID", " 77 "), 10, p -> { + asked[0] = p; + return true; + })); + assertEquals(77, asked[0]); + assertFalse(Joining.decide(env("KEPLOY_MOCK_AGENT", agent, "KEPLOY_TEST_PID", "77"), 10, p -> false).inert()); + assertTrue(Joining.decide(env("KEPLOY_MOCK_AGENT", agent, "KEPLOY_TEST_PID", "1"), 10, p -> true).inert()); + } + + @Test + void readsItsAncestorsFromProc() throws IOException { + long self = Long.parseLong(new File("/proc/self").getCanonicalFile().getName()); + assertEquals(self, Joining.pid()); + long parent = Joining.parent("self"); + assertTrue(parent > 0); + assertTrue(Joining.runsInsideProc(parent)); + assertTrue(Joining.runsInsideProc(Joining.parent(Long.toString(parent)))); + assertTrue(Joining.runsInsideProc(1)); + assertFalse(Joining.runsInsideProc(self)); + assertEquals(-1, Joining.parent("no-such-process")); + } + + @Test + void readsTheParentPastACommandNameWithParentheses() { + assertEquals(77, Joining.ppid("123 (a) b) c) S 77 1 1 0")); + assertEquals(-1, Joining.ppid("123 (a) S")); + assertEquals(-1, Joining.ppid("123 a S 77")); + assertEquals(-1, Joining.ppid("123 (a) S x 1")); + } +} diff --git a/keploy-junit5/src/test/java/io/keploy/junit5/JsonTest.java b/keploy-junit5/src/test/java/io/keploy/junit5/JsonTest.java new file mode 100644 index 0000000..9761428 --- /dev/null +++ b/keploy-junit5/src/test/java/io/keploy/junit5/JsonTest.java @@ -0,0 +1,67 @@ +package io.keploy.junit5; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class JsonTest { + @Test + void readsEveryKindOfValue() { + Map want = new LinkedHashMap<>(); + want.put("v", new BigDecimal("1")); + want.put("s", "a\"\\/\b\f\n\r\té\u00e9"); + want.put("t", true); + want.put("f", false); + want.put("n", null); + want.put("o", Collections.singletonMap("k", Arrays.asList(new BigDecimal("-0.5e+2"), new BigDecimal("2E-1"), new BigDecimal("0")))); + want.put("e", Collections.emptyMap()); + want.put("a", Collections.emptyList()); + assertEquals(want, Json.parse(" {\"v\" : 1 ,\t\"s\":\"a\\\"\\\\\\/\\b\\f\\n\\r\\té\\u00E9\",\"t\":true,\"f\":false,\"n\":null," + + "\"o\":{\"k\":[-0.5e+2, 2E-1 ,0]},\"e\":{ },\"a\":[ ]}\r\n")); + } + + @Test + void readsTheDeepestNestingItTakes() { + String s = "1"; + for (int k = 0; k < Json.MAX_DEPTH; k++) { + s = "[" + s + "]"; + } + Json.parse(s); + String deeper = "[" + s + "]"; + assertEquals("nested deeper than 64 at offset 65", assertThrows(IllegalArgumentException.class, () -> Json.parse(deeper)).getMessage()); + } + + @Test + void readsEveryDigit() { + assertEquals(new BigDecimal("1234567890.0987654321e9"), Json.parse("1234567890.0987654321e9")); + assertEquals(Collections.singletonMap("s", "a b"), Json.parse("{\"s\":\"a b\"}")); + } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "{", "{\"a\"", "{\"a\":", "{\"a\":1", "{\"a\":1,}", "{a:1}", "{a\":1}", "{\"a\" 1}", "[1", "[1,]", "[1 2]", + "\"abc", "\"a\\", "\"a\\x\"", "\"\\u12\"", "\"\\u12g4\"", "\"\\u+123\"", "\"\\u0123\"", "\"a\u0001\"", "\"a\u001f\"", "tru", "fals", "nul", "nulL", + "01", "+1", "-", ".5", "1.", "1e", "1e+", "--1", "1 2", "{}x", "x"}) + void refusesWhatIsNoJson(String s) { + assertThrows(IllegalArgumentException.class, () -> Json.parse(s)); + } + + @Test + void writesAFlatObject() { + Map m = new LinkedHashMap<>(); + m.put("name", "a\"b\\c\u0001\u001f "); + m.put("canSkip", true); + m.put("n", 2); + assertEquals("{\"name\":\"a\\\"b\\\\c\\u0001\\u001f \",\"canSkip\":true,\"n\":2}", Json.object(m)); + assertEquals("{}", Json.object(Collections.emptyMap())); + assertEquals("\"é/\"", Json.quote("é/")); + } +}