diff --git a/.github/workflows/verify-split-files.yml b/.github/workflows/verify-split-files.yml index 4ce72ee..88637f1 100644 --- a/.github/workflows/verify-split-files.yml +++ b/.github/workflows/verify-split-files.yml @@ -38,6 +38,7 @@ on: - "tests/exit-code-hints.sh" - "tests/forge-release-json.sh" - "tests/install-cleanup.sh" + - "tests/console-hangup.sh" - "tests/incus-target.sh" - "tests/incus-storage-space.sh" - "tests/release-stamp-pending.sh" @@ -72,6 +73,7 @@ on: - "tests/exit-code-hints.sh" - "tests/forge-release-json.sh" - "tests/install-cleanup.sh" + - "tests/console-hangup.sh" - "tests/incus-target.sh" - "tests/incus-storage-space.sh" - "tests/release-stamp-pending.sh" @@ -144,6 +146,8 @@ jobs: - name: Install cleanup and uv trap tests run: bash tests/install-cleanup.sh + - name: Console hangup tests + run: bash tests/console-hangup.sh - name: Incus target tests run: bash tests/incus-target.sh diff --git a/core/core.func b/core/core.func index 354865b..f207f85 100644 --- a/core/core.func +++ b/core/core.func @@ -939,7 +939,28 @@ log_section() { # for telemetry (the host builds the error trace from this file) # - Temporarily disables error trap to capture exit code correctly # ------------------------------------------------------------------------------ +# After a hangup the shell's stdout and stderr still point at the terminal that +# is gone, and every write to it fails. The trap redirects them to the log, but +# bash saves and restores fd 1 and 2 around a redirected call to a shell +# function -- silent() makes exactly such a call -- so the redirect is undone +# the moment that call returns. Re-apply it where no redirection is in effect. +_cs_reattach_output() { + [[ "${_CS_HANGUP:-}" == "1" ]] || return 0 + local log="${_CS_HANGUP_LOG:-}" + [[ -n "$log" ]] || log="$(get_active_logfile)" + [[ -n "$log" ]] || return 0 + exec >"$log" 2>&1 +} + +_silent_exec() { + ( + trap '' HUP + exec "$@" + ) +} + silent() { + _cs_reattach_output # A failure the caller handled (a retry loop) must not be reported for a later, unrelated error. unset _SILENT_FAILED_RC _SILENT_FAILED_CMD _SILENT_FAILED_LINE _SILENT_FAILED_LOG local cmd="$*" @@ -966,14 +987,21 @@ silent() { [[ -f "$logfile" ]] && start_bytes=$(stat -c%s "$logfile" 2>/dev/null || echo 0) [[ ! "$start_bytes" =~ ^[0-9]+$ ]] && start_bytes=0 - local rc + # A closed console sends SIGHUP to everything in the foreground. A function + # stays in this shell, which handles the signal itself; an external program + # inherits "ignore" and keeps running. + local rc run=("$@") + case "$(type -t "$1" 2>/dev/null)" in + function | builtin | "") ;; + *) run=(_silent_exec "$@") ;; + esac if [[ "${VERBOSE:-no}" == "yes" ]]; then # Not a pipe: the command must stay in this shell so functions keep their side effects. - "$@" > >(tee -a "$logfile") 2>&1 + "${run[@]}" > >(tee -a "$logfile") 2>&1 rc=$? wait $! 2>/dev/null else - "$@" >>"$logfile" 2>&1 + "${run[@]}" >>"$logfile" 2>&1 rc=$? fi @@ -1238,6 +1266,7 @@ fi # - Backgrounds spinner process and stores PID for later cleanup # ------------------------------------------------------------------------------ msg_info() { + _cs_reattach_output local msg="$1" [[ -z "$msg" ]] && return @@ -1296,6 +1325,7 @@ msg_info() { # - Uses green color for success indication # ------------------------------------------------------------------------------ msg_ok() { + _cs_reattach_output local msg="$1" [[ -z "$msg" ]] && return stop_spinner @@ -1318,6 +1348,7 @@ msg_ok() { # - Outputs to stderr # ------------------------------------------------------------------------------ msg_error() { + _cs_reattach_output stop_spinner _MSG_BLOCKS=() local msg="$1" @@ -1334,6 +1365,7 @@ msg_error() { # - Outputs to stderr # ------------------------------------------------------------------------------ msg_warn() { + _cs_reattach_output stop_spinner _MSG_BLOCKS=() local msg="$1" @@ -1350,6 +1382,7 @@ msg_warn() { # - Useful for specialized status messages # ------------------------------------------------------------------------------ msg_custom() { + _cs_reattach_output local symbol="${1:-"[*]"}" local color="${2:-"\e[36m"}" local msg="${3:-}" diff --git a/core/error_handler.func b/core/error_handler.func index 90dc2f8..dcba566 100644 --- a/core/error_handler.func +++ b/core/error_handler.func @@ -776,6 +776,7 @@ _stop_container_if_installing() { # ------------------------------------------------------------------------------ on_exit() { local exit_code=$? + [[ "${_CS_HANGUP:-}" == "1" ]] && echo "[INFO] $(date '+%Y-%m-%d %H:%M:%S') on_exit with code ${exit_code} (pid $$)" # Before anything else: the timings are only useful if they survive a failure. declare -f dev_mode_timing_summary >/dev/null 2>&1 && dev_mode_timing_summary @@ -796,6 +797,9 @@ on_exit() { if _is_container_context; then if [[ $exit_code -ne 0 ]]; then _container_write_failure "$exit_code" "${FAILED_COMMAND:-}" "${FAILED_LINE:-}" + elif [[ "${INSTALL_LOG:-}" == *.install-*.log ]]; then + # The host reads this when its lxc-attach did not live to see the end. + : >"${INSTALL_LOG%.log}.done" 2>/dev/null || true fi else if [[ "${POST_UPDATE_DONE:-}" != "true" ]] && declare -f post_update_to_api >/dev/null 2>&1; then @@ -874,11 +878,18 @@ on_terminate() { # ------------------------------------------------------------------------------ # on_hangup() - SIGHUP (SSH disconnect, terminal closed) # -# No cursor restore and no message: the terminal that would read them is gone. +# Not an abort: the run carries on with its output in the active log, and +# whoever opens a console later finds the result. Writes to the terminal that +# is gone would fail with EIO and end the script, hence the redirect. # ------------------------------------------------------------------------------ on_hangup() { declare -f stop_spinner >/dev/null 2>&1 && { stop_spinner 2>/dev/null || true; } - _eh_signal_exit 129 + local log="" + declare -f get_active_logfile >/dev/null 2>&1 && log="$(get_active_logfile 2>/dev/null)" + [[ -n "$log" ]] || log="${INSTALL_LOG:-${BUILD_LOG:-/dev/null}}" + exec >"$log" 2>&1 + export _CS_HANGUP=1 CS_ANIMATION_OK=0 _CS_HANGUP_LOG="$log" + echo "[WARN] $(date '+%Y-%m-%d %H:%M:%S') The terminal hung up (SIGHUP); continuing, output follows here. (pid $$, subshell ${BASH_SUBSHELL}, during: ${BASH_COMMAND:-?})" } # ============================================================================== diff --git a/docs/backends.md b/docs/backends.md index 79ccbbe..2d09cb5 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -16,6 +16,13 @@ wizard calls. | [`vm-core.func`](../pve/vm-core.func) | VM creation via `qm`, including disk import and template handling | | [`vm-app.func`](../pve/vm-app.func) | Deploys an application that normally goes into an LXC container into a full VM instead | +A closed console is not an abort. SIGHUP reaches the host script and the +install inside the container; both carry on with their output in their logs +(`on_hangup`), external commands under `$STD` ignore the signal, and when +lxc-attach dies with the terminal the host waits for the container's +`.done`/`.failed` markers (`_pve_wait_for_detached_install`) before it +reports. Ctrl-C still aborts. + ## `incus/` — Incus | File | Purpose | @@ -71,6 +78,15 @@ from `/etc/localtime`, or from `Etc/UTC`. [`core/core.func`](../core/core.func) renders "Incus Version 6.x" on an Incus host and "PVE Version …" on Proxmox VE, from the same `PVEVERSION` value. +A closed console is not an abort here either, but it takes a different route. +`incus exec` ends as soon as its client goes away and takes the command with +it, which `lxc-attach` does not do. The install therefore runs detached inside +the container, writing to its own log, and the host only watches it: input +reaches it through a fifo for as long as there is a console, and once there is +none the reads return EOF and the prompts take their defaults. +`_incus_wait_for_detached_install` then waits for the `.done`/`.failed` +markers before the host reports. + ### Driving a remote server The `incus` client always talks to a remote; `local:` is only the default. diff --git a/docs/core.md b/docs/core.md index 15710d1..dab3fd3 100644 --- a/docs/core.md +++ b/docs/core.md @@ -33,7 +33,8 @@ Worth knowing: - `silent()` runs a command with its output captured, and records the failing command, line and log path so the error handler can report the real failure - rather than the wrapper. + rather than the wrapper. External commands run with SIGHUP ignored, so a + closed console does not kill the apt or npm of the moment. - Several host facts are memoised in files under a runtime cache directory rather than in shell variables, because every call site is a `$( )` or a pipe — that is, a subshell, where a variable assignment would be discarded. diff --git a/incus/backend.func b/incus/backend.func index 1471fa0..9a1ca57 100644 --- a/incus/backend.func +++ b/incus/backend.func @@ -1061,6 +1061,30 @@ MOTDEOF msg_ok "MOTD configured" } +# ------------------------------------------------------------------------------ +# _incus_wait_for_detached_install +# +# The console closed while the install ran. incus ended the exec session that +# was watching it, but the install itself is detached and carries on. Wait for +# what it leaves behind: .failed (its exit code, read by the caller), .done, or +# the process gone without either. +# ------------------------------------------------------------------------------ +_incus_wait_for_detached_install() { + local base="$1" pid="" waited=0 + local poll="${_CS_HANGUP_POLL:-10}" limit="${_CS_HANGUP_TIMEOUT:-21600}" + while :; do + incus_ct_exec test -f "${base}.failed" >/dev/null 2>&1 && return 0 + incus_ct_exec test -f "${base}.done" >/dev/null 2>&1 && return 0 + pid="$(incus_ct_exec cat "${base}.pid" 2>/dev/null | tr -d '[:space:]')" + if [[ "$pid" =~ ^[0-9]+$ ]] && ! incus_ct_exec kill -0 "$pid" >/dev/null 2>&1; then + return 0 + fi + ((waited >= limit)) && return 0 + sleep "$poll" + waited=$((waited + poll)) + done +} + _incus_push_functions_and_run_install() { # Sets INCUS_LAST_INSTALL_EXIT (do not capture stdout — msg_* writes there). INCUS_LAST_INSTALL_EXIT=0 @@ -1095,15 +1119,53 @@ INNEREOF" >>"${LOGFILE:-$INCUS_BUILD_LOG}" 2>&1 || { msg_error "Failed to push f # No pty on purpose: under `incus exec -t` apt's tcsetattr fails with EIO and kills the shell. set +Eeuo pipefail trap - ERR + + # incus ends an exec session the moment its client goes away, so closing the + # console killed the install outright - the one thing lxc-attach does not do. + # Run it detached from that session and only watch it from here. The console + # keeps the view and the keyboard: input reaches the install through a fifo + # for as long as there is a console, and once there is none the reads return + # EOF and the prompts take their defaults. + local _base="/root/.install-${SESSION_ID}" + local _runner _watcher _body + _runner='setsid bash "$1".sh <"$1".in >>"$1".out 2>&1 & +echo $! >"$1".pid' + _watcher='tail -n +1 -f "$1".out & +_tp=$! +while [ ! -f "$1".done ] && [ ! -f "$1".failed ]; do sleep 2; done +sleep 1 +kill $_tp 2>/dev/null' # The trailing touch is a completion sentinel: no install script exits early. - incus exec "${CT_NAME}" -- bash -c "${_run_env} + _body="${_run_env} ${_install_script} -touch /root/.install-${SESSION_ID}.done" 2>&1 | +touch ${_base}.done" + + incus exec "${CT_NAME}" -- bash -c "cat >${_base}.sh <<'CSBODYEOF' +${_body} +CSBODYEOF +cat >${_base}.run <<'CSRUNEOF' +${_runner} +CSRUNEOF +cat >${_base}.watch <<'CSWATCHEOF' +${_watcher} +CSWATCHEOF +rm -f ${_base}.in ${_base}.pid ${_base}.done ${_base}.failed ${_base}.out +mkfifo ${_base}.in" >>"${LOGFILE:-$INCUS_BUILD_LOG}" 2>&1 + + # A writer has to hold the fifo open before the install opens it to read. + incus exec "${CT_NAME}" -- sh -c "cat >${_base}.in" & + local _feeder=$! + incus exec "${CT_NAME}" -- bash "${_base}.run" "${_base}" >/dev/null 2>&1 + + incus exec "${CT_NAME}" -- bash "${_base}.watch" "${_base}" 2>&1 | tee >(sed 's/.*\r//' >>"${LOGFILE:-$INCUS_BUILD_LOG}") - # PIPESTATUS, not $?: the line above turns pipefail off, so $? is tee's - # status - always 0. Every non-zero exit from the container was discarded - # here, which is why a half-finished install reported success. - lxc_exit=${PIPESTATUS[0]} + kill "$_feeder" 2>/dev/null || true + wait "$_feeder" 2>/dev/null || true + + # The watcher is gone either way - because the install ended, or because the + # console did. Only the markers say which. + _incus_wait_for_detached_install "$_base" + lxc_exit=0 set -Eeuo pipefail declare -f error_handler &>/dev/null && trap 'error_handler' ERR || true @@ -1125,10 +1187,12 @@ touch /root/.install-${SESSION_ID}.done" 2>&1 | incus_ct_exec rm -f "/root/.install-${SESSION_ID}.done" 2>/dev/null || true else msg_error "Install script stopped before it finished - nothing reported it" + [[ "${_CS_HANGUP:-}" == "1" ]] && msg_custom "ℹ️" "${YW}" "The console was closed; the install ran on detached, so look at its log for the reason" msg_custom "ℹ️" "${YW}" "Output: incus exec ${CT_NAME} -- tail -50 /root/.install-${SESSION_ID}.log" install_exit_code=1 fi fi + incus_ct_exec rm -f "${_base}.sh" "${_base}.run" "${_base}.watch" "${_base}.in" "${_base}.pid" "${_base}.out" >/dev/null 2>&1 || true INCUS_LAST_INSTALL_EXIT=$install_exit_code } diff --git a/incus/build.func b/incus/build.func index 30e93f8..a64cdb8 100644 --- a/incus/build.func +++ b/incus/build.func @@ -633,8 +633,20 @@ _incus_on_signal() { exit "$code" } +# A closed console is not an abort. The install runs detached inside the +# container, so losing the terminal costs the view and the keyboard, not the +# installation. Writes to the terminal that is gone fail with EIO and would +# end the run under errexit, hence the redirect. +_incus_on_hangup() { + local log="${LOGFILE:-${_INCUS_HOST_LOGFILE:-${INCUS_BUILD_LOG:-}}}" + [[ -n "$log" ]] || log="/tmp/incus-hangup-${SESSION_ID:-$$}.log" + exec >"$log" 2>&1 + export _CS_HANGUP=1 _CS_HANGUP_LOG="$log" CS_STDIN_OK=0 + echo "[WARN] $(date '+%Y-%m-%d %H:%M:%S') The terminal hung up (SIGHUP); continuing, output follows here." +} + trap 'api_exit_script' EXIT trap '_incus_on_err' ERR -trap '_incus_on_signal SIGHUP 129' SIGHUP +trap '_incus_on_hangup' SIGHUP trap '_incus_on_signal SIGINT 130' SIGINT trap '_incus_on_signal SIGTERM 143' SIGTERM diff --git a/incus/core.func b/incus/core.func index 9997431..5bf5a78 100644 --- a/incus/core.func +++ b/incus/core.func @@ -469,6 +469,7 @@ incus_stop_spinner() { # - Displays informational message with spinner animation # ------------------------------------------------------------------------------ incus_msg_info() { + declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output local msg="$1" [[ -z "$msg" ]] && return @@ -507,6 +508,7 @@ incus_msg_info() { # incus_msg_ok() # ------------------------------------------------------------------------------ incus_msg_ok() { + declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output local msg="$1" [[ -z "$msg" ]] && return incus_stop_spinner @@ -523,6 +525,7 @@ incus_msg_ok() { # incus_msg_error() # ------------------------------------------------------------------------------ incus_msg_error() { + declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output incus_stop_spinner local msg="$1" echo -e "${BFR:-}${CROSS:-✖️}${RD}${msg}${CL}" >&2 @@ -533,6 +536,7 @@ incus_msg_error() { # incus_msg_warn() # ------------------------------------------------------------------------------ incus_msg_warn() { + declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output incus_stop_spinner local msg="$1" # Same layout as core.func msg_warn (INFO already includes TAB padding). @@ -544,6 +548,7 @@ incus_msg_warn() { # incus_msg_custom() # ------------------------------------------------------------------------------ incus_msg_custom() { + declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output local symbol="${1:-"[*]"}" local color="${2:-"\e[36m"}" local msg="${3:-}" diff --git a/pve/backend.func b/pve/backend.func index 097a7d4..31b5feb 100644 --- a/pve/backend.func +++ b/pve/backend.func @@ -184,6 +184,46 @@ _pct_restart() { return 1 } +# ------------------------------------------------------------------------------ +# _pve_wait_for_detached_install +# +# The console closed mid-install: SIGHUP took lxc-attach, but the script inside +# ignores it and runs on. Wait for its markers: .failed (its exit code, read by +# the caller), .done (clean end), or the process gone without either. +# Returns 0 for .done, 1 for .failed, 199 when nothing reported, 124 on timeout. +# ------------------------------------------------------------------------------ +_pve_wait_for_detached_install() { + local base="/root/.install-${SESSION_ID}" pid="" waited=0 + local poll="${_CS_HANGUP_POLL:-15}" limit="${_CS_HANGUP_TIMEOUT:-21600}" + echo "[INFO] $(date '+%Y-%m-%d %H:%M:%S') Waiting for the install in container ${CTID} to finish on its own" + while :; do + if pct exec "$CTID" -- test -f "${base}.failed" 2>/dev/null; then + echo "[INFO] The install reported a failure" + return 1 + fi + if pct exec "$CTID" -- test -f "${base}.done" 2>/dev/null; then + echo "[INFO] The install finished" + return 0 + fi + if [[ "$(pct status "$CTID" 2>/dev/null)" != "status: running" ]]; then + echo "[WARN] Container ${CTID} is no longer running" + return 199 + fi + pid="$(pct exec "$CTID" -- cat "${base}.pid" 2>/dev/null | tr -d '[:space:]')" + if [[ "$pid" =~ ^[0-9]+$ ]] && ! pct exec "$CTID" -- kill -0 "$pid" 2>/dev/null; then + echo "[WARN] Install process ${pid} is gone without reporting; processes left in the container:" + pct exec "$CTID" -- ps -eo pid,ppid,stat,etime,cmd 2>/dev/null | head -n 40 + return 199 + fi + if ((waited >= limit)); then + echo "[WARN] Gave up waiting after ${waited}s" + return 124 + fi + sleep "$poll" + waited=$((waited + poll)) + done +} + # Where a template lands in $TEMPLATE_STORAGE, or the default dir layout when # pvesm cannot say. _template_path() { @@ -1404,9 +1444,20 @@ PROFILE fi exit 115 fi - lxc-attach -n "$CTID" -- bash -c "$_install_script" + msg_custom "ℹ️" "${YW}" "Closing this console does not abort the install; it finishes on its own (Ctrl-C does abort)." + msg_custom " " "${YW}" "Follow it with: pct exec ${CTID} -- tail -f /root/.install-${SESSION_ID}.log" + lxc-attach -n "$CTID" -- bash -c "printf '%s' \$\$ >/root/.install-${SESSION_ID}.pid +$_install_script" local lxc_exit=$? + # The terminal went and took lxc-attach with it; the install inside did + # not stop. Follow it through its markers instead of calling it failed. + if [[ "${_CS_HANGUP:-}" == "1" && $lxc_exit -ne 0 ]]; then + _pve_wait_for_detached_install + lxc_exit=$? + fi + pct exec "$CTID" -- rm -f "/root/.install-${SESSION_ID}.pid" "/root/.install-${SESSION_ID}.done" 2>/dev/null || true + unset CONTAINER_INSTALLING if [[ -n "${SESSION_ID:-}" ]]; then @@ -1692,9 +1743,15 @@ PROFILE local response="" local read_rc - reclaim_tty - read -t 60 -r response /root/.install-${SESSION_ID}.pid +$_install_script" local apt_retry_exit=$? + if [[ "${_CS_HANGUP:-}" == "1" && $apt_retry_exit -ne 0 ]]; then + _pve_wait_for_detached_install + apt_retry_exit=$? + fi + pct exec "$CTID" -- rm -f "/root/.install-${SESSION_ID}.pid" "/root/.install-${SESSION_ID}.done" 2>/dev/null || true set -Eeuo pipefail trap 'error_handler' ERR @@ -3840,8 +3903,8 @@ api_exit_script() { fi } -# All five traps report telemetry, so all five belong to the host. EXIT was -# already guarded; the other four were not, and core/build.func sources this +# These traps report telemetry, so they belong to the host. EXIT was already +# guarded; the others were not, and core/build.func sources this # file unconditionally -- including inside a container, where a ct script runs # again for an update. A Ctrl-C or any ERR during an update therefore posted a # failed record, typed lxc, indistinguishable from a failed install. Their pct @@ -3851,7 +3914,7 @@ api_exit_script() { if command -v pveversion >/dev/null 2>&1; then trap 'api_exit_script' EXIT trap '_ec=$?; if [[ $_ec -ne 0 ]]; then post_update_to_api "failed" "$_ec" 2>/dev/null || true; if declare -f ensure_log_on_host &>/dev/null; then ensure_log_on_host 2>/dev/null || true; fi; fi' ERR - trap 'post_update_to_api "failed" "129" 2>/dev/null || true; if [[ -n "${CTID:-}" ]] && command -v pct &>/dev/null; then pct stop "$CTID" 2>/dev/null || true; fi; exit 129' SIGHUP + # SIGHUP stays with on_hangup (core/error_handler.func): a closed console is not an abort. trap 'post_update_to_api "failed" "130" 2>/dev/null || true; if [[ -n "${CTID:-}" ]] && command -v pct &>/dev/null; then pct stop "$CTID" 2>/dev/null || true; fi; exit 130' SIGINT trap 'post_update_to_api "failed" "143" 2>/dev/null || true; if [[ -n "${CTID:-}" ]] && command -v pct &>/dev/null; then pct stop "$CTID" 2>/dev/null || true; fi; exit 143' SIGTERM fi diff --git a/tests/console-hangup.sh b/tests/console-hangup.sh new file mode 100644 index 0000000..ea447f1 --- /dev/null +++ b/tests/console-hangup.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# A closed console (SIGHUP) is not an abort: silent() shields the external +# command of the moment, on_hangup moves the output to the log and carries on, +# a clean install leaves a .done marker, and the host waits for it when +# lxc-attach died with the terminal. +set -uo pipefail + +CORE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +eval "$(awk '/^(silent|_silent_exec|_cs_reattach_output)[(][)] [{]/,/^}/' "$CORE_DIR/core/core.func")" +eval "$(awk '/^(on_hangup|on_exit|_eh_quiet_spinner)[(][)] [{]/,/^}/' "$CORE_DIR/core/error_handler.func")" +eval "$(awk '/^_pve_wait_for_detached_install[(][)] [{]/,/^}/' "$CORE_DIR/pve/backend.func")" + +TEST_DIR="$(mktemp -d)" +trap 'rm -rf -- "$TEST_DIR"' EXIT +LOG="$TEST_DIR/install.log" +ROOT="$TEST_DIR/root" +mkdir -p "$ROOT" +passed=0 failed=0 +TAB=" " + +get_active_logfile() { echo "$LOG"; } +debug_log() { :; } +stop_spinner() { :; } +_is_container_context() { return 0; } +_container_write_failure() { echo "failure $1" >>"$TEST_DIR/failure"; } + +expect() { + if [[ "$2" == "$3" ]]; then + echo "PASS $1" + passed=$((passed + 1)) + else + echo "FAIL $1: expected <$2>, got <$3>" >&2 + failed=$((failed + 1)) + fi +} + +# silent(): an external command ignores SIGHUP, a function keeps its side effects. +: >"$LOG" +silent sh -c 'kill -HUP $$; echo survived' +expect "an external command under silent survives SIGHUP" 0 "$?" +expect "and finished its work" 1 "$(grep -c survived "$LOG")" +marker() { MARK="set"; } +unset MARK +silent marker +expect "a function keeps its side effects" set "${MARK:-unset}" +silent definitely_missing_command_xyz +expect "a missing command still reports 127" 127 "$?" + +# on_hangup(): the run carries on, output moves to the log. +: >"$LOG" +out="$( (trap on_hangup HUP; kill -HUP $BASHPID; echo "after hangup"; echo "flag=${_CS_HANGUP:-}") 2>&1 )" +rc=$? +expect "the shell survives the hangup" 0 "$rc" +expect "nothing more reaches the terminal" "" "$out" +expect "later output lands in the log" 1 "$(grep -c 'after hangup' "$LOG")" +expect "the hangup is noted in the log" 1 "$(grep -c SIGHUP "$LOG")" +expect "the run knows it is detached" 1 "$(grep -c 'flag=1' "$LOG")" + +# on_exit(): a clean install leaves .done for the host; a failed one does not. +INSTALL_LOG="$ROOT/.install-abc.log" +export INSTALL_LOG +(trap on_exit EXIT; exit 0) +expect "a clean install leaves a .done marker" 1 "$(ls "$ROOT/.install-abc.done" 2>/dev/null | wc -l)" +rm -f "$ROOT/.install-abc.done" +(trap on_exit EXIT; exit 3) +expect "a failed install leaves no .done marker" 0 "$(ls "$ROOT/.install-abc.done" 2>/dev/null | wc -l)" +expect "but the failure artifact" 1 "$(grep -c 'failure 3' "$TEST_DIR/failure")" +(unset INSTALL_LOG; trap on_exit EXIT; exit 0) +expect "an update run leaves no marker" 0 "$(ls "$ROOT"/.install-*.done 2>/dev/null | wc -l)" + +# _pve_wait_for_detached_install(): the host follows the markers of a detached install. +CTID=100 SESSION_ID=abc CT_STATE="status: running" +_CS_HANGUP_POLL=1 _CS_HANGUP_TIMEOUT=3 +pct() { + case "$1" in + status) echo "$CT_STATE" ;; + exec) + shift 3 + case "$1" in + test) [[ -f "${TEST_DIR}${3}" ]] ;; + cat) cat "${TEST_DIR}${2}" 2>/dev/null ;; + kill) command kill "$2" "$3" 2>/dev/null ;; + esac + ;; + esac +} +sleep 30 & +worker=$! +echo "$worker" >"$ROOT/.install-abc.pid" +(sleep 2; : >"$ROOT/.install-abc.done") & +_pve_wait_for_detached_install >/dev/null +expect "a .done marker ends the wait cleanly" 0 "$?" +rm -f "$ROOT/.install-abc.done" + +: >"$ROOT/.install-abc.failed" +_pve_wait_for_detached_install >/dev/null +expect "a .failed marker hands the flag to the caller" 1 "$?" +rm -f "$ROOT/.install-abc.failed" + +_pve_wait_for_detached_install >/dev/null +expect "no marker before the timeout" 124 "$?" + +kill "$worker" 2>/dev/null +wait "$worker" 2>/dev/null +_pve_wait_for_detached_install >/dev/null +expect "a process gone without a marker is reported as such" 199 "$?" + +sleep 30 & +worker=$! +echo "$worker" >"$ROOT/.install-abc.pid" +CT_STATE="status: stopped" +_pve_wait_for_detached_install >/dev/null +expect "a stopped container ends the wait" 199 "$?" +kill "$worker" 2>/dev/null + +# A redirected call to a shell function makes bash save fd 1 and restore it +# afterwards, which undid the redirect the hangup trap had made. Every message +# after that wrote to the terminal that was gone, and the failed write ended +# the run under errexit. +REDIR="$TEST_DIR/redirected.log" +AFTER="$TEST_DIR/after.log" +inner() { echo "inner"; } +out="$( ( + exec >"$TEST_DIR/orig.log" 2>&1 + _CS_HANGUP=1 _CS_HANGUP_LOG="$AFTER" + export _CS_HANGUP _CS_HANGUP_LOG + inner >"$REDIR" 2>&1 + _cs_reattach_output + echo "after the redirected call" + readlink /proc/$$/fd/1 +) 2>&1 )" +expect "output after a redirected call goes to the hangup log" 1 "$(grep -c 'after the redirected call' "$AFTER" 2>/dev/null)" +expect "and not back to the original target" 0 "$(grep -c 'after the redirected call' "$TEST_DIR/orig.log" 2>/dev/null)" + +: >"$AFTER" +( unset _CS_HANGUP; _CS_HANGUP_LOG="$AFTER"; _cs_reattach_output; echo "no hangup" ) >"$TEST_DIR/orig2.log" 2>&1 +expect "without a hangup nothing is redirected" 0 "$(grep -c 'no hangup' "$AFTER" 2>/dev/null)" +expect "it stays where it was" 1 "$(grep -c 'no hangup' "$TEST_DIR/orig2.log" 2>/dev/null)" + +echo "$passed passed, $failed failed" +((failed == 0)) diff --git a/tests/failed-command-output.sh b/tests/failed-command-output.sh index 163eefd..e454509 100644 --- a/tests/failed-command-output.sh +++ b/tests/failed-command-output.sh @@ -2,7 +2,7 @@ set -uo pipefail CORE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -eval "$(awk '/^(silent|set_std_mode)[(][)] [{]/,/^}/' "$CORE_DIR/core/core.func")" +eval "$(awk '/^(silent|set_std_mode|_silent_exec|_cs_reattach_output)[(][)] [{]/,/^}/' "$CORE_DIR/core/core.func")" eval "$(awk '/^_eh_print_failed_output[(][)] [{]/,/^}/' "$CORE_DIR/core/error_handler.func")" TEST_DIR="$(mktemp -d)" diff --git a/ui/API.txt b/ui/API.txt index e878cb7..7366117 100644 --- a/ui/API.txt +++ b/ui/API.txt @@ -8,6 +8,7 @@ _cs_host_arch _cs_host_version_line _cs_live_base _cs_os_family +_cs_reattach_output _cs_ref_line _cs_run_os_section _cs_run_update @@ -49,6 +50,7 @@ _pve_version _pveam_available _pveam_update _sanitize_value +_silent_exec _stop_container_if_installing _telemetry_report_exit _tm_clean_log