From b6789412647e557aeb25259ac3708f80ecb0c902 Mon Sep 17 00:00:00 2001 From: MickLesk <47820557+MickLesk@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:47:48 +0200 Subject: [PATCH 1/7] Let an install survive a closed console Closing the Proxmox shell or losing the SSH session sent SIGHUP to the host script, which posted "aborted", stopped the container and left it half-built; the install inside was killed with it. Both now carry on: on_hangup moves the output to the active log instead of exiting, silent() starts external programs with SIGHUP ignored so the apt, npm or cargo of the moment keeps running, a clean install leaves a .done marker, and when lxc-attach dies with the terminal the host follows the container's markers (.failed, .done, the install's pid) until it can report. The failure prompt keeps the container when nobody is there to answer. Ctrl-C still aborts. --- .github/workflows/verify-split-files.yml | 4 + core/core.func | 20 +++- core/error_handler.func | 14 ++- docs/backends.md | 7 ++ docs/core.md | 3 +- pve/backend.func | 71 ++++++++++++-- tests/console-hangup.sh | 117 +++++++++++++++++++++++ ui/API.txt | 1 + 8 files changed, 223 insertions(+), 14 deletions(-) create mode 100644 tests/console-hangup.sh diff --git a/.github/workflows/verify-split-files.yml b/.github/workflows/verify-split-files.yml index 7e84463..231f7d7 100644 --- a/.github/workflows/verify-split-files.yml +++ b/.github/workflows/verify-split-files.yml @@ -37,6 +37,7 @@ on: - "tests/exit-code-hints.sh" - "tests/forge-release-json.sh" - "tests/install-cleanup.sh" + - "tests/console-hangup.sh" - "tests/incus-target.sh" - "tests/incus-storage-space.sh" - "tests/release-stamp-pending.sh" @@ -70,6 +71,7 @@ on: - "tests/exit-code-hints.sh" - "tests/forge-release-json.sh" - "tests/install-cleanup.sh" + - "tests/console-hangup.sh" - "tests/incus-target.sh" - "tests/incus-storage-space.sh" - "tests/release-stamp-pending.sh" @@ -139,6 +141,8 @@ jobs: - name: Install cleanup and uv trap tests run: bash tests/install-cleanup.sh + - name: Console hangup tests + run: bash tests/console-hangup.sh - name: Incus target tests run: bash tests/incus-target.sh diff --git a/core/core.func b/core/core.func index 354865b..f34e2e8 100644 --- a/core/core.func +++ b/core/core.func @@ -939,6 +939,13 @@ log_section() { # for telemetry (the host builds the error trace from this file) # - Temporarily disables error trap to capture exit code correctly # ------------------------------------------------------------------------------ +_silent_exec() { + ( + trap '' HUP + exec "$@" + ) +} + silent() { # A failure the caller handled (a retry loop) must not be reported for a later, unrelated error. unset _SILENT_FAILED_RC _SILENT_FAILED_CMD _SILENT_FAILED_LINE _SILENT_FAILED_LOG @@ -966,14 +973,21 @@ silent() { [[ -f "$logfile" ]] && start_bytes=$(stat -c%s "$logfile" 2>/dev/null || echo 0) [[ ! "$start_bytes" =~ ^[0-9]+$ ]] && start_bytes=0 - local rc + # A closed console sends SIGHUP to everything in the foreground. A function + # stays in this shell, which handles the signal itself; an external program + # inherits "ignore" and keeps running. + local rc run=("$@") + case "$(type -t "$1" 2>/dev/null)" in + function | builtin | "") ;; + *) run=(_silent_exec "$@") ;; + esac if [[ "${VERBOSE:-no}" == "yes" ]]; then # Not a pipe: the command must stay in this shell so functions keep their side effects. - "$@" > >(tee -a "$logfile") 2>&1 + "${run[@]}" > >(tee -a "$logfile") 2>&1 rc=$? wait $! 2>/dev/null else - "$@" >>"$logfile" 2>&1 + "${run[@]}" >>"$logfile" 2>&1 rc=$? fi diff --git a/core/error_handler.func b/core/error_handler.func index 90dc2f8..50f4327 100644 --- a/core/error_handler.func +++ b/core/error_handler.func @@ -796,6 +796,9 @@ on_exit() { if _is_container_context; then if [[ $exit_code -ne 0 ]]; then _container_write_failure "$exit_code" "${FAILED_COMMAND:-}" "${FAILED_LINE:-}" + elif [[ "${INSTALL_LOG:-}" == *.install-*.log ]]; then + # The host reads this when its lxc-attach did not live to see the end. + : >"${INSTALL_LOG%.log}.done" 2>/dev/null || true fi else if [[ "${POST_UPDATE_DONE:-}" != "true" ]] && declare -f post_update_to_api >/dev/null 2>&1; then @@ -874,11 +877,18 @@ on_terminate() { # ------------------------------------------------------------------------------ # on_hangup() - SIGHUP (SSH disconnect, terminal closed) # -# No cursor restore and no message: the terminal that would read them is gone. +# Not an abort: the run carries on with its output in the active log, and +# whoever opens a console later finds the result. Writes to the terminal that +# is gone would fail with EIO and end the script, hence the redirect. # ------------------------------------------------------------------------------ on_hangup() { declare -f stop_spinner >/dev/null 2>&1 && { stop_spinner 2>/dev/null || true; } - _eh_signal_exit 129 + local log="" + declare -f get_active_logfile >/dev/null 2>&1 && log="$(get_active_logfile 2>/dev/null)" + [[ -n "$log" ]] || log="${INSTALL_LOG:-${BUILD_LOG:-/dev/null}}" + exec >"$log" 2>&1 + export _CS_HANGUP=1 CS_ANIMATION_OK=0 + echo "[WARN] $(date '+%Y-%m-%d %H:%M:%S') The terminal hung up (SIGHUP); continuing, output follows here." } # ============================================================================== diff --git a/docs/backends.md b/docs/backends.md index 79ccbbe..1d94f46 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -16,6 +16,13 @@ wizard calls. | [`vm-core.func`](../pve/vm-core.func) | VM creation via `qm`, including disk import and template handling | | [`vm-app.func`](../pve/vm-app.func) | Deploys an application that normally goes into an LXC container into a full VM instead | +A closed console is not an abort. SIGHUP reaches the host script and the +install inside the container; both carry on with their output in their logs +(`on_hangup`), external commands under `$STD` ignore the signal, and when +lxc-attach dies with the terminal the host waits for the container's +`.done`/`.failed` markers (`_pve_wait_for_detached_install`) before it +reports. Ctrl-C still aborts. + ## `incus/` — Incus | File | Purpose | diff --git a/docs/core.md b/docs/core.md index 15710d1..dab3fd3 100644 --- a/docs/core.md +++ b/docs/core.md @@ -33,7 +33,8 @@ Worth knowing: - `silent()` runs a command with its output captured, and records the failing command, line and log path so the error handler can report the real failure - rather than the wrapper. + rather than the wrapper. External commands run with SIGHUP ignored, so a + closed console does not kill the apt or npm of the moment. - Several host facts are memoised in files under a runtime cache directory rather than in shell variables, because every call site is a `$( )` or a pipe — that is, a subshell, where a variable assignment would be discarded. diff --git a/pve/backend.func b/pve/backend.func index 097a7d4..2dc0b6b 100644 --- a/pve/backend.func +++ b/pve/backend.func @@ -184,6 +184,38 @@ _pct_restart() { return 1 } +# ------------------------------------------------------------------------------ +# _pve_wait_for_detached_install +# +# The console closed mid-install: SIGHUP took lxc-attach, but the script inside +# ignores it and runs on. Wait for its markers: .failed (its exit code, read by +# the caller), .done (clean end), or the process gone without either. +# Returns 0 for .done, 1 for .failed, 199 when nothing reported, 124 on timeout. +# ------------------------------------------------------------------------------ +_pve_wait_for_detached_install() { + local base="/root/.install-${SESSION_ID}" pid="" waited=0 + local poll="${_CS_HANGUP_POLL:-15}" limit="${_CS_HANGUP_TIMEOUT:-21600}" + echo "[INFO] $(date '+%Y-%m-%d %H:%M:%S') Waiting for the install in container ${CTID} to finish on its own" + while :; do + if pct exec "$CTID" -- test -f "${base}.failed" 2>/dev/null; then + return 1 + fi + if pct exec "$CTID" -- test -f "${base}.done" 2>/dev/null; then + return 0 + fi + [[ "$(pct status "$CTID" 2>/dev/null)" == "status: running" ]] || return 199 + pid="$(pct exec "$CTID" -- cat "${base}.pid" 2>/dev/null | tr -d '[:space:]')" + if [[ "$pid" =~ ^[0-9]+$ ]] && ! pct exec "$CTID" -- kill -0 "$pid" 2>/dev/null; then + return 199 + fi + if ((waited >= limit)); then + return 124 + fi + sleep "$poll" + waited=$((waited + poll)) + done +} + # Where a template lands in $TEMPLATE_STORAGE, or the default dir layout when # pvesm cannot say. _template_path() { @@ -1404,9 +1436,20 @@ PROFILE fi exit 115 fi - lxc-attach -n "$CTID" -- bash -c "$_install_script" + msg_custom "ℹ️" "${YW}" "Closing this console does not abort the install; it finishes on its own (Ctrl-C does abort)." + msg_custom " " "${YW}" "Follow it with: pct exec ${CTID} -- tail -f /root/.install-${SESSION_ID}.log" + lxc-attach -n "$CTID" -- bash -c "printf '%s' \$\$ >/root/.install-${SESSION_ID}.pid +$_install_script" local lxc_exit=$? + # The terminal went and took lxc-attach with it; the install inside did + # not stop. Follow it through its markers instead of calling it failed. + if [[ "${_CS_HANGUP:-}" == "1" && $lxc_exit -ne 0 ]]; then + _pve_wait_for_detached_install + lxc_exit=$? + fi + pct exec "$CTID" -- rm -f "/root/.install-${SESSION_ID}.pid" "/root/.install-${SESSION_ID}.done" 2>/dev/null || true + unset CONTAINER_INSTALLING if [[ -n "${SESSION_ID:-}" ]]; then @@ -1692,9 +1735,15 @@ PROFILE local response="" local read_rc - reclaim_tty - read -t 60 -r response /root/.install-${SESSION_ID}.pid +$_install_script" local apt_retry_exit=$? + if [[ "${_CS_HANGUP:-}" == "1" && $apt_retry_exit -ne 0 ]]; then + _pve_wait_for_detached_install + apt_retry_exit=$? + fi + pct exec "$CTID" -- rm -f "/root/.install-${SESSION_ID}.pid" "/root/.install-${SESSION_ID}.done" 2>/dev/null || true set -Eeuo pipefail trap 'error_handler' ERR @@ -3840,8 +3895,8 @@ api_exit_script() { fi } -# All five traps report telemetry, so all five belong to the host. EXIT was -# already guarded; the other four were not, and core/build.func sources this +# These traps report telemetry, so they belong to the host. EXIT was already +# guarded; the others were not, and core/build.func sources this # file unconditionally -- including inside a container, where a ct script runs # again for an update. A Ctrl-C or any ERR during an update therefore posted a # failed record, typed lxc, indistinguishable from a failed install. Their pct @@ -3851,7 +3906,7 @@ api_exit_script() { if command -v pveversion >/dev/null 2>&1; then trap 'api_exit_script' EXIT trap '_ec=$?; if [[ $_ec -ne 0 ]]; then post_update_to_api "failed" "$_ec" 2>/dev/null || true; if declare -f ensure_log_on_host &>/dev/null; then ensure_log_on_host 2>/dev/null || true; fi; fi' ERR - trap 'post_update_to_api "failed" "129" 2>/dev/null || true; if [[ -n "${CTID:-}" ]] && command -v pct &>/dev/null; then pct stop "$CTID" 2>/dev/null || true; fi; exit 129' SIGHUP + # SIGHUP stays with on_hangup (core/error_handler.func): a closed console is not an abort. trap 'post_update_to_api "failed" "130" 2>/dev/null || true; if [[ -n "${CTID:-}" ]] && command -v pct &>/dev/null; then pct stop "$CTID" 2>/dev/null || true; fi; exit 130' SIGINT trap 'post_update_to_api "failed" "143" 2>/dev/null || true; if [[ -n "${CTID:-}" ]] && command -v pct &>/dev/null; then pct stop "$CTID" 2>/dev/null || true; fi; exit 143' SIGTERM fi diff --git a/tests/console-hangup.sh b/tests/console-hangup.sh new file mode 100644 index 0000000..52b01c3 --- /dev/null +++ b/tests/console-hangup.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# A closed console (SIGHUP) is not an abort: silent() shields the external +# command of the moment, on_hangup moves the output to the log and carries on, +# a clean install leaves a .done marker, and the host waits for it when +# lxc-attach died with the terminal. +set -uo pipefail + +CORE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +eval "$(awk '/^(silent|_silent_exec)[(][)] [{]/,/^}/' "$CORE_DIR/core/core.func")" +eval "$(awk '/^(on_hangup|on_exit|_eh_quiet_spinner)[(][)] [{]/,/^}/' "$CORE_DIR/core/error_handler.func")" +eval "$(awk '/^_pve_wait_for_detached_install[(][)] [{]/,/^}/' "$CORE_DIR/pve/backend.func")" + +TEST_DIR="$(mktemp -d)" +trap 'rm -rf -- "$TEST_DIR"' EXIT +LOG="$TEST_DIR/install.log" +ROOT="$TEST_DIR/root" +mkdir -p "$ROOT" +passed=0 failed=0 +TAB=" " + +get_active_logfile() { echo "$LOG"; } +debug_log() { :; } +stop_spinner() { :; } +_is_container_context() { return 0; } +_container_write_failure() { echo "failure $1" >>"$TEST_DIR/failure"; } + +expect() { + if [[ "$2" == "$3" ]]; then + echo "PASS $1" + passed=$((passed + 1)) + else + echo "FAIL $1: expected <$2>, got <$3>" >&2 + failed=$((failed + 1)) + fi +} + +# silent(): an external command ignores SIGHUP, a function keeps its side effects. +: >"$LOG" +silent sh -c 'kill -HUP $$; echo survived' +expect "an external command under silent survives SIGHUP" 0 "$?" +expect "and finished its work" 1 "$(grep -c survived "$LOG")" +marker() { MARK="set"; } +unset MARK +silent marker +expect "a function keeps its side effects" set "${MARK:-unset}" +silent definitely_missing_command_xyz +expect "a missing command still reports 127" 127 "$?" + +# on_hangup(): the run carries on, output moves to the log. +: >"$LOG" +out="$( (trap on_hangup HUP; kill -HUP $BASHPID; echo "after hangup"; echo "flag=${_CS_HANGUP:-}") 2>&1 )" +rc=$? +expect "the shell survives the hangup" 0 "$rc" +expect "nothing more reaches the terminal" "" "$out" +expect "later output lands in the log" 1 "$(grep -c 'after hangup' "$LOG")" +expect "the hangup is noted in the log" 1 "$(grep -c SIGHUP "$LOG")" +expect "the run knows it is detached" 1 "$(grep -c 'flag=1' "$LOG")" + +# on_exit(): a clean install leaves .done for the host; a failed one does not. +INSTALL_LOG="$ROOT/.install-abc.log" +export INSTALL_LOG +(trap on_exit EXIT; exit 0) +expect "a clean install leaves a .done marker" 1 "$(ls "$ROOT/.install-abc.done" 2>/dev/null | wc -l)" +rm -f "$ROOT/.install-abc.done" +(trap on_exit EXIT; exit 3) +expect "a failed install leaves no .done marker" 0 "$(ls "$ROOT/.install-abc.done" 2>/dev/null | wc -l)" +expect "but the failure artifact" 1 "$(grep -c 'failure 3' "$TEST_DIR/failure")" +(unset INSTALL_LOG; trap on_exit EXIT; exit 0) +expect "an update run leaves no marker" 0 "$(ls "$ROOT"/.install-*.done 2>/dev/null | wc -l)" + +# _pve_wait_for_detached_install(): the host follows the markers of a detached install. +CTID=100 SESSION_ID=abc CT_STATE="status: running" +_CS_HANGUP_POLL=1 _CS_HANGUP_TIMEOUT=3 +pct() { + case "$1" in + status) echo "$CT_STATE" ;; + exec) + shift 3 + case "$1" in + test) [[ -f "${TEST_DIR}${3}" ]] ;; + cat) cat "${TEST_DIR}${2}" 2>/dev/null ;; + kill) command kill "$2" "$3" 2>/dev/null ;; + esac + ;; + esac +} +sleep 30 & +worker=$! +echo "$worker" >"$ROOT/.install-abc.pid" +(sleep 2; : >"$ROOT/.install-abc.done") & +_pve_wait_for_detached_install >/dev/null +expect "a .done marker ends the wait cleanly" 0 "$?" +rm -f "$ROOT/.install-abc.done" + +: >"$ROOT/.install-abc.failed" +_pve_wait_for_detached_install >/dev/null +expect "a .failed marker hands the flag to the caller" 1 "$?" +rm -f "$ROOT/.install-abc.failed" + +_pve_wait_for_detached_install >/dev/null +expect "no marker before the timeout" 124 "$?" + +kill "$worker" 2>/dev/null +wait "$worker" 2>/dev/null +_pve_wait_for_detached_install >/dev/null +expect "a process gone without a marker is reported as such" 199 "$?" + +sleep 30 & +worker=$! +echo "$worker" >"$ROOT/.install-abc.pid" +CT_STATE="status: stopped" +_pve_wait_for_detached_install >/dev/null +expect "a stopped container ends the wait" 199 "$?" +kill "$worker" 2>/dev/null + +echo "$passed passed, $failed failed" +((failed == 0)) diff --git a/ui/API.txt b/ui/API.txt index e878cb7..cea5bc3 100644 --- a/ui/API.txt +++ b/ui/API.txt @@ -49,6 +49,7 @@ _pve_version _pveam_available _pveam_update _sanitize_value +_silent_exec _stop_container_if_installing _telemetry_report_exit _tm_clean_log From 1e1ddd734cde4423c8ef633bd576567ea55ec838 Mon Sep 17 00:00:00 2001 From: MickLesk <47820557+MickLesk@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:05:34 +0200 Subject: [PATCH 2/7] Say what the hangup interrupted and why the wait ended --- core/error_handler.func | 3 ++- pve/backend.func | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/core/error_handler.func b/core/error_handler.func index 50f4327..e622292 100644 --- a/core/error_handler.func +++ b/core/error_handler.func @@ -776,6 +776,7 @@ _stop_container_if_installing() { # ------------------------------------------------------------------------------ on_exit() { local exit_code=$? + [[ "${_CS_HANGUP:-}" == "1" ]] && echo "[INFO] $(date '+%Y-%m-%d %H:%M:%S') on_exit with code ${exit_code} (pid $$)" # Before anything else: the timings are only useful if they survive a failure. declare -f dev_mode_timing_summary >/dev/null 2>&1 && dev_mode_timing_summary @@ -888,7 +889,7 @@ on_hangup() { [[ -n "$log" ]] || log="${INSTALL_LOG:-${BUILD_LOG:-/dev/null}}" exec >"$log" 2>&1 export _CS_HANGUP=1 CS_ANIMATION_OK=0 - echo "[WARN] $(date '+%Y-%m-%d %H:%M:%S') The terminal hung up (SIGHUP); continuing, output follows here." + echo "[WARN] $(date '+%Y-%m-%d %H:%M:%S') The terminal hung up (SIGHUP); continuing, output follows here. (pid $$, subshell ${BASH_SUBSHELL}, during: ${BASH_COMMAND:-?})" } # ============================================================================== diff --git a/pve/backend.func b/pve/backend.func index 2dc0b6b..31b5feb 100644 --- a/pve/backend.func +++ b/pve/backend.func @@ -198,17 +198,25 @@ _pve_wait_for_detached_install() { echo "[INFO] $(date '+%Y-%m-%d %H:%M:%S') Waiting for the install in container ${CTID} to finish on its own" while :; do if pct exec "$CTID" -- test -f "${base}.failed" 2>/dev/null; then + echo "[INFO] The install reported a failure" return 1 fi if pct exec "$CTID" -- test -f "${base}.done" 2>/dev/null; then + echo "[INFO] The install finished" return 0 fi - [[ "$(pct status "$CTID" 2>/dev/null)" == "status: running" ]] || return 199 + if [[ "$(pct status "$CTID" 2>/dev/null)" != "status: running" ]]; then + echo "[WARN] Container ${CTID} is no longer running" + return 199 + fi pid="$(pct exec "$CTID" -- cat "${base}.pid" 2>/dev/null | tr -d '[:space:]')" if [[ "$pid" =~ ^[0-9]+$ ]] && ! pct exec "$CTID" -- kill -0 "$pid" 2>/dev/null; then + echo "[WARN] Install process ${pid} is gone without reporting; processes left in the container:" + pct exec "$CTID" -- ps -eo pid,ppid,stat,etime,cmd 2>/dev/null | head -n 40 return 199 fi if ((waited >= limit)); then + echo "[WARN] Gave up waiting after ${waited}s" return 124 fi sleep "$poll" From 1c9b934227ee035f7b54989727707d9731d16323 Mon Sep 17 00:00:00 2001 From: MickLesk <47820557+MickLesk@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:17:07 +0100 Subject: [PATCH 3/7] Keep the output redirect after a hangup interrupts a redirected call A redirected call to a shell function makes bash save fd 1 and fd 2 and restore them when the call returns. silent() makes exactly that call, so the hangup trap's redirect was undone the moment the interrupted command finished, and stdout pointed at the terminal that was gone. The first message after it failed with EIO, which under errexit ended the install without reaching on_exit -- the half-built container and the missing status the telemetry reported as an abort. silent() and the msg_* helpers now re-apply the redirect where no redirection is in effect. --- core/core.func | 19 +++++++++++++++++++ core/error_handler.func | 2 +- tests/console-hangup.sh | 26 +++++++++++++++++++++++++- ui/API.txt | 1 + 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/core/core.func b/core/core.func index f34e2e8..f207f85 100644 --- a/core/core.func +++ b/core/core.func @@ -939,6 +939,19 @@ log_section() { # for telemetry (the host builds the error trace from this file) # - Temporarily disables error trap to capture exit code correctly # ------------------------------------------------------------------------------ +# After a hangup the shell's stdout and stderr still point at the terminal that +# is gone, and every write to it fails. The trap redirects them to the log, but +# bash saves and restores fd 1 and 2 around a redirected call to a shell +# function -- silent() makes exactly such a call -- so the redirect is undone +# the moment that call returns. Re-apply it where no redirection is in effect. +_cs_reattach_output() { + [[ "${_CS_HANGUP:-}" == "1" ]] || return 0 + local log="${_CS_HANGUP_LOG:-}" + [[ -n "$log" ]] || log="$(get_active_logfile)" + [[ -n "$log" ]] || return 0 + exec >"$log" 2>&1 +} + _silent_exec() { ( trap '' HUP @@ -947,6 +960,7 @@ _silent_exec() { } silent() { + _cs_reattach_output # A failure the caller handled (a retry loop) must not be reported for a later, unrelated error. unset _SILENT_FAILED_RC _SILENT_FAILED_CMD _SILENT_FAILED_LINE _SILENT_FAILED_LOG local cmd="$*" @@ -1252,6 +1266,7 @@ fi # - Backgrounds spinner process and stores PID for later cleanup # ------------------------------------------------------------------------------ msg_info() { + _cs_reattach_output local msg="$1" [[ -z "$msg" ]] && return @@ -1310,6 +1325,7 @@ msg_info() { # - Uses green color for success indication # ------------------------------------------------------------------------------ msg_ok() { + _cs_reattach_output local msg="$1" [[ -z "$msg" ]] && return stop_spinner @@ -1332,6 +1348,7 @@ msg_ok() { # - Outputs to stderr # ------------------------------------------------------------------------------ msg_error() { + _cs_reattach_output stop_spinner _MSG_BLOCKS=() local msg="$1" @@ -1348,6 +1365,7 @@ msg_error() { # - Outputs to stderr # ------------------------------------------------------------------------------ msg_warn() { + _cs_reattach_output stop_spinner _MSG_BLOCKS=() local msg="$1" @@ -1364,6 +1382,7 @@ msg_warn() { # - Useful for specialized status messages # ------------------------------------------------------------------------------ msg_custom() { + _cs_reattach_output local symbol="${1:-"[*]"}" local color="${2:-"\e[36m"}" local msg="${3:-}" diff --git a/core/error_handler.func b/core/error_handler.func index e622292..dcba566 100644 --- a/core/error_handler.func +++ b/core/error_handler.func @@ -888,7 +888,7 @@ on_hangup() { declare -f get_active_logfile >/dev/null 2>&1 && log="$(get_active_logfile 2>/dev/null)" [[ -n "$log" ]] || log="${INSTALL_LOG:-${BUILD_LOG:-/dev/null}}" exec >"$log" 2>&1 - export _CS_HANGUP=1 CS_ANIMATION_OK=0 + export _CS_HANGUP=1 CS_ANIMATION_OK=0 _CS_HANGUP_LOG="$log" echo "[WARN] $(date '+%Y-%m-%d %H:%M:%S') The terminal hung up (SIGHUP); continuing, output follows here. (pid $$, subshell ${BASH_SUBSHELL}, during: ${BASH_COMMAND:-?})" } diff --git a/tests/console-hangup.sh b/tests/console-hangup.sh index 52b01c3..ea447f1 100644 --- a/tests/console-hangup.sh +++ b/tests/console-hangup.sh @@ -6,7 +6,7 @@ set -uo pipefail CORE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -eval "$(awk '/^(silent|_silent_exec)[(][)] [{]/,/^}/' "$CORE_DIR/core/core.func")" +eval "$(awk '/^(silent|_silent_exec|_cs_reattach_output)[(][)] [{]/,/^}/' "$CORE_DIR/core/core.func")" eval "$(awk '/^(on_hangup|on_exit|_eh_quiet_spinner)[(][)] [{]/,/^}/' "$CORE_DIR/core/error_handler.func")" eval "$(awk '/^_pve_wait_for_detached_install[(][)] [{]/,/^}/' "$CORE_DIR/pve/backend.func")" @@ -113,5 +113,29 @@ _pve_wait_for_detached_install >/dev/null expect "a stopped container ends the wait" 199 "$?" kill "$worker" 2>/dev/null +# A redirected call to a shell function makes bash save fd 1 and restore it +# afterwards, which undid the redirect the hangup trap had made. Every message +# after that wrote to the terminal that was gone, and the failed write ended +# the run under errexit. +REDIR="$TEST_DIR/redirected.log" +AFTER="$TEST_DIR/after.log" +inner() { echo "inner"; } +out="$( ( + exec >"$TEST_DIR/orig.log" 2>&1 + _CS_HANGUP=1 _CS_HANGUP_LOG="$AFTER" + export _CS_HANGUP _CS_HANGUP_LOG + inner >"$REDIR" 2>&1 + _cs_reattach_output + echo "after the redirected call" + readlink /proc/$$/fd/1 +) 2>&1 )" +expect "output after a redirected call goes to the hangup log" 1 "$(grep -c 'after the redirected call' "$AFTER" 2>/dev/null)" +expect "and not back to the original target" 0 "$(grep -c 'after the redirected call' "$TEST_DIR/orig.log" 2>/dev/null)" + +: >"$AFTER" +( unset _CS_HANGUP; _CS_HANGUP_LOG="$AFTER"; _cs_reattach_output; echo "no hangup" ) >"$TEST_DIR/orig2.log" 2>&1 +expect "without a hangup nothing is redirected" 0 "$(grep -c 'no hangup' "$AFTER" 2>/dev/null)" +expect "it stays where it was" 1 "$(grep -c 'no hangup' "$TEST_DIR/orig2.log" 2>/dev/null)" + echo "$passed passed, $failed failed" ((failed == 0)) diff --git a/ui/API.txt b/ui/API.txt index cea5bc3..b02dff2 100644 --- a/ui/API.txt +++ b/ui/API.txt @@ -5,6 +5,7 @@ _cs_clear _cs_conffile_list _cs_engine_ref_line _cs_host_arch +_cs_reattach_output _cs_host_version_line _cs_live_base _cs_os_family From e38f837f65d620b1f66c7e97562104266af15d0b Mon Sep 17 00:00:00 2001 From: MickLesk <47820557+MickLesk@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:24:17 +0100 Subject: [PATCH 4/7] Let the silent test see the new output helper --- tests/failed-command-output.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/failed-command-output.sh b/tests/failed-command-output.sh index 163eefd..e454509 100644 --- a/tests/failed-command-output.sh +++ b/tests/failed-command-output.sh @@ -2,7 +2,7 @@ set -uo pipefail CORE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -eval "$(awk '/^(silent|set_std_mode)[(][)] [{]/,/^}/' "$CORE_DIR/core/core.func")" +eval "$(awk '/^(silent|set_std_mode|_silent_exec|_cs_reattach_output)[(][)] [{]/,/^}/' "$CORE_DIR/core/core.func")" eval "$(awk '/^_eh_print_failed_output[(][)] [{]/,/^}/' "$CORE_DIR/core/error_handler.func")" TEST_DIR="$(mktemp -d)" From 58a1a11abd91b4162bdb521f9105c80c5af34464 Mon Sep 17 00:00:00 2001 From: MickLesk <47820557+MickLesk@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:28:41 +0100 Subject: [PATCH 5/7] Sort the new helper into the ui API snapshot --- ui/API.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/API.txt b/ui/API.txt index b02dff2..7366117 100644 --- a/ui/API.txt +++ b/ui/API.txt @@ -5,10 +5,10 @@ _cs_clear _cs_conffile_list _cs_engine_ref_line _cs_host_arch -_cs_reattach_output _cs_host_version_line _cs_live_base _cs_os_family +_cs_reattach_output _cs_ref_line _cs_run_os_section _cs_run_update From a22f376850a272989e0f7ac7d28ec6e7d63f8578 Mon Sep 17 00:00:00 2001 From: MickLesk <47820557+MickLesk@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:48:15 +0100 Subject: [PATCH 6/7] Incus: run the install detached so a closed console cannot end it incus ends an exec session the moment its client goes away and takes the command with it -- measured on real hardware: the process was gone and its SIGHUP trap had never run, where the same test under lxc-attach survived and finished. The host made it worse by force-stopping the container on SIGHUP and reporting a failure. The install now runs detached from the exec session, with its input coming through a fifo while a console exists and its output followed from the host. Closing the console costs the view and the keyboard, not the installation; the host waits for the markers the install leaves. --- docs/backends.md | 9 ++++++ incus/backend.func | 76 ++++++++++++++++++++++++++++++++++++++++++---- incus/build.func | 13 +++++++- 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/docs/backends.md b/docs/backends.md index 1d94f46..2d09cb5 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -78,6 +78,15 @@ from `/etc/localtime`, or from `Etc/UTC`. [`core/core.func`](../core/core.func) renders "Incus Version 6.x" on an Incus host and "PVE Version …" on Proxmox VE, from the same `PVEVERSION` value. +A closed console is not an abort here either, but it takes a different route. +`incus exec` ends as soon as its client goes away and takes the command with +it, which `lxc-attach` does not do. The install therefore runs detached inside +the container, writing to its own log, and the host only watches it: input +reaches it through a fifo for as long as there is a console, and once there is +none the reads return EOF and the prompts take their defaults. +`_incus_wait_for_detached_install` then waits for the `.done`/`.failed` +markers before the host reports. + ### Driving a remote server The `incus` client always talks to a remote; `local:` is only the default. diff --git a/incus/backend.func b/incus/backend.func index 1471fa0..9a1ca57 100644 --- a/incus/backend.func +++ b/incus/backend.func @@ -1061,6 +1061,30 @@ MOTDEOF msg_ok "MOTD configured" } +# ------------------------------------------------------------------------------ +# _incus_wait_for_detached_install +# +# The console closed while the install ran. incus ended the exec session that +# was watching it, but the install itself is detached and carries on. Wait for +# what it leaves behind: .failed (its exit code, read by the caller), .done, or +# the process gone without either. +# ------------------------------------------------------------------------------ +_incus_wait_for_detached_install() { + local base="$1" pid="" waited=0 + local poll="${_CS_HANGUP_POLL:-10}" limit="${_CS_HANGUP_TIMEOUT:-21600}" + while :; do + incus_ct_exec test -f "${base}.failed" >/dev/null 2>&1 && return 0 + incus_ct_exec test -f "${base}.done" >/dev/null 2>&1 && return 0 + pid="$(incus_ct_exec cat "${base}.pid" 2>/dev/null | tr -d '[:space:]')" + if [[ "$pid" =~ ^[0-9]+$ ]] && ! incus_ct_exec kill -0 "$pid" >/dev/null 2>&1; then + return 0 + fi + ((waited >= limit)) && return 0 + sleep "$poll" + waited=$((waited + poll)) + done +} + _incus_push_functions_and_run_install() { # Sets INCUS_LAST_INSTALL_EXIT (do not capture stdout — msg_* writes there). INCUS_LAST_INSTALL_EXIT=0 @@ -1095,15 +1119,53 @@ INNEREOF" >>"${LOGFILE:-$INCUS_BUILD_LOG}" 2>&1 || { msg_error "Failed to push f # No pty on purpose: under `incus exec -t` apt's tcsetattr fails with EIO and kills the shell. set +Eeuo pipefail trap - ERR + + # incus ends an exec session the moment its client goes away, so closing the + # console killed the install outright - the one thing lxc-attach does not do. + # Run it detached from that session and only watch it from here. The console + # keeps the view and the keyboard: input reaches the install through a fifo + # for as long as there is a console, and once there is none the reads return + # EOF and the prompts take their defaults. + local _base="/root/.install-${SESSION_ID}" + local _runner _watcher _body + _runner='setsid bash "$1".sh <"$1".in >>"$1".out 2>&1 & +echo $! >"$1".pid' + _watcher='tail -n +1 -f "$1".out & +_tp=$! +while [ ! -f "$1".done ] && [ ! -f "$1".failed ]; do sleep 2; done +sleep 1 +kill $_tp 2>/dev/null' # The trailing touch is a completion sentinel: no install script exits early. - incus exec "${CT_NAME}" -- bash -c "${_run_env} + _body="${_run_env} ${_install_script} -touch /root/.install-${SESSION_ID}.done" 2>&1 | +touch ${_base}.done" + + incus exec "${CT_NAME}" -- bash -c "cat >${_base}.sh <<'CSBODYEOF' +${_body} +CSBODYEOF +cat >${_base}.run <<'CSRUNEOF' +${_runner} +CSRUNEOF +cat >${_base}.watch <<'CSWATCHEOF' +${_watcher} +CSWATCHEOF +rm -f ${_base}.in ${_base}.pid ${_base}.done ${_base}.failed ${_base}.out +mkfifo ${_base}.in" >>"${LOGFILE:-$INCUS_BUILD_LOG}" 2>&1 + + # A writer has to hold the fifo open before the install opens it to read. + incus exec "${CT_NAME}" -- sh -c "cat >${_base}.in" & + local _feeder=$! + incus exec "${CT_NAME}" -- bash "${_base}.run" "${_base}" >/dev/null 2>&1 + + incus exec "${CT_NAME}" -- bash "${_base}.watch" "${_base}" 2>&1 | tee >(sed 's/.*\r//' >>"${LOGFILE:-$INCUS_BUILD_LOG}") - # PIPESTATUS, not $?: the line above turns pipefail off, so $? is tee's - # status - always 0. Every non-zero exit from the container was discarded - # here, which is why a half-finished install reported success. - lxc_exit=${PIPESTATUS[0]} + kill "$_feeder" 2>/dev/null || true + wait "$_feeder" 2>/dev/null || true + + # The watcher is gone either way - because the install ended, or because the + # console did. Only the markers say which. + _incus_wait_for_detached_install "$_base" + lxc_exit=0 set -Eeuo pipefail declare -f error_handler &>/dev/null && trap 'error_handler' ERR || true @@ -1125,10 +1187,12 @@ touch /root/.install-${SESSION_ID}.done" 2>&1 | incus_ct_exec rm -f "/root/.install-${SESSION_ID}.done" 2>/dev/null || true else msg_error "Install script stopped before it finished - nothing reported it" + [[ "${_CS_HANGUP:-}" == "1" ]] && msg_custom "ℹ️" "${YW}" "The console was closed; the install ran on detached, so look at its log for the reason" msg_custom "ℹ️" "${YW}" "Output: incus exec ${CT_NAME} -- tail -50 /root/.install-${SESSION_ID}.log" install_exit_code=1 fi fi + incus_ct_exec rm -f "${_base}.sh" "${_base}.run" "${_base}.watch" "${_base}.in" "${_base}.pid" "${_base}.out" >/dev/null 2>&1 || true INCUS_LAST_INSTALL_EXIT=$install_exit_code } diff --git a/incus/build.func b/incus/build.func index 30e93f8..0b126f6 100644 --- a/incus/build.func +++ b/incus/build.func @@ -633,8 +633,19 @@ _incus_on_signal() { exit "$code" } +# A closed console is not an abort. The install runs detached inside the +# container, so losing the terminal costs the view and the keyboard, not the +# installation. Writes to the terminal that is gone fail with EIO and would +# end the run under errexit, hence the redirect. +_incus_on_hangup() { + local log="${LOGFILE:-${INCUS_BUILD_LOG:-/dev/null}}" + exec >"$log" 2>&1 + export _CS_HANGUP=1 _CS_HANGUP_LOG="$log" CS_STDIN_OK=0 + echo "[WARN] $(date '+%Y-%m-%d %H:%M:%S') The terminal hung up (SIGHUP); continuing, output follows here." +} + trap 'api_exit_script' EXIT trap '_incus_on_err' ERR -trap '_incus_on_signal SIGHUP 129' SIGHUP +trap '_incus_on_hangup' SIGHUP trap '_incus_on_signal SIGINT 130' SIGINT trap '_incus_on_signal SIGTERM 143' SIGTERM From 2df4be2e8a97807b1dad8c5be8956a5e7b7fd973 Mon Sep 17 00:00:00 2001 From: MickLesk <47820557+MickLesk@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:53:14 +0100 Subject: [PATCH 7/7] Incus: keep the host's own messages writable after a hangup --- incus/build.func | 3 ++- incus/core.func | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/incus/build.func b/incus/build.func index 0b126f6..a64cdb8 100644 --- a/incus/build.func +++ b/incus/build.func @@ -638,7 +638,8 @@ _incus_on_signal() { # installation. Writes to the terminal that is gone fail with EIO and would # end the run under errexit, hence the redirect. _incus_on_hangup() { - local log="${LOGFILE:-${INCUS_BUILD_LOG:-/dev/null}}" + local log="${LOGFILE:-${_INCUS_HOST_LOGFILE:-${INCUS_BUILD_LOG:-}}}" + [[ -n "$log" ]] || log="/tmp/incus-hangup-${SESSION_ID:-$$}.log" exec >"$log" 2>&1 export _CS_HANGUP=1 _CS_HANGUP_LOG="$log" CS_STDIN_OK=0 echo "[WARN] $(date '+%Y-%m-%d %H:%M:%S') The terminal hung up (SIGHUP); continuing, output follows here." diff --git a/incus/core.func b/incus/core.func index 9997431..5bf5a78 100644 --- a/incus/core.func +++ b/incus/core.func @@ -469,6 +469,7 @@ incus_stop_spinner() { # - Displays informational message with spinner animation # ------------------------------------------------------------------------------ incus_msg_info() { + declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output local msg="$1" [[ -z "$msg" ]] && return @@ -507,6 +508,7 @@ incus_msg_info() { # incus_msg_ok() # ------------------------------------------------------------------------------ incus_msg_ok() { + declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output local msg="$1" [[ -z "$msg" ]] && return incus_stop_spinner @@ -523,6 +525,7 @@ incus_msg_ok() { # incus_msg_error() # ------------------------------------------------------------------------------ incus_msg_error() { + declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output incus_stop_spinner local msg="$1" echo -e "${BFR:-}${CROSS:-✖️}${RD}${msg}${CL}" >&2 @@ -533,6 +536,7 @@ incus_msg_error() { # incus_msg_warn() # ------------------------------------------------------------------------------ incus_msg_warn() { + declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output incus_stop_spinner local msg="$1" # Same layout as core.func msg_warn (INFO already includes TAB padding). @@ -544,6 +548,7 @@ incus_msg_warn() { # incus_msg_custom() # ------------------------------------------------------------------------------ incus_msg_custom() { + declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output local symbol="${1:-"[*]"}" local color="${2:-"\e[36m"}" local msg="${3:-}"