Let an install survive a closed console

.github/workflows/verify-split-files.yml

@@ -38,6 +38,7 @@ on:
       - "tests/exit-code-hints.sh"
       - "tests/forge-release-json.sh"
       - "tests/install-cleanup.sh"
+      - "tests/console-hangup.sh"
       - "tests/incus-target.sh"
       - "tests/incus-storage-space.sh"
       - "tests/release-stamp-pending.sh"
@@ -72,6 +73,7 @@ on:
       - "tests/exit-code-hints.sh"
       - "tests/forge-release-json.sh"
       - "tests/install-cleanup.sh"
+      - "tests/console-hangup.sh"
       - "tests/incus-target.sh"
       - "tests/incus-storage-space.sh"
       - "tests/release-stamp-pending.sh"
@@ -144,6 +146,8 @@ jobs:
 
       - name: Install cleanup and uv trap tests
         run: bash tests/install-cleanup.sh
+      - name: Console hangup tests
+        run: bash tests/console-hangup.sh
 
       - name: Incus target tests
         run: bash tests/incus-target.sh

core/core.func

@@ -939,7 +939,28 @@ log_section() {
 #   for telemetry (the host builds the error trace from this file)
 # - Temporarily disables error trap to capture exit code correctly
 # ------------------------------------------------------------------------------
+# After a hangup the shell's stdout and stderr still point at the terminal that
+# is gone, and every write to it fails. The trap redirects them to the log, but
+# bash saves and restores fd 1 and 2 around a redirected call to a shell
+# function -- silent() makes exactly such a call -- so the redirect is undone
+# the moment that call returns. Re-apply it where no redirection is in effect.
+_cs_reattach_output() {
+  [[ "${_CS_HANGUP:-}" == "1" ]] || return 0
+  local log="${_CS_HANGUP_LOG:-}"
+  [[ -n "$log" ]] || log="$(get_active_logfile)"
+  [[ -n "$log" ]] || return 0
+  exec </dev/null >>"$log" 2>&1
+}
+
+_silent_exec() {
+  (
+    trap '' HUP
+    exec "$@"
+  )
+}
+
 silent() {
+  _cs_reattach_output
   # A failure the caller handled (a retry loop) must not be reported for a later, unrelated error.
   unset _SILENT_FAILED_RC _SILENT_FAILED_CMD _SILENT_FAILED_LINE _SILENT_FAILED_LOG
   local cmd="$*"
@@ -966,14 +987,21 @@ silent() {
   [[ -f "$logfile" ]] && start_bytes=$(stat -c%s "$logfile" 2>/dev/null || echo 0)
   [[ ! "$start_bytes" =~ ^[0-9]+$ ]] && start_bytes=0
 
-  local rc
+  # A closed console sends SIGHUP to everything in the foreground. A function
+  # stays in this shell, which handles the signal itself; an external program
+  # inherits "ignore" and keeps running.
+  local rc run=("$@")
+  case "$(type -t "$1" 2>/dev/null)" in
+  function | builtin | "") ;;
+  *) run=(_silent_exec "$@") ;;
+  esac
   if [[ "${VERBOSE:-no}" == "yes" ]]; then
     # Not a pipe: the command must stay in this shell so functions keep their side effects.
-    "$@" > >(tee -a "$logfile") 2>&1
+    "${run[@]}" > >(tee -a "$logfile") 2>&1
     rc=$?
     wait $! 2>/dev/null
   else
-    "$@" >>"$logfile" 2>&1
+    "${run[@]}" >>"$logfile" 2>&1
     rc=$?
   fi
 
@@ -1238,6 +1266,7 @@ fi
 # - Backgrounds spinner process and stores PID for later cleanup
 # ------------------------------------------------------------------------------
 msg_info() {
+  _cs_reattach_output
   local msg="$1"
   [[ -z "$msg" ]] && return
 
@@ -1296,6 +1325,7 @@ msg_info() {
 # - Uses green color for success indication
 # ------------------------------------------------------------------------------
 msg_ok() {
+  _cs_reattach_output
   local msg="$1"
   [[ -z "$msg" ]] && return
   stop_spinner
@@ -1318,6 +1348,7 @@ msg_ok() {
 # - Outputs to stderr
 # ------------------------------------------------------------------------------
 msg_error() {
+  _cs_reattach_output
   stop_spinner
   _MSG_BLOCKS=()
   local msg="$1"
@@ -1334,6 +1365,7 @@ msg_error() {
 # - Outputs to stderr
 # ------------------------------------------------------------------------------
 msg_warn() {
+  _cs_reattach_output
   stop_spinner
   _MSG_BLOCKS=()
   local msg="$1"
@@ -1350,6 +1382,7 @@ msg_warn() {
 # - Useful for specialized status messages
 # ------------------------------------------------------------------------------
 msg_custom() {
+  _cs_reattach_output
   local symbol="${1:-"[*]"}"
   local color="${2:-"\e[36m"}"
   local msg="${3:-}"

core/error_handler.func

@@ -776,6 +776,7 @@ _stop_container_if_installing() {
 # ------------------------------------------------------------------------------
 on_exit() {
   local exit_code=$?
+  [[ "${_CS_HANGUP:-}" == "1" ]] && echo "[INFO] $(date '+%Y-%m-%d %H:%M:%S') on_exit with code ${exit_code} (pid $$)"
 
   # Before anything else: the timings are only useful if they survive a failure.
   declare -f dev_mode_timing_summary >/dev/null 2>&1 && dev_mode_timing_summary
@@ -796,6 +797,9 @@ on_exit() {
   if _is_container_context; then
     if [[ $exit_code -ne 0 ]]; then
       _container_write_failure "$exit_code" "${FAILED_COMMAND:-}" "${FAILED_LINE:-}"
+    elif [[ "${INSTALL_LOG:-}" == *.install-*.log ]]; then
+      # The host reads this when its lxc-attach did not live to see the end.
+      : >"${INSTALL_LOG%.log}.done" 2>/dev/null || true
     fi
   else
     if [[ "${POST_UPDATE_DONE:-}" != "true" ]] && declare -f post_update_to_api >/dev/null 2>&1; then
@@ -874,11 +878,18 @@ on_terminate() {
 # ------------------------------------------------------------------------------
 # on_hangup() - SIGHUP (SSH disconnect, terminal closed)
 #
-# No cursor restore and no message: the terminal that would read them is gone.
+# Not an abort: the run carries on with its output in the active log, and
+# whoever opens a console later finds the result. Writes to the terminal that
+# is gone would fail with EIO and end the script, hence the redirect.
 # ------------------------------------------------------------------------------
 on_hangup() {
   declare -f stop_spinner >/dev/null 2>&1 && { stop_spinner 2>/dev/null || true; }
-  _eh_signal_exit 129
+  local log=""
+  declare -f get_active_logfile >/dev/null 2>&1 && log="$(get_active_logfile 2>/dev/null)"
+  [[ -n "$log" ]] || log="${INSTALL_LOG:-${BUILD_LOG:-/dev/null}}"
+  exec </dev/null >>"$log" 2>&1
+  export _CS_HANGUP=1 CS_ANIMATION_OK=0 _CS_HANGUP_LOG="$log"
+  echo "[WARN] $(date '+%Y-%m-%d %H:%M:%S') The terminal hung up (SIGHUP); continuing, output follows here. (pid $$, subshell ${BASH_SUBSHELL}, during: ${BASH_COMMAND:-?})"
 }
 
 # ==============================================================================

docs/backends.md

@@ -16,6 +16,13 @@ wizard calls.
 | [`vm-core.func`](../pve/vm-core.func) | VM creation via `qm`, including disk import and template handling |
 | [`vm-app.func`](../pve/vm-app.func) | Deploys an application that normally goes into an LXC container into a full VM instead |
 
+A closed console is not an abort. SIGHUP reaches the host script and the
+install inside the container; both carry on with their output in their logs
+(`on_hangup`), external commands under `$STD` ignore the signal, and when
+lxc-attach dies with the terminal the host waits for the container's
+`.done`/`.failed` markers (`_pve_wait_for_detached_install`) before it
+reports. Ctrl-C still aborts.
+
 ## `incus/` — Incus
 
 | File | Purpose |
@@ -71,6 +78,15 @@ from `/etc/localtime`, or from `Etc/UTC`.
 [`core/core.func`](../core/core.func) renders "Incus Version 6.x" on an Incus
 host and "PVE Version …" on Proxmox VE, from the same `PVEVERSION` value.
 
+A closed console is not an abort here either, but it takes a different route.
+`incus exec` ends as soon as its client goes away and takes the command with
+it, which `lxc-attach` does not do. The install therefore runs detached inside
+the container, writing to its own log, and the host only watches it: input
+reaches it through a fifo for as long as there is a console, and once there is
+none the reads return EOF and the prompts take their defaults.
+`_incus_wait_for_detached_install` then waits for the `.done`/`.failed`
+markers before the host reports.
+
 ### Driving a remote server
 
 The `incus` client always talks to a remote; `local:` is only the default.

docs/core.md

@@ -33,7 +33,8 @@ Worth knowing:
 
 - `silent()` runs a command with its output captured, and records the failing
   command, line and log path so the error handler can report the real failure
-  rather than the wrapper.
+  rather than the wrapper. External commands run with SIGHUP ignored, so a
+  closed console does not kill the apt or npm of the moment.
 - Several host facts are memoised in files under a runtime cache directory
   rather than in shell variables, because every call site is a `$( )` or a
   pipe — that is, a subshell, where a variable assignment would be discarded.

incus/backend.func

@@ -1061,6 +1061,30 @@ MOTDEOF
   msg_ok "MOTD configured"
 }
 
+# ------------------------------------------------------------------------------
+# _incus_wait_for_detached_install <base>
+#
+# The console closed while the install ran. incus ended the exec session that
+# was watching it, but the install itself is detached and carries on. Wait for
+# what it leaves behind: .failed (its exit code, read by the caller), .done, or
+# the process gone without either.
+# ------------------------------------------------------------------------------
+_incus_wait_for_detached_install() {
+  local base="$1" pid="" waited=0
+  local poll="${_CS_HANGUP_POLL:-10}" limit="${_CS_HANGUP_TIMEOUT:-21600}"
+  while :; do
+    incus_ct_exec test -f "${base}.failed" >/dev/null 2>&1 && return 0
+    incus_ct_exec test -f "${base}.done" >/dev/null 2>&1 && return 0
+    pid="$(incus_ct_exec cat "${base}.pid" 2>/dev/null | tr -d '[:space:]')"
+    if [[ "$pid" =~ ^[0-9]+$ ]] && ! incus_ct_exec kill -0 "$pid" >/dev/null 2>&1; then
+      return 0
+    fi
+    ((waited >= limit)) && return 0
+    sleep "$poll"
+    waited=$((waited + poll))
+  done
+}
+
 _incus_push_functions_and_run_install() {
   # Sets INCUS_LAST_INSTALL_EXIT (do not capture stdout — msg_* writes there).
   INCUS_LAST_INSTALL_EXIT=0
@@ -1095,15 +1119,53 @@ INNEREOF" >>"${LOGFILE:-$INCUS_BUILD_LOG}" 2>&1 || { msg_error "Failed to push f
   # No pty on purpose: under `incus exec -t` apt's tcsetattr fails with EIO and kills the shell.
   set +Eeuo pipefail
   trap - ERR
+
+  # incus ends an exec session the moment its client goes away, so closing the
+  # console killed the install outright - the one thing lxc-attach does not do.
+  # Run it detached from that session and only watch it from here. The console
+  # keeps the view and the keyboard: input reaches the install through a fifo
+  # for as long as there is a console, and once there is none the reads return
+  # EOF and the prompts take their defaults.
+  local _base="/root/.install-${SESSION_ID}"
+  local _runner _watcher _body
+  _runner='setsid bash "$1".sh <"$1".in >>"$1".out 2>&1 &
+echo $! >"$1".pid'
+  _watcher='tail -n +1 -f "$1".out &
+_tp=$!
+while [ ! -f "$1".done ] && [ ! -f "$1".failed ]; do sleep 2; done
+sleep 1
+kill $_tp 2>/dev/null'
   # The trailing touch is a completion sentinel: no install script exits early.
-  incus exec "${CT_NAME}" -- bash -c "${_run_env}
+  _body="${_run_env}
 ${_install_script}
-touch /root/.install-${SESSION_ID}.done" 2>&1 |
+touch ${_base}.done"
+
+  incus exec "${CT_NAME}" -- bash -c "cat >${_base}.sh <<'CSBODYEOF'
+${_body}
+CSBODYEOF
+cat >${_base}.run <<'CSRUNEOF'
+${_runner}
+CSRUNEOF
+cat >${_base}.watch <<'CSWATCHEOF'
+${_watcher}
+CSWATCHEOF
+rm -f ${_base}.in ${_base}.pid ${_base}.done ${_base}.failed ${_base}.out
+mkfifo ${_base}.in" >>"${LOGFILE:-$INCUS_BUILD_LOG}" 2>&1
+
+  # A writer has to hold the fifo open before the install opens it to read.
+  incus exec "${CT_NAME}" -- sh -c "cat >${_base}.in" &
+  local _feeder=$!
+  incus exec "${CT_NAME}" -- bash "${_base}.run" "${_base}" >/dev/null 2>&1
+
+  incus exec "${CT_NAME}" -- bash "${_base}.watch" "${_base}" 2>&1 |
     tee >(sed 's/.*\r//' >>"${LOGFILE:-$INCUS_BUILD_LOG}")
-  # PIPESTATUS, not $?: the line above turns pipefail off, so $? is tee's
-  # status - always 0. Every non-zero exit from the container was discarded
-  # here, which is why a half-finished install reported success.
-  lxc_exit=${PIPESTATUS[0]}
+  kill "$_feeder" 2>/dev/null || true
+  wait "$_feeder" 2>/dev/null || true
+
+  # The watcher is gone either way - because the install ended, or because the
+  # console did. Only the markers say which.
+  _incus_wait_for_detached_install "$_base"
+  lxc_exit=0
   set -Eeuo pipefail
   declare -f error_handler &>/dev/null && trap 'error_handler' ERR || true
 
@@ -1125,10 +1187,12 @@ touch /root/.install-${SESSION_ID}.done" 2>&1 |
       incus_ct_exec rm -f "/root/.install-${SESSION_ID}.done" 2>/dev/null || true
     else
       msg_error "Install script stopped before it finished - nothing reported it"
+      [[ "${_CS_HANGUP:-}" == "1" ]] && msg_custom "ℹ️" "${YW}" "The console was closed; the install ran on detached, so look at its log for the reason"
       msg_custom "ℹ️" "${YW}" "Output: incus exec ${CT_NAME} -- tail -50 /root/.install-${SESSION_ID}.log"
       install_exit_code=1
     fi
   fi
+  incus_ct_exec rm -f "${_base}.sh" "${_base}.run" "${_base}.watch" "${_base}.in" "${_base}.pid" "${_base}.out" >/dev/null 2>&1 || true
   INCUS_LAST_INSTALL_EXIT=$install_exit_code
 }
 

incus/build.func

@@ -633,8 +633,20 @@ _incus_on_signal() {
   exit "$code"
 }
 
+# A closed console is not an abort. The install runs detached inside the
+# container, so losing the terminal costs the view and the keyboard, not the
+# installation. Writes to the terminal that is gone fail with EIO and would
+# end the run under errexit, hence the redirect.
+_incus_on_hangup() {
+  local log="${LOGFILE:-${_INCUS_HOST_LOGFILE:-${INCUS_BUILD_LOG:-}}}"
+  [[ -n "$log" ]] || log="/tmp/incus-hangup-${SESSION_ID:-$$}.log"
+  exec </dev/null >>"$log" 2>&1
+  export _CS_HANGUP=1 _CS_HANGUP_LOG="$log" CS_STDIN_OK=0
+  echo "[WARN] $(date '+%Y-%m-%d %H:%M:%S') The terminal hung up (SIGHUP); continuing, output follows here."
+}
+
 trap 'api_exit_script' EXIT
 trap '_incus_on_err' ERR
-trap '_incus_on_signal SIGHUP 129' SIGHUP
+trap '_incus_on_hangup' SIGHUP
 trap '_incus_on_signal SIGINT 130' SIGINT
 trap '_incus_on_signal SIGTERM 143' SIGTERM

incus/core.func

@@ -469,6 +469,7 @@ incus_stop_spinner() {
 # - Displays informational message with spinner animation
 # ------------------------------------------------------------------------------
 incus_msg_info() {
+  declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output
   local msg="$1"
   [[ -z "$msg" ]] && return
 
@@ -507,6 +508,7 @@ incus_msg_info() {
 # incus_msg_ok()
 # ------------------------------------------------------------------------------
 incus_msg_ok() {
+  declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output
   local msg="$1"
   [[ -z "$msg" ]] && return
   incus_stop_spinner
@@ -523,6 +525,7 @@ incus_msg_ok() {
 # incus_msg_error()
 # ------------------------------------------------------------------------------
 incus_msg_error() {
+  declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output
   incus_stop_spinner
   local msg="$1"
   echo -e "${BFR:-}${CROSS:-✖️}${RD}${msg}${CL}" >&2
@@ -533,6 +536,7 @@ incus_msg_error() {
 # incus_msg_warn()
 # ------------------------------------------------------------------------------
 incus_msg_warn() {
+  declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output
   incus_stop_spinner
   local msg="$1"
   # Same layout as core.func msg_warn (INFO already includes TAB padding).
@@ -544,6 +548,7 @@ incus_msg_warn() {
 # incus_msg_custom()
 # ------------------------------------------------------------------------------
 incus_msg_custom() {
+  declare -f _cs_reattach_output >/dev/null 2>&1 && _cs_reattach_output
   local symbol="${1:-"[*]"}"
   local color="${2:-"\e[36m"}"
   local msg="${3:-}"