diff --git a/scripts/install-desktop-system-deps-ci.sh b/scripts/install-desktop-system-deps-ci.sh index e550e39cc5..10a50d1e29 100755 --- a/scripts/install-desktop-system-deps-ci.sh +++ b/scripts/install-desktop-system-deps-ci.sh @@ -3,9 +3,11 @@ # each receives 85 seconds to exit after TERM and is killed at 90 seconds. set -euo pipefail -readonly apt_config=/etc/apt/apt.conf.d/99buzz-ci-bounds +readonly apt_config=${BUZZ_APT_CONFIG:-/etc/apt/apt.conf.d/99buzz-ci-bounds} readonly max_attempts=2 -readonly retry_delay_seconds=5 +readonly attempt_timeout_seconds=${BUZZ_DEPENDENCY_TIMEOUT_SECONDS:-85} +readonly kill_grace_seconds=${BUZZ_DEPENDENCY_KILL_GRACE_SECONDS:-5} +readonly retry_delay_seconds=${BUZZ_DEPENDENCY_RETRY_DELAY_SECONDS:-5} cleanup() { sudo rm -f "$apt_config" @@ -22,15 +24,30 @@ APT_CONFIG export DEBIAN_FRONTEND=noninteractive for attempt in $(seq 1 "$max_attempts"); do echo "System dependency install attempt ${attempt}/${max_attempts}" + started_at=$SECONDS set +e - timeout --signal=TERM --kill-after=5s 85s "$@" + timeout \ + --signal=TERM \ + --kill-after="${kill_grace_seconds}s" \ + "${attempt_timeout_seconds}s" \ + "$@" status=$? set -e + elapsed_seconds=$((SECONDS - started_at)) if [ "$status" -eq 0 ]; then exit 0 fi - if [ "$status" -ne 124 ]; then + timed_out=false + if [ "$status" -eq 124 ]; then + timed_out=true + elif [ "$status" -eq 137 ] && [ "$elapsed_seconds" -ge "$attempt_timeout_seconds" ]; then + # GNU timeout returns 137 when its post-TERM grace expires. Elapsed time + # distinguishes that case from a command that exits 137 immediately. + timed_out=true + fi + + if [ "$timed_out" != true ]; then echo "System dependency installation failed (status ${status}); not retrying a non-timeout failure" >&2 exit "$status" fi diff --git a/scripts/test-install-desktop-system-deps-ci.sh b/scripts/test-install-desktop-system-deps-ci.sh new file mode 100755 index 0000000000..7eeae0a9c5 --- /dev/null +++ b/scripts/test-install-desktop-system-deps-ci.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Linux regression coverage for the outer timeout/retry contract. +set -euo pipefail + +if [ "$(uname -s)" != Linux ]; then + echo "This harness requires Linux and GNU timeout" >&2 + exit 1 +fi + +tmp=$(mktemp -d) +cleanup() { + rm -rf "$tmp" +} +trap cleanup EXIT + +cat > "$tmp/sudo" <<'SUDO' +#!/usr/bin/env bash +exec "$@" +SUDO +cat > "$tmp/attempt" <<'ATTEMPT' +#!/usr/bin/env bash +mode=$1 +count_file=$2 +count=0 +[ ! -f "$count_file" ] || count=$(cat "$count_file") +echo $((count + 1)) > "$count_file" +case "$mode" in + success) exit 0 ;; + quick-137) exit 137 ;; + cooperative-timeout) sleep 10 ;; + forced-kill-timeout) + trap '' TERM + while :; do sleep 1; done + ;; + *) exit 99 ;; +esac +ATTEMPT +chmod +x "$tmp/sudo" "$tmp/attempt" + +run_case() { + local mode=$1 + local expected_status=$2 + local expected_attempts=$3 + local count_file="$tmp/${mode}.count" + + set +e + PATH="$tmp:$PATH" \ + BUZZ_APT_CONFIG="$tmp/${mode}.apt.conf" \ + BUZZ_DEPENDENCY_TIMEOUT_SECONDS=1 \ + BUZZ_DEPENDENCY_KILL_GRACE_SECONDS=1 \ + BUZZ_DEPENDENCY_RETRY_DELAY_SECONDS=0 \ + scripts/install-desktop-system-deps-ci.sh "$tmp/attempt" "$mode" "$count_file" + local status=$? + set -e + + if [ "$status" -ne "$expected_status" ]; then + echo "$mode: expected status $expected_status, got $status" >&2 + exit 1 + fi + if [ "$(cat "$count_file")" -ne "$expected_attempts" ]; then + echo "$mode: expected $expected_attempts attempt(s), got $(cat "$count_file")" >&2 + exit 1 + fi + if [ -e "$tmp/${mode}.apt.conf" ]; then + echo "$mode: apt config was not cleaned up" >&2 + exit 1 + fi + echo "$mode: PASS (status $status, attempts $expected_attempts)" +} + +run_case success 0 1 +run_case quick-137 137 1 +run_case cooperative-timeout 124 2 +run_case forced-kill-timeout 137 2