diff --git a/bin/panama b/bin/panama index 6b0f18e..6db8338 100755 --- a/bin/panama +++ b/bin/panama @@ -89,6 +89,8 @@ ${BOLD}Commands:${RESET} pattern to run a subset. --safe selects hermetic contracts only. Plain terminal runs prompt before non-hermetic work. Automation must grant each required capability with a repeatable --allow. + Each non-hermetic contract announces its exact capabilities + before it starts. Failures print captured stdout/stderr. Successful stdout stays quiet; successful stderr is a warning. The default outer timeout is 180 seconds. Set PANAMA_TEST_TIMEOUT_SECONDS to a positive @@ -447,12 +449,20 @@ PROMPT CONTRACT_MANIFEST="tests/contracts.manifest" CONTRACT_CAPABILITIES=(hermetic live-host live-compositor live-desktop network privileged) +contract_paths() { + local candidate + while IFS= read -r candidate; do + [[ -x "$candidate" || "$candidate" == *_test.py ]] || continue + printf 'tests/%s\n' "${candidate#"$PANAMA_DIR/tests/"}" + done < <(find "$PANAMA_DIR/tests" -type f \ + -not -path '*/fixtures/*' -not -path '*__pycache__*' | sort) +} + contract_manifest_entries() { local line capabilities path while IFS= read -r line || [[ -n "$line" ]]; do - line="${line%%#*}" - read -r capabilities path _ <<<"$line" - [[ -n "${capabilities:-}" && -n "${path:-}" ]] || continue + [[ "$line" =~ ^[[:space:]]*(#|$) ]] && continue + IFS=$' \t' read -r capabilities path <<<"$line" printf '%s\t%s\n' "$path" "$capabilities" done < "$PANAMA_DIR/$CONTRACT_MANIFEST" } @@ -464,6 +474,98 @@ require_contract_manifest() { } } +validate_contract_manifest() { + require_contract_manifest || return 1 + + local manifest="$PANAMA_DIR/$CONTRACT_MANIFEST" + local line capabilities path extra previous_comment="" previous_was_comment=0 + local previous_path="" capability discovered + local -a capability_list=() findings=() + local -A expected_contracts=() manifest_paths=() + + while IFS= read -r discovered; do + expected_contracts["$discovered"]=1 + done < <(contract_paths) + + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$line" =~ ^[[:space:]]*# ]]; then + previous_comment="${line#*#}" + previous_comment="${previous_comment#"${previous_comment%%[![:space:]]*}"}" + previous_comment="${previous_comment%"${previous_comment##*[![:space:]]}"}" + previous_was_comment=1 + continue + fi + + if [[ "$line" =~ ^[[:space:]]*$ ]]; then + previous_comment="" + previous_was_comment=0 + continue + fi + + IFS=$' \t' read -r capabilities path extra <<<"$line" + if [[ -z "${capabilities:-}" || -z "${path:-}" || -n "${extra:-}" ]]; then + findings+=("manifest line is not exactly two fields: $line") + previous_comment="" + previous_was_comment=0 + continue + fi + + if [[ -n "$previous_path" && "$path" < "$previous_path" ]]; then + findings+=('paths are not lexicographically sorted') + fi + previous_path="$path" + + if [[ -n "${manifest_paths[$path]:-}" ]]; then + findings+=("duplicate path $path") + fi + manifest_paths["$path"]=1 + + local -A line_capabilities=() + if [[ "$capabilities" == ,* || "$capabilities" == *, || "$capabilities" == *,,* ]]; then + findings+=("empty capability on $path") + fi + IFS=',' read -r -a capability_list <<<"$capabilities" + for capability in "${capability_list[@]}"; do + [[ -n "$capability" ]] || continue + if [[ -n "${line_capabilities[$capability]:-}" ]]; then + findings+=("duplicate capability $capability on $path") + fi + line_capabilities["$capability"]=1 + is_contract_capability "$capability" \ + || findings+=("unknown capability $capability on $path") + done + + if [[ -n "${line_capabilities[hermetic]:-}" && ${#line_capabilities[@]} -ne 1 ]]; then + findings+=("hermetic must appear alone on $path") + fi + + if [[ "$capabilities" != hermetic ]]; then + if (( previous_was_comment != 1 )); then + findings+=("$path is non-hermetic but lacks a directly preceding comment") + elif [[ -z "$previous_comment" ]]; then + findings+=("$path is non-hermetic but lacks a non-empty directly preceding comment") + fi + fi + previous_comment="" + previous_was_comment=0 + done < "$manifest" + + for discovered in "${!expected_contracts[@]}"; do + [[ -n "${manifest_paths[$discovered]:-}" ]] \ + || findings+=("missing contract $discovered") + done + for path in "${!manifest_paths[@]}"; do + [[ -n "${expected_contracts[$path]:-}" ]] \ + || findings+=("stale manifest path $path") + done + + if (( ${#findings[@]} > 0 )); then + err "Contract manifest validation failed with ${#findings[@]} finding(s):" + printf ' - %s\n' "${findings[@]}" >&2 + return 1 + fi +} + test_usage() { err "Usage: ${BOLD}$PROGRAM test [--safe] [--allow ] [pattern]${RESET}" return 2 @@ -492,10 +594,59 @@ is_contract_capability() { # --safe runs only contracts the manifest classifies as hermetic and reports # each external capability it skipped. A plain terminal run asks before any # selected non-hermetic work. Automation must grant every required capability -# with repeatable --allow flags. Each contract gets an outer timeout, 180 +# with repeatable --allow flags. Non-hermetic contracts announce their exact +# capability list before execution. Each contract gets an outer timeout, 180 # seconds by default. PANAMA_TEST_TIMEOUT_SECONDS accepts a positive integer -# override. Failures include captured stdout and stderr. Successful stdout stays -# quiet, while successful stderr is surfaced as a warning. +# override. Failures include captured stdout and stderr. Successful stdout +# stays quiet, while successful stderr is surfaced as a warning. +PANAMA_ACTIVE_CONTRACT_PID="" +PANAMA_CONTRACT_CAPTURE_DIR="" + +cleanup_contract_capture() { + if [[ -n "$PANAMA_CONTRACT_CAPTURE_DIR" && -d "$PANAMA_CONTRACT_CAPTURE_DIR" ]]; then + rm -rf -- "$PANAMA_CONTRACT_CAPTURE_DIR" || true + fi + PANAMA_CONTRACT_CAPTURE_DIR="" +} + +terminate_active_contract() { + local pid="$PANAMA_ACTIVE_CONTRACT_PID" + PANAMA_ACTIVE_CONTRACT_PID="" + [[ "$pid" =~ ^[1-9][0-9]*$ && "$pid" != "$$" ]] || return 0 + + # GNU timeout owns a process group whose ID is its PID. Signal that complete + # group so a contract cannot leave descendants behind, with a direct-PID + # fallback for implementations that do not create the group. + kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true +} + +handle_contract_signal() { + local signal_status="$1" + trap - INT TERM + terminate_active_contract + cleanup_contract_capture + trap - EXIT + exit "$signal_status" +} + +prepare_contract_capture() { + local capture_dir="" + if ! capture_dir="$(mktemp -d)"; then + err 'Could not create contract capture directory.' + return 1 + fi + if [[ -z "$capture_dir" || ! -d "$capture_dir" ]]; then + err 'Could not create contract capture directory.' + return 1 + fi + + PANAMA_CONTRACT_CAPTURE_DIR="$capture_dir" + trap cleanup_contract_capture EXIT + trap 'handle_contract_signal 130' INT + trap 'handle_contract_signal 143' TERM +} + cmd_test() { local timeout_seconds="${PANAMA_TEST_TIMEOUT_SECONDS:-180}" [[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]] || { @@ -503,11 +654,11 @@ cmd_test() { return 2 } - require_contract_manifest || return 1 + validate_contract_manifest || return 1 cmd_test_impl "$timeout_seconds" "$@" } -cmd_test_impl() ( +cmd_test_impl() { local timeout_seconds="$1" shift local pattern="" safe=0 arg capability capabilities rel path @@ -605,12 +756,10 @@ cmd_test_impl() ( fi fi - local capture_dir stdout_file stderr_file name run_status index=0 + local capture_dir stdout_file stderr_file name run_status index=0 final_status=0 local -a failed=() runner=() - capture_dir="$(mktemp -d)" - trap 'rm -rf -- "$capture_dir"' EXIT - trap 'rm -rf -- "$capture_dir"; exit 130' INT - trap 'rm -rf -- "$capture_dir"; exit 143' TERM + prepare_contract_capture || return 1 + capture_dir="$PANAMA_CONTRACT_CAPTURE_DIR" info "Running ${#suite[@]} contract(s)" for index in "${!suite[@]}"; do @@ -624,9 +773,16 @@ cmd_test_impl() ( else runner=("$path") fi + capabilities="${manifest_capabilities[$rel]}" + if [[ "$capabilities" != hermetic ]]; then + info "Running $name [$capabilities]" + fi run_status=0 timeout --signal=TERM --kill-after=5 "$timeout_seconds" \ - "${runner[@]}" >"$stdout_file" 2>"$stderr_file" || run_status=$? + "${runner[@]}" >"$stdout_file" 2>"$stderr_file" & + PANAMA_ACTIVE_CONTRACT_PID=$! + wait "$PANAMA_ACTIVE_CONTRACT_PID" || run_status=$? + PANAMA_ACTIVE_CONTRACT_PID="" if (( run_status == 0 )); then ok "$name" if [[ -s "$stderr_file" ]]; then @@ -655,12 +811,16 @@ cmd_test_impl() ( header "Result" if (( ${#failed[@]} == 0 )); then ok "${#suite[@]} contract(s) passed" - return 0 + else + err "${#failed[@]} of ${#suite[@]} failed:" + printf ' %s\n' "${failed[@]}" >&2 + final_status=1 fi - err "${#failed[@]} of ${#suite[@]} failed:" - printf ' %s\n' "${failed[@]}" >&2 - return 1 -) + + cleanup_contract_capture + trap - EXIT INT TERM + return "$final_status" +} # ---------------------------------------------------------------------------- # Command: contracts @@ -721,7 +881,7 @@ cmd_contracts() { suffix="${suffix#*/}" done - require_contract_manifest || return 1 + validate_contract_manifest || return 1 local -A manifest_capabilities=() local capabilities while IFS=$'\t' read -r rel capabilities; do @@ -732,11 +892,11 @@ cmd_contracts() { # runner would actually execute. local -a hits=() local candidate rel - while IFS= read -r candidate; do - [[ -x "$candidate" || "$candidate" == *_test.py ]] || continue + while IFS= read -r rel; do + candidate="$PANAMA_DIR/$rel" grep -qF "${patterns[@]}" "$candidate" 2>/dev/null || continue - hits+=("tests/${candidate#"$PANAMA_DIR"/tests/}") - done < <(find "$PANAMA_DIR/tests" -type f -not -path '*/fixtures/*' -not -path '*__pycache__*' | sort) + hits+=("$rel") + done < <(contract_paths) if (( ${#hits[@]} == 0 )); then printf 'No contract mentions %s — coverage may be indirect (a harness or a generated artifact); nothing verified.\n' "$path" >&2 diff --git a/docs/superpowers/plans/2026-08-26-verification-gate-remediation.md b/docs/superpowers/plans/2026-08-26-verification-gate-remediation.md index 2835c0c..70fab75 100644 --- a/docs/superpowers/plans/2026-08-26-verification-gate-remediation.md +++ b/docs/superpowers/plans/2026-08-26-verification-gate-remediation.md @@ -242,7 +242,7 @@ contract_manifest_entries() { } ``` -Do not make a missing manifest mean “everything is safe.” `cmd_test` and `cmd_contracts` must fail clearly if the file cannot be read. The manifest contract owns deeper format validation; the CLI owns the runtime read failure. +Do not make a missing or malformed manifest mean “everything is safe.” Before either `cmd_test` or `cmd_contracts` consumes manifest entries, the public CLI independently validates the complete actual manifest: exactly two fields, known/non-empty/non-duplicate capabilities, exclusive `hermetic`, unique sorted paths, the executable/`*_test.py` fixture-excluded discovery set, and a non-empty directly preceding comment for every non-hermetic entry. Missing discovered contracts and stale manifest paths are fatal before selection or execution. Keep `tests/setup/contract-manifest-contract` as an independent validator rather than sourcing runtime code; a later contract cannot protect earlier execution. ### Step 3: Implement argument and capability policy @@ -271,7 +271,7 @@ local timeout_seconds="${PANAMA_TEST_TIMEOUT_SECONDS:-180}" } ``` -Run the implementation body in a subshell, create one capture directory with `mktemp -d`, and trap its removal on `EXIT`, `INT`, and `TERM` inside that subshell. This keeps cleanup reliable without leaking or overwriting traps in the parent CLI process. For each contract, run either `python3 path` or the executable through: +The public CLI process must own contract supervision. Check that `mktemp -d` succeeds before constructing any capture path, then install top-level `EXIT`, `INT`, and `TERM` cleanup around the checked directory. Launch the active `timeout` asynchronously and retain its PID/process-group ownership. On INT or TERM sent to the exact CLI PID, signal the active timeout/process group, wait for it, remove capture storage, and exit 130 or 143. On normal completion, remove capture storage and clear the temporary traps without changing the aggregate test status. For each contract, run either `python3 path` or the executable through: ```bash timeout --signal=TERM --kill-after=5 "$timeout_seconds" \ @@ -389,8 +389,10 @@ The leaked fixture must include: - a literal YAML credential `POSTGRES_PASSWORD: fixture-should-be-rejected`; - a literal env credential `API_TOKEN=fixture-should-be-rejected`. +- a plain-text PEM private-key header with an exact `path:line: private key` finding; +- plain-text synthetic `sk-ant-`, minimum-supported-length `ghp_`, and `xoxb-` signatures, each with an exact `path:line: provider token` finding. -Use intentionally invalid fixture strings, not realistic provider token formats. +Add clean plain-text near-misses for each signature. Keep semantic fixture credentials intentionally invalid and signature fixtures clearly synthetic while still matching the supported signature shapes. ### Step 2: Write the scanner with only the Python standard library diff --git a/tests/server/compose-secrets-contract b/tests/server/compose-secrets-contract index 6cdc36d..e0095c6 100755 --- a/tests/server/compose-secrets-contract +++ b/tests/server/compose-secrets-contract @@ -29,7 +29,8 @@ note() { findings+=("$1"); } scanner="$repo_dir/tests/server/scan-tracked-secrets.py" fixtures_dir="$repo_dir/tests/server/fixtures/secrets" -if ! python3 "$scanner" "$fixtures_dir/clean" compose.yml .env.example README.md; then +if ! python3 "$scanner" "$fixtures_dir/clean" \ + compose.yml .env.example README.md signature-near-misses.txt; then note 'the clean secret-scanning fixture was rejected' fi @@ -53,6 +54,10 @@ expect_leak .env.example '.env.example:1: API_TOKEN' expect_leak plain-list.yml 'plain-list.yml:4: API_TOKEN' expect_leak quoted-mapping.yml 'quoted-mapping.yml:4: API_TOKEN' expect_leak quoted-list.yml 'quoted-list.yml:4: API_TOKEN' +expect_leak pem-private-key.txt 'pem-private-key.txt:1: private key' +expect_leak anthropic-token.txt 'anthropic-token.txt:1: provider token' +expect_leak github-token.txt 'github-token.txt:1: provider token' +expect_leak slack-token.txt 'slack-token.txt:1: provider token' mapfile -t tracked_server_files < <(git -C "$repo_dir" ls-files 'server/**' 'server/*') if ! output="$(python3 "$scanner" "$repo_dir" "${tracked_server_files[@]}" 2>&1)"; then diff --git a/tests/server/fixtures/secrets/clean/signature-near-misses.txt b/tests/server/fixtures/secrets/clean/signature-near-misses.txt new file mode 100644 index 0000000..e7dd200 --- /dev/null +++ b/tests/server/fixtures/secrets/clean/signature-near-misses.txt @@ -0,0 +1,4 @@ +-----BEGIN SYNTHETIC PUBLIC KEY----- +sk-ant- +ghp_0123456789ABCDEFGHI +xoxb- diff --git a/tests/server/fixtures/secrets/leaked/anthropic-token.txt b/tests/server/fixtures/secrets/leaked/anthropic-token.txt new file mode 100644 index 0000000..4c4f134 --- /dev/null +++ b/tests/server/fixtures/secrets/leaked/anthropic-token.txt @@ -0,0 +1 @@ +sk-ant-SYNTHETIC_FIXTURE_TOKEN diff --git a/tests/server/fixtures/secrets/leaked/github-token.txt b/tests/server/fixtures/secrets/leaked/github-token.txt new file mode 100644 index 0000000..ac0c0af --- /dev/null +++ b/tests/server/fixtures/secrets/leaked/github-token.txt @@ -0,0 +1 @@ +ghp_0123456789ABCDEFGHIJ diff --git a/tests/server/fixtures/secrets/leaked/pem-private-key.txt b/tests/server/fixtures/secrets/leaked/pem-private-key.txt new file mode 100644 index 0000000..56d4831 --- /dev/null +++ b/tests/server/fixtures/secrets/leaked/pem-private-key.txt @@ -0,0 +1 @@ +-----BEGIN SYNTHETIC PRIVATE KEY----- diff --git a/tests/server/fixtures/secrets/leaked/slack-token.txt b/tests/server/fixtures/secrets/leaked/slack-token.txt new file mode 100644 index 0000000..5adc092 --- /dev/null +++ b/tests/server/fixtures/secrets/leaked/slack-token.txt @@ -0,0 +1 @@ +xoxb-SYNTHETIC_FIXTURE_TOKEN diff --git a/tests/setup/test-runner-contract b/tests/setup/test-runner-contract index b2de9e6..ef0f5c9 100755 --- a/tests/setup/test-runner-contract +++ b/tests/setup/test-runner-contract @@ -9,9 +9,23 @@ repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" fixture="$(mktemp -d)" output="" status=0 +background_cli_pid="" +background_contract_pgid="" -cleanup() { rm -rf -- "$fixture"; } -trap cleanup EXIT INT TERM +cleanup() { + trap - EXIT INT TERM + if [[ "$background_cli_pid" =~ ^[1-9][0-9]*$ ]]; then + kill -TERM "$background_cli_pid" 2>/dev/null || true + wait "$background_cli_pid" 2>/dev/null || true + fi + if [[ "$background_contract_pgid" =~ ^[1-9][0-9]*$ ]]; then + kill -KILL -- "-$background_contract_pgid" 2>/dev/null || true + fi + rm -rf -- "$fixture" +} +trap cleanup EXIT +trap 'cleanup; exit 130' INT +trap 'cleanup; exit 143' TERM fail() { printf 'test runner: %s\n' "$*" >&2; exit 1; } @@ -25,6 +39,12 @@ assert_not_contains() { [[ "$haystack" != *"$needle"* ]] || fail "expected output not to contain: $needle\n$haystack" } +assert_before() { + local first="$1" second="$2" haystack="$3" + [[ "$haystack" == *"$first"*"$second"* ]] \ + || fail "expected '$first' before '$second':\n$haystack" +} + assert_execution() { local expected="$1" actual actual="$(sort "$fixture/executions" 2>/dev/null || true)" @@ -33,6 +53,33 @@ assert_execution() { reset_executions() { : > "$fixture/executions"; } +wait_for_file() { + local path="$1" attempt + for (( attempt = 0; attempt < 100; attempt++ )); do + [[ -s "$path" ]] && return 0 + sleep 0.05 + done + return 1 +} + +wait_for_process_exit() { + local pid="$1" attempt + for (( attempt = 0; attempt < 100; attempt++ )); do + kill -0 "$pid" 2>/dev/null || return 0 + sleep 0.05 + done + return 1 +} + +wait_for_path_removal() { + local path="$1" attempt + for (( attempt = 0; attempt < 100; attempt++ )); do + [[ ! -e "$path" ]] && return 0 + sleep 0.05 + done + return 1 +} + run_panama() { output="$(cd "$fixture" && TMPDIR="$fixture" PANAMA_TEST_FIXTURE="$fixture" "$fixture/bin/panama" "$@" &1)" status=$? @@ -45,18 +92,39 @@ run_panama_with_timeout() { run_panama_tty_default_no() { local command tty_stdout="$fixture/tty.stdout" + local pty_state="$fixture/pty-state" + mkdir -p "$pty_state"/{config,state,cache,data,runtime} + chmod 700 "$pty_state/runtime" printf -v command 'cd %q && TMPDIR=%q PANAMA_TEST_FIXTURE=%q %q test composite > %q' \ "$fixture" "$fixture" "$fixture" "$fixture/bin/panama" "$tty_stdout" - output="$(python3 - "$command" <<'PY' + output="$( + HOME="$fixture/pty-home" \ + BASH_ENV="$fixture/pty-bash-env" \ + PANAMA_PTY_STARTUP_SENTINEL="$fixture/pty-startup-sourced" \ + python3 - "$command" "$fixture/pty-home" \ + "$pty_state/config" "$pty_state/state" "$pty_state/cache" \ + "$pty_state/data" "$pty_state/runtime" "$fixture" <<'PY' import errno import os import pty import sys command = sys.argv[1] +environment = os.environ.copy() +environment.pop('BASH_ENV', None) +environment.pop('ENV', None) +environment.update({ + 'HOME': sys.argv[2], + 'XDG_CONFIG_HOME': sys.argv[3], + 'XDG_STATE_HOME': sys.argv[4], + 'XDG_CACHE_HOME': sys.argv[5], + 'XDG_DATA_HOME': sys.argv[6], + 'XDG_RUNTIME_DIR': sys.argv[7], + 'TMPDIR': sys.argv[8], +}) pid, terminal = pty.fork() if pid == 0: - os.execv('/bin/bash', ['bash', '-lc', command]) + os.execve('/bin/bash', ['bash', '--noprofile', '--norc', '-c', command], environment) chunks = [] replied = False @@ -78,7 +146,7 @@ _, child_status = os.waitpid(pid, 0) sys.stdout.buffer.write(b''.join(chunks)) sys.exit(os.waitstatus_to_exitcode(child_status)) PY -)" + )" status=$? } @@ -88,6 +156,48 @@ assert_occurrences() { [[ "$actual" == "$expected" ]] || fail "expected $expected occurrence(s) of '$needle', got $actual\n$haystack" } +replace_manifest_line() { + local original="$1" replacement="$2" line + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$line" == "$original" ]]; then + [[ "$replacement" == __REMOVE__ ]] || printf '%s\n' "$replacement" + else + printf '%s\n' "$line" + fi + done <<<"$valid_manifest" +} + +swap_manifest_contract_paths() { + local line + while IFS= read -r line || [[ -n "$line" ]]; do + case "$line" in + 'live-compositor,live-desktop tests/composite-contract') + printf '%s\n' 'live-compositor,live-desktop tests/desktop-contract' + ;; + 'live-desktop tests/desktop-contract') + printf '%s\n' 'live-desktop tests/composite-contract' + ;; + *) printf '%s\n' "$line" ;; + esac + done <<<"$valid_manifest" +} + +expect_manifest_rejection() { + local label="$1" expected="$2" contents="$3" + + printf '%s\n' "$contents" >"$fixture/tests/contracts.manifest" + reset_executions + run_panama test pass + [[ $status -ne 0 ]] || fail "$label manifest unexpectedly allowed test execution" + assert_contains "$expected" "$output" + assert_execution '' + + run_panama contracts config/subject + [[ $status -ne 0 ]] || fail "$label manifest unexpectedly allowed contracts lookup" + assert_contains "$expected" "$output" + assert_execution '' +} + mkdir -p "$fixture/bin" "$fixture/tests" "$fixture/config" cp "$repo_dir/bin/panama" "$fixture/bin/panama" chmod +x "$fixture/bin/panama" @@ -111,6 +221,8 @@ privileged tests/privileged-contract hermetic tests/stderr-contract EOF +valid_manifest="$(<"$fixture/tests/contracts.manifest")" + cat > "$fixture/tests/pass-contract" <<'EOF' #!/usr/bin/env bash printf 'pass\n' >> "$PANAMA_TEST_FIXTURE/executions" @@ -135,7 +247,13 @@ EOF cat > "$fixture/tests/hang-contract" <<'EOF' #!/usr/bin/env bash printf 'hang\n' >> "$PANAMA_TEST_FIXTURE/executions" -trap 'printf terminated >"$PANAMA_TEST_FIXTURE/terminated"; exit 124' TERM +printf '%s\n' "$BASHPID" > "$PANAMA_TEST_FIXTURE/hang.pid" +finish() { + printf 'terminated\n' >"$PANAMA_TEST_FIXTURE/terminated" + exit "$1" +} +trap 'finish 130' INT +trap 'finish 143' TERM while :; do sleep 1; done EOF @@ -147,6 +265,7 @@ EOF cat > "$fixture/tests/desktop-contract" <<'EOF' #!/usr/bin/env bash printf 'desktop\n' >> "$PANAMA_TEST_FIXTURE/executions" +printf 'desktop fixture complete\n' >&2 # config/subject EOF @@ -172,6 +291,15 @@ chmod +x "$fixture/tests"/{composite,desktop,fail,hang,host,network,pass,privile # A PTY-backed default-no confirmation remains visible when stdout is redirected # but stdin and stderr are terminals. The fixture proves that one prompt gates # the selected composite capability set without running its contract. +mkdir -p "$fixture/pty-home" +for profile in .bash_profile .bashrc .profile; do + cat >"$fixture/pty-home/$profile" <<'EOF' +printf 'profile\n' >>"${PANAMA_PTY_STARTUP_SENTINEL:?}" +EOF +done +cat >"$fixture/pty-bash-env" <<'EOF' +printf 'BASH_ENV\n' >>"${PANAMA_PTY_STARTUP_SENTINEL:?}" +EOF run_panama_tty_default_no [[ $status -ne 0 ]] || fail 'TTY default-no prompt unexpectedly ran the fixture' assert_execution '' @@ -179,6 +307,8 @@ assert_contains 'Run 1 contract(s) requiring: live-compositor live-desktop?' "$o assert_occurrences 'Run 1 contract(s) requiring:' "$output" 1 assert_contains 'No contracts were run.' "$(<"$fixture/tty.stdout")" assert_not_contains 'Run 1 contract(s) requiring:' "$(<"$fixture/tty.stdout")" +[[ ! -e "$fixture/pty-startup-sourced" ]] \ + || fail 'PTY fixture sourced a shell profile or BASH_ENV' # --safe must select hermetic entries from the manifest, not merely omit a # legacy desktop list. The failing and timed-out fixtures make the command @@ -204,6 +334,8 @@ assert_contains 'pass --allow live-desktop' "$output" run_panama test --allow live-desktop desktop [[ $status -eq 0 ]] || fail "explicit desktop grant failed: $output" assert_execution 'desktop' +assert_contains 'Running desktop-contract [live-desktop]' "$output" +assert_before 'Running desktop-contract [live-desktop]' 'desktop fixture complete' "$output" reset_executions run_panama test --allow live-compositor --allow live-desktop composite @@ -229,6 +361,45 @@ assert_execution 'hang' [[ -f "$fixture/terminated" ]] || fail 'timed-out contract was not terminated with TERM' assert_contains 'timed out' "$output" +# INT/TERM ownership belongs to the exact public CLI PID, not a runner +# subshell. The CLI must wait for the timeout process group and remove its +# capture directory before returning the signal-derived status. +reset_executions +rm -f -- "$fixture/hang.pid" "$fixture/terminated" +( + cd "$fixture" || exit 1 + exec env TMPDIR="$fixture" PANAMA_TEST_FIXTURE="$fixture" \ + "$fixture/bin/panama" test hang +) >"$fixture/exact-term.out" 2>&1 & +background_cli_pid=$! +wait_for_file "$fixture/hang.pid" \ + || fail 'exact-PID TERM fixture never started the hang contract' +hang_pid="$(<"$fixture/hang.pid")" +background_contract_pgid="$(ps -o pgid= -p "$hang_pid" | tr -d '[:space:]')" +[[ "$background_contract_pgid" =~ ^[1-9][0-9]*$ ]] \ + || fail "could not resolve hang process group for PID $hang_pid" +mapfile -t active_capture_dirs < <( + find "$fixture" -mindepth 1 -maxdepth 1 -type d -name 'tmp.*' -print +) +(( ${#active_capture_dirs[@]} == 1 )) \ + || fail "expected one active capture directory, got ${#active_capture_dirs[@]}" +active_capture_dir="${active_capture_dirs[0]}" + +kill -TERM "$background_cli_pid" \ + || fail 'could not send TERM to the exact public CLI PID' +term_status=0 +wait "$background_cli_pid" || term_status=$? +background_cli_pid="" +[[ "$term_status" -eq 143 ]] \ + || fail "exact-PID TERM returned $term_status instead of 143: $(<"$fixture/exact-term.out")" +wait_for_process_exit "$hang_pid" \ + || fail "hang contract PID $hang_pid survived exact-PID TERM" +background_contract_pgid="" +wait_for_path_removal "$active_capture_dir" \ + || fail "capture directory survived exact-PID TERM: $active_capture_dir" +[[ -f "$fixture/terminated" ]] \ + || fail 'exact-PID TERM did not reach the hang contract cleanup trap' + reset_executions run_panama test fail [[ $status -ne 0 ]] || fail 'failed contract unexpectedly passed' @@ -244,6 +415,19 @@ reset_executions run_panama test pass [[ $status -eq 0 ]] || fail "pass contract failed: $output" assert_not_contains 'pass stdout' "$output" +assert_not_contains 'pass-contract [hermetic]' "$output" + +reset_executions +printf 'not a directory\n' >"$fixture/invalid-tmpdir" +output="$( + cd "$fixture" && \ + TMPDIR="$fixture/invalid-tmpdir" PANAMA_TEST_FIXTURE="$fixture" \ + "$fixture/bin/panama" test pass &1 +)" +status=$? +[[ $status -ne 0 ]] || fail 'invalid TMPDIR unexpectedly allowed contract execution' +assert_contains 'Could not create contract capture directory.' "$output" +assert_execution '' reset_executions run_panama test --safe desktop @@ -258,10 +442,52 @@ assert_contains 'tests/desktop-contract [live-desktop]' "$output" assert_contains 'tests/composite-contract [live-compositor,live-desktop]' "$output" assert_contains 'tests/pass-contract [hermetic]' "$output" +# Both public manifest consumers fail closed on the complete format and +# discovery set. Validation happens before selection, lookup, or contract +# execution, so even a malformed entry unrelated to the requested pattern is +# fatal and leaves the execution log empty. +expect_manifest_rejection unknown-capability \ + 'unknown capability hermetik on tests/pass-contract' \ + "$(replace_manifest_line 'hermetic tests/pass-contract' 'hermetik tests/pass-contract')" +expect_manifest_rejection mixed-hermetic \ + 'hermetic must appear alone on tests/pass-contract' \ + "$(replace_manifest_line 'hermetic tests/pass-contract' 'hermetic,network tests/pass-contract')" +expect_manifest_rejection duplicate-path \ + 'duplicate path tests/pass-contract' \ + "$(replace_manifest_line 'hermetic tests/pass-contract' $'hermetic tests/pass-contract\nhermetic tests/pass-contract')" +expect_manifest_rejection stale-path \ + 'stale manifest path tests/stale-contract' \ + "$(replace_manifest_line 'hermetic tests/pass-contract' 'hermetic tests/stale-contract')" +expect_manifest_rejection missing-contract \ + 'missing contract tests/pass-contract' \ + "$(replace_manifest_line 'hermetic tests/pass-contract' __REMOVE__)" +expect_manifest_rejection extra-field \ + 'manifest line is not exactly two fields' \ + "$(replace_manifest_line 'hermetic tests/pass-contract' 'hermetic tests/pass-contract unexpected')" +expect_manifest_rejection empty-capability \ + 'empty capability on tests/pass-contract' \ + "$(replace_manifest_line 'hermetic tests/pass-contract' 'hermetic, tests/pass-contract')" +expect_manifest_rejection duplicate-capability \ + 'duplicate capability hermetic on tests/pass-contract' \ + "$(replace_manifest_line 'hermetic tests/pass-contract' 'hermetic,hermetic tests/pass-contract')" +expect_manifest_rejection unsorted-paths \ + 'paths are not lexicographically sorted' \ + "$(swap_manifest_contract_paths)" +expect_manifest_rejection uncommented-non-hermetic \ + 'tests/desktop-contract is non-hermetic but lacks a directly preceding comment' \ + "$(replace_manifest_line '# Maps the live desktop.' __REMOVE__)" +expect_manifest_rejection blank-comment \ + 'tests/desktop-contract is non-hermetic but lacks a non-empty directly preceding comment' \ + "$(replace_manifest_line '# Maps the live desktop.' '#')" + +printf '%s\n' "$valid_manifest" >"$fixture/tests/contracts.manifest" + mv "$fixture/tests/contracts.manifest" "$fixture/tests/contracts.manifest.missing" +reset_executions run_panama test pass [[ $status -ne 0 ]] || fail 'missing manifest unexpectedly allowed test execution' assert_contains 'contracts.manifest' "$output" +assert_execution '' mv "$fixture/tests/contracts.manifest.missing" "$fixture/tests/contracts.manifest" capture_dirs="$(find "$fixture" -mindepth 1 -maxdepth 1 -type d -name 'tmp.*' -print)" diff --git a/tests/setup/update-command-contract b/tests/setup/update-command-contract index 05d7eb5..baa8f77 100755 --- a/tests/setup/update-command-contract +++ b/tests/setup/update-command-contract @@ -234,16 +234,116 @@ grep -qx 'install-packages' <<<"$ran_full" \ # The fixture also covers a clean fast-forward, installer status propagation, # and the boundary between update and sync before forcing the conflict below. +# Make every ambient configuration source hostile before constructing the Git +# fixtures. A hermetic fixture overrides these values with its own empty state; +# consuming any of them either leaves a sentinel or prevents a commit. +hostile="$tmp/hostile-environment" +mkdir -p "$hostile/home" "$hostile/xdg-config" "$hostile/xdg-state" \ + "$hostile/xdg-cache" "$hostile/xdg-data" "$hostile/hooks" \ + "$hostile/template/hooks" +for profile in .bash_profile .bashrc .profile; do + cat >"$hostile/home/$profile" <<'EOF' +printf 'profile\n' >>"${PANAMA_HOSTILE_PROFILE_SENTINEL:?}" +EOF +done +cat >"$hostile/bash-env" <<'EOF' +printf 'BASH_ENV\n' >>"${PANAMA_HOSTILE_BASH_ENV_SENTINEL:?}" +EOF +cat >"$hostile/hooks/pre-commit" <<'EOF' +#!/usr/bin/env bash +printf 'global hook\n' >>"${PANAMA_HOSTILE_GIT_SENTINEL:?}" +exit 97 +EOF +cat >"$hostile/template/hooks/pre-commit" <<'EOF' +#!/usr/bin/env bash +# PANAMA_HOSTILE_TEMPLATE_HOOK +printf 'template hook\n' >>"${PANAMA_HOSTILE_TEMPLATE_SENTINEL:?}" +exit 98 +EOF +chmod +x "$hostile/hooks/pre-commit" "$hostile/template/hooks/pre-commit" +cat >"$hostile/global.gitconfig" </dev/null || return 1 - git -C "$root/upstream" config user.email contract@panama || return 1 - git -C "$root/upstream" config user.name contract || return 1 + prepare_cli_fixture_environment "$root" || return 1 + fixture_git "$root" init -q --bare "$root/origin.git" || return 1 + fixture_git "$root" -C "$root/origin.git" config core.hooksPath "$root/empty-hooks" || return 1 + fixture_git "$root" clone -q "$root/origin.git" "$root/upstream" 2>/dev/null || return 1 + configure_fixture_repo "$root" "$root/upstream" || return 1 mkdir -p "$root/upstream/bin" || return 1 cp "$panama" "$root/upstream/bin/panama" || return 1 @@ -254,21 +354,20 @@ exit "${PANAMA_UPDATE_INSTALL_RC:-0}" EOF chmod +x "$root/upstream/bin/panama" "$root/upstream/install" || return 1 printf 'one\n' >"$root/upstream/f" || return 1 - git -C "$root/upstream" add -A || return 1 - git -C "$root/upstream" commit -qm initial || return 1 - git -C "$root/upstream" push -qu origin HEAD || return 1 + fixture_git "$root" -C "$root/upstream" add -A || return 1 + fixture_git "$root" -C "$root/upstream" commit -qm initial || return 1 + fixture_git "$root" -C "$root/upstream" push -qu origin HEAD || return 1 - git clone -q "$root/origin.git" "$root/machine" || return 1 - git -C "$root/machine" config user.email contract@panama || return 1 - git -C "$root/machine" config user.name contract || return 1 + fixture_git "$root" clone -q "$root/origin.git" "$root/machine" || return 1 + configure_fixture_repo "$root" "$root/machine" || return 1 ) advance_upstream() ( local root="$1" file="$2" contents="$3" printf '%s\n' "$contents" >"$root/upstream/$file" || return 1 - git -C "$root/upstream" add "$file" || return 1 - git -C "$root/upstream" commit -qm "update $file" || return 1 - git -C "$root/upstream" push -q || return 1 + fixture_git "$root" -C "$root/upstream" add "$file" || return 1 + fixture_git "$root" -C "$root/upstream" commit -qm "update $file" || return 1 + fixture_git "$root" -C "$root/upstream" push -q || return 1 ) # This write fails before the later Git commands. The helper must return that @@ -285,26 +384,28 @@ fi clean="$tmp/clean-update" if build_cli_fixture "$clean" && advance_upstream "$clean" release new; then - machine_before="$(git -C "$clean/machine" rev-parse HEAD)" - upstream_after="$(git -C "$clean/upstream" rev-parse HEAD)" + machine_before="$(fixture_git "$clean" -C "$clean/machine" rev-parse HEAD)" + upstream_after="$(fixture_git "$clean" -C "$clean/upstream" rev-parse HEAD)" [[ "$machine_before" != "$upstream_after" ]] \ || note 'the clean update fixture started current, so it cannot prove a fast-forward' : >"$clean/install.log" update_status=0 - PANAMA_UPDATE_FIXTURE_LOG="$clean/install.log" \ + run_cli_fixture_environment "$clean" \ + PANAMA_UPDATE_FIXTURE_LOG="$clean/install.log" \ "$clean/machine/bin/panama" update >"$clean/update.out" 2>&1 \ || update_status=$? [[ "$update_status" -eq 0 ]] \ || note "panama update failed on a clean clone with status $update_status" - [[ "$(git -C "$clean/machine" rev-parse HEAD)" == "$upstream_after" ]] \ + [[ "$(fixture_git "$clean" -C "$clean/machine" rev-parse HEAD)" == "$upstream_after" ]] \ || note 'panama update did not fast-forward the clean machine clone' grep -qx -- '--upgrade' "$clean/install.log" \ || note 'panama update did not invoke the installer with --upgrade' : >"$clean/install.log" update_status=0 - PANAMA_UPDATE_FIXTURE_LOG="$clean/install.log" PANAMA_UPDATE_INSTALL_RC=23 \ + run_cli_fixture_environment "$clean" \ + PANAMA_UPDATE_FIXTURE_LOG="$clean/install.log" PANAMA_UPDATE_INSTALL_RC=23 \ "$clean/machine/bin/panama" update >"$clean/failing-update.out" 2>&1 \ || update_status=$? [[ "$update_status" -eq 23 ]] \ @@ -316,7 +417,8 @@ if build_cli_fixture "$clean" && advance_upstream "$clean" release new; then printf 'local sync\n' >"$clean/machine/synced" sync_status=0 printf 'y\ncontract sync\n' \ - | PANAMA_UPDATE_FIXTURE_LOG="$clean/install.log" \ + | run_cli_fixture_environment "$clean" \ + PANAMA_UPDATE_FIXTURE_LOG="$clean/install.log" \ "$clean/machine/bin/panama" sync >"$clean/sync.out" 2>&1 \ || sync_status=$? [[ "$sync_status" -eq 0 ]] \ @@ -335,19 +437,22 @@ if build_cli_fixture "$conflict"; then if advance_upstream "$conflict" f upstream; then : >"$conflict/install.log" conflict_status=0 - PANAMA_UPDATE_FIXTURE_LOG="$conflict/install.log" \ + run_cli_fixture_environment "$conflict" \ + PANAMA_UPDATE_FIXTURE_LOG="$conflict/install.log" \ "$conflict/machine/bin/panama" update >"$conflict/update.out" 2>&1 \ || conflict_status=$? [[ "$conflict_status" -eq 0 ]] \ || note "panama update failed while recovering a stash conflict with status $conflict_status" [[ "$(<"$conflict/machine/f")" == upstream ]] \ || note 'panama update did not reset the conflicted file to the upstream version' - if git -C "$conflict/machine" grep -qE '^(<<<<<<<|=======|>>>>>>>)' -- .; then + if fixture_git "$conflict" -C "$conflict/machine" \ + grep -qE '^(<<<<<<<|=======|>>>>>>>)' -- .; then note 'panama update left conflict markers in the machine checkout' fi - [[ -n "$(git -C "$conflict/machine" stash list)" ]] \ + [[ -n "$(fixture_git "$conflict" -C "$conflict/machine" stash list)" ]] \ || note 'panama update dropped the stash after its conflicted pop' - recovered="$(git -C "$conflict/machine" show 'stash@{0}:f' 2>/dev/null)" + recovered="$(fixture_git "$conflict" -C "$conflict/machine" \ + show 'stash@{0}:f' 2>/dev/null)" [[ "$recovered" == local ]] \ || note 'the stash left by panama update does not contain the local version' else @@ -357,6 +462,18 @@ else note 'the conflict update fixture could not be built' fi +for sentinel in profile-sourced bash-env-sourced global-config-sourced template-hook-sourced; do + [[ ! -e "$hostile/$sentinel" ]] \ + || note "the update Git fixture consumed hostile state: $sentinel" +done +template_copy="$( + find "$tmp" -path "$hostile" -prune -o \ + -type f -path '*/hooks/pre-commit' \ + -exec grep -lF 'PANAMA_HOSTILE_TEMPLATE_HOOK' {} + 2>/dev/null +)" +[[ -z "$template_copy" ]] \ + || note "the update Git fixture copied a hostile template hook: $template_copy" + if (( ${#findings[@]} > 0 )); then printf 'update command contract: %d finding(s)\n' "${#findings[@]}" >&2 printf ' - %s\n' "${findings[@]}" >&2