Fix: Close verification gate review findings

This commit is contained in:
Gabriel Brown
2026-08-27 00:21:45 -04:00
parent 77f625d7bb
commit 12b858371e
10 changed files with 575 additions and 57 deletions
+184 -24
View File
@@ -89,6 +89,8 @@ ${BOLD}Commands:${RESET}
pattern to run a subset. --safe selects hermetic contracts only.
Plain terminal runs prompt before non-hermetic work. Automation
must grant each required capability with a repeatable --allow.
Each non-hermetic contract announces its exact capabilities
before it starts.
Failures print captured stdout/stderr. Successful stdout stays
quiet; successful stderr is a warning. The default outer timeout
is 180 seconds. Set PANAMA_TEST_TIMEOUT_SECONDS to a positive
@@ -447,12 +449,20 @@ PROMPT
CONTRACT_MANIFEST="tests/contracts.manifest"
CONTRACT_CAPABILITIES=(hermetic live-host live-compositor live-desktop network privileged)
contract_paths() {
local candidate
while IFS= read -r candidate; do
[[ -x "$candidate" || "$candidate" == *_test.py ]] || continue
printf 'tests/%s\n' "${candidate#"$PANAMA_DIR/tests/"}"
done < <(find "$PANAMA_DIR/tests" -type f \
-not -path '*/fixtures/*' -not -path '*__pycache__*' | sort)
}
contract_manifest_entries() {
local line capabilities path
while IFS= read -r line || [[ -n "$line" ]]; do
line="${line%%#*}"
read -r capabilities path _ <<<"$line"
[[ -n "${capabilities:-}" && -n "${path:-}" ]] || continue
[[ "$line" =~ ^[[:space:]]*(#|$) ]] && continue
IFS=$' \t' read -r capabilities path <<<"$line"
printf '%s\t%s\n' "$path" "$capabilities"
done < "$PANAMA_DIR/$CONTRACT_MANIFEST"
}
@@ -464,6 +474,98 @@ require_contract_manifest() {
}
}
validate_contract_manifest() {
require_contract_manifest || return 1
local manifest="$PANAMA_DIR/$CONTRACT_MANIFEST"
local line capabilities path extra previous_comment="" previous_was_comment=0
local previous_path="" capability discovered
local -a capability_list=() findings=()
local -A expected_contracts=() manifest_paths=()
while IFS= read -r discovered; do
expected_contracts["$discovered"]=1
done < <(contract_paths)
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ "$line" =~ ^[[:space:]]*# ]]; then
previous_comment="${line#*#}"
previous_comment="${previous_comment#"${previous_comment%%[![:space:]]*}"}"
previous_comment="${previous_comment%"${previous_comment##*[![:space:]]}"}"
previous_was_comment=1
continue
fi
if [[ "$line" =~ ^[[:space:]]*$ ]]; then
previous_comment=""
previous_was_comment=0
continue
fi
IFS=$' \t' read -r capabilities path extra <<<"$line"
if [[ -z "${capabilities:-}" || -z "${path:-}" || -n "${extra:-}" ]]; then
findings+=("manifest line is not exactly two fields: $line")
previous_comment=""
previous_was_comment=0
continue
fi
if [[ -n "$previous_path" && "$path" < "$previous_path" ]]; then
findings+=('paths are not lexicographically sorted')
fi
previous_path="$path"
if [[ -n "${manifest_paths[$path]:-}" ]]; then
findings+=("duplicate path $path")
fi
manifest_paths["$path"]=1
local -A line_capabilities=()
if [[ "$capabilities" == ,* || "$capabilities" == *, || "$capabilities" == *,,* ]]; then
findings+=("empty capability on $path")
fi
IFS=',' read -r -a capability_list <<<"$capabilities"
for capability in "${capability_list[@]}"; do
[[ -n "$capability" ]] || continue
if [[ -n "${line_capabilities[$capability]:-}" ]]; then
findings+=("duplicate capability $capability on $path")
fi
line_capabilities["$capability"]=1
is_contract_capability "$capability" \
|| findings+=("unknown capability $capability on $path")
done
if [[ -n "${line_capabilities[hermetic]:-}" && ${#line_capabilities[@]} -ne 1 ]]; then
findings+=("hermetic must appear alone on $path")
fi
if [[ "$capabilities" != hermetic ]]; then
if (( previous_was_comment != 1 )); then
findings+=("$path is non-hermetic but lacks a directly preceding comment")
elif [[ -z "$previous_comment" ]]; then
findings+=("$path is non-hermetic but lacks a non-empty directly preceding comment")
fi
fi
previous_comment=""
previous_was_comment=0
done < "$manifest"
for discovered in "${!expected_contracts[@]}"; do
[[ -n "${manifest_paths[$discovered]:-}" ]] \
|| findings+=("missing contract $discovered")
done
for path in "${!manifest_paths[@]}"; do
[[ -n "${expected_contracts[$path]:-}" ]] \
|| findings+=("stale manifest path $path")
done
if (( ${#findings[@]} > 0 )); then
err "Contract manifest validation failed with ${#findings[@]} finding(s):"
printf ' - %s\n' "${findings[@]}" >&2
return 1
fi
}
test_usage() {
err "Usage: ${BOLD}$PROGRAM test [--safe] [--allow <capability>] [pattern]${RESET}"
return 2
@@ -492,10 +594,59 @@ is_contract_capability() {
# --safe runs only contracts the manifest classifies as hermetic and reports
# each external capability it skipped. A plain terminal run asks before any
# selected non-hermetic work. Automation must grant every required capability
# with repeatable --allow flags. Each contract gets an outer timeout, 180
# with repeatable --allow flags. Non-hermetic contracts announce their exact
# capability list before execution. Each contract gets an outer timeout, 180
# seconds by default. PANAMA_TEST_TIMEOUT_SECONDS accepts a positive integer
# override. Failures include captured stdout and stderr. Successful stdout stays
# quiet, while successful stderr is surfaced as a warning.
# override. Failures include captured stdout and stderr. Successful stdout
# stays quiet, while successful stderr is surfaced as a warning.
PANAMA_ACTIVE_CONTRACT_PID=""
PANAMA_CONTRACT_CAPTURE_DIR=""
cleanup_contract_capture() {
if [[ -n "$PANAMA_CONTRACT_CAPTURE_DIR" && -d "$PANAMA_CONTRACT_CAPTURE_DIR" ]]; then
rm -rf -- "$PANAMA_CONTRACT_CAPTURE_DIR" || true
fi
PANAMA_CONTRACT_CAPTURE_DIR=""
}
terminate_active_contract() {
local pid="$PANAMA_ACTIVE_CONTRACT_PID"
PANAMA_ACTIVE_CONTRACT_PID=""
[[ "$pid" =~ ^[1-9][0-9]*$ && "$pid" != "$$" ]] || return 0
# GNU timeout owns a process group whose ID is its PID. Signal that complete
# group so a contract cannot leave descendants behind, with a direct-PID
# fallback for implementations that do not create the group.
kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
}
handle_contract_signal() {
local signal_status="$1"
trap - INT TERM
terminate_active_contract
cleanup_contract_capture
trap - EXIT
exit "$signal_status"
}
prepare_contract_capture() {
local capture_dir=""
if ! capture_dir="$(mktemp -d)"; then
err 'Could not create contract capture directory.'
return 1
fi
if [[ -z "$capture_dir" || ! -d "$capture_dir" ]]; then
err 'Could not create contract capture directory.'
return 1
fi
PANAMA_CONTRACT_CAPTURE_DIR="$capture_dir"
trap cleanup_contract_capture EXIT
trap 'handle_contract_signal 130' INT
trap 'handle_contract_signal 143' TERM
}
cmd_test() {
local timeout_seconds="${PANAMA_TEST_TIMEOUT_SECONDS:-180}"
[[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]] || {
@@ -503,11 +654,11 @@ cmd_test() {
return 2
}
require_contract_manifest || return 1
validate_contract_manifest || return 1
cmd_test_impl "$timeout_seconds" "$@"
}
cmd_test_impl() (
cmd_test_impl() {
local timeout_seconds="$1"
shift
local pattern="" safe=0 arg capability capabilities rel path
@@ -605,12 +756,10 @@ cmd_test_impl() (
fi
fi
local capture_dir stdout_file stderr_file name run_status index=0
local capture_dir stdout_file stderr_file name run_status index=0 final_status=0
local -a failed=() runner=()
capture_dir="$(mktemp -d)"
trap 'rm -rf -- "$capture_dir"' EXIT
trap 'rm -rf -- "$capture_dir"; exit 130' INT
trap 'rm -rf -- "$capture_dir"; exit 143' TERM
prepare_contract_capture || return 1
capture_dir="$PANAMA_CONTRACT_CAPTURE_DIR"
info "Running ${#suite[@]} contract(s)"
for index in "${!suite[@]}"; do
@@ -624,9 +773,16 @@ cmd_test_impl() (
else
runner=("$path")
fi
capabilities="${manifest_capabilities[$rel]}"
if [[ "$capabilities" != hermetic ]]; then
info "Running $name [$capabilities]"
fi
run_status=0
timeout --signal=TERM --kill-after=5 "$timeout_seconds" \
"${runner[@]}" >"$stdout_file" 2>"$stderr_file" || run_status=$?
"${runner[@]}" >"$stdout_file" 2>"$stderr_file" &
PANAMA_ACTIVE_CONTRACT_PID=$!
wait "$PANAMA_ACTIVE_CONTRACT_PID" || run_status=$?
PANAMA_ACTIVE_CONTRACT_PID=""
if (( run_status == 0 )); then
ok "$name"
if [[ -s "$stderr_file" ]]; then
@@ -655,12 +811,16 @@ cmd_test_impl() (
header "Result"
if (( ${#failed[@]} == 0 )); then
ok "${#suite[@]} contract(s) passed"
return 0
else
err "${#failed[@]} of ${#suite[@]} failed:"
printf ' %s\n' "${failed[@]}" >&2
final_status=1
fi
err "${#failed[@]} of ${#suite[@]} failed:"
printf ' %s\n' "${failed[@]}" >&2
return 1
)
cleanup_contract_capture
trap - EXIT INT TERM
return "$final_status"
}
# ----------------------------------------------------------------------------
# Command: contracts
@@ -721,7 +881,7 @@ cmd_contracts() {
suffix="${suffix#*/}"
done
require_contract_manifest || return 1
validate_contract_manifest || return 1
local -A manifest_capabilities=()
local capabilities
while IFS=$'\t' read -r rel capabilities; do
@@ -732,11 +892,11 @@ cmd_contracts() {
# runner would actually execute.
local -a hits=()
local candidate rel
while IFS= read -r candidate; do
[[ -x "$candidate" || "$candidate" == *_test.py ]] || continue
while IFS= read -r rel; do
candidate="$PANAMA_DIR/$rel"
grep -qF "${patterns[@]}" "$candidate" 2>/dev/null || continue
hits+=("tests/${candidate#"$PANAMA_DIR"/tests/}")
done < <(find "$PANAMA_DIR/tests" -type f -not -path '*/fixtures/*' -not -path '*__pycache__*' | sort)
hits+=("$rel")
done < <(contract_paths)
if (( ${#hits[@]} == 0 )); then
printf 'No contract mentions %s — coverage may be indirect (a harness or a generated artifact); nothing verified.\n' "$path" >&2
@@ -242,7 +242,7 @@ contract_manifest_entries() {
}
```
Do not make a missing manifest mean “everything is safe.” `cmd_test` and `cmd_contracts` must fail clearly if the file cannot be read. The manifest contract owns deeper format validation; the CLI owns the runtime read failure.
Do not make a missing or malformed manifest mean “everything is safe.” Before either `cmd_test` or `cmd_contracts` consumes manifest entries, the public CLI independently validates the complete actual manifest: exactly two fields, known/non-empty/non-duplicate capabilities, exclusive `hermetic`, unique sorted paths, the executable/`*_test.py` fixture-excluded discovery set, and a non-empty directly preceding comment for every non-hermetic entry. Missing discovered contracts and stale manifest paths are fatal before selection or execution. Keep `tests/setup/contract-manifest-contract` as an independent validator rather than sourcing runtime code; a later contract cannot protect earlier execution.
### Step 3: Implement argument and capability policy
@@ -271,7 +271,7 @@ local timeout_seconds="${PANAMA_TEST_TIMEOUT_SECONDS:-180}"
}
```
Run the implementation body in a subshell, create one capture directory with `mktemp -d`, and trap its removal on `EXIT`, `INT`, and `TERM` inside that subshell. This keeps cleanup reliable without leaking or overwriting traps in the parent CLI process. For each contract, run either `python3 path` or the executable through:
The public CLI process must own contract supervision. Check that `mktemp -d` succeeds before constructing any capture path, then install top-level `EXIT`, `INT`, and `TERM` cleanup around the checked directory. Launch the active `timeout` asynchronously and retain its PID/process-group ownership. On INT or TERM sent to the exact CLI PID, signal the active timeout/process group, wait for it, remove capture storage, and exit 130 or 143. On normal completion, remove capture storage and clear the temporary traps without changing the aggregate test status. For each contract, run either `python3 path` or the executable through:
```bash
timeout --signal=TERM --kill-after=5 "$timeout_seconds" \
@@ -389,8 +389,10 @@ The leaked fixture must include:
- a literal YAML credential `POSTGRES_PASSWORD: fixture-should-be-rejected`;
- a literal env credential `API_TOKEN=fixture-should-be-rejected`.
- a plain-text PEM private-key header with an exact `path:line: private key` finding;
- plain-text synthetic `sk-ant-`, minimum-supported-length `ghp_`, and `xoxb-` signatures, each with an exact `path:line: provider token` finding.
Use intentionally invalid fixture strings, not realistic provider token formats.
Add clean plain-text near-misses for each signature. Keep semantic fixture credentials intentionally invalid and signature fixtures clearly synthetic while still matching the supported signature shapes.
### Step 2: Write the scanner with only the Python standard library
+6 -1
View File
@@ -29,7 +29,8 @@ note() { findings+=("$1"); }
scanner="$repo_dir/tests/server/scan-tracked-secrets.py"
fixtures_dir="$repo_dir/tests/server/fixtures/secrets"
if ! python3 "$scanner" "$fixtures_dir/clean" compose.yml .env.example README.md; then
if ! python3 "$scanner" "$fixtures_dir/clean" \
compose.yml .env.example README.md signature-near-misses.txt; then
note 'the clean secret-scanning fixture was rejected'
fi
@@ -53,6 +54,10 @@ expect_leak .env.example '.env.example:1: API_TOKEN'
expect_leak plain-list.yml 'plain-list.yml:4: API_TOKEN'
expect_leak quoted-mapping.yml 'quoted-mapping.yml:4: API_TOKEN'
expect_leak quoted-list.yml 'quoted-list.yml:4: API_TOKEN'
expect_leak pem-private-key.txt 'pem-private-key.txt:1: private key'
expect_leak anthropic-token.txt 'anthropic-token.txt:1: provider token'
expect_leak github-token.txt 'github-token.txt:1: provider token'
expect_leak slack-token.txt 'slack-token.txt:1: provider token'
mapfile -t tracked_server_files < <(git -C "$repo_dir" ls-files 'server/**' 'server/*')
if ! output="$(python3 "$scanner" "$repo_dir" "${tracked_server_files[@]}" 2>&1)"; then
@@ -0,0 +1,4 @@
-----BEGIN SYNTHETIC PUBLIC KEY-----
sk-ant-
ghp_0123456789ABCDEFGHI
xoxb-
@@ -0,0 +1 @@
sk-ant-SYNTHETIC_FIXTURE_TOKEN
@@ -0,0 +1 @@
ghp_0123456789ABCDEFGHIJ
@@ -0,0 +1 @@
-----BEGIN SYNTHETIC PRIVATE KEY-----
@@ -0,0 +1 @@
xoxb-SYNTHETIC_FIXTURE_TOKEN
+232 -6
View File
@@ -9,9 +9,23 @@ repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
fixture="$(mktemp -d)"
output=""
status=0
background_cli_pid=""
background_contract_pgid=""
cleanup() { rm -rf -- "$fixture"; }
trap cleanup EXIT INT TERM
cleanup() {
trap - EXIT INT TERM
if [[ "$background_cli_pid" =~ ^[1-9][0-9]*$ ]]; then
kill -TERM "$background_cli_pid" 2>/dev/null || true
wait "$background_cli_pid" 2>/dev/null || true
fi
if [[ "$background_contract_pgid" =~ ^[1-9][0-9]*$ ]]; then
kill -KILL -- "-$background_contract_pgid" 2>/dev/null || true
fi
rm -rf -- "$fixture"
}
trap cleanup EXIT
trap 'cleanup; exit 130' INT
trap 'cleanup; exit 143' TERM
fail() { printf 'test runner: %s\n' "$*" >&2; exit 1; }
@@ -25,6 +39,12 @@ assert_not_contains() {
[[ "$haystack" != *"$needle"* ]] || fail "expected output not to contain: $needle\n$haystack"
}
assert_before() {
local first="$1" second="$2" haystack="$3"
[[ "$haystack" == *"$first"*"$second"* ]] \
|| fail "expected '$first' before '$second':\n$haystack"
}
assert_execution() {
local expected="$1" actual
actual="$(sort "$fixture/executions" 2>/dev/null || true)"
@@ -33,6 +53,33 @@ assert_execution() {
reset_executions() { : > "$fixture/executions"; }
wait_for_file() {
local path="$1" attempt
for (( attempt = 0; attempt < 100; attempt++ )); do
[[ -s "$path" ]] && return 0
sleep 0.05
done
return 1
}
wait_for_process_exit() {
local pid="$1" attempt
for (( attempt = 0; attempt < 100; attempt++ )); do
kill -0 "$pid" 2>/dev/null || return 0
sleep 0.05
done
return 1
}
wait_for_path_removal() {
local path="$1" attempt
for (( attempt = 0; attempt < 100; attempt++ )); do
[[ ! -e "$path" ]] && return 0
sleep 0.05
done
return 1
}
run_panama() {
output="$(cd "$fixture" && TMPDIR="$fixture" PANAMA_TEST_FIXTURE="$fixture" "$fixture/bin/panama" "$@" </dev/null 2>&1)"
status=$?
@@ -45,18 +92,39 @@ run_panama_with_timeout() {
run_panama_tty_default_no() {
local command tty_stdout="$fixture/tty.stdout"
local pty_state="$fixture/pty-state"
mkdir -p "$pty_state"/{config,state,cache,data,runtime}
chmod 700 "$pty_state/runtime"
printf -v command 'cd %q && TMPDIR=%q PANAMA_TEST_FIXTURE=%q %q test composite > %q' \
"$fixture" "$fixture" "$fixture" "$fixture/bin/panama" "$tty_stdout"
output="$(python3 - "$command" <<'PY'
output="$(
HOME="$fixture/pty-home" \
BASH_ENV="$fixture/pty-bash-env" \
PANAMA_PTY_STARTUP_SENTINEL="$fixture/pty-startup-sourced" \
python3 - "$command" "$fixture/pty-home" \
"$pty_state/config" "$pty_state/state" "$pty_state/cache" \
"$pty_state/data" "$pty_state/runtime" "$fixture" <<'PY'
import errno
import os
import pty
import sys
command = sys.argv[1]
environment = os.environ.copy()
environment.pop('BASH_ENV', None)
environment.pop('ENV', None)
environment.update({
'HOME': sys.argv[2],
'XDG_CONFIG_HOME': sys.argv[3],
'XDG_STATE_HOME': sys.argv[4],
'XDG_CACHE_HOME': sys.argv[5],
'XDG_DATA_HOME': sys.argv[6],
'XDG_RUNTIME_DIR': sys.argv[7],
'TMPDIR': sys.argv[8],
})
pid, terminal = pty.fork()
if pid == 0:
os.execv('/bin/bash', ['bash', '-lc', command])
os.execve('/bin/bash', ['bash', '--noprofile', '--norc', '-c', command], environment)
chunks = []
replied = False
@@ -78,7 +146,7 @@ _, child_status = os.waitpid(pid, 0)
sys.stdout.buffer.write(b''.join(chunks))
sys.exit(os.waitstatus_to_exitcode(child_status))
PY
)"
)"
status=$?
}
@@ -88,6 +156,48 @@ assert_occurrences() {
[[ "$actual" == "$expected" ]] || fail "expected $expected occurrence(s) of '$needle', got $actual\n$haystack"
}
replace_manifest_line() {
local original="$1" replacement="$2" line
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ "$line" == "$original" ]]; then
[[ "$replacement" == __REMOVE__ ]] || printf '%s\n' "$replacement"
else
printf '%s\n' "$line"
fi
done <<<"$valid_manifest"
}
swap_manifest_contract_paths() {
local line
while IFS= read -r line || [[ -n "$line" ]]; do
case "$line" in
'live-compositor,live-desktop tests/composite-contract')
printf '%s\n' 'live-compositor,live-desktop tests/desktop-contract'
;;
'live-desktop tests/desktop-contract')
printf '%s\n' 'live-desktop tests/composite-contract'
;;
*) printf '%s\n' "$line" ;;
esac
done <<<"$valid_manifest"
}
expect_manifest_rejection() {
local label="$1" expected="$2" contents="$3"
printf '%s\n' "$contents" >"$fixture/tests/contracts.manifest"
reset_executions
run_panama test pass
[[ $status -ne 0 ]] || fail "$label manifest unexpectedly allowed test execution"
assert_contains "$expected" "$output"
assert_execution ''
run_panama contracts config/subject
[[ $status -ne 0 ]] || fail "$label manifest unexpectedly allowed contracts lookup"
assert_contains "$expected" "$output"
assert_execution ''
}
mkdir -p "$fixture/bin" "$fixture/tests" "$fixture/config"
cp "$repo_dir/bin/panama" "$fixture/bin/panama"
chmod +x "$fixture/bin/panama"
@@ -111,6 +221,8 @@ privileged tests/privileged-contract
hermetic tests/stderr-contract
EOF
valid_manifest="$(<"$fixture/tests/contracts.manifest")"
cat > "$fixture/tests/pass-contract" <<'EOF'
#!/usr/bin/env bash
printf 'pass\n' >> "$PANAMA_TEST_FIXTURE/executions"
@@ -135,7 +247,13 @@ EOF
cat > "$fixture/tests/hang-contract" <<'EOF'
#!/usr/bin/env bash
printf 'hang\n' >> "$PANAMA_TEST_FIXTURE/executions"
trap 'printf terminated >"$PANAMA_TEST_FIXTURE/terminated"; exit 124' TERM
printf '%s\n' "$BASHPID" > "$PANAMA_TEST_FIXTURE/hang.pid"
finish() {
printf 'terminated\n' >"$PANAMA_TEST_FIXTURE/terminated"
exit "$1"
}
trap 'finish 130' INT
trap 'finish 143' TERM
while :; do sleep 1; done
EOF
@@ -147,6 +265,7 @@ EOF
cat > "$fixture/tests/desktop-contract" <<'EOF'
#!/usr/bin/env bash
printf 'desktop\n' >> "$PANAMA_TEST_FIXTURE/executions"
printf 'desktop fixture complete\n' >&2
# config/subject
EOF
@@ -172,6 +291,15 @@ chmod +x "$fixture/tests"/{composite,desktop,fail,hang,host,network,pass,privile
# A PTY-backed default-no confirmation remains visible when stdout is redirected
# but stdin and stderr are terminals. The fixture proves that one prompt gates
# the selected composite capability set without running its contract.
mkdir -p "$fixture/pty-home"
for profile in .bash_profile .bashrc .profile; do
cat >"$fixture/pty-home/$profile" <<'EOF'
printf 'profile\n' >>"${PANAMA_PTY_STARTUP_SENTINEL:?}"
EOF
done
cat >"$fixture/pty-bash-env" <<'EOF'
printf 'BASH_ENV\n' >>"${PANAMA_PTY_STARTUP_SENTINEL:?}"
EOF
run_panama_tty_default_no
[[ $status -ne 0 ]] || fail 'TTY default-no prompt unexpectedly ran the fixture'
assert_execution ''
@@ -179,6 +307,8 @@ assert_contains 'Run 1 contract(s) requiring: live-compositor live-desktop?' "$o
assert_occurrences 'Run 1 contract(s) requiring:' "$output" 1
assert_contains 'No contracts were run.' "$(<"$fixture/tty.stdout")"
assert_not_contains 'Run 1 contract(s) requiring:' "$(<"$fixture/tty.stdout")"
[[ ! -e "$fixture/pty-startup-sourced" ]] \
|| fail 'PTY fixture sourced a shell profile or BASH_ENV'
# --safe must select hermetic entries from the manifest, not merely omit a
# legacy desktop list. The failing and timed-out fixtures make the command
@@ -204,6 +334,8 @@ assert_contains 'pass --allow live-desktop' "$output"
run_panama test --allow live-desktop desktop
[[ $status -eq 0 ]] || fail "explicit desktop grant failed: $output"
assert_execution 'desktop'
assert_contains 'Running desktop-contract [live-desktop]' "$output"
assert_before 'Running desktop-contract [live-desktop]' 'desktop fixture complete' "$output"
reset_executions
run_panama test --allow live-compositor --allow live-desktop composite
@@ -229,6 +361,45 @@ assert_execution 'hang'
[[ -f "$fixture/terminated" ]] || fail 'timed-out contract was not terminated with TERM'
assert_contains 'timed out' "$output"
# INT/TERM ownership belongs to the exact public CLI PID, not a runner
# subshell. The CLI must wait for the timeout process group and remove its
# capture directory before returning the signal-derived status.
reset_executions
rm -f -- "$fixture/hang.pid" "$fixture/terminated"
(
cd "$fixture" || exit 1
exec env TMPDIR="$fixture" PANAMA_TEST_FIXTURE="$fixture" \
"$fixture/bin/panama" test hang
) >"$fixture/exact-term.out" 2>&1 &
background_cli_pid=$!
wait_for_file "$fixture/hang.pid" \
|| fail 'exact-PID TERM fixture never started the hang contract'
hang_pid="$(<"$fixture/hang.pid")"
background_contract_pgid="$(ps -o pgid= -p "$hang_pid" | tr -d '[:space:]')"
[[ "$background_contract_pgid" =~ ^[1-9][0-9]*$ ]] \
|| fail "could not resolve hang process group for PID $hang_pid"
mapfile -t active_capture_dirs < <(
find "$fixture" -mindepth 1 -maxdepth 1 -type d -name 'tmp.*' -print
)
(( ${#active_capture_dirs[@]} == 1 )) \
|| fail "expected one active capture directory, got ${#active_capture_dirs[@]}"
active_capture_dir="${active_capture_dirs[0]}"
kill -TERM "$background_cli_pid" \
|| fail 'could not send TERM to the exact public CLI PID'
term_status=0
wait "$background_cli_pid" || term_status=$?
background_cli_pid=""
[[ "$term_status" -eq 143 ]] \
|| fail "exact-PID TERM returned $term_status instead of 143: $(<"$fixture/exact-term.out")"
wait_for_process_exit "$hang_pid" \
|| fail "hang contract PID $hang_pid survived exact-PID TERM"
background_contract_pgid=""
wait_for_path_removal "$active_capture_dir" \
|| fail "capture directory survived exact-PID TERM: $active_capture_dir"
[[ -f "$fixture/terminated" ]] \
|| fail 'exact-PID TERM did not reach the hang contract cleanup trap'
reset_executions
run_panama test fail
[[ $status -ne 0 ]] || fail 'failed contract unexpectedly passed'
@@ -244,6 +415,19 @@ reset_executions
run_panama test pass
[[ $status -eq 0 ]] || fail "pass contract failed: $output"
assert_not_contains 'pass stdout' "$output"
assert_not_contains 'pass-contract [hermetic]' "$output"
reset_executions
printf 'not a directory\n' >"$fixture/invalid-tmpdir"
output="$(
cd "$fixture" && \
TMPDIR="$fixture/invalid-tmpdir" PANAMA_TEST_FIXTURE="$fixture" \
"$fixture/bin/panama" test pass </dev/null 2>&1
)"
status=$?
[[ $status -ne 0 ]] || fail 'invalid TMPDIR unexpectedly allowed contract execution'
assert_contains 'Could not create contract capture directory.' "$output"
assert_execution ''
reset_executions
run_panama test --safe desktop
@@ -258,10 +442,52 @@ assert_contains 'tests/desktop-contract [live-desktop]' "$output"
assert_contains 'tests/composite-contract [live-compositor,live-desktop]' "$output"
assert_contains 'tests/pass-contract [hermetic]' "$output"
# Both public manifest consumers fail closed on the complete format and
# discovery set. Validation happens before selection, lookup, or contract
# execution, so even a malformed entry unrelated to the requested pattern is
# fatal and leaves the execution log empty.
expect_manifest_rejection unknown-capability \
'unknown capability hermetik on tests/pass-contract' \
"$(replace_manifest_line 'hermetic tests/pass-contract' 'hermetik tests/pass-contract')"
expect_manifest_rejection mixed-hermetic \
'hermetic must appear alone on tests/pass-contract' \
"$(replace_manifest_line 'hermetic tests/pass-contract' 'hermetic,network tests/pass-contract')"
expect_manifest_rejection duplicate-path \
'duplicate path tests/pass-contract' \
"$(replace_manifest_line 'hermetic tests/pass-contract' $'hermetic tests/pass-contract\nhermetic tests/pass-contract')"
expect_manifest_rejection stale-path \
'stale manifest path tests/stale-contract' \
"$(replace_manifest_line 'hermetic tests/pass-contract' 'hermetic tests/stale-contract')"
expect_manifest_rejection missing-contract \
'missing contract tests/pass-contract' \
"$(replace_manifest_line 'hermetic tests/pass-contract' __REMOVE__)"
expect_manifest_rejection extra-field \
'manifest line is not exactly two fields' \
"$(replace_manifest_line 'hermetic tests/pass-contract' 'hermetic tests/pass-contract unexpected')"
expect_manifest_rejection empty-capability \
'empty capability on tests/pass-contract' \
"$(replace_manifest_line 'hermetic tests/pass-contract' 'hermetic, tests/pass-contract')"
expect_manifest_rejection duplicate-capability \
'duplicate capability hermetic on tests/pass-contract' \
"$(replace_manifest_line 'hermetic tests/pass-contract' 'hermetic,hermetic tests/pass-contract')"
expect_manifest_rejection unsorted-paths \
'paths are not lexicographically sorted' \
"$(swap_manifest_contract_paths)"
expect_manifest_rejection uncommented-non-hermetic \
'tests/desktop-contract is non-hermetic but lacks a directly preceding comment' \
"$(replace_manifest_line '# Maps the live desktop.' __REMOVE__)"
expect_manifest_rejection blank-comment \
'tests/desktop-contract is non-hermetic but lacks a non-empty directly preceding comment' \
"$(replace_manifest_line '# Maps the live desktop.' '#')"
printf '%s\n' "$valid_manifest" >"$fixture/tests/contracts.manifest"
mv "$fixture/tests/contracts.manifest" "$fixture/tests/contracts.manifest.missing"
reset_executions
run_panama test pass
[[ $status -ne 0 ]] || fail 'missing manifest unexpectedly allowed test execution'
assert_contains 'contracts.manifest' "$output"
assert_execution ''
mv "$fixture/tests/contracts.manifest.missing" "$fixture/tests/contracts.manifest"
capture_dirs="$(find "$fixture" -mindepth 1 -maxdepth 1 -type d -name 'tmp.*' -print)"
+140 -23
View File
@@ -234,16 +234,116 @@ grep -qx 'install-packages' <<<"$ran_full" \
# The fixture also covers a clean fast-forward, installer status propagation,
# and the boundary between update and sync before forcing the conflict below.
# Make every ambient configuration source hostile before constructing the Git
# fixtures. A hermetic fixture overrides these values with its own empty state;
# consuming any of them either leaves a sentinel or prevents a commit.
hostile="$tmp/hostile-environment"
mkdir -p "$hostile/home" "$hostile/xdg-config" "$hostile/xdg-state" \
"$hostile/xdg-cache" "$hostile/xdg-data" "$hostile/hooks" \
"$hostile/template/hooks"
for profile in .bash_profile .bashrc .profile; do
cat >"$hostile/home/$profile" <<'EOF'
printf 'profile\n' >>"${PANAMA_HOSTILE_PROFILE_SENTINEL:?}"
EOF
done
cat >"$hostile/bash-env" <<'EOF'
printf 'BASH_ENV\n' >>"${PANAMA_HOSTILE_BASH_ENV_SENTINEL:?}"
EOF
cat >"$hostile/hooks/pre-commit" <<'EOF'
#!/usr/bin/env bash
printf 'global hook\n' >>"${PANAMA_HOSTILE_GIT_SENTINEL:?}"
exit 97
EOF
cat >"$hostile/template/hooks/pre-commit" <<'EOF'
#!/usr/bin/env bash
# PANAMA_HOSTILE_TEMPLATE_HOOK
printf 'template hook\n' >>"${PANAMA_HOSTILE_TEMPLATE_SENTINEL:?}"
exit 98
EOF
chmod +x "$hostile/hooks/pre-commit" "$hostile/template/hooks/pre-commit"
cat >"$hostile/global.gitconfig" <<EOF
[core]
hooksPath = $hostile/hooks
[commit]
gpgSign = true
[init]
templateDir = $hostile/template
EOF
cp "$hostile/global.gitconfig" "$hostile/system.gitconfig"
export HOME="$hostile/home"
export XDG_CONFIG_HOME="$hostile/xdg-config"
export XDG_STATE_HOME="$hostile/xdg-state"
export XDG_CACHE_HOME="$hostile/xdg-cache"
export XDG_DATA_HOME="$hostile/xdg-data"
export BASH_ENV="$hostile/bash-env"
export PANAMA_HOSTILE_PROFILE_SENTINEL="$hostile/profile-sourced"
export PANAMA_HOSTILE_BASH_ENV_SENTINEL="$hostile/bash-env-sourced"
export PANAMA_HOSTILE_GIT_SENTINEL="$hostile/global-config-sourced"
export PANAMA_HOSTILE_TEMPLATE_SENTINEL="$hostile/template-hook-sourced"
export GIT_CONFIG_NOSYSTEM=0
export GIT_CONFIG_SYSTEM="$hostile/system.gitconfig"
export GIT_CONFIG_GLOBAL="$hostile/global.gitconfig"
export GIT_CONFIG_COUNT=1
export GIT_CONFIG_KEY_0=core.hooksPath
export GIT_CONFIG_VALUE_0="$hostile/hooks"
export GIT_TEMPLATE_DIR="$hostile/template"
prepare_cli_fixture_environment() {
local root="$1"
mkdir -p "$root/home" "$root/xdg-config" "$root/xdg-state" \
"$root/xdg-cache" "$root/xdg-data" "$root/xdg-runtime" \
"$root/empty-templates" "$root/empty-hooks"
chmod 700 "$root/xdg-runtime"
}
run_cli_fixture_environment() {
local root="$1"
shift
env -u BASH_ENV -u ENV -u GIT_CONFIG_PARAMETERS \
-u GIT_CONFIG_KEY_0 -u GIT_CONFIG_VALUE_0 \
HOME="$root/home" \
XDG_CONFIG_HOME="$root/xdg-config" \
XDG_STATE_HOME="$root/xdg-state" \
XDG_CACHE_HOME="$root/xdg-cache" \
XDG_DATA_HOME="$root/xdg-data" \
XDG_RUNTIME_DIR="$root/xdg-runtime" \
GIT_CONFIG_NOSYSTEM=1 \
GIT_CONFIG_SYSTEM=/dev/null \
GIT_CONFIG_GLOBAL=/dev/null \
GIT_CONFIG_COUNT=0 \
GIT_TEMPLATE_DIR="$root/empty-templates" \
"$@"
}
fixture_git() {
local root="$1"
shift
run_cli_fixture_environment "$root" \
git -c commit.gpgSign=false -c tag.gpgSign=false \
-c core.hooksPath="$root/empty-hooks" "$@"
}
configure_fixture_repo() {
local root="$1" repository="$2"
fixture_git "$root" -C "$repository" config user.email contract@panama || return 1
fixture_git "$root" -C "$repository" config user.name contract || return 1
fixture_git "$root" -C "$repository" config commit.gpgSign false || return 1
fixture_git "$root" -C "$repository" config tag.gpgSign false || return 1
fixture_git "$root" -C "$repository" config core.hooksPath "$root/empty-hooks" || return 1
}
# Each fixture has the same three repositories as a real update: a bare remote,
# a clone that publishes upstream changes, and the machine clone being updated.
build_cli_fixture() (
local root="$1"
rm -rf "$root" || return 1
mkdir -p "$root" || return 1
git init -q --bare "$root/origin.git" || return 1
git clone -q "$root/origin.git" "$root/upstream" 2>/dev/null || return 1
git -C "$root/upstream" config user.email contract@panama || return 1
git -C "$root/upstream" config user.name contract || return 1
prepare_cli_fixture_environment "$root" || return 1
fixture_git "$root" init -q --bare "$root/origin.git" || return 1
fixture_git "$root" -C "$root/origin.git" config core.hooksPath "$root/empty-hooks" || return 1
fixture_git "$root" clone -q "$root/origin.git" "$root/upstream" 2>/dev/null || return 1
configure_fixture_repo "$root" "$root/upstream" || return 1
mkdir -p "$root/upstream/bin" || return 1
cp "$panama" "$root/upstream/bin/panama" || return 1
@@ -254,21 +354,20 @@ exit "${PANAMA_UPDATE_INSTALL_RC:-0}"
EOF
chmod +x "$root/upstream/bin/panama" "$root/upstream/install" || return 1
printf 'one\n' >"$root/upstream/f" || return 1
git -C "$root/upstream" add -A || return 1
git -C "$root/upstream" commit -qm initial || return 1
git -C "$root/upstream" push -qu origin HEAD || return 1
fixture_git "$root" -C "$root/upstream" add -A || return 1
fixture_git "$root" -C "$root/upstream" commit -qm initial || return 1
fixture_git "$root" -C "$root/upstream" push -qu origin HEAD || return 1
git clone -q "$root/origin.git" "$root/machine" || return 1
git -C "$root/machine" config user.email contract@panama || return 1
git -C "$root/machine" config user.name contract || return 1
fixture_git "$root" clone -q "$root/origin.git" "$root/machine" || return 1
configure_fixture_repo "$root" "$root/machine" || return 1
)
advance_upstream() (
local root="$1" file="$2" contents="$3"
printf '%s\n' "$contents" >"$root/upstream/$file" || return 1
git -C "$root/upstream" add "$file" || return 1
git -C "$root/upstream" commit -qm "update $file" || return 1
git -C "$root/upstream" push -q || return 1
fixture_git "$root" -C "$root/upstream" add "$file" || return 1
fixture_git "$root" -C "$root/upstream" commit -qm "update $file" || return 1
fixture_git "$root" -C "$root/upstream" push -q || return 1
)
# This write fails before the later Git commands. The helper must return that
@@ -285,26 +384,28 @@ fi
clean="$tmp/clean-update"
if build_cli_fixture "$clean" && advance_upstream "$clean" release new; then
machine_before="$(git -C "$clean/machine" rev-parse HEAD)"
upstream_after="$(git -C "$clean/upstream" rev-parse HEAD)"
machine_before="$(fixture_git "$clean" -C "$clean/machine" rev-parse HEAD)"
upstream_after="$(fixture_git "$clean" -C "$clean/upstream" rev-parse HEAD)"
[[ "$machine_before" != "$upstream_after" ]] \
|| note 'the clean update fixture started current, so it cannot prove a fast-forward'
: >"$clean/install.log"
update_status=0
PANAMA_UPDATE_FIXTURE_LOG="$clean/install.log" \
run_cli_fixture_environment "$clean" \
PANAMA_UPDATE_FIXTURE_LOG="$clean/install.log" \
"$clean/machine/bin/panama" update >"$clean/update.out" 2>&1 \
|| update_status=$?
[[ "$update_status" -eq 0 ]] \
|| note "panama update failed on a clean clone with status $update_status"
[[ "$(git -C "$clean/machine" rev-parse HEAD)" == "$upstream_after" ]] \
[[ "$(fixture_git "$clean" -C "$clean/machine" rev-parse HEAD)" == "$upstream_after" ]] \
|| note 'panama update did not fast-forward the clean machine clone'
grep -qx -- '--upgrade' "$clean/install.log" \
|| note 'panama update did not invoke the installer with --upgrade'
: >"$clean/install.log"
update_status=0
PANAMA_UPDATE_FIXTURE_LOG="$clean/install.log" PANAMA_UPDATE_INSTALL_RC=23 \
run_cli_fixture_environment "$clean" \
PANAMA_UPDATE_FIXTURE_LOG="$clean/install.log" PANAMA_UPDATE_INSTALL_RC=23 \
"$clean/machine/bin/panama" update >"$clean/failing-update.out" 2>&1 \
|| update_status=$?
[[ "$update_status" -eq 23 ]] \
@@ -316,7 +417,8 @@ if build_cli_fixture "$clean" && advance_upstream "$clean" release new; then
printf 'local sync\n' >"$clean/machine/synced"
sync_status=0
printf 'y\ncontract sync\n' \
| PANAMA_UPDATE_FIXTURE_LOG="$clean/install.log" \
| run_cli_fixture_environment "$clean" \
PANAMA_UPDATE_FIXTURE_LOG="$clean/install.log" \
"$clean/machine/bin/panama" sync >"$clean/sync.out" 2>&1 \
|| sync_status=$?
[[ "$sync_status" -eq 0 ]] \
@@ -335,19 +437,22 @@ if build_cli_fixture "$conflict"; then
if advance_upstream "$conflict" f upstream; then
: >"$conflict/install.log"
conflict_status=0
PANAMA_UPDATE_FIXTURE_LOG="$conflict/install.log" \
run_cli_fixture_environment "$conflict" \
PANAMA_UPDATE_FIXTURE_LOG="$conflict/install.log" \
"$conflict/machine/bin/panama" update >"$conflict/update.out" 2>&1 \
|| conflict_status=$?
[[ "$conflict_status" -eq 0 ]] \
|| note "panama update failed while recovering a stash conflict with status $conflict_status"
[[ "$(<"$conflict/machine/f")" == upstream ]] \
|| note 'panama update did not reset the conflicted file to the upstream version'
if git -C "$conflict/machine" grep -qE '^(<<<<<<<|=======|>>>>>>>)' -- .; then
if fixture_git "$conflict" -C "$conflict/machine" \
grep -qE '^(<<<<<<<|=======|>>>>>>>)' -- .; then
note 'panama update left conflict markers in the machine checkout'
fi
[[ -n "$(git -C "$conflict/machine" stash list)" ]] \
[[ -n "$(fixture_git "$conflict" -C "$conflict/machine" stash list)" ]] \
|| note 'panama update dropped the stash after its conflicted pop'
recovered="$(git -C "$conflict/machine" show 'stash@{0}:f' 2>/dev/null)"
recovered="$(fixture_git "$conflict" -C "$conflict/machine" \
show 'stash@{0}:f' 2>/dev/null)"
[[ "$recovered" == local ]] \
|| note 'the stash left by panama update does not contain the local version'
else
@@ -357,6 +462,18 @@ else
note 'the conflict update fixture could not be built'
fi
for sentinel in profile-sourced bash-env-sourced global-config-sourced template-hook-sourced; do
[[ ! -e "$hostile/$sentinel" ]] \
|| note "the update Git fixture consumed hostile state: $sentinel"
done
template_copy="$(
find "$tmp" -path "$hostile" -prune -o \
-type f -path '*/hooks/pre-commit' \
-exec grep -lF 'PANAMA_HOSTILE_TEMPLATE_HOOK' {} + 2>/dev/null
)"
[[ -z "$template_copy" ]] \
|| note "the update Git fixture copied a hostile template hook: $template_copy"
if (( ${#findings[@]} > 0 )); then
printf 'update command contract: %d finding(s)\n' "${#findings[@]}" >&2
printf ' - %s\n' "${findings[@]}" >&2