Fix: Make contract execution safe and diagnostic
This commit is contained in:
Gabriel Brown
+183
-88
@@ -10,7 +10,7 @@
|
||||
# edit Open the Panama repo in Neovim
|
||||
# doctor Report what is actually running on this machine
|
||||
# diagnose Hand this machine's health and recent errors to your agent
|
||||
# test Run every contract under tests/ (--safe skips the hijacking ones)
|
||||
# test Run contract manifest entries under tests/ (--safe is hermetic only)
|
||||
# contracts Name the contracts that mention a given file
|
||||
# upgrade Re-run the installer from anywhere, interview included
|
||||
# migrate Apply repairs this machine has not had yet
|
||||
@@ -87,10 +87,10 @@ ${BOLD}Commands:${RESET}
|
||||
Needs an agent chosen on Settings › System › Agents.
|
||||
${GREEN}test${RESET} Run every contract under tests/. Give it a pattern to run
|
||||
a subset: 'panama test dock' runs the ones matching 'dock'.
|
||||
--safe skips the ones that take over the live desktop; what
|
||||
they are and why is tests/desktop-hijacking.
|
||||
${GREEN}contracts${RESET} Name the contracts that mention a given file, each marked
|
||||
safe or desktop. A heuristic over the text of tests/, so it
|
||||
--safe runs hermetic contracts only. External capabilities need
|
||||
explicit --allow grants when automation invokes them.
|
||||
${GREEN}contracts${RESET} Name the contracts that mention a given file, each labeled
|
||||
with manifest capabilities. A heuristic over the text of tests/, so it
|
||||
answers "what should I run" rather than "what covers this".
|
||||
${GREEN}upgrade${RESET} Re-run ./install from anywhere, interview and all. For a new
|
||||
machine, or to change an answer you gave. Routine updates are
|
||||
@@ -431,29 +431,43 @@ PROMPT
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# The desktop-hijacking ledger
|
||||
# Contract manifest
|
||||
# ----------------------------------------------------------------------------
|
||||
#
|
||||
# tests/desktop-hijacking lists, one repo-relative path per line with a '#'
|
||||
# comment saying what it does to the live session, the contracts that drive the
|
||||
# real shell, compositor or machine rather than a harness. Read by `test --safe`
|
||||
# to decide what to skip, and by `contracts` to mark each hit.
|
||||
#
|
||||
# Prints the paths, comments and blank lines stripped. A missing ledger prints
|
||||
# nothing: no ledger means nothing is known to hijack, which is the honest
|
||||
# reading of an absent file and keeps `--safe` from failing on a fresh checkout.
|
||||
DESKTOP_HIJACKING_LEDGER="tests/desktop-hijacking"
|
||||
# The manifest is the runtime authority for every collected contract. An absent
|
||||
# manifest is unsafe: this command must never infer that unclassified tests are
|
||||
# hermetic.
|
||||
CONTRACT_MANIFEST="tests/contracts.manifest"
|
||||
CONTRACT_CAPABILITIES=(hermetic live-host live-compositor live-desktop network privileged)
|
||||
|
||||
hijacking_entries() {
|
||||
local ledger="$PANAMA_DIR/$DESKTOP_HIJACKING_LEDGER" line
|
||||
[[ -r "$ledger" ]] || return 0
|
||||
contract_manifest_entries() {
|
||||
local line capabilities path
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
line="${line%%#*}"
|
||||
line="${line#"${line%%[![:space:]]*}"}"
|
||||
line="${line%"${line##*[![:space:]]}"}"
|
||||
[[ -n "$line" ]] && printf '%s\n' "$line"
|
||||
done < "$ledger"
|
||||
return 0
|
||||
read -r capabilities path _ <<<"$line"
|
||||
[[ -n "${capabilities:-}" && -n "${path:-}" ]] || continue
|
||||
printf '%s\t%s\n' "$path" "$capabilities"
|
||||
done < "$PANAMA_DIR/$CONTRACT_MANIFEST"
|
||||
}
|
||||
|
||||
require_contract_manifest() {
|
||||
[[ -r "$PANAMA_DIR/$CONTRACT_MANIFEST" ]] || {
|
||||
err "Contract manifest is missing or unreadable: $PANAMA_DIR/$CONTRACT_MANIFEST"
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
test_usage() {
|
||||
err "Usage: ${BOLD}$PROGRAM test [--safe] [--allow <capability>] [pattern]${RESET}"
|
||||
return 2
|
||||
}
|
||||
|
||||
is_contract_capability() {
|
||||
local capability="$1" known
|
||||
for known in "${CONTRACT_CAPABILITIES[@]}"; do
|
||||
[[ "$capability" == "$known" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
@@ -472,93 +486,173 @@ hijacking_entries() {
|
||||
# open overlays, restart the shell, move your windows. Running the suite while
|
||||
# sitting in front of the machine used to mean losing the session for a few
|
||||
# minutes, so the honest options were "run everything" or "run nothing". --safe
|
||||
# is the third: skip exactly what tests/desktop-hijacking names, and say how
|
||||
# many were skipped, so the gap is stated rather than implied.
|
||||
# is the third: run only contracts the manifest classifies as hermetic, and say
|
||||
# which external capability coverage it skipped.
|
||||
cmd_test() {
|
||||
local pattern="" safe=0 arg
|
||||
# Position-independent, because 'panama test --safe dock' and
|
||||
# 'panama test dock --safe' are the same intent and nobody should have to
|
||||
# remember which one this accepts.
|
||||
for arg in "$@"; do
|
||||
local timeout_seconds="${PANAMA_TEST_TIMEOUT_SECONDS:-180}"
|
||||
[[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]] || {
|
||||
err 'PANAMA_TEST_TIMEOUT_SECONDS must be a positive integer.'
|
||||
return 2
|
||||
}
|
||||
|
||||
require_contract_manifest || return 1
|
||||
cmd_test_impl "$timeout_seconds" "$@"
|
||||
}
|
||||
|
||||
cmd_test_impl() (
|
||||
local timeout_seconds="$1"
|
||||
shift
|
||||
local pattern="" safe=0 arg capability capabilities rel path
|
||||
local -A grants=() manifest_capabilities=() skipped_counts=() missing_grants=() selected_capabilities=()
|
||||
local -a suite=() missing_capability_list=() selected_capability_list=() capability_list=()
|
||||
|
||||
# Position-independent: flags can precede or follow the optional pattern.
|
||||
while (( $# > 0 )); do
|
||||
arg="$1"
|
||||
shift
|
||||
case "$arg" in
|
||||
--safe) safe=1 ;;
|
||||
*) pattern="$arg" ;;
|
||||
--allow)
|
||||
(( $# > 0 )) || { test_usage; return 2; }
|
||||
capability="$1"
|
||||
shift
|
||||
is_contract_capability "$capability" || {
|
||||
err "Unknown contract capability: $capability"
|
||||
return 2
|
||||
}
|
||||
[[ "$capability" != hermetic ]] || {
|
||||
err 'hermetic contracts do not need --allow.'
|
||||
return 2
|
||||
}
|
||||
grants["$capability"]=1
|
||||
;;
|
||||
--*) test_usage; return 2 ;;
|
||||
*)
|
||||
[[ -z "$pattern" ]] || { test_usage; return 2; }
|
||||
pattern="$arg"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
local -a suite=()
|
||||
local -A hijacking=()
|
||||
local skipped=0 entry rel
|
||||
(( safe == 0 || ${#grants[@]} == 0 )) || {
|
||||
err '--safe cannot be combined with --allow.'
|
||||
return 2
|
||||
}
|
||||
|
||||
if (( safe )); then
|
||||
while IFS= read -r entry; do
|
||||
hijacking["$entry"]=1
|
||||
done < <(hijacking_entries)
|
||||
fi
|
||||
|
||||
# Executables, plus the Python suites. Those are unittest files rather than
|
||||
# executables, and collecting only what has the executable bit would skip them
|
||||
# without saying so -- which is how all three came to be run by nothing at all.
|
||||
# A runner with a blind spot is worse than no runner, because it reports PASS.
|
||||
while IFS= read -r path; do
|
||||
[[ -x "$path" || "$path" == *_test.py ]] || continue
|
||||
[[ -z "$pattern" || "$path" == *"$pattern"* ]] || continue
|
||||
rel="tests/${path#"$PANAMA_DIR"/tests/}"
|
||||
if (( safe )) && [[ -n "${hijacking[$rel]:-}" ]]; then
|
||||
(( ++skipped ))
|
||||
while IFS=$'\t' read -r rel capabilities; do
|
||||
manifest_capabilities["$rel"]="$capabilities"
|
||||
[[ -z "$pattern" || "$rel" == *"$pattern"* ]] || continue
|
||||
if (( safe )) && [[ "$capabilities" != hermetic ]]; then
|
||||
IFS=',' read -r -a capability_list <<<"$capabilities"
|
||||
for capability in "${capability_list[@]}"; do
|
||||
(( ++skipped_counts["$capability"] ))
|
||||
done
|
||||
continue
|
||||
fi
|
||||
suite+=("$path")
|
||||
done < <(find "$PANAMA_DIR/tests" -type f -not -path '*/fixtures/*' -not -path '*__pycache__*' | sort)
|
||||
suite+=("$rel")
|
||||
done < <(contract_manifest_entries)
|
||||
|
||||
if (( ${#suite[@]} == 0 )); then
|
||||
# "Nothing matched" and "everything that matched was skipped" are different
|
||||
# answers, and reporting the first for the second is how --safe would come
|
||||
# to look like a broken pattern.
|
||||
if (( skipped > 0 )); then
|
||||
err "Every contract matching '${pattern}' is desktop-hijacking; --safe skipped all ${skipped}."
|
||||
printf ' What they do to the session: %s/%s\n' "$PANAMA_DIR" "$DESKTOP_HIJACKING_LEDGER" >&2
|
||||
if (( safe )) && (( ${#skipped_counts[@]} > 0 )); then
|
||||
err "Every contract matching '${pattern}' needs an external capability; --safe skipped all of them."
|
||||
else
|
||||
err "No contracts match '${pattern}'"
|
||||
fi
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
if (( safe )); then
|
||||
for capability in "${CONTRACT_CAPABILITIES[@]}"; do
|
||||
[[ "$capability" == hermetic ]] && continue
|
||||
printf 'Skipped %d %s contract(s).\n' "${skipped_counts[$capability]:-0}" "$capability"
|
||||
done
|
||||
else
|
||||
for rel in "${suite[@]}"; do
|
||||
capabilities="${manifest_capabilities[$rel]}"
|
||||
[[ "$capabilities" == hermetic ]] && continue
|
||||
IFS=',' read -r -a capability_list <<<"$capabilities"
|
||||
for capability in "${capability_list[@]}"; do
|
||||
selected_capabilities["$capability"]=1
|
||||
[[ -n "${grants[$capability]:-}" ]] || missing_grants["$capability"]=1
|
||||
done
|
||||
done
|
||||
for capability in "${CONTRACT_CAPABILITIES[@]}"; do
|
||||
[[ "$capability" == hermetic ]] && continue
|
||||
[[ -n "${selected_capabilities[$capability]:-}" ]] && selected_capability_list+=("$capability")
|
||||
[[ -n "${missing_grants[$capability]:-}" ]] && missing_capability_list+=("$capability")
|
||||
done
|
||||
if (( ${#missing_capability_list[@]} > 0 )); then
|
||||
if [[ -t 0 && -t 2 ]]; then
|
||||
confirm "Run ${#suite[@]} contract(s) requiring: ${selected_capability_list[*]}?" || {
|
||||
warn 'No contracts were run.'
|
||||
return 1
|
||||
}
|
||||
else
|
||||
err "Selected contracts require: ${missing_capability_list[*]}."
|
||||
for capability in "${missing_capability_list[@]}"; do
|
||||
printf ' Automation: pass --allow %s\n' "$capability" >&2
|
||||
done
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
local capture_dir stdout_file stderr_file name run_status index=0
|
||||
local -a failed=() runner=()
|
||||
capture_dir="$(mktemp -d)"
|
||||
trap 'rm -rf -- "$capture_dir"' EXIT
|
||||
trap 'rm -rf -- "$capture_dir"; exit 130' INT
|
||||
trap 'rm -rf -- "$capture_dir"; exit 143' TERM
|
||||
|
||||
info "Running ${#suite[@]} contract(s)"
|
||||
local -a failed=()
|
||||
local path name
|
||||
local -a runner
|
||||
for path in "${suite[@]}"; do
|
||||
name="${path#"$PANAMA_DIR"/tests/}"
|
||||
for index in "${!suite[@]}"; do
|
||||
rel="${suite[$index]}"
|
||||
path="$PANAMA_DIR/$rel"
|
||||
name="${rel#tests/}"
|
||||
stdout_file="$capture_dir/$index.stdout"
|
||||
stderr_file="$capture_dir/$index.stderr"
|
||||
if [[ "$path" == *_test.py ]]; then
|
||||
runner=(python3 "$path")
|
||||
else
|
||||
runner=("$path")
|
||||
fi
|
||||
if "${runner[@]}" >/dev/null 2>&1; then
|
||||
run_status=0
|
||||
timeout --signal=TERM --kill-after=5 "$timeout_seconds" \
|
||||
"${runner[@]}" >"$stdout_file" 2>"$stderr_file" || run_status=$?
|
||||
if (( run_status == 0 )); then
|
||||
ok "$name"
|
||||
else
|
||||
err "$name"
|
||||
failed+=("$name")
|
||||
if [[ -s "$stderr_file" ]]; then
|
||||
warn "$name wrote to stderr:"
|
||||
cat "$stderr_file" >&2
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if (( run_status == 124 || run_status == 137 )); then
|
||||
err "$name timed out after ${timeout_seconds}s"
|
||||
else
|
||||
err "$name failed (exit $run_status)"
|
||||
fi
|
||||
[[ -s "$stdout_file" ]] && {
|
||||
printf '%s stdout:\n' "$name" >&2
|
||||
cat "$stdout_file" >&2
|
||||
}
|
||||
[[ -s "$stderr_file" ]] && {
|
||||
printf '%s stderr:\n' "$name" >&2
|
||||
cat "$stderr_file" >&2
|
||||
}
|
||||
failed+=("$name")
|
||||
done
|
||||
|
||||
header "Result"
|
||||
if (( ${#failed[@]} == 0 )); then
|
||||
ok "${#suite[@]} contract(s) passed"
|
||||
if (( safe )); then
|
||||
printf 'Skipped %d desktop-hijacking contract(s) (%s).\n' "$skipped" "$DESKTOP_HIJACKING_LEDGER"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
err "${#failed[@]} of ${#suite[@]} failed:"
|
||||
printf ' %s\n' "${failed[@]}" >&2
|
||||
warn "Run one on its own to see why: ${BOLD}${PANAMA_DIR}/tests/<name>${RESET}"
|
||||
if (( safe )); then
|
||||
printf 'Skipped %d desktop-hijacking contract(s) (%s).\n' "$skipped" "$DESKTOP_HIJACKING_LEDGER"
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
)
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Command: contracts
|
||||
@@ -577,8 +671,8 @@ cmd_test() {
|
||||
# a real answer -- exit 1 so a script can tell the difference -- but it is a
|
||||
# statement about this search, not about the file.
|
||||
#
|
||||
# Each hit is marked from tests/desktop-hijacking, so the output also answers
|
||||
# "and can I run them right now".
|
||||
# Each hit is labeled from tests/contracts.manifest, so the output also answers
|
||||
# which boundary the matching contract reaches.
|
||||
cmd_contracts() {
|
||||
local target="${1:-}"
|
||||
if [[ -z "$target" ]]; then
|
||||
@@ -621,11 +715,12 @@ cmd_contracts() {
|
||||
suffix="${suffix#*/}"
|
||||
done
|
||||
|
||||
local -A hijacking=()
|
||||
local entry
|
||||
while IFS= read -r entry; do
|
||||
hijacking["$entry"]=1
|
||||
done < <(hijacking_entries)
|
||||
require_contract_manifest || return 1
|
||||
local -A manifest_capabilities=()
|
||||
local capabilities
|
||||
while IFS=$'\t' read -r rel capabilities; do
|
||||
manifest_capabilities["$rel"]="$capabilities"
|
||||
done < <(contract_manifest_entries)
|
||||
|
||||
# The same collection `test` runs, so anything named here is something the
|
||||
# runner would actually execute.
|
||||
@@ -643,11 +738,11 @@ cmd_contracts() {
|
||||
fi
|
||||
|
||||
for rel in "${hits[@]}"; do
|
||||
if [[ -n "${hijacking[$rel]:-}" ]]; then
|
||||
printf '%s [desktop]\n' "$rel"
|
||||
else
|
||||
printf '%s [safe]\n' "$rel"
|
||||
fi
|
||||
[[ -n "${manifest_capabilities[$rel]:-}" ]] || {
|
||||
err "Contract has no manifest capability label: $rel"
|
||||
return 1
|
||||
}
|
||||
printf '%s [%s]\n' "$rel" "${manifest_capabilities[$rel]}"
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user