Test: Classify every contract capability
This commit is contained in:
Gabriel Brown
Executable
+172
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Every runner-visible contract has one capability classification in
|
||||
# tests/contracts.manifest. The manifest is deliberately complete: callers can
|
||||
# decide what is safe to run without rediscovering test behaviour themselves.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
manifest="$repo_dir/tests/contracts.manifest"
|
||||
|
||||
discover_contracts() {
|
||||
discovered_contracts=()
|
||||
while IFS= read -r path; do
|
||||
[[ -x "$path" || "$path" == *_test.py ]] || continue
|
||||
discovered_contracts+=("tests/${path#"$repo_dir/tests/"}")
|
||||
done < <(find "$repo_dir/tests" -type f \
|
||||
-not -path '*/fixtures/*' -not -path '*__pycache__*' | sort)
|
||||
}
|
||||
|
||||
validate_manifest() {
|
||||
local candidate="$1"
|
||||
local -n expected_contracts="$2"
|
||||
local line capabilities path extra previous_was_comment=0
|
||||
local -a capability_list=()
|
||||
local -A manifest_paths=() capability_counts=()
|
||||
local previous_path=""
|
||||
|
||||
validation_findings=()
|
||||
validation_note() { validation_findings+=("$1"); }
|
||||
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
if [[ "$line" =~ ^[[:space:]]*# ]]; then
|
||||
previous_was_comment=1
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$line" =~ ^[[:space:]]*$ ]]; then
|
||||
previous_was_comment=0
|
||||
continue
|
||||
fi
|
||||
|
||||
IFS=$' \t' read -r capabilities path extra <<<"$line"
|
||||
if [[ -z "${capabilities:-}" || -z "${path:-}" || -n "${extra:-}" ]]; then
|
||||
validation_note "manifest line is not exactly two fields: $line"
|
||||
previous_was_comment=0
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -n "$previous_path" && "$path" < "$previous_path" ]]; then
|
||||
validation_note 'paths are not lexicographically sorted'
|
||||
fi
|
||||
previous_path="$path"
|
||||
|
||||
if [[ -n "${manifest_paths[$path]:-}" ]]; then
|
||||
validation_note "duplicate path $path"
|
||||
fi
|
||||
manifest_paths["$path"]=1
|
||||
|
||||
IFS=',' read -r -a capability_list <<<"$capabilities"
|
||||
local -A line_capabilities=()
|
||||
local capability
|
||||
for capability in "${capability_list[@]}"; do
|
||||
if [[ -z "$capability" ]]; then
|
||||
validation_note "empty capability on $path"
|
||||
continue
|
||||
fi
|
||||
if [[ -n "${line_capabilities[$capability]:-}" ]]; then
|
||||
validation_note "duplicate capability $capability on $path"
|
||||
fi
|
||||
line_capabilities["$capability"]=1
|
||||
case "$capability" in
|
||||
hermetic|live-host|live-compositor|live-desktop|network|privileged)
|
||||
capability_counts["$capability"]=1
|
||||
;;
|
||||
*) validation_note "unknown capability $capability on $path" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -n "${line_capabilities[hermetic]:-}" && ${#line_capabilities[@]} -ne 1 ]]; then
|
||||
validation_note "hermetic must appear alone on $path"
|
||||
fi
|
||||
|
||||
if [[ "$capabilities" != hermetic && "$previous_was_comment" -ne 1 ]]; then
|
||||
validation_note "$path is non-hermetic but lacks a directly preceding comment"
|
||||
fi
|
||||
previous_was_comment=0
|
||||
done < "$candidate"
|
||||
|
||||
local expected
|
||||
for expected in "${!expected_contracts[@]}"; do
|
||||
[[ -n "${manifest_paths[$expected]:-}" ]] || validation_note "missing contract $expected"
|
||||
done
|
||||
for path in "${!manifest_paths[@]}"; do
|
||||
[[ -n "${expected_contracts[$path]:-}" ]] || validation_note "stale manifest path $path"
|
||||
done
|
||||
|
||||
for capability in live-host live-compositor live-desktop network; do
|
||||
[[ -n "${capability_counts[$capability]:-}" ]] || validation_note "manifest has no $capability contract"
|
||||
done
|
||||
|
||||
if (( ${#validation_findings[@]} > 0 )); then
|
||||
printf 'contract manifest: %d finding(s)\n' "${#validation_findings[@]}" >&2
|
||||
printf ' - %s\n' "${validation_findings[@]}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
run_parser_fixture() {
|
||||
local label="$1" expected_message="$2" contents="$3" output fixture
|
||||
shift 3
|
||||
local -A fixture_paths=()
|
||||
local fixture_path
|
||||
for fixture_path in "$@"; do
|
||||
fixture_paths["$fixture_path"]=1
|
||||
done
|
||||
|
||||
fixture="$(mktemp)"
|
||||
printf '%s' "$contents" > "$fixture"
|
||||
if output="$(validate_manifest "$fixture" fixture_paths 2>&1)"; then
|
||||
printf 'contract manifest: parser fixture %s unexpectedly passed\n' "$label" >&2
|
||||
rm -f "$fixture"
|
||||
return 1
|
||||
fi
|
||||
rm -f "$fixture"
|
||||
|
||||
if ! grep -Fq "$expected_message" <<<"$output"; then
|
||||
printf 'contract manifest: parser fixture %s did not name %q: %s\n' \
|
||||
"$label" "$expected_message" "$output" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_parser_fixtures() {
|
||||
run_parser_fixture missing-contract 'missing contract tests/b' \
|
||||
$'hermetic tests/a\n' tests/a tests/b || return 1
|
||||
run_parser_fixture stale-path 'stale manifest path tests/stale' \
|
||||
$'hermetic tests/a\nhermetic tests/stale\n' tests/a || return 1
|
||||
run_parser_fixture duplicate-path 'duplicate path tests/a' \
|
||||
$'hermetic tests/a\nhermetic tests/a\n' tests/a || return 1
|
||||
run_parser_fixture unknown-capability 'unknown capability unknown on tests/a' \
|
||||
$'# Reads an external thing.\nunknown tests/a\n' tests/a || return 1
|
||||
run_parser_fixture mixed-hermetic 'hermetic must appear alone on tests/a' \
|
||||
$'# Uses the network.\nhermetic,network tests/a\n' tests/a || return 1
|
||||
run_parser_fixture unsorted-paths 'paths are not lexicographically sorted' \
|
||||
$'hermetic tests/b\nhermetic tests/a\n' tests/a tests/b || return 1
|
||||
run_parser_fixture uncommented-non-hermetic \
|
||||
'tests/a is non-hermetic but lacks a directly preceding comment' \
|
||||
$'network tests/a\n' tests/a || return 1
|
||||
}
|
||||
|
||||
[[ -r "$manifest" ]] || {
|
||||
printf 'contract manifest: %s is missing\n' "$manifest" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
discover_contracts
|
||||
declare -A discovered_paths=()
|
||||
for path in "${discovered_contracts[@]}"; do
|
||||
discovered_paths["$path"]=1
|
||||
done
|
||||
|
||||
status=0
|
||||
validate_manifest "$manifest" discovered_paths || status=1
|
||||
run_parser_fixtures || status=1
|
||||
|
||||
(( status == 0 )) || exit 1
|
||||
|
||||
printf 'contract manifest: PASS (%d discovered; %d manifested)\n' \
|
||||
"${#discovered_contracts[@]}" "${#discovered_paths[@]}"
|
||||
@@ -1,184 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# The desktop-hijacking ledger is complete, and `panama test --safe` obeys it.
|
||||
#
|
||||
# `panama test --safe` exists so the suite can be run from inside the session it
|
||||
# tests. That promise is only as good as tests/desktop-hijacking: a contract
|
||||
# that takes over the live shell and is not listed there is run by --safe, and
|
||||
# the desktop goes away in the middle of somebody's work -- with the command
|
||||
# line having just claimed it would not.
|
||||
#
|
||||
# A hand-kept list decays, so it is not trusted on its own. This sweeps tests/
|
||||
# for the shapes a hijacking contract has and fails on any that are missing from
|
||||
# the ledger. The heuristics live here, in the thing that runs, so a new
|
||||
# hijacking contract cannot stay unlisted quietly:
|
||||
#
|
||||
# * it calls `qs ipc call` without booting its own `qs -p` harness, so the
|
||||
# instance answering is the shell you are looking at;
|
||||
# * it restarts panama-quickshell.service;
|
||||
# * it calls a bare `qs kill`, which kills that same shell.
|
||||
#
|
||||
# The sweep is one direction only. The ledger is deliberately larger than what
|
||||
# these three shapes find -- a contract that rotates the real monitor or
|
||||
# rewrites a real xdg-mime default hijacks the session just as thoroughly and
|
||||
# looks like nothing in particular from the outside -- so an entry the sweep
|
||||
# does not reach is not a finding.
|
||||
#
|
||||
# Nothing here runs a listed contract. The one live check drives `--safe` with a
|
||||
# pattern that matches only ledger entries, so the run selects them, skips them
|
||||
# all, and executes nothing.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
ledger="$repo_dir/tests/desktop-hijacking"
|
||||
panama="$repo_dir/bin/panama"
|
||||
|
||||
findings=()
|
||||
note() { findings+=("$1"); }
|
||||
|
||||
[[ -r "$ledger" ]] || { printf 'desktop hijacking contract: %s is missing\n' "$ledger" >&2; exit 1; }
|
||||
|
||||
# ── The entries ──────────────────────────────────────────────────────────────
|
||||
#
|
||||
# Every line names a contract that exists AND that `panama test` would collect.
|
||||
# A listed path the runner never picks up (no executable bit, not a *_test.py)
|
||||
# is skipped by --safe in name only, which reads as protection and is not.
|
||||
|
||||
entries=()
|
||||
commented=()
|
||||
pending_comment=0
|
||||
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
case "$line" in
|
||||
'#'*) pending_comment=1; continue ;;
|
||||
''|[[:space:]]*'') ;;
|
||||
esac
|
||||
trimmed="${line%%#*}"
|
||||
trimmed="${trimmed#"${trimmed%%[![:space:]]*}"}"
|
||||
trimmed="${trimmed%"${trimmed##*[![:space:]]}"}"
|
||||
if [[ -z "$trimmed" ]]; then
|
||||
[[ -z "$line" ]] && pending_comment=0
|
||||
continue
|
||||
fi
|
||||
entries+=("$trimmed")
|
||||
commented+=("$pending_comment")
|
||||
pending_comment=0
|
||||
done < "$ledger"
|
||||
|
||||
(( ${#entries[@]} > 0 )) || note 'the ledger lists no contracts at all'
|
||||
|
||||
for index in "${!entries[@]}"; do
|
||||
entry="${entries[$index]}"
|
||||
path="$repo_dir/$entry"
|
||||
|
||||
[[ "$entry" == tests/* ]] \
|
||||
|| note "\"$entry\" is not a repo-relative path under tests/"
|
||||
|
||||
if [[ ! -e "$path" ]]; then
|
||||
note "the ledger lists $entry, which does not exist"
|
||||
continue
|
||||
fi
|
||||
|
||||
[[ -x "$path" || "$entry" == *_test.py ]] \
|
||||
|| note "$entry is listed but 'panama test' would never collect it, so skipping it protects nothing"
|
||||
|
||||
# The ledger's whole job is saying what a contract does to the session. An
|
||||
# entry with no comment is a path somebody has to go and read.
|
||||
(( commented[index] )) \
|
||||
|| note "$entry is listed with no comment saying what it does to the live session"
|
||||
done
|
||||
|
||||
duplicates="$(printf '%s\n' "${entries[@]}" | sort | uniq -d)"
|
||||
[[ -z "$duplicates" ]] || note "the ledger lists these twice: ${duplicates//$'\n'/, }"
|
||||
|
||||
# ── The honesty sweep ────────────────────────────────────────────────────────
|
||||
#
|
||||
# Whole-line comments are dropped first: contracts discuss `qs ipc call` in
|
||||
# their headers, and a header is not a call. What survives is matched only at a
|
||||
# command position -- start of line, or after a pipe, semicolon, &&, (, or ! --
|
||||
# so the same words quoted inside a grep pattern or a failure message do not
|
||||
# count as driving anything.
|
||||
|
||||
command_position='(^|[|;&({!]|\$\()[[:space:]]*'
|
||||
|
||||
is_listed() {
|
||||
local candidate="$1" listed
|
||||
for listed in "${entries[@]}"; do
|
||||
[[ "$listed" == "$candidate" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
while IFS= read -r file; do
|
||||
[[ -x "$file" || "$file" == *_test.py ]] || continue
|
||||
|
||||
code="$(grep -v '^[[:space:]]*#' "$file")"
|
||||
rel="tests/${file#"$repo_dir"/tests/}"
|
||||
reason=""
|
||||
|
||||
if grep -qE "${command_position}qs[[:space:]]+ipc[[:space:]]+call" <<<"$code"; then
|
||||
# Its own harness means its own Quickshell instance: `qs -p <entry>`
|
||||
# addresses that root, not the shell running the desktop.
|
||||
grep -qE "${command_position}[A-Za-z_]*[[:space:]]*=?[[:space:]]*.*qs[[:space:]]+-p" <<<"$code" \
|
||||
|| reason='calls `qs ipc call` without booting its own `qs -p` harness'
|
||||
fi
|
||||
|
||||
if [[ -z "$reason" ]] && grep -qE "${command_position}systemctl.*restart.*panama-quickshell\.service" <<<"$code"; then
|
||||
reason='restarts panama-quickshell.service'
|
||||
fi
|
||||
|
||||
if [[ -z "$reason" ]] && grep -qE "${command_position}qs[[:space:]]+kill" <<<"$code"; then
|
||||
reason='calls a bare `qs kill`, which stops the live shell'
|
||||
fi
|
||||
|
||||
[[ -n "$reason" ]] || continue
|
||||
is_listed "$rel" \
|
||||
|| note "$rel $reason, but is not in tests/desktop-hijacking"
|
||||
done < <(find "$repo_dir/tests" -type f -not -path '*/fixtures/*' -not -path '*__pycache__*' | sort)
|
||||
|
||||
# ── --safe actually reads it ─────────────────────────────────────────────────
|
||||
#
|
||||
# Static first, because the summary line is the only thing telling a reader that
|
||||
# anything was left out, and a --safe run that silently skips is worse than one
|
||||
# that does not skip at all.
|
||||
|
||||
if [[ ! -r "$panama" ]]; then
|
||||
note 'bin/panama is missing'
|
||||
elif ! test_body="$(sed -n '/^cmd_test()/,/^}/p' "$panama")" || [[ -z "$test_body" ]]; then
|
||||
note 'cmd_test could not be found in bin/panama'
|
||||
else
|
||||
grep -q -- '--safe' <<<"$test_body" \
|
||||
|| note 'cmd_test does not handle --safe'
|
||||
grep -qE 'DESKTOP_HIJACKING_LEDGER|desktop-hijacking' <<<"$test_body" \
|
||||
|| note 'cmd_test never consults the desktop-hijacking ledger, so --safe skips nothing'
|
||||
grep -qF 'desktop-hijacking contract(s)' <<<"$test_body" \
|
||||
|| note '--safe no longer reports how many contracts it skipped'
|
||||
fi
|
||||
|
||||
grep -qF 'tests/desktop-hijacking' "$panama" \
|
||||
|| note 'bin/panama never names tests/desktop-hijacking'
|
||||
|
||||
# Then for real. The pattern is the first ledger entry with its 'tests/' prefix
|
||||
# removed, which cmd_test matches against the full path -- so it selects that
|
||||
# one contract, --safe removes it, and nothing is left to run. A --safe that
|
||||
# ignored the ledger would run it instead, which is the failure this catches.
|
||||
if (( ${#entries[@]} > 0 )) && [[ -x "$panama" ]]; then
|
||||
probe="${entries[0]#tests/}"
|
||||
output="$("$panama" test --safe "$probe" 2>&1)"
|
||||
status=$?
|
||||
if (( status == 0 )); then
|
||||
note "'panama test --safe $probe' ran a ledger-listed contract instead of skipping it"
|
||||
elif ! grep -qF 'desktop-hijacking' <<<"$output"; then
|
||||
note "'panama test --safe $probe' refused without mentioning the ledger: $output"
|
||||
fi
|
||||
fi
|
||||
|
||||
if (( ${#findings[@]} > 0 )); then
|
||||
mapfile -t findings < <(printf '%s\n' "${findings[@]}" | sort -u)
|
||||
printf 'desktop hijacking contract: %d finding(s)\n' "${#findings[@]}" >&2
|
||||
printf ' - %s\n' "${findings[@]}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf 'desktop hijacking contract: PASS (%d contracts listed; the sweep found none unlisted)\n' "${#entries[@]}"
|
||||
Reference in New Issue
Block a user