Test: Classify every contract capability
This commit is contained in:
Gabriel Brown
Executable
+172
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Every runner-visible contract has one capability classification in
|
||||
# tests/contracts.manifest. The manifest is deliberately complete: callers can
|
||||
# decide what is safe to run without rediscovering test behaviour themselves.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
repo_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
manifest="$repo_dir/tests/contracts.manifest"
|
||||
|
||||
discover_contracts() {
|
||||
discovered_contracts=()
|
||||
while IFS= read -r path; do
|
||||
[[ -x "$path" || "$path" == *_test.py ]] || continue
|
||||
discovered_contracts+=("tests/${path#"$repo_dir/tests/"}")
|
||||
done < <(find "$repo_dir/tests" -type f \
|
||||
-not -path '*/fixtures/*' -not -path '*__pycache__*' | sort)
|
||||
}
|
||||
|
||||
validate_manifest() {
|
||||
local candidate="$1"
|
||||
local -n expected_contracts="$2"
|
||||
local line capabilities path extra previous_was_comment=0
|
||||
local -a capability_list=()
|
||||
local -A manifest_paths=() capability_counts=()
|
||||
local previous_path=""
|
||||
|
||||
validation_findings=()
|
||||
validation_note() { validation_findings+=("$1"); }
|
||||
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
if [[ "$line" =~ ^[[:space:]]*# ]]; then
|
||||
previous_was_comment=1
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$line" =~ ^[[:space:]]*$ ]]; then
|
||||
previous_was_comment=0
|
||||
continue
|
||||
fi
|
||||
|
||||
IFS=$' \t' read -r capabilities path extra <<<"$line"
|
||||
if [[ -z "${capabilities:-}" || -z "${path:-}" || -n "${extra:-}" ]]; then
|
||||
validation_note "manifest line is not exactly two fields: $line"
|
||||
previous_was_comment=0
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -n "$previous_path" && "$path" < "$previous_path" ]]; then
|
||||
validation_note 'paths are not lexicographically sorted'
|
||||
fi
|
||||
previous_path="$path"
|
||||
|
||||
if [[ -n "${manifest_paths[$path]:-}" ]]; then
|
||||
validation_note "duplicate path $path"
|
||||
fi
|
||||
manifest_paths["$path"]=1
|
||||
|
||||
IFS=',' read -r -a capability_list <<<"$capabilities"
|
||||
local -A line_capabilities=()
|
||||
local capability
|
||||
for capability in "${capability_list[@]}"; do
|
||||
if [[ -z "$capability" ]]; then
|
||||
validation_note "empty capability on $path"
|
||||
continue
|
||||
fi
|
||||
if [[ -n "${line_capabilities[$capability]:-}" ]]; then
|
||||
validation_note "duplicate capability $capability on $path"
|
||||
fi
|
||||
line_capabilities["$capability"]=1
|
||||
case "$capability" in
|
||||
hermetic|live-host|live-compositor|live-desktop|network|privileged)
|
||||
capability_counts["$capability"]=1
|
||||
;;
|
||||
*) validation_note "unknown capability $capability on $path" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -n "${line_capabilities[hermetic]:-}" && ${#line_capabilities[@]} -ne 1 ]]; then
|
||||
validation_note "hermetic must appear alone on $path"
|
||||
fi
|
||||
|
||||
if [[ "$capabilities" != hermetic && "$previous_was_comment" -ne 1 ]]; then
|
||||
validation_note "$path is non-hermetic but lacks a directly preceding comment"
|
||||
fi
|
||||
previous_was_comment=0
|
||||
done < "$candidate"
|
||||
|
||||
local expected
|
||||
for expected in "${!expected_contracts[@]}"; do
|
||||
[[ -n "${manifest_paths[$expected]:-}" ]] || validation_note "missing contract $expected"
|
||||
done
|
||||
for path in "${!manifest_paths[@]}"; do
|
||||
[[ -n "${expected_contracts[$path]:-}" ]] || validation_note "stale manifest path $path"
|
||||
done
|
||||
|
||||
for capability in live-host live-compositor live-desktop network; do
|
||||
[[ -n "${capability_counts[$capability]:-}" ]] || validation_note "manifest has no $capability contract"
|
||||
done
|
||||
|
||||
if (( ${#validation_findings[@]} > 0 )); then
|
||||
printf 'contract manifest: %d finding(s)\n' "${#validation_findings[@]}" >&2
|
||||
printf ' - %s\n' "${validation_findings[@]}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
run_parser_fixture() {
|
||||
local label="$1" expected_message="$2" contents="$3" output fixture
|
||||
shift 3
|
||||
local -A fixture_paths=()
|
||||
local fixture_path
|
||||
for fixture_path in "$@"; do
|
||||
fixture_paths["$fixture_path"]=1
|
||||
done
|
||||
|
||||
fixture="$(mktemp)"
|
||||
printf '%s' "$contents" > "$fixture"
|
||||
if output="$(validate_manifest "$fixture" fixture_paths 2>&1)"; then
|
||||
printf 'contract manifest: parser fixture %s unexpectedly passed\n' "$label" >&2
|
||||
rm -f "$fixture"
|
||||
return 1
|
||||
fi
|
||||
rm -f "$fixture"
|
||||
|
||||
if ! grep -Fq "$expected_message" <<<"$output"; then
|
||||
printf 'contract manifest: parser fixture %s did not name %q: %s\n' \
|
||||
"$label" "$expected_message" "$output" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_parser_fixtures() {
|
||||
run_parser_fixture missing-contract 'missing contract tests/b' \
|
||||
$'hermetic tests/a\n' tests/a tests/b || return 1
|
||||
run_parser_fixture stale-path 'stale manifest path tests/stale' \
|
||||
$'hermetic tests/a\nhermetic tests/stale\n' tests/a || return 1
|
||||
run_parser_fixture duplicate-path 'duplicate path tests/a' \
|
||||
$'hermetic tests/a\nhermetic tests/a\n' tests/a || return 1
|
||||
run_parser_fixture unknown-capability 'unknown capability unknown on tests/a' \
|
||||
$'# Reads an external thing.\nunknown tests/a\n' tests/a || return 1
|
||||
run_parser_fixture mixed-hermetic 'hermetic must appear alone on tests/a' \
|
||||
$'# Uses the network.\nhermetic,network tests/a\n' tests/a || return 1
|
||||
run_parser_fixture unsorted-paths 'paths are not lexicographically sorted' \
|
||||
$'hermetic tests/b\nhermetic tests/a\n' tests/a tests/b || return 1
|
||||
run_parser_fixture uncommented-non-hermetic \
|
||||
'tests/a is non-hermetic but lacks a directly preceding comment' \
|
||||
$'network tests/a\n' tests/a || return 1
|
||||
}
|
||||
|
||||
[[ -r "$manifest" ]] || {
|
||||
printf 'contract manifest: %s is missing\n' "$manifest" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
discover_contracts
|
||||
declare -A discovered_paths=()
|
||||
for path in "${discovered_contracts[@]}"; do
|
||||
discovered_paths["$path"]=1
|
||||
done
|
||||
|
||||
status=0
|
||||
validate_manifest "$manifest" discovered_paths || status=1
|
||||
run_parser_fixtures || status=1
|
||||
|
||||
(( status == 0 )) || exit 1
|
||||
|
||||
printf 'contract manifest: PASS (%d discovered; %d manifested)\n' \
|
||||
"${#discovered_contracts[@]}" "${#discovered_paths[@]}"
|
||||
Reference in New Issue
Block a user