eric/ansible-role-notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify the Notarius integration tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-09 04:30:54 +00:00
parent eb4f5110b2
commit cfecf37261
5 changed files with 63 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
README.md
View File

@@ -67,15 +67,15 @@ outside this role.
## Validation ## Validation
The repository-local integration suite installs its pinned collection, runs The repository-local integration suite installs its pinned collection, runs
syntax checking and production-profile lint, verifies the default release, syntax checking and production-profile lint, and exercises the default CLI and
ownership, executable output, minimal configuration, native configuration native configuration validation before requiring clean repeated convergence.
validation, and immediate idempotence. It also covers missing-binary and It verifies an explicitly selected Notarius release, combines missing-binary
checkout rebuilds, caller-supplied Go with dependency management disabled, recovery with a caller-supplied Go executable while dependency management is
metadata repair without rebuilding, caller-side pipeline composition, disabled, and covers one representative configuration with null omission,
field-specific configuration rendering and falsey values, and platform guards. intentional falsey values, caller-side pipeline composition, and native
It mutates only the managed source, binary, configuration, and a private validation. It mutates only the managed source, binary, configuration, and a
temporary directory, then leaves the default release and minimal configuration private temporary directory, then restores the default deployment during
installed. cleanup.
```sh ```sh
tests/integration tests/integration

212
tests/integration
View File

@@ -11,13 +11,8 @@ export ANSIBLE_ROLES_PATH="$(dirname "$repository_root")"
inventory="$repository_root/tests/inventory" inventory="$repository_root/tests/inventory"
converge="$repository_root/tests/converge.yml" converge="$repository_root/tests/converge.yml"
verify="$repository_root/tests/verify.yml"
verify_config="$repository_root/tests/verify-config.yml" verify_config="$repository_root/tests/verify-config.yml"
unsupported="$repository_root/tests/unsupported.yml"
ansible_playbook="$workspace_venv/bin/ansible-playbook" ansible_playbook="$workspace_venv/bin/ansible-playbook"
default_commit=77ae132e5609916da40dcd4fcc749c000a63cbac
previous_commit=61436d7c186af57b06004e090ceac898e83cdb71
source_dir=/usr/local/src/notarius
binary_path=/usr/local/bin/notarius binary_path=/usr/local/bin/notarius
config_path=/usr/local/etc/notarius/config.yml config_path=/usr/local/etc/notarius/config.yml
@@ -34,34 +29,14 @@ require_clean_recap() {
grep -Eq 'changed=0 .*failed=0' "$1" grep -Eq 'changed=0 .*failed=0' "$1"
} }
require_changed_recap() {
grep -Eq 'changed=[1-9][0-9]* .*failed=0' "$1"
}
run_converge() { run_converge() {
output_file=$1 output_file=$1
shift shift
"$ansible_playbook" -i "$inventory" "$@" "$converge" | tee "$output_file" "$ansible_playbook" -i "$inventory" "$@" "$converge" | tee "$output_file"
} }
verify_release() { validate_config() {
expected_version=$1 "$binary_path" config validate --config "$config_path"
expected_commit=$2
"$ansible_playbook" -i "$inventory" \
-e "notarius_expected_version=$expected_version" \
-e "notarius_expected_commit=$expected_commit" \
"$verify"
}
verify_rendered_config() {
"$ansible_playbook" -i "$inventory" \
-e "{\"notarius_expected_config\":$1}" \
"$verify_config"
}
require_configuration_change_without_build() {
grep -A1 -E 'TASK .*Build Notarius' "$1" | grep -q '^skipping:'
grep -A1 -E 'TASK .*Write Notarius configuration' "$1" | grep -q '^changed:'
} }
cd "$repository_root" cd "$repository_root"
@@ -69,172 +44,49 @@ cd "$repository_root"
-r collections/requirements.yml \ -r collections/requirements.yml \
-p "$ANSIBLE_COLLECTIONS_PATH" -p "$ANSIBLE_COLLECTIONS_PATH"
"$ansible_playbook" -i "$inventory" --syntax-check "$converge" "$ansible_playbook" -i "$inventory" --syntax-check "$converge"
"$workspace_venv/bin/ansible-lint" \ "$workspace_venv/bin/ansible-lint" "$converge" "$verify_config"
"$converge" "$verify" "$verify_config" "$unsupported"
baseline_log="$test_tmp/baseline.log" baseline_log="$test_tmp/baseline.log"
run_converge "$baseline_log" run_converge "$baseline_log"
verify_release v0.6.0 "$default_commit" "$binary_path" --version
verify_rendered_config '{"version":4}' validate_config
repeat_log="$test_tmp/repeat.log" baseline_repeat_log="$test_tmp/baseline-repeat.log"
run_converge "$repeat_log" run_converge "$baseline_repeat_log"
require_clean_recap "$repeat_log" require_clean_recap "$baseline_repeat_log"
sudo rm -f -- "$binary_path" selected_release_log="$test_tmp/selected-release.log"
missing_binary_log="$test_tmp/missing-binary.log" run_converge "$selected_release_log" -e notarius_version=v0.5.0
run_converge "$missing_binary_log" [[ "$("$binary_path" --version)" == 'notarius v0.5.0' ]]
require_changed_recap "$missing_binary_log"
grep -A1 -E 'TASK .*Build Notarius' "$missing_binary_log" | grep -q '^changed:'
missing_binary_repeat_log="$test_tmp/missing-binary-repeat.log"
run_converge "$missing_binary_repeat_log"
require_clean_recap "$missing_binary_repeat_log"
custom_go_binary="$test_tmp/go" custom_go_binary="$test_tmp/go"
custom_go_invocation_log="$test_tmp/custom-go-invocation.log" custom_go_invocations="$test_tmp/go-invocations.log"
printf '%s\n' \ printf '%s\n' \
'#!/usr/bin/env bash' \ '#!/usr/bin/env bash' \
"printf 'invoked\\n' >> '$custom_go_invocation_log'" \ 'set -euo pipefail' \
"printf '%s\\n' \"\$*\" >> '$custom_go_invocations'" \
'exec /snap/bin/go "$@"' > "$custom_go_binary" 'exec /snap/bin/go "$@"' > "$custom_go_binary"
chmod 0755 "$custom_go_binary" chmod 0755 "$custom_go_binary"
sudo rm -f -- "$binary_path" sudo rm -f -- "$binary_path"
custom_go_log="$test_tmp/custom-go.log" recovery_log="$test_tmp/recovery.log"
run_converge "$custom_go_log" \ run_converge "$recovery_log" \
-e '{"notarius_manage_build_dependencies":false}' \ -e notarius_version=v0.5.0 \
-e '{"notarius_manage_build_dependencies": false}' \
-e "notarius_go_binary=$custom_go_binary" -e "notarius_go_binary=$custom_go_binary"
require_changed_recap "$custom_go_log" grep -q . "$custom_go_invocations"
grep -A1 -E 'TASK .*Build Notarius' "$custom_go_log" | grep -q '^changed:' [[ "$("$binary_path" --version)" == 'notarius v0.5.0' ]]
[[ -s "$custom_go_invocation_log" ]] recovery_repeat_log="$test_tmp/recovery-repeat.log"
custom_go_repeat_log="$test_tmp/custom-go-repeat.log" run_converge "$recovery_repeat_log" \
run_converge "$custom_go_repeat_log" \ -e notarius_version=v0.5.0 \
-e '{"notarius_manage_build_dependencies":false}' \ -e '{"notarius_manage_build_dependencies": false}' \
-e "notarius_go_binary=$custom_go_binary" -e "notarius_go_binary=$custom_go_binary"
require_clean_recap "$custom_go_repeat_log" require_clean_recap "$recovery_repeat_log"
sudo chmod 0700 "$binary_path"
metadata_log="$test_tmp/metadata.log"
run_converge "$metadata_log"
require_changed_recap "$metadata_log"
grep -A1 -E 'TASK .*Build Notarius' "$metadata_log" | grep -q '^skipping:'
grep -A3 -E 'TASK .*Converge installed metadata' "$metadata_log" | \
grep -q "^changed:.*'path': '$binary_path'"
metadata_repeat_log="$test_tmp/metadata-repeat.log"
run_converge "$metadata_repeat_log"
require_clean_recap "$metadata_repeat_log"
previous_release_log="$test_tmp/previous-release.log"
run_converge "$previous_release_log" -e notarius_version=v0.5.0
require_changed_recap "$previous_release_log"
verify_release v0.5.0 "$previous_commit"
previous_repeat_log="$test_tmp/previous-repeat.log"
run_converge "$previous_repeat_log" -e notarius_version=v0.5.0
require_clean_recap "$previous_repeat_log"
restored_release_log="$test_tmp/restored-release.log"
run_converge "$restored_release_log"
require_changed_recap "$restored_release_log"
verify_release v0.6.0 "$default_commit"
restored_repeat_log="$test_tmp/restored-repeat.log"
run_converge "$restored_repeat_log"
require_clean_recap "$restored_repeat_log"
representative_config='{"notarius_promptkit_local_backend_endpoint":"http://localhost:8000/v1","notarius_promptkit_local_backend_concurrency_limit":2,"notarius_pipelines":{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}},"notarius_concurrency_total_llm":2,"notarius_concurrency_stage_workers_extract":2,"notarius_output_directory":"/tmp/notarius-output","notarius_cache_chunk_plans_mode":"refresh","notarius_cache_chunk_plans_directory":"/tmp/notarius-cache/chunk-plans","notarius_cache_checkpoints_enabled":true,"notarius_cache_checkpoints_directory":"/tmp/notarius-cache/checkpoints","notarius_debug_directory":"/tmp/notarius-debug"}'
representative_expected='{"version":4,"promptkit":{"local_backend":{"endpoint":"http://localhost:8000/v1","concurrency_limit":2}},"pipelines":{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}},"concurrency":{"total_llm":2,"stage_workers":{"extract":2}},"output":{"directory":"/tmp/notarius-output"},"cache":{"chunk_plans":{"mode":"refresh","directory":"/tmp/notarius-cache/chunk-plans"},"checkpoints":{"enabled":true,"directory":"/tmp/notarius-cache/checkpoints"}},"debug":{"directory":"/tmp/notarius-debug"}}'
representative_log="$test_tmp/representative.log"
run_converge "$representative_log" -e "$representative_config"
require_changed_recap "$representative_log"
require_configuration_change_without_build "$representative_log"
verify_rendered_config "$representative_expected"
representative_repeat_log="$test_tmp/representative-repeat.log"
run_converge "$representative_repeat_log" -e "$representative_config"
require_clean_recap "$representative_repeat_log"
profile_dir_config='{"notarius_promptkit_profile_dir":"/usr/local/src/notarius/examples/profiles"}'
profile_dir_expected='{"version":4,"promptkit":{"profile_dir":"/usr/local/src/notarius/examples/profiles"}}'
profile_dir_log="$test_tmp/profile-dir.log"
run_converge "$profile_dir_log" -e "$profile_dir_config"
require_changed_recap "$profile_dir_log"
require_configuration_change_without_build "$profile_dir_log"
verify_rendered_config "$profile_dir_expected"
profile_dir_repeat_log="$test_tmp/profile-dir-repeat.log"
run_converge "$profile_dir_repeat_log" -e "$profile_dir_config"
require_clean_recap "$profile_dir_repeat_log"
profile_file_config='{"notarius_promptkit_profile_file":"/usr/local/src/notarius/examples/profiles/dnd-extraction.yml"}'
profile_file_expected='{"version":4,"promptkit":{"profile_file":"/usr/local/src/notarius/examples/profiles/dnd-extraction.yml"}}'
profile_file_log="$test_tmp/profile-file.log"
run_converge "$profile_file_log" -e "$profile_file_config"
require_changed_recap "$profile_file_log"
require_configuration_change_without_build "$profile_file_log"
verify_rendered_config "$profile_file_expected"
profile_file_repeat_log="$test_tmp/profile-file-repeat.log"
run_converge "$profile_file_repeat_log" -e "$profile_file_config"
require_clean_recap "$profile_file_repeat_log"
falsey_config='{"notarius_promptkit_local_backend_endpoint":"http://localhost:8000/v1","notarius_promptkit_local_backend_concurrency_limit":0,"notarius_pipelines":{},"notarius_cache_chunk_plans_directory":"","notarius_cache_checkpoints_enabled":false}'
falsey_expected='{"version":4,"promptkit":{"local_backend":{"endpoint":"http://localhost:8000/v1","concurrency_limit":0}},"pipelines":{},"cache":{"chunk_plans":{"directory":""},"checkpoints":{"enabled":false}}}'
falsey_log="$test_tmp/falsey.log"
run_converge "$falsey_log" -e "$falsey_config"
require_changed_recap "$falsey_log"
require_configuration_change_without_build "$falsey_log"
verify_rendered_config "$falsey_expected"
falsey_repeat_log="$test_tmp/falsey-repeat.log"
run_converge "$falsey_repeat_log" -e "$falsey_config"
require_clean_recap "$falsey_repeat_log"
pipeline_group_one='{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}' pipeline_group_one='{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}'
pipeline_group_two='{"dnd-archive":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}' pipeline_group_two='{"dnd-archive":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}'
pipeline_composition_config="{\"notarius_pipeline_group_one\":$pipeline_group_one,\"notarius_pipeline_group_two\":$pipeline_group_two,\"notarius_pipelines\":\"{{ notarius_pipeline_group_one | combine(notarius_pipeline_group_two) }}\"}" representative_config="{\"notarius_pipeline_group_one\":$pipeline_group_one,\"notarius_pipeline_group_two\":$pipeline_group_two,\"notarius_pipelines\":\"{{ notarius_pipeline_group_one | combine(notarius_pipeline_group_two) }}\",\"notarius_config_version\":4,\"notarius_promptkit_local_backend_endpoint\":\"http://localhost:8000/v1\",\"notarius_promptkit_local_backend_concurrency_limit\":0,\"notarius_cache_checkpoints_enabled\":false,\"notarius_cache_chunk_plans_directory\":\"\",\"notarius_output_directory\":\"/tmp/notarius-output\",\"notarius_promptkit_profile_dir\":null,\"notarius_promptkit_profile_file\":null,\"notarius_debug_directory\":null}"
pipeline_composition_expected='{"version":4,"pipelines":{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}},"dnd-archive":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}}' representative_log="$test_tmp/representative.log"
pipeline_composition_log="$test_tmp/pipeline-composition.log" run_converge "$representative_log" -e "$representative_config"
run_converge "$pipeline_composition_log" -e "$pipeline_composition_config" "$ansible_playbook" -i "$inventory" "$verify_config"
require_changed_recap "$pipeline_composition_log" representative_repeat_log="$test_tmp/representative-repeat.log"
require_configuration_change_without_build "$pipeline_composition_log" run_converge "$representative_repeat_log" -e "$representative_config"
verify_rendered_config "$pipeline_composition_expected" require_clean_recap "$representative_repeat_log"
pipeline_composition_repeat_log="$test_tmp/pipeline-composition-repeat.log"
run_converge "$pipeline_composition_repeat_log" -e "$pipeline_composition_config"
require_clean_recap "$pipeline_composition_repeat_log"
default_config_log="$test_tmp/default-config.log"
run_converge "$default_config_log"
require_changed_recap "$default_config_log"
require_configuration_change_without_build "$default_config_log"
verify_release v0.6.0 "$default_commit"
verify_rendered_config '{"version":4}'
default_config_repeat_log="$test_tmp/default-config-repeat.log"
run_converge "$default_config_repeat_log"
require_clean_recap "$default_config_repeat_log"
platform_cases=(
'Debian 26.04 x86_64 distribution'
'Ubuntu 24.04 x86_64 release'
'Ubuntu 26.04 riscv64 architecture'
)
for platform_case in "${platform_cases[@]}"; do
read -r distribution release architecture label <<<"$platform_case"
case_root="$test_tmp/unsupported-$label"
case_source="$case_root/source"
case_binary="$case_root/bin/notarius"
case_config="$case_root/etc/config.yml"
case_log="$test_tmp/unsupported-$label.log"
if "$ansible_playbook" -i "$inventory" \
-e "test_distribution=$distribution" \
-e "test_distribution_version=$release" \
-e "test_architecture=$architecture" \
-e "notarius_source_dir=$case_source" \
-e "notarius_binary_path=$case_binary" \
-e "notarius_config_path=$case_config" \
"$unsupported" | tee "$case_log"; then
echo "Unsupported $label case unexpectedly succeeded" >&2
exit 1
fi
grep -Eq 'failed=1' "$case_log"
[[ ! -e "$case_source" ]]
[[ ! -e "$case_binary" ]]
[[ ! -e "$case_config" ]]
done
[[ "$(sudo git -C "$source_dir" rev-parse HEAD)" == "$default_commit" ]]
[[ "$(sudo stat -c '%U:%G %a' "$binary_path")" == 'root:root 755' ]]
[[ "$(sudo stat -c '%U:%G %a' "$config_path")" == 'root:root 644' ]]
[[ "$("$binary_path" --version)" == 'notarius v0.6.0' ]]
"$binary_path" config validate --config "$config_path"

11
tests/unsupported.yml
View File

@@ -1,11 +0,0 @@
---
- name: Exercise Notarius platform mutation guard
hosts: localhost
gather_facts: false
vars:
ansible_facts:
distribution: "{{ test_distribution }}"
distribution_version: "{{ test_distribution_version }}"
architecture: "{{ test_architecture }}"
roles:
- role: ansible-role-notarius

25
tests/verify-config.yml
View File

@@ -7,12 +7,31 @@
become: true become: true
ansible.builtin.slurp: ansible.builtin.slurp:
src: /usr/local/etc/notarius/config.yml src: /usr/local/etc/notarius/config.yml
register: notarius_rendered_config register: notarius_rendered_config_file
- name: Check rendered configuration - name: Check representative rendered configuration
ansible.builtin.assert: ansible.builtin.assert:
that: that:
- '(notarius_rendered_config.content | b64decode | from_yaml) == notarius_expected_config' - notarius_rendered_config.version == 4
- notarius_rendered_config.promptkit.local_backend.endpoint == "http://localhost:8000/v1"
- notarius_rendered_config.promptkit.local_backend.concurrency_limit == 0
- notarius_rendered_config.cache.checkpoints.enabled is false
- notarius_rendered_config.cache.chunk_plans.directory == ""
- notarius_rendered_config.output.directory == "/tmp/notarius-output"
- notarius_rendered_config.pipelines["dnd-session"] == notarius_expected_pipeline
- notarius_rendered_config.pipelines["dnd-archive"] == notarius_expected_pipeline
- notarius_rendered_config.promptkit.profile_dir is not defined
- notarius_rendered_config.promptkit.profile_file is not defined
- notarius_rendered_config.debug is not defined
vars:
notarius_rendered_config: "{{ notarius_rendered_config_file.content | b64decode | from_yaml }}"
notarius_expected_pipeline:
llm_profile: dnd-extraction
input: seriatim
artifacts:
spells:
extract: dnd/spells
normalize: dnd/spells
- name: Validate rendered configuration - name: Validate rendered configuration
ansible.builtin.command: ansible.builtin.command:

83
tests/verify.yml
View File

@@ -1,83 +0,0 @@
---
- name: Verify Notarius installation
hosts: localhost
gather_facts: false
tasks:
- name: Read the selected source commit
become: true
ansible.builtin.command:
argv:
- git
- -C
- /usr/local/src/notarius
- rev-parse
- HEAD
register: notarius_commit
changed_when: false
- name: Inspect source ownership
become: true
ansible.builtin.command:
argv:
- find
- /usr/local/src/notarius
- -not
- -user
- root
- -o
- -not
- -group
- root
register: notarius_source_ownership
changed_when: false
- name: Inspect binary metadata
become: true
ansible.builtin.stat:
path: /usr/local/bin/notarius
register: notarius_binary_metadata
- name: Inspect configuration metadata
become: true
ansible.builtin.stat:
path: /usr/local/etc/notarius/config.yml
register: notarius_config_metadata
- name: Read default configuration
become: true
ansible.builtin.slurp:
src: /usr/local/etc/notarius/config.yml
register: notarius_default_config
- name: Read the installed version
ansible.builtin.command:
argv:
- /usr/local/bin/notarius
- --version
register: notarius_version_output
changed_when: false
- name: Validate Notarius configuration
ansible.builtin.command:
argv:
- /usr/local/bin/notarius
- config
- validate
- --config
- /usr/local/etc/notarius/config.yml
changed_when: false
- name: Assert the installation contract
ansible.builtin.assert:
that:
- notarius_commit.stdout == notarius_expected_commit
- notarius_source_ownership.stdout == ""
- notarius_binary_metadata.stat.isreg
- notarius_binary_metadata.stat.pw_name == "root"
- notarius_binary_metadata.stat.gr_name == "root"
- notarius_binary_metadata.stat.mode == "0755"
- notarius_config_metadata.stat.pw_name == "root"
- notarius_config_metadata.stat.gr_name == "root"
- notarius_config_metadata.stat.mode == "0644"
- '(notarius_default_config.content | b64decode | from_yaml) == {"version": 4}'
- notarius_version_output.stdout == "notarius " ~ notarius_expected_version