eric/ansible-role-notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify the Notarius integration tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-09 04:30:54 +00:00
parent eb4f5110b2
commit cfecf37261
5 changed files with 63 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
tests/integration
View File

@@ -11,13 +11,8 @@ export ANSIBLE_ROLES_PATH="$(dirname "$repository_root")"
inventory="$repository_root/tests/inventory"
converge="$repository_root/tests/converge.yml"
verify="$repository_root/tests/verify.yml"
verify_config="$repository_root/tests/verify-config.yml"
unsupported="$repository_root/tests/unsupported.yml"
ansible_playbook="$workspace_venv/bin/ansible-playbook"
default_commit=77ae132e5609916da40dcd4fcc749c000a63cbac
previous_commit=61436d7c186af57b06004e090ceac898e83cdb71
source_dir=/usr/local/src/notarius
binary_path=/usr/local/bin/notarius
config_path=/usr/local/etc/notarius/config.yml
@@ -34,34 +29,14 @@ require_clean_recap() {
grep -Eq 'changed=0 .*failed=0' "$1"
}
require_changed_recap() {
grep -Eq 'changed=[1-9][0-9]* .*failed=0' "$1"
}
run_converge() {
output_file=$1
shift
"$ansible_playbook" -i "$inventory" "$@" "$converge" | tee "$output_file"
}
verify_release() {
expected_version=$1
expected_commit=$2
"$ansible_playbook" -i "$inventory" \
-e "notarius_expected_version=$expected_version" \
-e "notarius_expected_commit=$expected_commit" \
"$verify"
}
verify_rendered_config() {
"$ansible_playbook" -i "$inventory" \
-e "{\"notarius_expected_config\":$1}" \
"$verify_config"
}
require_configuration_change_without_build() {
grep -A1 -E 'TASK .*Build Notarius' "$1" | grep -q '^skipping:'
grep -A1 -E 'TASK .*Write Notarius configuration' "$1" | grep -q '^changed:'
validate_config() {
"$binary_path" config validate --config "$config_path"
}
cd "$repository_root"
@@ -69,172 +44,49 @@ cd "$repository_root"
-r collections/requirements.yml \
-p "$ANSIBLE_COLLECTIONS_PATH"
"$ansible_playbook" -i "$inventory" --syntax-check "$converge"
"$workspace_venv/bin/ansible-lint" \
"$converge" "$verify" "$verify_config" "$unsupported"
"$workspace_venv/bin/ansible-lint" "$converge" "$verify_config"
baseline_log="$test_tmp/baseline.log"
run_converge "$baseline_log"
verify_release v0.6.0 "$default_commit"
verify_rendered_config '{"version":4}'
repeat_log="$test_tmp/repeat.log"
run_converge "$repeat_log"
require_clean_recap "$repeat_log"
"$binary_path" --version
validate_config
baseline_repeat_log="$test_tmp/baseline-repeat.log"
run_converge "$baseline_repeat_log"
require_clean_recap "$baseline_repeat_log"
sudo rm -f -- "$binary_path"
missing_binary_log="$test_tmp/missing-binary.log"
run_converge "$missing_binary_log"
require_changed_recap "$missing_binary_log"
grep -A1 -E 'TASK .*Build Notarius' "$missing_binary_log" | grep -q '^changed:'
missing_binary_repeat_log="$test_tmp/missing-binary-repeat.log"
run_converge "$missing_binary_repeat_log"
require_clean_recap "$missing_binary_repeat_log"
selected_release_log="$test_tmp/selected-release.log"
run_converge "$selected_release_log" -e notarius_version=v0.5.0
[[ "$("$binary_path" --version)" == 'notarius v0.5.0' ]]
custom_go_binary="$test_tmp/go"
custom_go_invocation_log="$test_tmp/custom-go-invocation.log"
custom_go_invocations="$test_tmp/go-invocations.log"
printf '%s\n' \
'#!/usr/bin/env bash' \
"printf 'invoked\\n' >> '$custom_go_invocation_log'" \
'set -euo pipefail' \
"printf '%s\\n' \"\$*\" >> '$custom_go_invocations'" \
'exec /snap/bin/go "$@"' > "$custom_go_binary"
chmod 0755 "$custom_go_binary"
sudo rm -f -- "$binary_path"
custom_go_log="$test_tmp/custom-go.log"
run_converge "$custom_go_log" \
-e '{"notarius_manage_build_dependencies":false}' \
recovery_log="$test_tmp/recovery.log"
run_converge "$recovery_log" \
-e notarius_version=v0.5.0 \
-e '{"notarius_manage_build_dependencies": false}' \
-e "notarius_go_binary=$custom_go_binary"
require_changed_recap "$custom_go_log"
grep -A1 -E 'TASK .*Build Notarius' "$custom_go_log" | grep -q '^changed:'
[[ -s "$custom_go_invocation_log" ]]
custom_go_repeat_log="$test_tmp/custom-go-repeat.log"
run_converge "$custom_go_repeat_log" \
-e '{"notarius_manage_build_dependencies":false}' \
grep -q . "$custom_go_invocations"
[[ "$("$binary_path" --version)" == 'notarius v0.5.0' ]]
recovery_repeat_log="$test_tmp/recovery-repeat.log"
run_converge "$recovery_repeat_log" \
-e notarius_version=v0.5.0 \
-e '{"notarius_manage_build_dependencies": false}' \
-e "notarius_go_binary=$custom_go_binary"
require_clean_recap "$custom_go_repeat_log"
sudo chmod 0700 "$binary_path"
metadata_log="$test_tmp/metadata.log"
run_converge "$metadata_log"
require_changed_recap "$metadata_log"
grep -A1 -E 'TASK .*Build Notarius' "$metadata_log" | grep -q '^skipping:'
grep -A3 -E 'TASK .*Converge installed metadata' "$metadata_log" | \
grep -q "^changed:.*'path': '$binary_path'"
metadata_repeat_log="$test_tmp/metadata-repeat.log"
run_converge "$metadata_repeat_log"
require_clean_recap "$metadata_repeat_log"
previous_release_log="$test_tmp/previous-release.log"
run_converge "$previous_release_log" -e notarius_version=v0.5.0
require_changed_recap "$previous_release_log"
verify_release v0.5.0 "$previous_commit"
previous_repeat_log="$test_tmp/previous-repeat.log"
run_converge "$previous_repeat_log" -e notarius_version=v0.5.0
require_clean_recap "$previous_repeat_log"
restored_release_log="$test_tmp/restored-release.log"
run_converge "$restored_release_log"
require_changed_recap "$restored_release_log"
verify_release v0.6.0 "$default_commit"
restored_repeat_log="$test_tmp/restored-repeat.log"
run_converge "$restored_repeat_log"
require_clean_recap "$restored_repeat_log"
representative_config='{"notarius_promptkit_local_backend_endpoint":"http://localhost:8000/v1","notarius_promptkit_local_backend_concurrency_limit":2,"notarius_pipelines":{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}},"notarius_concurrency_total_llm":2,"notarius_concurrency_stage_workers_extract":2,"notarius_output_directory":"/tmp/notarius-output","notarius_cache_chunk_plans_mode":"refresh","notarius_cache_chunk_plans_directory":"/tmp/notarius-cache/chunk-plans","notarius_cache_checkpoints_enabled":true,"notarius_cache_checkpoints_directory":"/tmp/notarius-cache/checkpoints","notarius_debug_directory":"/tmp/notarius-debug"}'
representative_expected='{"version":4,"promptkit":{"local_backend":{"endpoint":"http://localhost:8000/v1","concurrency_limit":2}},"pipelines":{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}},"concurrency":{"total_llm":2,"stage_workers":{"extract":2}},"output":{"directory":"/tmp/notarius-output"},"cache":{"chunk_plans":{"mode":"refresh","directory":"/tmp/notarius-cache/chunk-plans"},"checkpoints":{"enabled":true,"directory":"/tmp/notarius-cache/checkpoints"}},"debug":{"directory":"/tmp/notarius-debug"}}'
representative_log="$test_tmp/representative.log"
run_converge "$representative_log" -e "$representative_config"
require_changed_recap "$representative_log"
require_configuration_change_without_build "$representative_log"
verify_rendered_config "$representative_expected"
representative_repeat_log="$test_tmp/representative-repeat.log"
run_converge "$representative_repeat_log" -e "$representative_config"
require_clean_recap "$representative_repeat_log"
profile_dir_config='{"notarius_promptkit_profile_dir":"/usr/local/src/notarius/examples/profiles"}'
profile_dir_expected='{"version":4,"promptkit":{"profile_dir":"/usr/local/src/notarius/examples/profiles"}}'
profile_dir_log="$test_tmp/profile-dir.log"
run_converge "$profile_dir_log" -e "$profile_dir_config"
require_changed_recap "$profile_dir_log"
require_configuration_change_without_build "$profile_dir_log"
verify_rendered_config "$profile_dir_expected"
profile_dir_repeat_log="$test_tmp/profile-dir-repeat.log"
run_converge "$profile_dir_repeat_log" -e "$profile_dir_config"
require_clean_recap "$profile_dir_repeat_log"
profile_file_config='{"notarius_promptkit_profile_file":"/usr/local/src/notarius/examples/profiles/dnd-extraction.yml"}'
profile_file_expected='{"version":4,"promptkit":{"profile_file":"/usr/local/src/notarius/examples/profiles/dnd-extraction.yml"}}'
profile_file_log="$test_tmp/profile-file.log"
run_converge "$profile_file_log" -e "$profile_file_config"
require_changed_recap "$profile_file_log"
require_configuration_change_without_build "$profile_file_log"
verify_rendered_config "$profile_file_expected"
profile_file_repeat_log="$test_tmp/profile-file-repeat.log"
run_converge "$profile_file_repeat_log" -e "$profile_file_config"
require_clean_recap "$profile_file_repeat_log"
falsey_config='{"notarius_promptkit_local_backend_endpoint":"http://localhost:8000/v1","notarius_promptkit_local_backend_concurrency_limit":0,"notarius_pipelines":{},"notarius_cache_chunk_plans_directory":"","notarius_cache_checkpoints_enabled":false}'
falsey_expected='{"version":4,"promptkit":{"local_backend":{"endpoint":"http://localhost:8000/v1","concurrency_limit":0}},"pipelines":{},"cache":{"chunk_plans":{"directory":""},"checkpoints":{"enabled":false}}}'
falsey_log="$test_tmp/falsey.log"
run_converge "$falsey_log" -e "$falsey_config"
require_changed_recap "$falsey_log"
require_configuration_change_without_build "$falsey_log"
verify_rendered_config "$falsey_expected"
falsey_repeat_log="$test_tmp/falsey-repeat.log"
run_converge "$falsey_repeat_log" -e "$falsey_config"
require_clean_recap "$falsey_repeat_log"
require_clean_recap "$recovery_repeat_log"
pipeline_group_one='{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}'
pipeline_group_two='{"dnd-archive":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}'
pipeline_composition_config="{\"notarius_pipeline_group_one\":$pipeline_group_one,\"notarius_pipeline_group_two\":$pipeline_group_two,\"notarius_pipelines\":\"{{ notarius_pipeline_group_one | combine(notarius_pipeline_group_two) }}\"}"
pipeline_composition_expected='{"version":4,"pipelines":{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}},"dnd-archive":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}}'
pipeline_composition_log="$test_tmp/pipeline-composition.log"
run_converge "$pipeline_composition_log" -e "$pipeline_composition_config"
require_changed_recap "$pipeline_composition_log"
require_configuration_change_without_build "$pipeline_composition_log"
verify_rendered_config "$pipeline_composition_expected"
pipeline_composition_repeat_log="$test_tmp/pipeline-composition-repeat.log"
run_converge "$pipeline_composition_repeat_log" -e "$pipeline_composition_config"
require_clean_recap "$pipeline_composition_repeat_log"
default_config_log="$test_tmp/default-config.log"
run_converge "$default_config_log"
require_changed_recap "$default_config_log"
require_configuration_change_without_build "$default_config_log"
verify_release v0.6.0 "$default_commit"
verify_rendered_config '{"version":4}'
default_config_repeat_log="$test_tmp/default-config-repeat.log"
run_converge "$default_config_repeat_log"
require_clean_recap "$default_config_repeat_log"
platform_cases=(
'Debian 26.04 x86_64 distribution'
'Ubuntu 24.04 x86_64 release'
'Ubuntu 26.04 riscv64 architecture'
)
for platform_case in "${platform_cases[@]}"; do
read -r distribution release architecture label <<<"$platform_case"
case_root="$test_tmp/unsupported-$label"
case_source="$case_root/source"
case_binary="$case_root/bin/notarius"
case_config="$case_root/etc/config.yml"
case_log="$test_tmp/unsupported-$label.log"
if "$ansible_playbook" -i "$inventory" \
-e "test_distribution=$distribution" \
-e "test_distribution_version=$release" \
-e "test_architecture=$architecture" \
-e "notarius_source_dir=$case_source" \
-e "notarius_binary_path=$case_binary" \
-e "notarius_config_path=$case_config" \
"$unsupported" | tee "$case_log"; then
echo "Unsupported $label case unexpectedly succeeded" >&2
exit 1
fi
grep -Eq 'failed=1' "$case_log"
[[ ! -e "$case_source" ]]
[[ ! -e "$case_binary" ]]
[[ ! -e "$case_config" ]]
done
[[ "$(sudo git -C "$source_dir" rev-parse HEAD)" == "$default_commit" ]]
[[ "$(sudo stat -c '%U:%G %a' "$binary_path")" == 'root:root 755' ]]
[[ "$(sudo stat -c '%U:%G %a' "$config_path")" == 'root:root 644' ]]
[[ "$("$binary_path" --version)" == 'notarius v0.6.0' ]]
"$binary_path" config validate --config "$config_path"
representative_config="{\"notarius_pipeline_group_one\":$pipeline_group_one,\"notarius_pipeline_group_two\":$pipeline_group_two,\"notarius_pipelines\":\"{{ notarius_pipeline_group_one | combine(notarius_pipeline_group_two) }}\",\"notarius_config_version\":4,\"notarius_promptkit_local_backend_endpoint\":\"http://localhost:8000/v1\",\"notarius_promptkit_local_backend_concurrency_limit\":0,\"notarius_cache_checkpoints_enabled\":false,\"notarius_cache_chunk_plans_directory\":\"\",\"notarius_output_directory\":\"/tmp/notarius-output\",\"notarius_promptkit_profile_dir\":null,\"notarius_promptkit_profile_file\":null,\"notarius_debug_directory\":null}"
representative_log="$test_tmp/representative.log"
run_converge "$representative_log" -e "$representative_config"
"$ansible_playbook" -i "$inventory" "$verify_config"
representative_repeat_log="$test_tmp/representative-repeat.log"
run_converge "$representative_repeat_log" -e "$representative_config"
require_clean_recap "$representative_repeat_log"

11
tests/unsupported.yml
View File

@@ -1,11 +0,0 @@
---
- name: Exercise Notarius platform mutation guard
hosts: localhost
gather_facts: false
vars:
ansible_facts:
distribution: "{{ test_distribution }}"
distribution_version: "{{ test_distribution_version }}"
architecture: "{{ test_architecture }}"
roles:
- role: ansible-role-notarius

25
tests/verify-config.yml
View File

@@ -7,12 +7,31 @@
become: true
ansible.builtin.slurp:
src: /usr/local/etc/notarius/config.yml
register: notarius_rendered_config
register: notarius_rendered_config_file
- name: Check rendered configuration
- name: Check representative rendered configuration
ansible.builtin.assert:
that:
- '(notarius_rendered_config.content | b64decode | from_yaml) == notarius_expected_config'
- notarius_rendered_config.version == 4
- notarius_rendered_config.promptkit.local_backend.endpoint == "http://localhost:8000/v1"
- notarius_rendered_config.promptkit.local_backend.concurrency_limit == 0
- notarius_rendered_config.cache.checkpoints.enabled is false
- notarius_rendered_config.cache.chunk_plans.directory == ""
- notarius_rendered_config.output.directory == "/tmp/notarius-output"
- notarius_rendered_config.pipelines["dnd-session"] == notarius_expected_pipeline
- notarius_rendered_config.pipelines["dnd-archive"] == notarius_expected_pipeline
- notarius_rendered_config.promptkit.profile_dir is not defined
- notarius_rendered_config.promptkit.profile_file is not defined
- notarius_rendered_config.debug is not defined
vars:
notarius_rendered_config: "{{ notarius_rendered_config_file.content | b64decode | from_yaml }}"
notarius_expected_pipeline:
llm_profile: dnd-extraction
input: seriatim
artifacts:
spells:
extract: dnd/spells
normalize: dnd/spells
- name: Validate rendered configuration
ansible.builtin.command:

83
tests/verify.yml
View File

@@ -1,83 +0,0 @@
---
- name: Verify Notarius installation
hosts: localhost
gather_facts: false
tasks:
- name: Read the selected source commit
become: true
ansible.builtin.command:
argv:
- git
- -C
- /usr/local/src/notarius
- rev-parse
- HEAD
register: notarius_commit
changed_when: false
- name: Inspect source ownership
become: true
ansible.builtin.command:
argv:
- find
- /usr/local/src/notarius
- -not
- -user
- root
- -o
- -not
- -group
- root
register: notarius_source_ownership
changed_when: false
- name: Inspect binary metadata
become: true
ansible.builtin.stat:
path: /usr/local/bin/notarius
register: notarius_binary_metadata
- name: Inspect configuration metadata
become: true
ansible.builtin.stat:
path: /usr/local/etc/notarius/config.yml
register: notarius_config_metadata
- name: Read default configuration
become: true
ansible.builtin.slurp:
src: /usr/local/etc/notarius/config.yml
register: notarius_default_config
- name: Read the installed version
ansible.builtin.command:
argv:
- /usr/local/bin/notarius
- --version
register: notarius_version_output
changed_when: false
- name: Validate Notarius configuration
ansible.builtin.command:
argv:
- /usr/local/bin/notarius
- config
- validate
- --config
- /usr/local/etc/notarius/config.yml
changed_when: false
- name: Assert the installation contract
ansible.builtin.assert:
that:
- notarius_commit.stdout == notarius_expected_commit
- notarius_source_ownership.stdout == ""
- notarius_binary_metadata.stat.isreg
- notarius_binary_metadata.stat.pw_name == "root"
- notarius_binary_metadata.stat.gr_name == "root"
- notarius_binary_metadata.stat.mode == "0755"
- notarius_config_metadata.stat.pw_name == "root"
- notarius_config_metadata.stat.gr_name == "root"
- notarius_config_metadata.stat.mode == "0644"
- '(notarius_default_config.content | b64decode | from_yaml) == {"version": 4}'
- notarius_version_output.stdout == "notarius " ~ notarius_expected_version