From 733a1944d5e1bc2d7686b38210170514d1b017fd Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 8 Sep 2026 18:53:01 +0000 Subject: [PATCH] Add focused Notarius integration coverage --- README.md | 10 +- tests/integration | 202 ++++++++++++++++++++++++++++++++++++++-- tests/unsupported.yml | 11 +++ tests/verify-config.yml | 25 +++++ 4 files changed, 238 insertions(+), 10 deletions(-) create mode 100644 tests/unsupported.yml create mode 100644 tests/verify-config.yml diff --git a/README.md b/README.md index 5aac508..6aa4249 100644 --- a/README.md +++ b/README.md @@ -69,9 +69,13 @@ outside this role. The repository-local integration suite installs its pinned collection, runs syntax checking and production-profile lint, verifies the default release, ownership, executable output, minimal configuration, native configuration -validation, and immediate idempotence. It mutates only the managed source, -binary, configuration, and a private temporary directory, then leaves the -default release and minimal configuration installed. +validation, and immediate idempotence. It also covers missing-binary and +checkout rebuilds, caller-supplied Go with dependency management disabled, +metadata repair without rebuilding, caller-side pipeline composition, +field-specific configuration rendering and falsey values, and platform guards. +It mutates only the managed source, binary, configuration, and a private +temporary directory, then leaves the default release and minimal configuration +installed. ```sh tests/integration diff --git a/tests/integration b/tests/integration index ba9d872..bfec83d 100755 --- a/tests/integration +++ b/tests/integration @@ -12,8 +12,14 @@ export ANSIBLE_ROLES_PATH="$(dirname "$repository_root")" inventory="$repository_root/tests/inventory" converge="$repository_root/tests/converge.yml" verify="$repository_root/tests/verify.yml" +verify_config="$repository_root/tests/verify-config.yml" +unsupported="$repository_root/tests/unsupported.yml" ansible_playbook="$workspace_venv/bin/ansible-playbook" default_commit=77ae132e5609916da40dcd4fcc749c000a63cbac +previous_commit=61436d7c186af57b06004e090ceac898e83cdb71 +source_dir=/usr/local/src/notarius +binary_path=/usr/local/bin/notarius +config_path=/usr/local/etc/notarius/config.yml cleanup() { exit_status=$? @@ -28,19 +34,201 @@ require_clean_recap() { grep -Eq 'changed=0 .*failed=0' "$1" } +require_changed_recap() { + grep -Eq 'changed=[1-9][0-9]* .*failed=0' "$1" +} + +run_converge() { + output_file=$1 + shift + "$ansible_playbook" -i "$inventory" "$@" "$converge" | tee "$output_file" +} + +verify_release() { + expected_version=$1 + expected_commit=$2 + "$ansible_playbook" -i "$inventory" \ + -e "notarius_expected_version=$expected_version" \ + -e "notarius_expected_commit=$expected_commit" \ + "$verify" +} + +verify_rendered_config() { + "$ansible_playbook" -i "$inventory" \ + -e "{\"notarius_expected_config\":$1}" \ + "$verify_config" +} + +require_configuration_change_without_build() { + grep -A1 -E 'TASK .*Build Notarius' "$1" | grep -q '^skipping:' + grep -A1 -E 'TASK .*Write Notarius configuration' "$1" | grep -q '^changed:' +} + cd "$repository_root" "$workspace_venv/bin/ansible-galaxy" collection install \ -r collections/requirements.yml \ -p "$ANSIBLE_COLLECTIONS_PATH" "$ansible_playbook" -i "$inventory" --syntax-check "$converge" -"$workspace_venv/bin/ansible-lint" "$converge" "$verify" +"$workspace_venv/bin/ansible-lint" \ + "$converge" "$verify" "$verify_config" "$unsupported" baseline_log="$test_tmp/baseline.log" -"$ansible_playbook" -i "$inventory" "$converge" | tee "$baseline_log" -"$ansible_playbook" -i "$inventory" \ - -e "notarius_expected_version=v0.6.0" \ - -e "notarius_expected_commit=$default_commit" \ - "$verify" +run_converge "$baseline_log" +verify_release v0.6.0 "$default_commit" +verify_rendered_config '{"version":4}' repeat_log="$test_tmp/repeat.log" -"$ansible_playbook" -i "$inventory" "$converge" | tee "$repeat_log" +run_converge "$repeat_log" require_clean_recap "$repeat_log" + +sudo rm -f -- "$binary_path" +missing_binary_log="$test_tmp/missing-binary.log" +run_converge "$missing_binary_log" +require_changed_recap "$missing_binary_log" +grep -A1 -E 'TASK .*Build Notarius' "$missing_binary_log" | grep -q '^changed:' +missing_binary_repeat_log="$test_tmp/missing-binary-repeat.log" +run_converge "$missing_binary_repeat_log" +require_clean_recap "$missing_binary_repeat_log" + +custom_go_binary="$test_tmp/go" +ln -s /snap/bin/go "$custom_go_binary" +sudo rm -f -- "$binary_path" +custom_go_log="$test_tmp/custom-go.log" +run_converge "$custom_go_log" \ + -e '{"notarius_manage_build_dependencies":false}' \ + -e "notarius_go_binary=$custom_go_binary" +require_changed_recap "$custom_go_log" +grep -A1 -E 'TASK .*Build Notarius' "$custom_go_log" | grep -q '^changed:' +custom_go_repeat_log="$test_tmp/custom-go-repeat.log" +run_converge "$custom_go_repeat_log" \ + -e '{"notarius_manage_build_dependencies":false}' \ + -e "notarius_go_binary=$custom_go_binary" +require_clean_recap "$custom_go_repeat_log" + +sudo chmod 0700 "$binary_path" +metadata_log="$test_tmp/metadata.log" +run_converge "$metadata_log" +require_changed_recap "$metadata_log" +grep -A1 -E 'TASK .*Build Notarius' "$metadata_log" | grep -q '^skipping:' +grep -A3 -E 'TASK .*Converge installed metadata' "$metadata_log" | \ + grep -q "^changed:.*'path': '$binary_path'" +metadata_repeat_log="$test_tmp/metadata-repeat.log" +run_converge "$metadata_repeat_log" +require_clean_recap "$metadata_repeat_log" + +previous_release_log="$test_tmp/previous-release.log" +run_converge "$previous_release_log" -e notarius_version=v0.5.0 +require_changed_recap "$previous_release_log" +verify_release v0.5.0 "$previous_commit" +previous_repeat_log="$test_tmp/previous-repeat.log" +run_converge "$previous_repeat_log" -e notarius_version=v0.5.0 +require_clean_recap "$previous_repeat_log" + +restored_release_log="$test_tmp/restored-release.log" +run_converge "$restored_release_log" +require_changed_recap "$restored_release_log" +verify_release v0.6.0 "$default_commit" +restored_repeat_log="$test_tmp/restored-repeat.log" +run_converge "$restored_repeat_log" +require_clean_recap "$restored_repeat_log" + +representative_config='{"notarius_promptkit_local_backend_endpoint":"http://localhost:8000/v1","notarius_promptkit_local_backend_concurrency_limit":2,"notarius_pipelines":{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}},"notarius_concurrency_total_llm":2,"notarius_concurrency_stage_workers_extract":2,"notarius_output_directory":"/tmp/notarius-output","notarius_cache_chunk_plans_mode":"refresh","notarius_cache_chunk_plans_directory":"/tmp/notarius-cache/chunk-plans","notarius_cache_checkpoints_enabled":true,"notarius_cache_checkpoints_directory":"/tmp/notarius-cache/checkpoints","notarius_debug_directory":"/tmp/notarius-debug"}' +representative_expected='{"version":4,"promptkit":{"local_backend":{"endpoint":"http://localhost:8000/v1","concurrency_limit":2}},"pipelines":{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}},"concurrency":{"total_llm":2,"stage_workers":{"extract":2}},"output":{"directory":"/tmp/notarius-output"},"cache":{"chunk_plans":{"mode":"refresh","directory":"/tmp/notarius-cache/chunk-plans"},"checkpoints":{"enabled":true,"directory":"/tmp/notarius-cache/checkpoints"}},"debug":{"directory":"/tmp/notarius-debug"}}' +representative_log="$test_tmp/representative.log" +run_converge "$representative_log" -e "$representative_config" +require_changed_recap "$representative_log" +require_configuration_change_without_build "$representative_log" +verify_rendered_config "$representative_expected" +representative_repeat_log="$test_tmp/representative-repeat.log" +run_converge "$representative_repeat_log" -e "$representative_config" +require_clean_recap "$representative_repeat_log" + +profile_dir_config='{"notarius_promptkit_profile_dir":"/usr/local/src/notarius/examples/profiles"}' +profile_dir_expected='{"version":4,"promptkit":{"profile_dir":"/usr/local/src/notarius/examples/profiles"}}' +profile_dir_log="$test_tmp/profile-dir.log" +run_converge "$profile_dir_log" -e "$profile_dir_config" +require_changed_recap "$profile_dir_log" +require_configuration_change_without_build "$profile_dir_log" +verify_rendered_config "$profile_dir_expected" +profile_dir_repeat_log="$test_tmp/profile-dir-repeat.log" +run_converge "$profile_dir_repeat_log" -e "$profile_dir_config" +require_clean_recap "$profile_dir_repeat_log" + +profile_file_config='{"notarius_promptkit_profile_file":"/usr/local/src/notarius/examples/profiles/dnd-extraction.yml"}' +profile_file_expected='{"version":4,"promptkit":{"profile_file":"/usr/local/src/notarius/examples/profiles/dnd-extraction.yml"}}' +profile_file_log="$test_tmp/profile-file.log" +run_converge "$profile_file_log" -e "$profile_file_config" +require_changed_recap "$profile_file_log" +require_configuration_change_without_build "$profile_file_log" +verify_rendered_config "$profile_file_expected" +profile_file_repeat_log="$test_tmp/profile-file-repeat.log" +run_converge "$profile_file_repeat_log" -e "$profile_file_config" +require_clean_recap "$profile_file_repeat_log" + +falsey_config='{"notarius_promptkit_local_backend_endpoint":"http://localhost:8000/v1","notarius_promptkit_local_backend_concurrency_limit":0,"notarius_pipelines":{},"notarius_cache_chunk_plans_directory":"","notarius_cache_checkpoints_enabled":false}' +falsey_expected='{"version":4,"promptkit":{"local_backend":{"endpoint":"http://localhost:8000/v1","concurrency_limit":0}},"pipelines":{},"cache":{"chunk_plans":{"directory":""},"checkpoints":{"enabled":false}}}' +falsey_log="$test_tmp/falsey.log" +run_converge "$falsey_log" -e "$falsey_config" +require_changed_recap "$falsey_log" +require_configuration_change_without_build "$falsey_log" +verify_rendered_config "$falsey_expected" +falsey_repeat_log="$test_tmp/falsey-repeat.log" +run_converge "$falsey_repeat_log" -e "$falsey_config" +require_clean_recap "$falsey_repeat_log" + +pipeline_group_one='{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}' +pipeline_group_two='{"dnd-archive":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}' +pipeline_composition_config="{\"notarius_pipeline_group_one\":$pipeline_group_one,\"notarius_pipeline_group_two\":$pipeline_group_two,\"notarius_pipelines\":\"{{ notarius_pipeline_group_one | combine(notarius_pipeline_group_two) }}\"}" +pipeline_composition_expected='{"version":4,"pipelines":{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}},"dnd-archive":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}}' +pipeline_composition_log="$test_tmp/pipeline-composition.log" +run_converge "$pipeline_composition_log" -e "$pipeline_composition_config" +require_changed_recap "$pipeline_composition_log" +require_configuration_change_without_build "$pipeline_composition_log" +verify_rendered_config "$pipeline_composition_expected" +pipeline_composition_repeat_log="$test_tmp/pipeline-composition-repeat.log" +run_converge "$pipeline_composition_repeat_log" -e "$pipeline_composition_config" +require_clean_recap "$pipeline_composition_repeat_log" + +default_config_log="$test_tmp/default-config.log" +run_converge "$default_config_log" +require_changed_recap "$default_config_log" +require_configuration_change_without_build "$default_config_log" +verify_release v0.6.0 "$default_commit" +verify_rendered_config '{"version":4}' +default_config_repeat_log="$test_tmp/default-config-repeat.log" +run_converge "$default_config_repeat_log" +require_clean_recap "$default_config_repeat_log" + +platform_cases=( + 'Debian 26.04 x86_64 distribution' + 'Ubuntu 24.04 x86_64 release' + 'Ubuntu 26.04 riscv64 architecture' +) +for platform_case in "${platform_cases[@]}"; do + read -r distribution release architecture label <<<"$platform_case" + case_root="$test_tmp/unsupported-$label" + case_source="$case_root/source" + case_binary="$case_root/bin/notarius" + case_config="$case_root/etc/config.yml" + case_log="$test_tmp/unsupported-$label.log" + if "$ansible_playbook" -i "$inventory" \ + -e "test_distribution=$distribution" \ + -e "test_distribution_version=$release" \ + -e "test_architecture=$architecture" \ + -e "notarius_source_dir=$case_source" \ + -e "notarius_binary_path=$case_binary" \ + -e "notarius_config_path=$case_config" \ + "$unsupported" | tee "$case_log"; then + echo "Unsupported $label case unexpectedly succeeded" >&2 + exit 1 + fi + grep -Eq 'failed=1' "$case_log" + [[ ! -e "$case_source" ]] + [[ ! -e "$case_binary" ]] + [[ ! -e "$case_config" ]] +done + +[[ "$(sudo git -C "$source_dir" rev-parse HEAD)" == "$default_commit" ]] +[[ "$(sudo stat -c '%U:%G %a' "$binary_path")" == 'root:root 755' ]] +[[ "$(sudo stat -c '%U:%G %a' "$config_path")" == 'root:root 644' ]] +[[ "$("$binary_path" --version)" == 'notarius v0.6.0' ]] +"$binary_path" config validate --config "$config_path" diff --git a/tests/unsupported.yml b/tests/unsupported.yml new file mode 100644 index 0000000..1dd8c6c --- /dev/null +++ b/tests/unsupported.yml @@ -0,0 +1,11 @@ +--- +- name: Exercise Notarius platform mutation guard + hosts: localhost + gather_facts: false + vars: + ansible_facts: + distribution: "{{ test_distribution }}" + distribution_version: "{{ test_distribution_version }}" + architecture: "{{ test_architecture }}" + roles: + - role: ansible-role-notarius diff --git a/tests/verify-config.yml b/tests/verify-config.yml new file mode 100644 index 0000000..0c55ac7 --- /dev/null +++ b/tests/verify-config.yml @@ -0,0 +1,25 @@ +--- +- name: Verify rendered Notarius configuration + hosts: localhost + gather_facts: false + tasks: + - name: Read rendered configuration + become: true + ansible.builtin.slurp: + src: /usr/local/etc/notarius/config.yml + register: notarius_rendered_config + + - name: Check rendered configuration + ansible.builtin.assert: + that: + - '(notarius_rendered_config.content | b64decode | from_yaml) == notarius_expected_config' + + - name: Validate rendered configuration + ansible.builtin.command: + argv: + - /usr/local/bin/notarius + - config + - validate + - --config + - /usr/local/etc/notarius/config.yml + changed_when: false