From cfecf37261f2595d45113235c591ee102d6f181f Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 9 Sep 2026 04:30:54 +0000 Subject: [PATCH] Simplify the Notarius integration tests --- README.md | 18 ++-- tests/integration | 212 ++++++---------------------------------- tests/unsupported.yml | 11 --- tests/verify-config.yml | 25 ++++- tests/verify.yml | 83 ---------------- 5 files changed, 63 insertions(+), 286 deletions(-) delete mode 100644 tests/unsupported.yml delete mode 100644 tests/verify.yml diff --git a/README.md b/README.md index 6aa4249..38771f6 100644 --- a/README.md +++ b/README.md @@ -67,15 +67,15 @@ outside this role. ## Validation The repository-local integration suite installs its pinned collection, runs -syntax checking and production-profile lint, verifies the default release, -ownership, executable output, minimal configuration, native configuration -validation, and immediate idempotence. It also covers missing-binary and -checkout rebuilds, caller-supplied Go with dependency management disabled, -metadata repair without rebuilding, caller-side pipeline composition, -field-specific configuration rendering and falsey values, and platform guards. -It mutates only the managed source, binary, configuration, and a private -temporary directory, then leaves the default release and minimal configuration -installed. +syntax checking and production-profile lint, and exercises the default CLI and +native configuration validation before requiring clean repeated convergence. +It verifies an explicitly selected Notarius release, combines missing-binary +recovery with a caller-supplied Go executable while dependency management is +disabled, and covers one representative configuration with null omission, +intentional falsey values, caller-side pipeline composition, and native +validation. It mutates only the managed source, binary, configuration, and a +private temporary directory, then restores the default deployment during +cleanup. ```sh tests/integration diff --git a/tests/integration b/tests/integration index 65ca766..fc44bf2 100755 --- a/tests/integration +++ b/tests/integration @@ -11,13 +11,8 @@ export ANSIBLE_ROLES_PATH="$(dirname "$repository_root")" inventory="$repository_root/tests/inventory" converge="$repository_root/tests/converge.yml" -verify="$repository_root/tests/verify.yml" verify_config="$repository_root/tests/verify-config.yml" -unsupported="$repository_root/tests/unsupported.yml" ansible_playbook="$workspace_venv/bin/ansible-playbook" -default_commit=77ae132e5609916da40dcd4fcc749c000a63cbac -previous_commit=61436d7c186af57b06004e090ceac898e83cdb71 -source_dir=/usr/local/src/notarius binary_path=/usr/local/bin/notarius config_path=/usr/local/etc/notarius/config.yml @@ -34,34 +29,14 @@ require_clean_recap() { grep -Eq 'changed=0 .*failed=0' "$1" } -require_changed_recap() { - grep -Eq 'changed=[1-9][0-9]* .*failed=0' "$1" -} - run_converge() { output_file=$1 shift "$ansible_playbook" -i "$inventory" "$@" "$converge" | tee "$output_file" } -verify_release() { - expected_version=$1 - expected_commit=$2 - "$ansible_playbook" -i "$inventory" \ - -e "notarius_expected_version=$expected_version" \ - -e "notarius_expected_commit=$expected_commit" \ - "$verify" -} - -verify_rendered_config() { - "$ansible_playbook" -i "$inventory" \ - -e "{\"notarius_expected_config\":$1}" \ - "$verify_config" -} - -require_configuration_change_without_build() { - grep -A1 -E 'TASK .*Build Notarius' "$1" | grep -q '^skipping:' - grep -A1 -E 'TASK .*Write Notarius configuration' "$1" | grep -q '^changed:' +validate_config() { + "$binary_path" config validate --config "$config_path" } cd "$repository_root" @@ -69,172 +44,49 @@ cd "$repository_root" -r collections/requirements.yml \ -p "$ANSIBLE_COLLECTIONS_PATH" "$ansible_playbook" -i "$inventory" --syntax-check "$converge" -"$workspace_venv/bin/ansible-lint" \ - "$converge" "$verify" "$verify_config" "$unsupported" +"$workspace_venv/bin/ansible-lint" "$converge" "$verify_config" baseline_log="$test_tmp/baseline.log" run_converge "$baseline_log" -verify_release v0.6.0 "$default_commit" -verify_rendered_config '{"version":4}' -repeat_log="$test_tmp/repeat.log" -run_converge "$repeat_log" -require_clean_recap "$repeat_log" +"$binary_path" --version +validate_config +baseline_repeat_log="$test_tmp/baseline-repeat.log" +run_converge "$baseline_repeat_log" +require_clean_recap "$baseline_repeat_log" -sudo rm -f -- "$binary_path" -missing_binary_log="$test_tmp/missing-binary.log" -run_converge "$missing_binary_log" -require_changed_recap "$missing_binary_log" -grep -A1 -E 'TASK .*Build Notarius' "$missing_binary_log" | grep -q '^changed:' -missing_binary_repeat_log="$test_tmp/missing-binary-repeat.log" -run_converge "$missing_binary_repeat_log" -require_clean_recap "$missing_binary_repeat_log" +selected_release_log="$test_tmp/selected-release.log" +run_converge "$selected_release_log" -e notarius_version=v0.5.0 +[[ "$("$binary_path" --version)" == 'notarius v0.5.0' ]] custom_go_binary="$test_tmp/go" -custom_go_invocation_log="$test_tmp/custom-go-invocation.log" +custom_go_invocations="$test_tmp/go-invocations.log" printf '%s\n' \ '#!/usr/bin/env bash' \ - "printf 'invoked\\n' >> '$custom_go_invocation_log'" \ + 'set -euo pipefail' \ + "printf '%s\\n' \"\$*\" >> '$custom_go_invocations'" \ 'exec /snap/bin/go "$@"' > "$custom_go_binary" chmod 0755 "$custom_go_binary" sudo rm -f -- "$binary_path" -custom_go_log="$test_tmp/custom-go.log" -run_converge "$custom_go_log" \ - -e '{"notarius_manage_build_dependencies":false}' \ +recovery_log="$test_tmp/recovery.log" +run_converge "$recovery_log" \ + -e notarius_version=v0.5.0 \ + -e '{"notarius_manage_build_dependencies": false}' \ -e "notarius_go_binary=$custom_go_binary" -require_changed_recap "$custom_go_log" -grep -A1 -E 'TASK .*Build Notarius' "$custom_go_log" | grep -q '^changed:' -[[ -s "$custom_go_invocation_log" ]] -custom_go_repeat_log="$test_tmp/custom-go-repeat.log" -run_converge "$custom_go_repeat_log" \ - -e '{"notarius_manage_build_dependencies":false}' \ +grep -q . "$custom_go_invocations" +[[ "$("$binary_path" --version)" == 'notarius v0.5.0' ]] +recovery_repeat_log="$test_tmp/recovery-repeat.log" +run_converge "$recovery_repeat_log" \ + -e notarius_version=v0.5.0 \ + -e '{"notarius_manage_build_dependencies": false}' \ -e "notarius_go_binary=$custom_go_binary" -require_clean_recap "$custom_go_repeat_log" - -sudo chmod 0700 "$binary_path" -metadata_log="$test_tmp/metadata.log" -run_converge "$metadata_log" -require_changed_recap "$metadata_log" -grep -A1 -E 'TASK .*Build Notarius' "$metadata_log" | grep -q '^skipping:' -grep -A3 -E 'TASK .*Converge installed metadata' "$metadata_log" | \ - grep -q "^changed:.*'path': '$binary_path'" -metadata_repeat_log="$test_tmp/metadata-repeat.log" -run_converge "$metadata_repeat_log" -require_clean_recap "$metadata_repeat_log" - -previous_release_log="$test_tmp/previous-release.log" -run_converge "$previous_release_log" -e notarius_version=v0.5.0 -require_changed_recap "$previous_release_log" -verify_release v0.5.0 "$previous_commit" -previous_repeat_log="$test_tmp/previous-repeat.log" -run_converge "$previous_repeat_log" -e notarius_version=v0.5.0 -require_clean_recap "$previous_repeat_log" - -restored_release_log="$test_tmp/restored-release.log" -run_converge "$restored_release_log" -require_changed_recap "$restored_release_log" -verify_release v0.6.0 "$default_commit" -restored_repeat_log="$test_tmp/restored-repeat.log" -run_converge "$restored_repeat_log" -require_clean_recap "$restored_repeat_log" - -representative_config='{"notarius_promptkit_local_backend_endpoint":"http://localhost:8000/v1","notarius_promptkit_local_backend_concurrency_limit":2,"notarius_pipelines":{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}},"notarius_concurrency_total_llm":2,"notarius_concurrency_stage_workers_extract":2,"notarius_output_directory":"/tmp/notarius-output","notarius_cache_chunk_plans_mode":"refresh","notarius_cache_chunk_plans_directory":"/tmp/notarius-cache/chunk-plans","notarius_cache_checkpoints_enabled":true,"notarius_cache_checkpoints_directory":"/tmp/notarius-cache/checkpoints","notarius_debug_directory":"/tmp/notarius-debug"}' -representative_expected='{"version":4,"promptkit":{"local_backend":{"endpoint":"http://localhost:8000/v1","concurrency_limit":2}},"pipelines":{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}},"concurrency":{"total_llm":2,"stage_workers":{"extract":2}},"output":{"directory":"/tmp/notarius-output"},"cache":{"chunk_plans":{"mode":"refresh","directory":"/tmp/notarius-cache/chunk-plans"},"checkpoints":{"enabled":true,"directory":"/tmp/notarius-cache/checkpoints"}},"debug":{"directory":"/tmp/notarius-debug"}}' -representative_log="$test_tmp/representative.log" -run_converge "$representative_log" -e "$representative_config" -require_changed_recap "$representative_log" -require_configuration_change_without_build "$representative_log" -verify_rendered_config "$representative_expected" -representative_repeat_log="$test_tmp/representative-repeat.log" -run_converge "$representative_repeat_log" -e "$representative_config" -require_clean_recap "$representative_repeat_log" - -profile_dir_config='{"notarius_promptkit_profile_dir":"/usr/local/src/notarius/examples/profiles"}' -profile_dir_expected='{"version":4,"promptkit":{"profile_dir":"/usr/local/src/notarius/examples/profiles"}}' -profile_dir_log="$test_tmp/profile-dir.log" -run_converge "$profile_dir_log" -e "$profile_dir_config" -require_changed_recap "$profile_dir_log" -require_configuration_change_without_build "$profile_dir_log" -verify_rendered_config "$profile_dir_expected" -profile_dir_repeat_log="$test_tmp/profile-dir-repeat.log" -run_converge "$profile_dir_repeat_log" -e "$profile_dir_config" -require_clean_recap "$profile_dir_repeat_log" - -profile_file_config='{"notarius_promptkit_profile_file":"/usr/local/src/notarius/examples/profiles/dnd-extraction.yml"}' -profile_file_expected='{"version":4,"promptkit":{"profile_file":"/usr/local/src/notarius/examples/profiles/dnd-extraction.yml"}}' -profile_file_log="$test_tmp/profile-file.log" -run_converge "$profile_file_log" -e "$profile_file_config" -require_changed_recap "$profile_file_log" -require_configuration_change_without_build "$profile_file_log" -verify_rendered_config "$profile_file_expected" -profile_file_repeat_log="$test_tmp/profile-file-repeat.log" -run_converge "$profile_file_repeat_log" -e "$profile_file_config" -require_clean_recap "$profile_file_repeat_log" - -falsey_config='{"notarius_promptkit_local_backend_endpoint":"http://localhost:8000/v1","notarius_promptkit_local_backend_concurrency_limit":0,"notarius_pipelines":{},"notarius_cache_chunk_plans_directory":"","notarius_cache_checkpoints_enabled":false}' -falsey_expected='{"version":4,"promptkit":{"local_backend":{"endpoint":"http://localhost:8000/v1","concurrency_limit":0}},"pipelines":{},"cache":{"chunk_plans":{"directory":""},"checkpoints":{"enabled":false}}}' -falsey_log="$test_tmp/falsey.log" -run_converge "$falsey_log" -e "$falsey_config" -require_changed_recap "$falsey_log" -require_configuration_change_without_build "$falsey_log" -verify_rendered_config "$falsey_expected" -falsey_repeat_log="$test_tmp/falsey-repeat.log" -run_converge "$falsey_repeat_log" -e "$falsey_config" -require_clean_recap "$falsey_repeat_log" +require_clean_recap "$recovery_repeat_log" pipeline_group_one='{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}' pipeline_group_two='{"dnd-archive":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}' -pipeline_composition_config="{\"notarius_pipeline_group_one\":$pipeline_group_one,\"notarius_pipeline_group_two\":$pipeline_group_two,\"notarius_pipelines\":\"{{ notarius_pipeline_group_one | combine(notarius_pipeline_group_two) }}\"}" -pipeline_composition_expected='{"version":4,"pipelines":{"dnd-session":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}},"dnd-archive":{"llm_profile":"dnd-extraction","input":"seriatim","artifacts":{"spells":{"extract":"dnd/spells","normalize":"dnd/spells"}}}}}' -pipeline_composition_log="$test_tmp/pipeline-composition.log" -run_converge "$pipeline_composition_log" -e "$pipeline_composition_config" -require_changed_recap "$pipeline_composition_log" -require_configuration_change_without_build "$pipeline_composition_log" -verify_rendered_config "$pipeline_composition_expected" -pipeline_composition_repeat_log="$test_tmp/pipeline-composition-repeat.log" -run_converge "$pipeline_composition_repeat_log" -e "$pipeline_composition_config" -require_clean_recap "$pipeline_composition_repeat_log" - -default_config_log="$test_tmp/default-config.log" -run_converge "$default_config_log" -require_changed_recap "$default_config_log" -require_configuration_change_without_build "$default_config_log" -verify_release v0.6.0 "$default_commit" -verify_rendered_config '{"version":4}' -default_config_repeat_log="$test_tmp/default-config-repeat.log" -run_converge "$default_config_repeat_log" -require_clean_recap "$default_config_repeat_log" - -platform_cases=( - 'Debian 26.04 x86_64 distribution' - 'Ubuntu 24.04 x86_64 release' - 'Ubuntu 26.04 riscv64 architecture' -) -for platform_case in "${platform_cases[@]}"; do - read -r distribution release architecture label <<<"$platform_case" - case_root="$test_tmp/unsupported-$label" - case_source="$case_root/source" - case_binary="$case_root/bin/notarius" - case_config="$case_root/etc/config.yml" - case_log="$test_tmp/unsupported-$label.log" - if "$ansible_playbook" -i "$inventory" \ - -e "test_distribution=$distribution" \ - -e "test_distribution_version=$release" \ - -e "test_architecture=$architecture" \ - -e "notarius_source_dir=$case_source" \ - -e "notarius_binary_path=$case_binary" \ - -e "notarius_config_path=$case_config" \ - "$unsupported" | tee "$case_log"; then - echo "Unsupported $label case unexpectedly succeeded" >&2 - exit 1 - fi - grep -Eq 'failed=1' "$case_log" - [[ ! -e "$case_source" ]] - [[ ! -e "$case_binary" ]] - [[ ! -e "$case_config" ]] -done - -[[ "$(sudo git -C "$source_dir" rev-parse HEAD)" == "$default_commit" ]] -[[ "$(sudo stat -c '%U:%G %a' "$binary_path")" == 'root:root 755' ]] -[[ "$(sudo stat -c '%U:%G %a' "$config_path")" == 'root:root 644' ]] -[[ "$("$binary_path" --version)" == 'notarius v0.6.0' ]] -"$binary_path" config validate --config "$config_path" +representative_config="{\"notarius_pipeline_group_one\":$pipeline_group_one,\"notarius_pipeline_group_two\":$pipeline_group_two,\"notarius_pipelines\":\"{{ notarius_pipeline_group_one | combine(notarius_pipeline_group_two) }}\",\"notarius_config_version\":4,\"notarius_promptkit_local_backend_endpoint\":\"http://localhost:8000/v1\",\"notarius_promptkit_local_backend_concurrency_limit\":0,\"notarius_cache_checkpoints_enabled\":false,\"notarius_cache_chunk_plans_directory\":\"\",\"notarius_output_directory\":\"/tmp/notarius-output\",\"notarius_promptkit_profile_dir\":null,\"notarius_promptkit_profile_file\":null,\"notarius_debug_directory\":null}" +representative_log="$test_tmp/representative.log" +run_converge "$representative_log" -e "$representative_config" +"$ansible_playbook" -i "$inventory" "$verify_config" +representative_repeat_log="$test_tmp/representative-repeat.log" +run_converge "$representative_repeat_log" -e "$representative_config" +require_clean_recap "$representative_repeat_log" diff --git a/tests/unsupported.yml b/tests/unsupported.yml deleted file mode 100644 index 1dd8c6c..0000000 --- a/tests/unsupported.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Exercise Notarius platform mutation guard - hosts: localhost - gather_facts: false - vars: - ansible_facts: - distribution: "{{ test_distribution }}" - distribution_version: "{{ test_distribution_version }}" - architecture: "{{ test_architecture }}" - roles: - - role: ansible-role-notarius diff --git a/tests/verify-config.yml b/tests/verify-config.yml index 0c55ac7..39bcbee 100644 --- a/tests/verify-config.yml +++ b/tests/verify-config.yml @@ -7,12 +7,31 @@ become: true ansible.builtin.slurp: src: /usr/local/etc/notarius/config.yml - register: notarius_rendered_config + register: notarius_rendered_config_file - - name: Check rendered configuration + - name: Check representative rendered configuration ansible.builtin.assert: that: - - '(notarius_rendered_config.content | b64decode | from_yaml) == notarius_expected_config' + - notarius_rendered_config.version == 4 + - notarius_rendered_config.promptkit.local_backend.endpoint == "http://localhost:8000/v1" + - notarius_rendered_config.promptkit.local_backend.concurrency_limit == 0 + - notarius_rendered_config.cache.checkpoints.enabled is false + - notarius_rendered_config.cache.chunk_plans.directory == "" + - notarius_rendered_config.output.directory == "/tmp/notarius-output" + - notarius_rendered_config.pipelines["dnd-session"] == notarius_expected_pipeline + - notarius_rendered_config.pipelines["dnd-archive"] == notarius_expected_pipeline + - notarius_rendered_config.promptkit.profile_dir is not defined + - notarius_rendered_config.promptkit.profile_file is not defined + - notarius_rendered_config.debug is not defined + vars: + notarius_rendered_config: "{{ notarius_rendered_config_file.content | b64decode | from_yaml }}" + notarius_expected_pipeline: + llm_profile: dnd-extraction + input: seriatim + artifacts: + spells: + extract: dnd/spells + normalize: dnd/spells - name: Validate rendered configuration ansible.builtin.command: diff --git a/tests/verify.yml b/tests/verify.yml deleted file mode 100644 index c094db9..0000000 --- a/tests/verify.yml +++ /dev/null @@ -1,83 +0,0 @@ ---- -- name: Verify Notarius installation - hosts: localhost - gather_facts: false - tasks: - - name: Read the selected source commit - become: true - ansible.builtin.command: - argv: - - git - - -C - - /usr/local/src/notarius - - rev-parse - - HEAD - register: notarius_commit - changed_when: false - - - name: Inspect source ownership - become: true - ansible.builtin.command: - argv: - - find - - /usr/local/src/notarius - - -not - - -user - - root - - -o - - -not - - -group - - root - register: notarius_source_ownership - changed_when: false - - - name: Inspect binary metadata - become: true - ansible.builtin.stat: - path: /usr/local/bin/notarius - register: notarius_binary_metadata - - - name: Inspect configuration metadata - become: true - ansible.builtin.stat: - path: /usr/local/etc/notarius/config.yml - register: notarius_config_metadata - - - name: Read default configuration - become: true - ansible.builtin.slurp: - src: /usr/local/etc/notarius/config.yml - register: notarius_default_config - - - name: Read the installed version - ansible.builtin.command: - argv: - - /usr/local/bin/notarius - - --version - register: notarius_version_output - changed_when: false - - - name: Validate Notarius configuration - ansible.builtin.command: - argv: - - /usr/local/bin/notarius - - config - - validate - - --config - - /usr/local/etc/notarius/config.yml - changed_when: false - - - name: Assert the installation contract - ansible.builtin.assert: - that: - - notarius_commit.stdout == notarius_expected_commit - - notarius_source_ownership.stdout == "" - - notarius_binary_metadata.stat.isreg - - notarius_binary_metadata.stat.pw_name == "root" - - notarius_binary_metadata.stat.gr_name == "root" - - notarius_binary_metadata.stat.mode == "0755" - - notarius_config_metadata.stat.pw_name == "root" - - notarius_config_metadata.stat.gr_name == "root" - - notarius_config_metadata.stat.mode == "0644" - - '(notarius_default_config.content | b64decode | from_yaml) == {"version": 4}' - - notarius_version_output.stdout == "notarius " ~ notarius_expected_version