From d2d0b9d00b3b491ce7613ccfc8f686f3d1fa0896 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 9 Sep 2026 04:27:44 +0000 Subject: [PATCH] Simplify the Audita integration tests --- README.md | 19 ++--- tests/integration | 171 +++++++--------------------------------- tests/unsupported.yml | 11 --- tests/verify-config.yml | 22 ++++-- tests/verify.yml | 97 ----------------------- 5 files changed, 51 insertions(+), 269 deletions(-) delete mode 100644 tests/unsupported.yml delete mode 100644 tests/verify.yml diff --git a/README.md b/README.md index c67e3f8..8ba7be9 100644 --- a/README.md +++ b/README.md @@ -83,19 +83,20 @@ production convergence. ## Validation The integration driver installs pinned collection dependencies, runs syntax -checking and ansible-lint, verifies the default release, file ownership and -modes, executable behavior, and native configuration validation. It covers -missing-binary and checkout rebuilds, caller-supplied Go with dependency -management disabled, metadata repair without rebuilding, representative -field-specific configuration rendering, explicitly selected falsey values, -restoration to the minimal default configuration, and platform failures before -isolated application state exists. +checking and ansible-lint, and validates the default configuration with Audita +before requiring clean repeated convergence. It also covers one representative +field-specific configuration with nested values, null omission, and intentional +falsey values, then validates it natively. A combined recovery scenario proves +that a missing binary is rebuilt with a caller-supplied Go executable while +dependency management is disabled. Audita currently has no native +release-identity command, so the suite intentionally does not use a source +commit as a version-test proxy. On a fresh host, default convergence may also install Git through APT, refresh the APT cache, and install the Go Snap. Application state is limited to `/usr/local/src/audita`, `/usr/local/bin/audita`, and `/usr/local/etc/audita/config.yml`; temporary test-specific files remain inside -the suite's private temporary directory. The suite leaves the default release -and configuration installed: +the suite's private temporary directory. The suite restores the default +deployment during cleanup: ```sh tests/integration diff --git a/tests/integration b/tests/integration index 87dec7e..a4be11a 100755 --- a/tests/integration +++ b/tests/integration @@ -11,19 +11,17 @@ export ANSIBLE_ROLES_PATH="$(dirname "$repository_root")" inventory="$repository_root/tests/inventory" converge="$repository_root/tests/converge.yml" -verify="$repository_root/tests/verify.yml" verify_config="$repository_root/tests/verify-config.yml" -unsupported="$repository_root/tests/unsupported.yml" ansible_playbook="$workspace_venv/bin/ansible-playbook" -default_commit=a84941d68168a1ed9ea7423b6477ae8218be73f9 -alternate_commit=46b7356a3b54a2771ce9a617e4fa3e8d13728cac -source_dir=/usr/local/src/audita binary_path=/usr/local/bin/audita config_path=/usr/local/etc/audita/config.yml cleanup() { + exit_status=$? + trap - EXIT "$ansible_playbook" -i "$inventory" "$converge" >/dev/null 2>&1 || true rm -rf -- "$test_tmp" + exit "$exit_status" } trap cleanup EXIT @@ -31,32 +29,14 @@ require_clean_recap() { grep -Eq 'changed=0 .*failed=0' "$1" } -require_changed_recap() { - grep -Eq 'changed=[1-9][0-9]* .*failed=0' "$1" -} - run_converge() { output_file=$1 shift "$ansible_playbook" -i "$inventory" "$@" "$converge" | tee "$output_file" } -verify_release() { - expected_commit=$1 - "$ansible_playbook" -i "$inventory" \ - -e "audita_expected_commit=$expected_commit" \ - "$verify" -} - -verify_rendered_config() { - expected_config=$1 - "$ansible_playbook" -i "$inventory" \ - -e "{\"audita_expected_config\": $expected_config}" \ - "$verify_config" -} - validate_config() { - [[ "$("$binary_path" config validate --config "$config_path")" == 'config is valid' ]] + "$binary_path" config validate --config "$config_path" } cd "$repository_root" @@ -64,137 +44,40 @@ cd "$repository_root" -r collections/requirements.yml \ -p "$ANSIBLE_COLLECTIONS_PATH" "$ansible_playbook" -i "$inventory" --syntax-check "$converge" -"$workspace_venv/bin/ansible-lint" "$converge" "$verify" "$verify_config" "$unsupported" +"$workspace_venv/bin/ansible-lint" "$converge" "$verify_config" baseline_log="$test_tmp/baseline.log" run_converge "$baseline_log" -verify_release "$default_commit" +validate_config baseline_repeat_log="$test_tmp/baseline-repeat.log" run_converge "$baseline_repeat_log" require_clean_recap "$baseline_repeat_log" -sudo rm -f -- "$binary_path" -missing_binary_log="$test_tmp/missing-binary.log" -run_converge "$missing_binary_log" -require_changed_recap "$missing_binary_log" -grep -A1 -E 'TASK .*Build Audita' "$missing_binary_log" | grep -q '^changed:' -missing_binary_repeat_log="$test_tmp/missing-binary-repeat.log" -run_converge "$missing_binary_repeat_log" -require_clean_recap "$missing_binary_repeat_log" - -custom_go_binary="$test_tmp/go" -custom_go_invocation_log="$test_tmp/custom-go-invocation.log" -printf '%s\n' \ - '#!/usr/bin/env bash' \ - "printf 'invoked\\n' >> '$custom_go_invocation_log'" \ - 'exec /snap/bin/go "$@"' > "$custom_go_binary" -chmod 0755 "$custom_go_binary" -sudo rm -f -- "$binary_path" -custom_go_log="$test_tmp/custom-go.log" -run_converge "$custom_go_log" \ - -e '{"audita_manage_build_dependencies": false}' \ - -e "audita_go_binary=$custom_go_binary" -require_changed_recap "$custom_go_log" -grep -A1 -E 'TASK .*Build Audita' "$custom_go_log" | grep -q '^changed:' -[[ -s "$custom_go_invocation_log" ]] -custom_go_repeat_log="$test_tmp/custom-go-repeat.log" -run_converge "$custom_go_repeat_log" \ - -e '{"audita_manage_build_dependencies": false}' \ - -e "audita_go_binary=$custom_go_binary" -require_clean_recap "$custom_go_repeat_log" - -sudo chmod 0700 "$binary_path" -metadata_log="$test_tmp/metadata.log" -run_converge "$metadata_log" -require_changed_recap "$metadata_log" -grep -A1 -E 'TASK .*Build Audita' "$metadata_log" | grep -q '^skipping:' -grep -A3 -E 'TASK .*Converge installed metadata' "$metadata_log" | \ - grep -q "^changed:.*'path': '$binary_path'" -metadata_repeat_log="$test_tmp/metadata-repeat.log" -run_converge "$metadata_repeat_log" -require_clean_recap "$metadata_repeat_log" - -representative_config='{"audita_pipeline_modules": ["glossary"], "audita_output_schema": "bare-segments", "audita_llm_proposal_base_url": "https://openrouter.ai/api/v1", "audita_llm_proposal_model": "openrouter/google/gemma-4-31b-it", "audita_llm_proposal_timeout": "10s", "audita_llm_proposal_max_retries": 1, "audita_llm_validation_base_url": "https://openrouter.ai/api/v1", "audita_llm_validation_model": "openrouter/google/gemma-4-31b-it", "audita_llm_validation_timeout": 10, "audita_llm_validation_max_retries": 1, "audita_concurrency_total_llm": 2, "audita_concurrency_proposal_llm": 1, "audita_concurrency_validation_llm": 1, "audita_chunking_target_sections": 2, "audita_chunking_max_section_tokens": 8192, "audita_chunking_min_section_tokens": 2048, "audita_normalization_max_segment_gap": "4s", "audita_normalization_ellipsis_gap": 3.5, "audita_normalization_max_segment_duration": 60, "audita_normalization_max_segment_tokens": 2048, "audita_thresholds_glossary": 0.8, "audita_thresholds_homophones": 0.8, "audita_thresholds_spoken_word": 0.8, "audita_thresholds_grammar": 0.8, "audita_context_description": "Focused integration coverage", "audita_diagnostics_work_dir": "/tmp/audita-integration", "audita_diagnostics_retention": "never"}' -representative_expected='{"version": 1, "pipeline": {"modules": ["glossary"]}, "output": {"schema": "bare-segments"}, "llm": {"proposal": {"base_url": "https://openrouter.ai/api/v1", "model": "openrouter/google/gemma-4-31b-it", "timeout": "10s", "max_retries": 1}, "validation": {"base_url": "https://openrouter.ai/api/v1", "model": "openrouter/google/gemma-4-31b-it", "timeout": 10, "max_retries": 1}}, "concurrency": {"total_llm": 2, "proposal_llm": 1, "validation_llm": 1}, "chunking": {"target_sections": 2, "max_section_tokens": 8192, "min_section_tokens": 2048}, "normalization": {"max_segment_gap": "4s", "ellipsis_gap": 3.5, "max_segment_duration": 60, "max_segment_tokens": 2048}, "thresholds": {"glossary": 0.8, "homophones": 0.8, "spoken_word": 0.8, "grammar": 0.8}, "context": {"description": "Focused integration coverage"}, "diagnostics": {"work_dir": "/tmp/audita-integration", "retention": "never"}}' +representative_config='{"audita_config_version": 1, "audita_pipeline_modules": ["glossary"], "audita_llm_proposal_base_url": "https://openrouter.ai/api/v1", "audita_llm_proposal_model": "openrouter/google/gemma-4-31b-it", "audita_llm_proposal_timeout": "10s", "audita_llm_proposal_max_retries": 0, "audita_context_description": "", "audita_diagnostics_retention": "never", "audita_llm_validation_model": null, "audita_output_schema": null, "audita_diagnostics_work_dir": null}' representative_log="$test_tmp/representative-config.log" run_converge "$representative_log" -e "$representative_config" -require_changed_recap "$representative_log" -grep -A1 -E 'TASK .*Build Audita' "$representative_log" | grep -q '^skipping:' -grep -A1 -E 'TASK .*Write Audita configuration' "$representative_log" | grep -q '^changed:' -verify_rendered_config "$representative_expected" -validate_config +"$ansible_playbook" -i "$inventory" "$verify_config" representative_repeat_log="$test_tmp/representative-config-repeat.log" run_converge "$representative_repeat_log" -e "$representative_config" require_clean_recap "$representative_repeat_log" -falsey_config='{"audita_pipeline_modules": [], "audita_context_description": ""}' -falsey_expected='{"version": 1, "pipeline": {"modules": []}, "context": {"description": ""}}' -falsey_log="$test_tmp/falsey-config.log" -run_converge "$falsey_log" -e "$falsey_config" -require_changed_recap "$falsey_log" -grep -A1 -E 'TASK .*Build Audita' "$falsey_log" | grep -q '^skipping:' -grep -A1 -E 'TASK .*Write Audita configuration' "$falsey_log" | grep -q '^changed:' -verify_rendered_config "$falsey_expected" -falsey_repeat_log="$test_tmp/falsey-config-repeat.log" -run_converge "$falsey_repeat_log" -e "$falsey_config" -require_clean_recap "$falsey_repeat_log" - -default_config_log="$test_tmp/default-config.log" -run_converge "$default_config_log" -require_changed_recap "$default_config_log" -grep -A1 -E 'TASK .*Build Audita' "$default_config_log" | grep -q '^skipping:' -grep -A1 -E 'TASK .*Write Audita configuration' "$default_config_log" | grep -q '^changed:' -verify_release "$default_commit" -default_config_repeat_log="$test_tmp/default-config-repeat.log" -run_converge "$default_config_repeat_log" -require_clean_recap "$default_config_repeat_log" - -alternate_release_log="$test_tmp/alternate-release.log" -run_converge "$alternate_release_log" -e "audita_version=$alternate_commit" -require_changed_recap "$alternate_release_log" -grep -A1 -E 'TASK .*Build Audita' "$alternate_release_log" | grep -q '^changed:' -verify_release "$alternate_commit" -alternate_repeat_log="$test_tmp/alternate-repeat.log" -run_converge "$alternate_repeat_log" -e "audita_version=$alternate_commit" -require_clean_recap "$alternate_repeat_log" - -restored_release_log="$test_tmp/restored-release.log" -run_converge "$restored_release_log" -require_changed_recap "$restored_release_log" -grep -A1 -E 'TASK .*Build Audita' "$restored_release_log" | grep -q '^changed:' -verify_release "$default_commit" -restored_repeat_log="$test_tmp/restored-repeat.log" -run_converge "$restored_repeat_log" -require_clean_recap "$restored_repeat_log" - -platform_cases=( - 'Debian 26.04 x86_64 distribution' - 'Ubuntu 24.04 x86_64 release' - 'Ubuntu 26.04 riscv64 architecture' -) -for platform_case in "${platform_cases[@]}"; do - read -r distribution release architecture label <<<"$platform_case" - case_root="$test_tmp/unsupported-$label" - case_source="$case_root/source" - case_binary="$case_root/bin/audita" - case_config="$case_root/etc/config.yml" - case_log="$test_tmp/unsupported-$label.log" - if "$ansible_playbook" -i "$inventory" \ - -e "test_distribution=$distribution" \ - -e "test_distribution_version=$release" \ - -e "test_architecture=$architecture" \ - -e "audita_source_dir=$case_source" \ - -e "audita_binary_path=$case_binary" \ - -e "audita_config_path=$case_config" \ - "$unsupported" | tee "$case_log"; then - echo "Unsupported $label case unexpectedly succeeded" >&2 - exit 1 - fi - grep -Eq 'failed=1' "$case_log" - [[ ! -e "$case_root" ]] -done - -[[ "$(sudo git -C "$source_dir" rev-parse HEAD)" == "$default_commit" ]] -[[ "$(sudo stat -c '%U:%G %a' "$binary_path")" == 'root:root 755' ]] -[[ "$(sudo stat -c '%U:%G %a' "$config_path")" == 'root:root 644' ]] +custom_go_binary="$test_tmp/go" +custom_go_invocations="$test_tmp/go-invocations.log" +printf '%s\n' \ + '#!/usr/bin/env bash' \ + 'set -euo pipefail' \ + "printf '%s\\n' \"\$*\" >> '$custom_go_invocations'" \ + 'exec /snap/bin/go "$@"' > "$custom_go_binary" +chmod 0755 "$custom_go_binary" +sudo rm -f -- "$binary_path" +recovery_log="$test_tmp/recovery.log" +run_converge "$recovery_log" \ + -e '{"audita_manage_build_dependencies": false}' \ + -e "audita_go_binary=$custom_go_binary" +grep -q . "$custom_go_invocations" validate_config +recovery_repeat_log="$test_tmp/recovery-repeat.log" +run_converge "$recovery_repeat_log" \ + -e '{"audita_manage_build_dependencies": false}' \ + -e "audita_go_binary=$custom_go_binary" +require_clean_recap "$recovery_repeat_log" diff --git a/tests/unsupported.yml b/tests/unsupported.yml deleted file mode 100644 index 4a6e72c..0000000 --- a/tests/unsupported.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Exercise Audita platform mutation guard - hosts: localhost - gather_facts: false - vars: - ansible_facts: - distribution: "{{ test_distribution }}" - distribution_version: "{{ test_distribution_version }}" - architecture: "{{ test_architecture }}" - roles: - - role: ansible-role-audita diff --git a/tests/verify-config.yml b/tests/verify-config.yml index 5fc4403..ab5240e 100644 --- a/tests/verify-config.yml +++ b/tests/verify-config.yml @@ -9,10 +9,22 @@ src: /usr/local/etc/audita/config.yml register: audita_rendered_config_file - - name: Check rendered configuration + - name: Check representative rendered configuration ansible.builtin.assert: that: - - '(audita_rendered_config_file.content | b64decode | from_yaml) == audita_expected_config' + - audita_rendered_config.version == 1 + - audita_rendered_config.pipeline.modules == ["glossary"] + - audita_rendered_config.llm.proposal.base_url == "https://openrouter.ai/api/v1" + - audita_rendered_config.llm.proposal.model == "openrouter/google/gemma-4-31b-it" + - audita_rendered_config.llm.proposal.timeout == "10s" + - audita_rendered_config.llm.proposal.max_retries == 0 + - audita_rendered_config.context.description == "" + - audita_rendered_config.diagnostics.retention == "never" + - audita_rendered_config.llm.validation is not defined + - audita_rendered_config.output is not defined + - audita_rendered_config.diagnostics.work_dir is not defined + vars: + audita_rendered_config: "{{ audita_rendered_config_file.content | b64decode | from_yaml }}" - name: Validate rendered configuration ansible.builtin.command: @@ -22,10 +34,4 @@ - validate - --config - /usr/local/etc/audita/config.yml - register: audita_rendered_config_validation changed_when: false - - - name: Check rendered configuration validation output - ansible.builtin.assert: - that: - - audita_rendered_config_validation.stdout == "config is valid" diff --git a/tests/verify.yml b/tests/verify.yml deleted file mode 100644 index 4335905..0000000 --- a/tests/verify.yml +++ /dev/null @@ -1,97 +0,0 @@ ---- -- name: Verify Audita installation - hosts: localhost - gather_facts: false - tasks: - - name: Confirm selected source commit - become: true - ansible.builtin.command: - argv: - - git - - -C - - /usr/local/src/audita - - rev-parse - - HEAD - register: audita_commit - changed_when: false - - - name: Check selected source commit - ansible.builtin.assert: - that: - - audita_commit.stdout == audita_expected_commit - - - name: Inspect source ownership - become: true - ansible.builtin.command: - argv: - - find - - /usr/local/src/audita - - -not - - -user - - root - - -o - - -not - - -group - - root - register: audita_source_ownership - changed_when: false - - - name: Check source ownership - ansible.builtin.assert: - that: - - audita_source_ownership.stdout == "" - - - name: Inspect binary metadata - become: true - ansible.builtin.stat: - path: /usr/local/bin/audita - register: audita_binary_metadata - - - name: Check binary metadata - ansible.builtin.assert: - that: - - audita_binary_metadata.stat.isreg - - audita_binary_metadata.stat.pw_name == "root" - - audita_binary_metadata.stat.gr_name == "root" - - audita_binary_metadata.stat.mode == "0755" - - audita_binary_metadata.stat.executable - - - name: Inspect configuration metadata - become: true - ansible.builtin.stat: - path: /usr/local/etc/audita/config.yml - register: audita_config_metadata - - - name: Check configuration metadata - ansible.builtin.assert: - that: - - audita_config_metadata.stat.pw_name == "root" - - audita_config_metadata.stat.gr_name == "root" - - audita_config_metadata.stat.mode == "0644" - - - name: Read default configuration - become: true - ansible.builtin.slurp: - src: /usr/local/etc/audita/config.yml - register: audita_default_config - - - name: Check default configuration - ansible.builtin.assert: - that: - - '(audita_default_config.content | b64decode | from_yaml) == {"version": 1}' - - - name: Validate Audita configuration - ansible.builtin.command: - argv: - - /usr/local/bin/audita - - config - - validate - - --config - - /usr/local/etc/audita/config.yml - register: audita_config_validation - changed_when: false - - - name: Check configuration validation output - ansible.builtin.assert: - that: - - audita_config_validation.stdout == "config is valid"