Simplify the Audita integration tests

2026-09-09 04:27:44 +00:00
parent 847c4c3811
commit d2d0b9d00b
5 changed files with 51 additions and 269 deletions
--- a/README.md
+++ b/README.md
@@ -83,19 +83,20 @@ production convergence.
 ## Validation

 The integration driver installs pinned collection dependencies, runs syntax
-checking and ansible-lint, verifies the default release, file ownership and
-modes, executable behavior, and native configuration validation. It covers
-missing-binary and checkout rebuilds, caller-supplied Go with dependency
-management disabled, metadata repair without rebuilding, representative
-field-specific configuration rendering, explicitly selected falsey values,
-restoration to the minimal default configuration, and platform failures before
-isolated application state exists.
+checking and ansible-lint, and validates the default configuration with Audita
+before requiring clean repeated convergence. It also covers one representative
+field-specific configuration with nested values, null omission, and intentional
+falsey values, then validates it natively. A combined recovery scenario proves
+that a missing binary is rebuilt with a caller-supplied Go executable while
+dependency management is disabled. Audita currently has no native
+release-identity command, so the suite intentionally does not use a source
+commit as a version-test proxy.
 On a fresh host, default convergence may also install Git through APT, refresh
 the APT cache, and install the Go Snap. Application state is limited to
 `/usr/local/src/audita`, `/usr/local/bin/audita`, and
 `/usr/local/etc/audita/config.yml`; temporary test-specific files remain inside
-the suite's private temporary directory. The suite leaves the default release
-and configuration installed:
+the suite's private temporary directory. The suite restores the default
+deployment during cleanup:

 ```sh
 tests/integration
--- a/tests/integration
+++ b/tests/integration
@@ -11,19 +11,17 @@ export ANSIBLE_ROLES_PATH="$(dirname "$repository_root")"

 inventory="$repository_root/tests/inventory"
 converge="$repository_root/tests/converge.yml"
-verify="$repository_root/tests/verify.yml"
 verify_config="$repository_root/tests/verify-config.yml"
-unsupported="$repository_root/tests/unsupported.yml"
 ansible_playbook="$workspace_venv/bin/ansible-playbook"
-default_commit=a84941d68168a1ed9ea7423b6477ae8218be73f9
-alternate_commit=46b7356a3b54a2771ce9a617e4fa3e8d13728cac
-source_dir=/usr/local/src/audita
 binary_path=/usr/local/bin/audita
 config_path=/usr/local/etc/audita/config.yml

 cleanup() {
+  exit_status=$?
+  trap - EXIT
  "$ansible_playbook" -i "$inventory" "$converge" >/dev/null 2>&1 || true
  rm -rf -- "$test_tmp"
+  exit "$exit_status"
 }
 trap cleanup EXIT

@@ -31,32 +29,14 @@ require_clean_recap() {
  grep -Eq 'changed=0 .*failed=0' "$1"
 }

-require_changed_recap() {
-  grep -Eq 'changed=[1-9][0-9]* .*failed=0' "$1"
-}
-
 run_converge() {
  output_file=$1
  shift
  "$ansible_playbook" -i "$inventory" "$@" "$converge" | tee "$output_file"
 }

-verify_release() {
-  expected_commit=$1
-  "$ansible_playbook" -i "$inventory" \
-    -e "audita_expected_commit=$expected_commit" \
-    "$verify"
-}
-
-verify_rendered_config() {
-  expected_config=$1
-  "$ansible_playbook" -i "$inventory" \
-    -e "{\"audita_expected_config\": $expected_config}" \
-    "$verify_config"
-}
-
 validate_config() {
-  [[ "$("$binary_path" config validate --config "$config_path")" == 'config is valid' ]]
+  "$binary_path" config validate --config "$config_path"
 }

 cd "$repository_root"
@@ -64,137 +44,40 @@ cd "$repository_root"
  -r collections/requirements.yml \
  -p "$ANSIBLE_COLLECTIONS_PATH"
 "$ansible_playbook" -i "$inventory" --syntax-check "$converge"
-"$workspace_venv/bin/ansible-lint" "$converge" "$verify" "$verify_config" "$unsupported"
+"$workspace_venv/bin/ansible-lint" "$converge" "$verify_config"

 baseline_log="$test_tmp/baseline.log"
 run_converge "$baseline_log"
-verify_release "$default_commit"
+validate_config
 baseline_repeat_log="$test_tmp/baseline-repeat.log"
 run_converge "$baseline_repeat_log"
 require_clean_recap "$baseline_repeat_log"

-sudo rm -f -- "$binary_path"
-missing_binary_log="$test_tmp/missing-binary.log"
-run_converge "$missing_binary_log"
-require_changed_recap "$missing_binary_log"
-grep -A1 -E 'TASK .*Build Audita' "$missing_binary_log" | grep -q '^changed:'
-missing_binary_repeat_log="$test_tmp/missing-binary-repeat.log"
-run_converge "$missing_binary_repeat_log"
-require_clean_recap "$missing_binary_repeat_log"
-
-custom_go_binary="$test_tmp/go"
-custom_go_invocation_log="$test_tmp/custom-go-invocation.log"
-printf '%s\n' \
-  '#!/usr/bin/env bash' \
-  "printf 'invoked\\n' >> '$custom_go_invocation_log'" \
-  'exec /snap/bin/go "$@"' > "$custom_go_binary"
-chmod 0755 "$custom_go_binary"
-sudo rm -f -- "$binary_path"
-custom_go_log="$test_tmp/custom-go.log"
-run_converge "$custom_go_log" \
-  -e '{"audita_manage_build_dependencies": false}' \
-  -e "audita_go_binary=$custom_go_binary"
-require_changed_recap "$custom_go_log"
-grep -A1 -E 'TASK .*Build Audita' "$custom_go_log" | grep -q '^changed:'
-[[ -s "$custom_go_invocation_log" ]]
-custom_go_repeat_log="$test_tmp/custom-go-repeat.log"
-run_converge "$custom_go_repeat_log" \
-  -e '{"audita_manage_build_dependencies": false}' \
-  -e "audita_go_binary=$custom_go_binary"
-require_clean_recap "$custom_go_repeat_log"
-
-sudo chmod 0700 "$binary_path"
-metadata_log="$test_tmp/metadata.log"
-run_converge "$metadata_log"
-require_changed_recap "$metadata_log"
-grep -A1 -E 'TASK .*Build Audita' "$metadata_log" | grep -q '^skipping:'
-grep -A3 -E 'TASK .*Converge installed metadata' "$metadata_log" | \
-  grep -q "^changed:.*'path': '$binary_path'"
-metadata_repeat_log="$test_tmp/metadata-repeat.log"
-run_converge "$metadata_repeat_log"
-require_clean_recap "$metadata_repeat_log"
-
-representative_config='{"audita_pipeline_modules": ["glossary"], "audita_output_schema": "bare-segments", "audita_llm_proposal_base_url": "https://openrouter.ai/api/v1", "audita_llm_proposal_model": "openrouter/google/gemma-4-31b-it", "audita_llm_proposal_timeout": "10s", "audita_llm_proposal_max_retries": 1, "audita_llm_validation_base_url": "https://openrouter.ai/api/v1", "audita_llm_validation_model": "openrouter/google/gemma-4-31b-it", "audita_llm_validation_timeout": 10, "audita_llm_validation_max_retries": 1, "audita_concurrency_total_llm": 2, "audita_concurrency_proposal_llm": 1, "audita_concurrency_validation_llm": 1, "audita_chunking_target_sections": 2, "audita_chunking_max_section_tokens": 8192, "audita_chunking_min_section_tokens": 2048, "audita_normalization_max_segment_gap": "4s", "audita_normalization_ellipsis_gap": 3.5, "audita_normalization_max_segment_duration": 60, "audita_normalization_max_segment_tokens": 2048, "audita_thresholds_glossary": 0.8, "audita_thresholds_homophones": 0.8, "audita_thresholds_spoken_word": 0.8, "audita_thresholds_grammar": 0.8, "audita_context_description": "Focused integration coverage", "audita_diagnostics_work_dir": "/tmp/audita-integration", "audita_diagnostics_retention": "never"}'
-representative_expected='{"version": 1, "pipeline": {"modules": ["glossary"]}, "output": {"schema": "bare-segments"}, "llm": {"proposal": {"base_url": "https://openrouter.ai/api/v1", "model": "openrouter/google/gemma-4-31b-it", "timeout": "10s", "max_retries": 1}, "validation": {"base_url": "https://openrouter.ai/api/v1", "model": "openrouter/google/gemma-4-31b-it", "timeout": 10, "max_retries": 1}}, "concurrency": {"total_llm": 2, "proposal_llm": 1, "validation_llm": 1}, "chunking": {"target_sections": 2, "max_section_tokens": 8192, "min_section_tokens": 2048}, "normalization": {"max_segment_gap": "4s", "ellipsis_gap": 3.5, "max_segment_duration": 60, "max_segment_tokens": 2048}, "thresholds": {"glossary": 0.8, "homophones": 0.8, "spoken_word": 0.8, "grammar": 0.8}, "context": {"description": "Focused integration coverage"}, "diagnostics": {"work_dir": "/tmp/audita-integration", "retention": "never"}}'
+representative_config='{"audita_config_version": 1, "audita_pipeline_modules": ["glossary"], "audita_llm_proposal_base_url": "https://openrouter.ai/api/v1", "audita_llm_proposal_model": "openrouter/google/gemma-4-31b-it", "audita_llm_proposal_timeout": "10s", "audita_llm_proposal_max_retries": 0, "audita_context_description": "", "audita_diagnostics_retention": "never", "audita_llm_validation_model": null, "audita_output_schema": null, "audita_diagnostics_work_dir": null}'
 representative_log="$test_tmp/representative-config.log"
 run_converge "$representative_log" -e "$representative_config"
-require_changed_recap "$representative_log"
-grep -A1 -E 'TASK .*Build Audita' "$representative_log" | grep -q '^skipping:'
-grep -A1 -E 'TASK .*Write Audita configuration' "$representative_log" | grep -q '^changed:'
-verify_rendered_config "$representative_expected"
-validate_config
+"$ansible_playbook" -i "$inventory" "$verify_config"
 representative_repeat_log="$test_tmp/representative-config-repeat.log"
 run_converge "$representative_repeat_log" -e "$representative_config"
 require_clean_recap "$representative_repeat_log"

-falsey_config='{"audita_pipeline_modules": [], "audita_context_description": ""}'
-falsey_expected='{"version": 1, "pipeline": {"modules": []}, "context": {"description": ""}}'
-falsey_log="$test_tmp/falsey-config.log"
-run_converge "$falsey_log" -e "$falsey_config"
-require_changed_recap "$falsey_log"
-grep -A1 -E 'TASK .*Build Audita' "$falsey_log" | grep -q '^skipping:'
-grep -A1 -E 'TASK .*Write Audita configuration' "$falsey_log" | grep -q '^changed:'
-verify_rendered_config "$falsey_expected"
-falsey_repeat_log="$test_tmp/falsey-config-repeat.log"
-run_converge "$falsey_repeat_log" -e "$falsey_config"
-require_clean_recap "$falsey_repeat_log"
-
-default_config_log="$test_tmp/default-config.log"
-run_converge "$default_config_log"
-require_changed_recap "$default_config_log"
-grep -A1 -E 'TASK .*Build Audita' "$default_config_log" | grep -q '^skipping:'
-grep -A1 -E 'TASK .*Write Audita configuration' "$default_config_log" | grep -q '^changed:'
-verify_release "$default_commit"
-default_config_repeat_log="$test_tmp/default-config-repeat.log"
-run_converge "$default_config_repeat_log"
-require_clean_recap "$default_config_repeat_log"
-
-alternate_release_log="$test_tmp/alternate-release.log"
-run_converge "$alternate_release_log" -e "audita_version=$alternate_commit"
-require_changed_recap "$alternate_release_log"
-grep -A1 -E 'TASK .*Build Audita' "$alternate_release_log" | grep -q '^changed:'
-verify_release "$alternate_commit"
-alternate_repeat_log="$test_tmp/alternate-repeat.log"
-run_converge "$alternate_repeat_log" -e "audita_version=$alternate_commit"
-require_clean_recap "$alternate_repeat_log"
-
-restored_release_log="$test_tmp/restored-release.log"
-run_converge "$restored_release_log"
-require_changed_recap "$restored_release_log"
-grep -A1 -E 'TASK .*Build Audita' "$restored_release_log" | grep -q '^changed:'
-verify_release "$default_commit"
-restored_repeat_log="$test_tmp/restored-repeat.log"
-run_converge "$restored_repeat_log"
-require_clean_recap "$restored_repeat_log"
-
-platform_cases=(
-  'Debian 26.04 x86_64 distribution'
-  'Ubuntu 24.04 x86_64 release'
-  'Ubuntu 26.04 riscv64 architecture'
-)
-for platform_case in "${platform_cases[@]}"; do
-  read -r distribution release architecture label <<<"$platform_case"
-  case_root="$test_tmp/unsupported-$label"
-  case_source="$case_root/source"
-  case_binary="$case_root/bin/audita"
-  case_config="$case_root/etc/config.yml"
-  case_log="$test_tmp/unsupported-$label.log"
-  if "$ansible_playbook" -i "$inventory" \
-    -e "test_distribution=$distribution" \
-    -e "test_distribution_version=$release" \
-    -e "test_architecture=$architecture" \
-    -e "audita_source_dir=$case_source" \
-    -e "audita_binary_path=$case_binary" \
-    -e "audita_config_path=$case_config" \
-    "$unsupported" | tee "$case_log"; then
-    echo "Unsupported $label case unexpectedly succeeded" >&2
-    exit 1
-  fi
-  grep -Eq 'failed=1' "$case_log"
-  [[ ! -e "$case_root" ]]
-done
-
-[[ "$(sudo git -C "$source_dir" rev-parse HEAD)" == "$default_commit" ]]
-[[ "$(sudo stat -c '%U:%G %a' "$binary_path")" == 'root:root 755' ]]
-[[ "$(sudo stat -c '%U:%G %a' "$config_path")" == 'root:root 644' ]]
+custom_go_binary="$test_tmp/go"
+custom_go_invocations="$test_tmp/go-invocations.log"
+printf '%s\n' \
+  '#!/usr/bin/env bash' \
+  'set -euo pipefail' \
+  "printf '%s\\n' \"\$*\" >> '$custom_go_invocations'" \
+  'exec /snap/bin/go "$@"' > "$custom_go_binary"
+chmod 0755 "$custom_go_binary"
+sudo rm -f -- "$binary_path"
+recovery_log="$test_tmp/recovery.log"
+run_converge "$recovery_log" \
+  -e '{"audita_manage_build_dependencies": false}' \
+  -e "audita_go_binary=$custom_go_binary"
+grep -q . "$custom_go_invocations"
 validate_config
+recovery_repeat_log="$test_tmp/recovery-repeat.log"
+run_converge "$recovery_repeat_log" \
+  -e '{"audita_manage_build_dependencies": false}' \
+  -e "audita_go_binary=$custom_go_binary"
+require_clean_recap "$recovery_repeat_log"
--- a/tests/unsupported.yml
+++ b/tests/unsupported.yml
@@ -1,11 +0,0 @@
---
- name: Exercise Audita platform mutation guard
-  hosts: localhost
-  gather_facts: false
-  vars:
-    ansible_facts:
-      distribution: "{{ test_distribution }}"
-      distribution_version: "{{ test_distribution_version }}"
-      architecture: "{{ test_architecture }}"
-  roles:
-    - role: ansible-role-audita
--- a/tests/verify-config.yml
+++ b/tests/verify-config.yml
@@ -9,10 +9,22 @@
        src: /usr/local/etc/audita/config.yml
      register: audita_rendered_config_file

-    - name: Check rendered configuration
+    - name: Check representative rendered configuration
      ansible.builtin.assert:
        that:
-          - '(audita_rendered_config_file.content | b64decode | from_yaml) == audita_expected_config'
+          - audita_rendered_config.version == 1
+          - audita_rendered_config.pipeline.modules == ["glossary"]
+          - audita_rendered_config.llm.proposal.base_url == "https://openrouter.ai/api/v1"
+          - audita_rendered_config.llm.proposal.model == "openrouter/google/gemma-4-31b-it"
+          - audita_rendered_config.llm.proposal.timeout == "10s"
+          - audita_rendered_config.llm.proposal.max_retries == 0
+          - audita_rendered_config.context.description == ""
+          - audita_rendered_config.diagnostics.retention == "never"
+          - audita_rendered_config.llm.validation is not defined
+          - audita_rendered_config.output is not defined
+          - audita_rendered_config.diagnostics.work_dir is not defined
+      vars:
+        audita_rendered_config: "{{ audita_rendered_config_file.content | b64decode | from_yaml }}"

    - name: Validate rendered configuration
      ansible.builtin.command:
@@ -22,10 +34,4 @@
          - validate
          - --config
          - /usr/local/etc/audita/config.yml
-      register: audita_rendered_config_validation
      changed_when: false
-
-    - name: Check rendered configuration validation output
-      ansible.builtin.assert:
-        that:
-          - audita_rendered_config_validation.stdout == "config is valid"
--- a/tests/verify.yml
+++ b/tests/verify.yml
@@ -1,97 +0,0 @@
---
- name: Verify Audita installation
-  hosts: localhost
-  gather_facts: false
-  tasks:
-    - name: Confirm selected source commit
-      become: true
-      ansible.builtin.command:
-        argv:
-          - git
-          - -C
-          - /usr/local/src/audita
-          - rev-parse
-          - HEAD
-      register: audita_commit
-      changed_when: false
-
-    - name: Check selected source commit
-      ansible.builtin.assert:
-        that:
-          - audita_commit.stdout == audita_expected_commit
-
-    - name: Inspect source ownership
-      become: true
-      ansible.builtin.command:
-        argv:
-          - find
-          - /usr/local/src/audita
-          - -not
-          - -user
-          - root
-          - -o
-          - -not
-          - -group
-          - root
-      register: audita_source_ownership
-      changed_when: false
-
-    - name: Check source ownership
-      ansible.builtin.assert:
-        that:
-          - audita_source_ownership.stdout == ""
-
-    - name: Inspect binary metadata
-      become: true
-      ansible.builtin.stat:
-        path: /usr/local/bin/audita
-      register: audita_binary_metadata
-
-    - name: Check binary metadata
-      ansible.builtin.assert:
-        that:
-          - audita_binary_metadata.stat.isreg
-          - audita_binary_metadata.stat.pw_name == "root"
-          - audita_binary_metadata.stat.gr_name == "root"
-          - audita_binary_metadata.stat.mode == "0755"
-          - audita_binary_metadata.stat.executable
-
-    - name: Inspect configuration metadata
-      become: true
-      ansible.builtin.stat:
-        path: /usr/local/etc/audita/config.yml
-      register: audita_config_metadata
-
-    - name: Check configuration metadata
-      ansible.builtin.assert:
-        that:
-          - audita_config_metadata.stat.pw_name == "root"
-          - audita_config_metadata.stat.gr_name == "root"
-          - audita_config_metadata.stat.mode == "0644"
-
-    - name: Read default configuration
-      become: true
-      ansible.builtin.slurp:
-        src: /usr/local/etc/audita/config.yml
-      register: audita_default_config
-
-    - name: Check default configuration
-      ansible.builtin.assert:
-        that:
-          - '(audita_default_config.content | b64decode | from_yaml) == {"version": 1}'
-
-    - name: Validate Audita configuration
-      ansible.builtin.command:
-        argv:
-          - /usr/local/bin/audita
-          - config
-          - validate
-          - --config
-          - /usr/local/etc/audita/config.yml
-      register: audita_config_validation
-      changed_when: false
-
-    - name: Check configuration validation output
-      ansible.builtin.assert:
-        that:
-          - audita_config_validation.stdout == "config is valid"