From 52f67fe389b644eebd97f006517dd2f28f0f791d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 28 Sep 2026 13:45:44 +0000 Subject: [PATCH] Verify Audita version 2 configuration choices --- tests/integration | 70 +++++++++++++++++++++++++++++++++++++++-- tests/verify-config.yml | 39 ++++++++++++++++++++--- 2 files changed, 101 insertions(+), 8 deletions(-) diff --git a/tests/integration b/tests/integration index 67d5bbd..52578c0 100755 --- a/tests/integration +++ b/tests/integration @@ -40,12 +40,37 @@ validate_config() { "$binary_path" config validate --config "$config_path" } +verify_profiles() { + expected_validation_id=$1 + expected_validation_model=$2 + expected_validation_timeout=$3 + "$binary_path" config print-effective --config "$config_path" --view profiles | \ + "$workspace_venv/bin/python" -c ' +import sys +import yaml +profiles = yaml.safe_load(sys.stdin) +assert profiles["proposal"]["id"] == "fixture-proposal" +assert profiles["proposal"]["backend"] == "fixture" +assert profiles["proposal"]["model"] == "offline-fixture" +assert profiles["proposal"]["timeout_seconds"] == 30 +assert profiles["validation"]["id"] == sys.argv[1] +assert profiles["validation"]["backend"] == "fixture" +assert profiles["validation"]["model"] == sys.argv[2] +assert profiles["validation"]["timeout_seconds"] == int(sys.argv[3]) +' "$expected_validation_id" "$expected_validation_model" "$expected_validation_timeout" +} + mkdir -p "$profile_dir" printf '%s\n' \ 'id: fixture-proposal' \ 'backend: fixture' \ 'model: offline-fixture' \ 'timeout_seconds: 30' > "$profile_dir/fixture-proposal.yaml" +printf '%s\n' \ + 'id: fixture-validation' \ + 'backend: fixture' \ + 'model: offline-validation' \ + 'timeout_seconds: 45' > "$profile_dir/fixture-validation.yaml" baseline_args=( -e "audita_config_path=$config_path" @@ -64,16 +89,55 @@ cd "$repository_root" baseline_log="$test_tmp/baseline.log" run_converge "$baseline_log" validate_config -"$binary_path" config print-effective --config "$config_path" --view profiles > "$test_tmp/baseline-profiles.yml" +verify_profiles fixture-proposal offline-fixture 30 +"$ansible_playbook" -i "$inventory" \ + -e "audita_test_config_path=$config_path" \ + -e 'audita_test_case=baseline' "$verify_config" baseline_repeat_log="$test_tmp/baseline-repeat.log" run_converge "$baseline_repeat_log" require_clean_recap "$baseline_repeat_log" -representative_config='{"audita_pipeline_modules": ["glossary"], "audita_context_description": "", "audita_output_schema": null, "audita_llm_validation_profile": null}' +representative_config='{ + "audita_backend_base": {"fixture": {"endpoint": "http://127.0.0.1:11434/v1", "credential": null}}, + "audita_backend_capacity": {"fixture": {"concurrency": 2, "queue_capacity": 0}}, + "audita_backends": "{{ audita_backend_base | combine(audita_backend_capacity, recursive=true) }}", + "audita_profiles_dir": "profiles", + "audita_llm_validation_profile": "fixture-validation", + "audita_llm_transient_retries": 0, + "audita_llm_repair_attempts": 0, + "audita_llm_operation_timeout": "45s", + "audita_validation_max_prompt_tokens": 128, + "audita_pipeline_modules": [], + "audita_normalization_max_segment_gap": 1.25, + "audita_normalization_ellipsis_gap": 2.5, + "audita_normalization_max_segment_duration": 30.75, + "audita_context_description": "", + "audita_reporting_enabled": false, + "audita_reporting_stdout_dir": "", + "audita_diagnostics_debug": false +}' representative_log="$test_tmp/representative-config.log" run_converge "$representative_log" -e "$representative_config" validate_config -"$ansible_playbook" -i "$inventory" -e "audita_test_config_path=$config_path" "$verify_config" +verify_profiles fixture-validation offline-validation 45 +"$binary_path" config print-effective --config "$config_path" --view settings | \ + "$workspace_venv/bin/python" -c ' +import sys +import yaml +settings = yaml.safe_load(sys.stdin) +assert settings["backends"]["fixture"]["concurrency"] == 2 +assert settings["backends"]["fixture"]["queue_capacity"] == 0 +assert settings["llm"]["validation_profile"] == "fixture-validation" +assert settings["llm"]["transient_retries"] == 0 +assert settings["llm"]["repair_attempts"] == 0 +assert settings["llm"]["operation_timeout"] == "45s" +assert settings["normalization"]["max_segment_gap"] == 1.25 +assert settings["reporting"]["enabled"] is False +assert settings["diagnostics"]["debug"] is False +' +"$ansible_playbook" -i "$inventory" \ + -e "audita_test_config_path=$config_path" \ + -e 'audita_test_case=representative' "$verify_config" representative_repeat_log="$test_tmp/representative-config-repeat.log" run_converge "$representative_repeat_log" -e "$representative_config" require_clean_recap "$representative_repeat_log" diff --git a/tests/verify-config.yml b/tests/verify-config.yml index d1fc9e8..f9c4e65 100644 --- a/tests/verify-config.yml +++ b/tests/verify-config.yml @@ -2,23 +2,52 @@ - name: Verify rendered Audita configuration hosts: localhost gather_facts: false + vars: + audita_rendered_config: "{{ audita_rendered_config_file.content | b64decode | from_yaml }}" tasks: - name: Read rendered configuration ansible.builtin.slurp: src: "{{ audita_test_config_path }}" register: audita_rendered_config_file + - name: Check omitted baseline settings + ansible.builtin.assert: + that: + - audita_rendered_config.profiles_dir is not defined + - audita_rendered_config.llm.validation_profile is not defined + - audita_rendered_config.llm.transient_retries is not defined + - audita_rendered_config.llm.repair_attempts is not defined + - audita_rendered_config.llm.operation_timeout is not defined + - audita_rendered_config.reporting is not defined + - audita_rendered_config.diagnostics is not defined + when: audita_test_case == "baseline" + - name: Check representative rendered configuration ansible.builtin.assert: that: - audita_rendered_config.version == 2 + - audita_rendered_config.profiles_dir == "profiles" - audita_rendered_config.backends.fixture.endpoint == "http://127.0.0.1:11434/v1" - audita_rendered_config.backends.fixture.credential is none + - audita_rendered_config.backends.fixture.concurrency == 2 + - audita_rendered_config.backends.fixture.queue_capacity == 0 - audita_rendered_config.llm.proposal_profile == "fixture-proposal" - - audita_rendered_config.pipeline.modules == ["glossary"] + - audita_rendered_config.llm.validation_profile == "fixture-validation" + - audita_rendered_config.llm.transient_retries == 0 + - audita_rendered_config.llm.repair_attempts == 0 + - audita_rendered_config.llm.operation_timeout == "45s" + - audita_rendered_config.validation.max_prompt_tokens == 128 + - audita_rendered_config.pipeline.modules == [] + - audita_rendered_config.normalization.max_segment_gap == 1.25 + - audita_rendered_config.normalization.max_segment_gap is number + - audita_rendered_config.normalization.ellipsis_gap == 2.5 + - audita_rendered_config.normalization.ellipsis_gap is number + - audita_rendered_config.normalization.max_segment_duration == 30.75 + - audita_rendered_config.normalization.max_segment_duration is number - audita_rendered_config.context.description == "" - - audita_rendered_config.llm.validation_profile is not defined + - audita_rendered_config.reporting.enabled == false + - audita_rendered_config.reporting.stdout_dir == "" + - audita_rendered_config.diagnostics.debug == false - audita_rendered_config.output is not defined - - audita_rendered_config.diagnostics is not defined - vars: - audita_rendered_config: "{{ audita_rendered_config_file.content | b64decode | from_yaml }}" + - audita_rendered_config.diagnostics.work_dir is not defined + when: audita_test_case == "representative"