From ea8b09e190419586ede6df3815d1ea999a0f4391 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 29 Sep 2026 14:46:10 +0000 Subject: [PATCH] Verify suite configuration overrides and document migration --- README.md | 29 +++++++++++++++++ tests/integration | 82 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/README.md b/README.md index 22bd29c..4914fd7 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,28 @@ Replace `dnd_pipeline_narratio_notarius_llm_api_key` with `dnd_pipeline_narratio_audita_credentials` for Audita's backend credential names. The old diagnostics retention variable is no longer forwarded. +The old per-purpose Audita endpoint and API-key names now belong to the +backend map; both default profiles share the `dnd` backend and its one +concurrency limit. Old proposal and validation models and per-attempt timeouts +belong in complete profile mappings. The default 900-second value is retained +per attempt; `dnd_pipeline_audita_llm_operation_timeout` is an independent +whole-operation deadline and defaults to `null`, leaving Audita's native +policy in charge. The former `*_max_retries` variables do not have a direct +one-to-one replacement: use the optional Audita transient retry and repair +attempt settings according to the desired native behavior. Optional retry +pass-throughs for Notarius PromptKit and Scriptorium likewise default to +`null`; an explicit `0` is forwarded, not treated as omission. + +Inventory must supply each named backend credential through +`dnd_pipeline_secrets` and include its name in +`dnd_pipeline_narratio_audita_credentials`. Narratio forwards only names in +that list and does not discover them from Audita's configuration. The +Notarius list is independent. For profile overrides, use filename-safe map +keys equal to each profile's native `id`; the suite writes each map value +unchanged and does not merge individual fields. See [Audita's v2 manifest +migration](https://gitea.maximumdirect.net/eric/audita/src/tag/v2.0.0/docs/migrations/v2.md) +for application-owned manifests; this role does not convert them. + | Variable | Default | | --- | --- | | `dnd_pipeline_narratio_seriatim_output_schema` | `seriatim-intermediate` | @@ -260,6 +282,13 @@ dnd_pipeline_narratio_testing_scriptorium_profile: endpoint-profile dnd_pipeline_notarius_testing_llm_profile: alternate-economy-profile ``` +Narratio selects it with `--overlay testing`, and direct Notarius commands +use `--config-overlay testing`; the former `--profile` and +`--config-profile` selectors are retired. Previously deployed +`profiles/testing.yml` files are ignored, not pruned. Renaming the selected +overlay normally changes configuration identity, so cached work tied to the +old named selection may not be reused. + ### Notarius extraction policy The Notarius root configuration imports the fixed diff --git a/tests/integration b/tests/integration index ced365e..467dd00 100755 --- a/tests/integration +++ b/tests/integration @@ -46,6 +46,7 @@ baseline_args=( -e '{"dnd_pipeline_narratio_publish_outputs":[{"source":"narratio.transcript.final","required":true},{"source":"narratio.artifact.session_recap","required":true}]}' -e 'dnd_pipeline_narratio_testing_scriptorium_profile=endpoint-profile' -e 'dnd_pipeline_notarius_testing_llm_profile=gpt-terra-latest' + -e '{"dnd_pipeline_audita_llm_transient_retries":null,"dnd_pipeline_audita_llm_repair_attempts":null,"dnd_pipeline_audita_llm_operation_timeout":null,"dnd_pipeline_notarius_promptkit_transient_retries":null,"dnd_pipeline_scriptorium_transient_retries":null}' ) cleanup() { @@ -92,6 +93,22 @@ run_converge "$baseline_log" "${baseline_args[@]}" /usr/local/bin/seriatim --version AUDITA_LLM_API_KEY="$synthetic_audita_api_key" \ /usr/local/bin/audita config validate --config /usr/local/etc/audita/config.yml +"$workspace_venv/bin/python" - <<'PY' +import yaml + +with open("/usr/local/etc/audita/config.yml", encoding="utf-8") as config_file: + audita = yaml.safe_load(config_file) +with open("/usr/local/etc/notarius/config.yml", encoding="utf-8") as config_file: + notarius = yaml.safe_load(config_file) +with open("/usr/local/etc/scriptorium/config.yml", encoding="utf-8") as config_file: + scriptorium = yaml.safe_load(config_file) + +assert "transient_retries" not in audita["llm"] +assert "repair_attempts" not in audita["llm"] +assert "operation_timeout" not in audita["llm"] +assert "transient_retries" not in notarius.get("promptkit", {}) +assert "transient_retries" not in scriptorium +PY /usr/local/bin/notarius config validate --config /usr/local/etc/notarius/config.yml \ --pipeline dnd-session /usr/local/bin/notarius config validate --config /usr/local/etc/notarius/config.yml \ @@ -302,3 +319,68 @@ PY alternate_repeat_log="$test_tmp/alternate-repeat.log" run_converge "$alternate_repeat_log" "${baseline_args[@]}" -e "$alternate_config" require_clean_recap "$alternate_repeat_log" + +synthetic_fixture_audita_key='audita-override-synthetic-value' +synthetic_fixture_notarius_key='notarius-override-synthetic-value' +override_config="{\"dnd_pipeline_audita_backends\":{\"fixture\":{\"endpoint\":\"https://inference.fixture.invalid/v1\",\"credential\":\"FIXTURE_AUDITA_KEY\",\"concurrency\":2}},"\ +"\"dnd_pipeline_audita_profiles\":{\"fixture-proposal\":{\"id\":\"fixture-proposal\",\"backend\":\"fixture\",\"model\":\"fixture/proposal\",\"timeout_seconds\":900},"\ +"\"fixture-validation\":{\"id\":\"fixture-validation\",\"backend\":\"fixture\",\"model\":\"fixture/validation\",\"timeout_seconds\":900}},"\ +"\"dnd_pipeline_audita_llm_proposal_profile\":\"fixture-proposal\",\"dnd_pipeline_audita_llm_validation_profile\":\"fixture-validation\","\ +"\"dnd_pipeline_narratio_audita_credentials\":[\"FIXTURE_AUDITA_KEY\"],\"dnd_pipeline_narratio_notarius_credentials\":[\"FIXTURE_NOTARIUS_KEY\"],"\ +"\"dnd_pipeline_audita_llm_transient_retries\":0,\"dnd_pipeline_audita_llm_repair_attempts\":0,\"dnd_pipeline_audita_llm_operation_timeout\":\"20m\","\ +"\"dnd_pipeline_notarius_promptkit_transient_retries\":0,\"dnd_pipeline_scriptorium_transient_retries\":0,"\ +"\"dnd_pipeline_secrets\":{\"AUDITA_LLM_API_KEY\":\"$synthetic_audita_api_key\",\"OPENROUTER_API_KEY\":\"$synthetic_notarius_api_key\","\ +"\"FIXTURE_AUDITA_KEY\":\"$synthetic_fixture_audita_key\",\"FIXTURE_NOTARIUS_KEY\":\"$synthetic_fixture_notarius_key\","\ +"\"OBJECT_STORAGE_KEY_ID\":\"$synthetic_object_storage_key_id\",\"OBJECT_STORAGE_KEY\":\"$synthetic_object_storage_key\"}}" +override_log="$test_tmp/override.log" +run_converge "$override_log" "${baseline_args[@]}" -e "$override_config" +FIXTURE_AUDITA_KEY="$synthetic_fixture_audita_key" \ + /usr/local/bin/audita config validate --config /usr/local/etc/audita/config.yml +audita_override_settings="$test_tmp/audita-override-settings.yml" +audita_override_profiles="$test_tmp/audita-override-profiles.yml" +FIXTURE_AUDITA_KEY="$synthetic_fixture_audita_key" \ + /usr/local/bin/audita config print-effective --config /usr/local/etc/audita/config.yml \ + --view settings > "$audita_override_settings" +FIXTURE_AUDITA_KEY="$synthetic_fixture_audita_key" \ + /usr/local/bin/audita config print-effective --config /usr/local/etc/audita/config.yml \ + --view profiles > "$audita_override_profiles" +notarius_override_config="$test_tmp/notarius-override.yml" +scriptorium_override_config="$test_tmp/scriptorium-override.yml" +narratio_override_config="$test_tmp/narratio-override.yml" +/usr/local/bin/notarius config effective --config /usr/local/etc/notarius/config.yml \ + > "$notarius_override_config" +/usr/local/bin/scriptorium config show --config /usr/local/etc/scriptorium/config.yml \ + > "$scriptorium_override_config" +/usr/bin/sudo -u narratio /usr/local/bin/narratio config show \ + --config /usr/local/etc/narratio/pipeline.yml --campaign centaurs \ + > "$narratio_override_config" +"$workspace_venv/bin/python" - \ + "$audita_override_settings" "$audita_override_profiles" \ + "$notarius_override_config" "$scriptorium_override_config" \ + "$narratio_override_config" <<'PY' +import sys + +import yaml + +settings, profiles, notarius, scriptorium, narratio = ( + yaml.safe_load(open(path, encoding="utf-8")) for path in sys.argv[1:] +) +assert settings["backends"]["fixture"]["endpoint"] == "https://inference.fixture.invalid/v1" +assert settings["backends"]["fixture"]["concurrency"] == 2 +assert settings["llm"]["proposal_profile"] == "fixture-proposal" +assert settings["llm"]["validation_profile"] == "fixture-validation" +assert settings["llm"]["transient_retries"] == 0 +assert settings["llm"]["repair_attempts"] == 0 +assert settings["llm"]["operation_timeout"] == "20m0s" +assert profiles["proposal"]["model"] == "fixture/proposal" +assert profiles["validation"]["model"] == "fixture/validation" +assert notarius["promptkit"]["transient_retries"] == 0 +assert scriptorium["transient_retries"] == 0 +assert narratio["audita"]["credentials"] == ["FIXTURE_AUDITA_KEY"] +assert narratio["notarius"]["credentials"] == ["FIXTURE_NOTARIUS_KEY"] +PY +! grep -Fq "$synthetic_fixture_audita_key" "$override_log" "$narratio_override_config" +! grep -Fq "$synthetic_fixture_notarius_key" "$override_log" "$narratio_override_config" +override_repeat_log="$test_tmp/override-repeat.log" +run_converge "$override_repeat_log" "${baseline_args[@]}" -e "$override_config" +require_clean_recap "$override_repeat_log"