eric/ansible-collection-dnd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Squashed 'roles/pipeline/' changes from 8ceb3f4..c56a942

Browse Source 
c56a942 Add the Notarius testing overlay and production model defaults

git-subtree-dir: roles/pipeline
git-subtree-split: c56a9425d30ef714166f1ff7675b202590f86425
This commit is contained in:
Eric Rakestraw
2026-09-22 18:53:13 +00:00
parent 68a8bbc462
commit e1238eb4fa
8 changed files with 127 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
tests/integration
View File

@@ -44,6 +44,8 @@ baseline_args=(
-e 'dnd_pipeline_narratio_storage_s3_secret_access_key=OBJECT_STORAGE_KEY'
-e '{"dnd_pipeline_narratio_workspace_cleanup_after_publish":false,"dnd_pipeline_narratio_storage_s3_force_path_style":false,"dnd_pipeline_narratio_cache_s3_audio":false,"dnd_pipeline_narratio_publish_enabled":true,"dnd_pipeline_narratio_publish_upload_run":true}'
-e '{"dnd_pipeline_narratio_publish_outputs":[{"source":"narratio.transcript.final","required":true},{"source":"narratio.artifact.session_recap","required":true}]}'
-e 'dnd_pipeline_narratio_testing_scriptorium_profile=endpoint-profile'
-e 'dnd_pipeline_notarius_testing_llm_profile=gpt-terra-latest'
)
cleanup() {
@@ -91,9 +93,14 @@ run_converge "$baseline_log" "${baseline_args[@]}"
/usr/local/bin/audita config validate --config /usr/local/etc/audita/config.yml
/usr/local/bin/notarius config validate --config /usr/local/etc/notarius/config.yml \
--pipeline dnd-session
/usr/local/bin/notarius config validate --config /usr/local/etc/notarius/config.yml \
--config-profile testing --pipeline dnd-session
notarius_effective_config="$test_tmp/notarius-effective.yml"
/usr/local/bin/notarius config effective --config /usr/local/etc/notarius/config.yml \
> "$notarius_effective_config"
notarius_testing_effective_config="$test_tmp/notarius-testing-effective.yml"
/usr/local/bin/notarius config effective --config /usr/local/etc/notarius/config.yml \
--config-profile testing > "$notarius_testing_effective_config"
grep -Fq 'mode: auto' "$notarius_effective_config"
grep -Fq 'step: describe-session' "$notarius_effective_config"
/usr/local/bin/scriptorium inspect profile --config /usr/local/etc/scriptorium/config.yml \
@@ -108,6 +115,10 @@ narratio_effective_config="$test_tmp/narratio-effective.yml"
/usr/bin/sudo -u narratio /usr/local/bin/narratio config show \
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs \
> "$narratio_effective_config"
narratio_testing_effective_config="$test_tmp/narratio-testing-effective.yml"
/usr/bin/sudo -u narratio /usr/local/bin/narratio config show \
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs --profile testing \
> "$narratio_testing_effective_config"
grep -Fq 'enabled: true' "$narratio_effective_config"
grep -Fq 'llm_api_key: AUDITA_LLM_API_KEY' "$narratio_effective_config"
grep -Fq 'pipeline_id: dnd-session' "$narratio_effective_config"
@@ -116,13 +127,16 @@ grep -Fq 'resume: true' "$narratio_effective_config"
! grep -Fq 'chunk_cache:' "$narratio_effective_config"
grep -Fq 'party: narratio.input.party' "$narratio_effective_config"
grep -Fq 'lane_id: item-registry' "$narratio_effective_config"
"$workspace_venv/bin/python" - "$narratio_effective_config" <<'PY'
"$workspace_venv/bin/python" - \
"$narratio_effective_config" "$narratio_testing_effective_config" <<'PY'
import sys
import yaml
with open(sys.argv[1], encoding="utf-8") as config_file:
config = yaml.safe_load(config_file)
with open(sys.argv[2], encoding="utf-8") as config_file:
testing_config = yaml.safe_load(config_file)
assert config["workspace"]["cleanup_after_publish"] is False
assert config["storage"] == {
@@ -154,11 +168,21 @@ assert config["publish"] == {
},
],
}
assert "config_profile" not in config["notarius"]
assert testing_config["notarius"]["config_profile"] == "testing"
assert all(
artifact["profile_id"] == "endpoint-profile"
for artifact in testing_config["scriptorium"]["artifacts"].values()
)
assert any(
artifact_id.startswith("character_meta_analysis_")
for artifact_id in testing_config["scriptorium"]["artifacts"]
)
PY
! grep -Fq "$synthetic_audita_api_key" "$baseline_log" "$narratio_effective_config" "$notarius_effective_config"
! grep -Fq "$synthetic_notarius_api_key" "$baseline_log" "$narratio_effective_config" "$notarius_effective_config"
! grep -Fq "$synthetic_object_storage_key_id" "$baseline_log" "$narratio_effective_config" "$notarius_effective_config"
! grep -Fq "$synthetic_object_storage_key" "$baseline_log" "$narratio_effective_config" "$notarius_effective_config"
! grep -Fq "$synthetic_audita_api_key" "$baseline_log" "$narratio_effective_config" "$narratio_testing_effective_config" "$notarius_effective_config" "$notarius_testing_effective_config"
! grep -Fq "$synthetic_notarius_api_key" "$baseline_log" "$narratio_effective_config" "$narratio_testing_effective_config" "$notarius_effective_config" "$notarius_testing_effective_config"
! grep -Fq "$synthetic_object_storage_key_id" "$baseline_log" "$narratio_effective_config" "$narratio_testing_effective_config" "$notarius_effective_config" "$notarius_testing_effective_config"
! grep -Fq "$synthetic_object_storage_key" "$baseline_log" "$narratio_effective_config" "$narratio_testing_effective_config" "$notarius_effective_config" "$notarius_testing_effective_config"
/usr/bin/sudo -u narratio /usr/local/bin/narratio config artifacts \
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs | \
awk -F '\t' '$1 == "session_recap" && $4 == "true" { found = 1 } END { exit !found }'
@@ -168,9 +192,6 @@ PY
/usr/bin/sudo -u narratio /usr/local/bin/narratio config artifacts \
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs | \
awk -F '\t' 'index($1, "character_meta_analysis_") == 1 && $4 == "false" && $7 == "character_meta_analysis" { found = 1 } END { exit !found }'
/usr/bin/sudo -u narratio /usr/local/bin/narratio config diff default testing \
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs | \
awk -F '\t' '$1 != "changed" || $2 !~ /\.profile_id$/ { invalid = 1 } END { exit invalid || NR == 0 }'
/usr/bin/sudo -u "$operator_user" /usr/local/bin/narratio config validate \
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs
/usr/bin/sudo -u "$operator_user" /usr/local/bin/notarius config validate \
@@ -218,21 +239,6 @@ if run_converge "$invalid_key_log" "${baseline_args[@]}" \
exit 1
fi
testing_profile_log="$test_tmp/testing-profile.log"
run_converge "$testing_profile_log" "${baseline_args[@]}" \
-e 'dnd_pipeline_narratio_testing_scriptorium_profile=endpoint-profile'
/usr/bin/sudo -u narratio /usr/local/bin/narratio config validate \
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs --profile testing
/usr/bin/sudo -u narratio /usr/local/bin/narratio config diff default testing \
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs | \
grep -F '"endpoint-profile"'
/usr/local/bin/scriptorium inspect profile --config /usr/local/etc/scriptorium/config.yml \
--profile-dir "$profile_dir" --profile endpoint-profile
testing_profile_repeat_log="$test_tmp/testing-profile-repeat.log"
run_converge "$testing_profile_repeat_log" "${baseline_args[@]}" \
-e 'dnd_pipeline_narratio_testing_scriptorium_profile=endpoint-profile'
require_clean_recap "$testing_profile_repeat_log"
selected_version_log="$test_tmp/selected-version.log"
run_converge "$selected_version_log" "${baseline_args[@]}" \
-e 'dnd_pipeline_seriatim_version=v1.4.0'
@@ -242,24 +248,55 @@ run_converge "$selected_version_repeat_log" "${baseline_args[@]}" \
-e 'dnd_pipeline_seriatim_version=v1.4.0'
require_clean_recap "$selected_version_repeat_log"
pipeline_group_one='{"archive-session":{"llm_profile":"dnd-extraction","input":{"module":"seriatim"},"artifacts":{"spells":{"extract":{"module":"dnd/spells"},"normalize":{"module":"dnd/spells"}}}}}'
pipeline_group_two='{"research-session":{"llm_profile":"dnd-extraction","input":{"module":"seriatim"},"artifacts":{"spells":{"extract":{"module":"dnd/spells"},"normalize":{"module":"dnd/spells"}}}}}'
alternate_config="{\"notarius_pipeline_group_one\":$pipeline_group_one,\"notarius_pipeline_group_two\":$pipeline_group_two,\"dnd_pipeline_notarius_pipelines\":\"{{ notarius_pipeline_group_one | combine(notarius_pipeline_group_two) }}\",\"dnd_pipeline_narratio_notarius_pipeline_id\":\"archive-session\",\"dnd_pipeline_notarius_cache_chunk_plans_mode\":\"bypass\"}"
pipeline_group_one='{"archive-session":{"llm_profile":"gpt-luna-latest","input":{"module":"seriatim"},"artifacts":{"spells":{"extract":{"module":"dnd/spells"},"normalize":{"module":"dnd/spells"}}}}}'
pipeline_group_two='{"research-session":{"llm_profile":"gpt-luna-latest","input":{"module":"seriatim"},"artifacts":{"spells":{"extract":{"module":"dnd/spells"},"normalize":{"module":"dnd/spells"}}}}}'
alternate_config="{\"notarius_pipeline_group_one\":$pipeline_group_one,\"notarius_pipeline_group_two\":$pipeline_group_two,\"dnd_pipeline_notarius_pipelines\":\"{{ notarius_pipeline_group_one | combine(notarius_pipeline_group_two) }}\",\"dnd_pipeline_narratio_notarius_pipeline_id\":\"archive-session\",\"dnd_pipeline_notarius_testing_llm_profile\":\"gpt-terra-latest\",\"dnd_pipeline_notarius_cache_chunk_plans_mode\":\"bypass\"}"
alternate_log="$test_tmp/alternate.log"
run_converge "$alternate_log" "${baseline_args[@]}" -e "$alternate_config"
/usr/local/bin/notarius config validate --config /usr/local/etc/notarius/config.yml \
--pipeline archive-session
/usr/local/bin/notarius config validate --config /usr/local/etc/notarius/config.yml \
--config-profile testing --pipeline archive-session
alternate_narratio_config="$test_tmp/alternate-narratio.yml"
/usr/bin/sudo -u narratio /usr/local/bin/narratio config show \
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs \
> "$alternate_narratio_config"
grep -Fq 'pipeline_id: archive-session' "$alternate_narratio_config"
alternate_testing_narratio_config="$test_tmp/alternate-testing-narratio.yml"
/usr/bin/sudo -u narratio /usr/local/bin/narratio config show \
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs --profile testing \
> "$alternate_testing_narratio_config"
alternate_effective_config="$test_tmp/alternate-effective.yml"
/usr/local/bin/notarius config effective --config /usr/local/etc/notarius/config.yml \
> "$alternate_effective_config"
grep -Fq 'archive-session:' "$alternate_effective_config"
grep -Fq 'research-session:' "$alternate_effective_config"
grep -Fq 'mode: bypass' "$alternate_effective_config"
alternate_testing_effective_config="$test_tmp/alternate-testing-effective.yml"
/usr/local/bin/notarius config effective --config /usr/local/etc/notarius/config.yml \
--config-profile testing > "$alternate_testing_effective_config"
"$workspace_venv/bin/python" - \
"$alternate_narratio_config" "$alternate_testing_narratio_config" \
"$alternate_effective_config" "$alternate_testing_effective_config" <<'PY'
import sys
import yaml
with open(sys.argv[1], encoding="utf-8") as config_file:
narratio_base = yaml.safe_load(config_file)
with open(sys.argv[2], encoding="utf-8") as config_file:
narratio_testing = yaml.safe_load(config_file)
with open(sys.argv[3], encoding="utf-8") as config_file:
notarius_base = yaml.safe_load(config_file)
with open(sys.argv[4], encoding="utf-8") as config_file:
notarius_testing = yaml.safe_load(config_file)
assert narratio_base["notarius"]["pipeline_id"] == "archive-session"
assert "config_profile" not in narratio_base["notarius"]
assert narratio_testing["notarius"]["pipeline_id"] == "archive-session"
assert narratio_testing["notarius"]["config_profile"] == "testing"
assert notarius_base["pipelines"]["archive-session"]["llm_profile"] == "gpt-luna-latest"
assert notarius_testing["pipelines"]["archive-session"]["llm_profile"] == "gpt-terra-latest"
assert notarius_testing["pipelines"]["research-session"]["llm_profile"] == "gpt-luna-latest"
assert notarius_base["cache"]["chunk_plans"]["mode"] == "bypass"
PY
alternate_repeat_log="$test_tmp/alternate-repeat.log"
run_converge "$alternate_repeat_log" "${baseline_args[@]}" -e "$alternate_config"
require_clean_recap "$alternate_repeat_log"