213 lines
12 KiB
Bash
Executable File
213 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
repository_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
|
workspace_venv="$repository_root/../.venv"
|
|
test_tmp=$(mktemp -d)
|
|
chmod 0700 "$test_tmp"
|
|
|
|
export ANSIBLE_COLLECTIONS_PATH="$test_tmp/collections"
|
|
export ANSIBLE_ROLES_PATH="$test_tmp/roles:$(dirname "$repository_root")"
|
|
|
|
inventory="$repository_root/tests/inventory"
|
|
converge="$repository_root/tests/converge.yml"
|
|
ansible_playbook="$workspace_venv/bin/ansible-playbook"
|
|
profile_dir="$test_tmp/profiles"
|
|
campaigns_repository=$(cd "$repository_root/../assets/campaigns" && pwd)
|
|
prompt_library_repository=$(cd "$repository_root/../assets/narratio-prompts" && pwd)
|
|
prompt_library_root=/usr/local/share/narratio/prompt-library
|
|
campaigns_revision=$(git -C "$campaigns_repository" rev-parse HEAD)
|
|
prompt_library_revision=$(git -C "$prompt_library_repository" rev-parse HEAD)
|
|
operator_user="dnd_f0009_${$}"
|
|
shared_fixture_root="/var/cache/notarius/${operator_user}"
|
|
service_fixture_file="$shared_fixture_root/service.txt"
|
|
operator_fixture_file="$shared_fixture_root/operator.txt"
|
|
synthetic_audita_api_key='audita-pipeline-synthetic-value'
|
|
synthetic_notarius_api_key='notarius-pipeline-synthetic-value'
|
|
baseline_args=(
|
|
-e 'dnd_pipeline_whisperx_transcribe_url=http://127.0.0.1:8000/transcribe'
|
|
-e "dnd_pipeline_campaigns_repository_url=file://$campaigns_repository"
|
|
-e "dnd_pipeline_campaigns_revision=$campaigns_revision"
|
|
-e "dnd_pipeline_prompt_library_repository_url=file://$prompt_library_repository"
|
|
-e "dnd_pipeline_prompt_library_revision=$prompt_library_revision"
|
|
-e "dnd_pipeline_prompt_library_root=$prompt_library_root"
|
|
-e "{\"dnd_pipeline_operator_users\":[\"$operator_user\"]}"
|
|
-e "{\"dnd_pipeline_secrets\":{\"AUDITA_LLM_API_KEY\":\"$synthetic_audita_api_key\",\"OPENROUTER_API_KEY\":\"$synthetic_notarius_api_key\"}}"
|
|
)
|
|
|
|
cleanup() {
|
|
exit_status=$?
|
|
trap - EXIT
|
|
/usr/bin/sudo /bin/rm -rf -- "$shared_fixture_root" || true
|
|
"$ansible_playbook" -i "$inventory" "${baseline_args[@]}" "$converge" >/dev/null 2>&1 || true
|
|
/usr/bin/sudo /usr/sbin/userdel --remove "$operator_user" 2>/dev/null || true
|
|
rm -rf -- "$test_tmp"
|
|
exit "$exit_status"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
require_clean_recap() {
|
|
grep -Eq 'changed=0 .*failed=0' "$1"
|
|
}
|
|
|
|
run_converge() {
|
|
output_file=$1
|
|
shift
|
|
env -u AUDITA_LLM_API_KEY -u OPENROUTER_API_KEY \
|
|
"$ansible_playbook" -i "$inventory" "$@" "$converge" | tee "$output_file"
|
|
}
|
|
|
|
cd "$repository_root"
|
|
mkdir -p "$profile_dir"
|
|
printf '%s\n' \
|
|
'id: endpoint-profile' \
|
|
'endpoint: http://127.0.0.1:8000/v1' \
|
|
'model: offline-fixture' > "$profile_dir/endpoint-profile.yml"
|
|
"$workspace_venv/bin/ansible-galaxy" collection install \
|
|
-r collections/requirements.yml \
|
|
-p "$ANSIBLE_COLLECTIONS_PATH"
|
|
"$workspace_venv/bin/ansible-galaxy" role install \
|
|
-r requirements.yml \
|
|
-p "$test_tmp/roles"
|
|
"$ansible_playbook" -i "$inventory" --syntax-check "$converge"
|
|
"$workspace_venv/bin/ansible-lint" --offline --profile production "$converge"
|
|
/usr/bin/sudo /usr/sbin/useradd --create-home --shell /usr/sbin/nologin "$operator_user"
|
|
|
|
baseline_log="$test_tmp/baseline.log"
|
|
run_converge "$baseline_log" "${baseline_args[@]}"
|
|
/usr/local/bin/seriatim --version
|
|
/usr/local/bin/audita config validate --config /usr/local/etc/audita/config.yml
|
|
/usr/local/bin/notarius config validate --config /usr/local/etc/notarius/config.yml \
|
|
--pipeline dnd-session
|
|
notarius_effective_config="$test_tmp/notarius-effective.yml"
|
|
/usr/local/bin/notarius config effective --config /usr/local/etc/notarius/config.yml \
|
|
> "$notarius_effective_config"
|
|
grep -Fq 'mode: auto' "$notarius_effective_config"
|
|
grep -Fq 'step: describe-session' "$notarius_effective_config"
|
|
/usr/local/bin/scriptorium inspect profile --config /usr/local/etc/scriptorium/config.yml \
|
|
--profile-dir "$profile_dir" --profile endpoint-profile
|
|
/usr/bin/sudo -u narratio /usr/local/bin/narratio config validate \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs
|
|
/usr/bin/sudo -u narratio /usr/local/bin/narratio config validate \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign dilfs
|
|
/usr/bin/sudo -u narratio /usr/local/bin/narratio config validate \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs --profile testing
|
|
narratio_effective_config="$test_tmp/narratio-effective.yml"
|
|
/usr/bin/sudo -u narratio /usr/local/bin/narratio config show \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs \
|
|
> "$narratio_effective_config"
|
|
grep -Fq 'enabled: true' "$narratio_effective_config"
|
|
grep -Fq 'llm_api_key: AUDITA_LLM_API_KEY' "$narratio_effective_config"
|
|
grep -Fq 'pipeline_id: dnd-session' "$narratio_effective_config"
|
|
grep -Fq 'llm_api_key: OPENROUTER_API_KEY' "$narratio_effective_config"
|
|
grep -Fq 'resume: true' "$narratio_effective_config"
|
|
! grep -Fq 'chunk_cache:' "$narratio_effective_config"
|
|
grep -Fq 'party: narratio.input.party' "$narratio_effective_config"
|
|
grep -Fq 'lane_id: item-registry' "$narratio_effective_config"
|
|
! grep -Fq "$synthetic_audita_api_key" "$baseline_log" "$narratio_effective_config" "$notarius_effective_config"
|
|
! grep -Fq "$synthetic_notarius_api_key" "$baseline_log" "$narratio_effective_config" "$notarius_effective_config"
|
|
/usr/bin/sudo -u narratio /usr/local/bin/narratio config artifacts \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs | \
|
|
awk -F '\t' '$1 == "session_recap" && $4 == "true" { found = 1 } END { exit !found }'
|
|
/usr/bin/sudo -u narratio /usr/local/bin/narratio config artifacts \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs | \
|
|
awk -F '\t' '$1 == "hostile_rumors" && $6 == "session_recap" { found = 1 } END { exit !found }'
|
|
/usr/bin/sudo -u narratio /usr/local/bin/narratio config artifacts \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs | \
|
|
awk -F '\t' 'index($1, "character_meta_analysis_") == 1 && $4 == "false" && $7 == "character_meta_analysis" { found = 1 } END { exit !found }'
|
|
/usr/bin/sudo -u narratio /usr/local/bin/narratio config diff default testing \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs | \
|
|
awk -F '\t' '$1 != "changed" || $2 !~ /\.profile_id$/ { invalid = 1 } END { exit invalid || NR == 0 }'
|
|
/usr/bin/sudo -u "$operator_user" /usr/local/bin/narratio config validate \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs
|
|
/usr/bin/sudo -u "$operator_user" /usr/local/bin/notarius config validate \
|
|
--config /usr/local/etc/notarius/config.yml --pipeline dnd-session
|
|
/usr/bin/sudo -u nobody /bin/cat /usr/local/etc/narratio/pipeline.yml >/dev/null
|
|
if /usr/bin/sudo -u nobody /usr/local/bin/narratio config validate \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs; then
|
|
exit 1
|
|
fi
|
|
/usr/bin/sudo -u narratio /bin/sh -c \
|
|
"umask 077; mkdir '$shared_fixture_root'; printf '%s\\n' service-created > '$service_fixture_file'"
|
|
/usr/bin/sudo -u "$operator_user" /bin/sh -c \
|
|
"grep -Fx service-created '$service_fixture_file'; printf '%s\\n' operator-appended >> '$service_fixture_file'"
|
|
/usr/bin/sudo -u narratio /bin/grep -Fx operator-appended "$service_fixture_file"
|
|
/usr/bin/sudo -u "$operator_user" /bin/sh -c \
|
|
"umask 077; printf '%s\\n' operator-created > '$operator_fixture_file'"
|
|
/usr/bin/sudo -u narratio /bin/grep -Fx operator-created "$operator_fixture_file"
|
|
session_recap_prompt_json=$(/usr/local/bin/scriptorium inspect prompt \
|
|
--config /usr/local/etc/scriptorium/config.yml \
|
|
--prompt dnd.session_recap --prompt-version 2.0.0 --format json)
|
|
session_recap_profile=$(printf '%s\n' "$session_recap_prompt_json" | \
|
|
"$workspace_venv/bin/python" -c 'import json, sys; print(json.load(sys.stdin)["default_profile_id"])')
|
|
/usr/local/bin/scriptorium inspect profile --config /usr/local/etc/scriptorium/config.yml \
|
|
--profile "$session_recap_profile"
|
|
/usr/local/bin/scriptorium inspect prompt --config /usr/local/etc/scriptorium/config.yml \
|
|
--prompt dnd.scene_map --prompt-version 1.0.0 --format json | \
|
|
"$workspace_venv/bin/python" -c '
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
schema_root = Path(sys.argv[1]).resolve()
|
|
schema_path = json.load(sys.stdin)["output_contract"]["schema_path"]
|
|
schema = (schema_root / schema_path).resolve()
|
|
if not schema.is_relative_to(schema_root) or not schema.is_file():
|
|
raise SystemExit(f"invalid deployed schema path: {schema}")
|
|
' "$prompt_library_root/schemas"
|
|
baseline_repeat_log="$test_tmp/baseline-repeat.log"
|
|
run_converge "$baseline_repeat_log" "${baseline_args[@]}"
|
|
require_clean_recap "$baseline_repeat_log"
|
|
|
|
invalid_key_log="$test_tmp/invalid-key.log"
|
|
if run_converge "$invalid_key_log" "${baseline_args[@]}" \
|
|
-e '{"dnd_pipeline_secrets":{"INVALID-KEY":"invalid-key-synthetic-value"}}'; then
|
|
exit 1
|
|
fi
|
|
|
|
testing_profile_log="$test_tmp/testing-profile.log"
|
|
run_converge "$testing_profile_log" "${baseline_args[@]}" \
|
|
-e 'dnd_pipeline_narratio_testing_scriptorium_profile=endpoint-profile'
|
|
/usr/bin/sudo -u narratio /usr/local/bin/narratio config validate \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs --profile testing
|
|
/usr/bin/sudo -u narratio /usr/local/bin/narratio config diff default testing \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs | \
|
|
grep -F '"endpoint-profile"'
|
|
/usr/local/bin/scriptorium inspect profile --config /usr/local/etc/scriptorium/config.yml \
|
|
--profile-dir "$profile_dir" --profile endpoint-profile
|
|
testing_profile_repeat_log="$test_tmp/testing-profile-repeat.log"
|
|
run_converge "$testing_profile_repeat_log" "${baseline_args[@]}" \
|
|
-e 'dnd_pipeline_narratio_testing_scriptorium_profile=endpoint-profile'
|
|
require_clean_recap "$testing_profile_repeat_log"
|
|
|
|
selected_version_log="$test_tmp/selected-version.log"
|
|
run_converge "$selected_version_log" "${baseline_args[@]}" \
|
|
-e 'dnd_pipeline_seriatim_version=v1.4.0'
|
|
/usr/local/bin/seriatim --version | grep -Fx 'seriatim version v1.4.0'
|
|
selected_version_repeat_log="$test_tmp/selected-version-repeat.log"
|
|
run_converge "$selected_version_repeat_log" "${baseline_args[@]}" \
|
|
-e 'dnd_pipeline_seriatim_version=v1.4.0'
|
|
require_clean_recap "$selected_version_repeat_log"
|
|
|
|
pipeline_group_one='{"archive-session":{"llm_profile":"dnd-extraction","input":{"module":"seriatim"},"artifacts":{"spells":{"extract":{"module":"dnd/spells"},"normalize":{"module":"dnd/spells"}}}}}'
|
|
pipeline_group_two='{"research-session":{"llm_profile":"dnd-extraction","input":{"module":"seriatim"},"artifacts":{"spells":{"extract":{"module":"dnd/spells"},"normalize":{"module":"dnd/spells"}}}}}'
|
|
alternate_config="{\"notarius_pipeline_group_one\":$pipeline_group_one,\"notarius_pipeline_group_two\":$pipeline_group_two,\"dnd_pipeline_notarius_pipelines\":\"{{ notarius_pipeline_group_one | combine(notarius_pipeline_group_two) }}\",\"dnd_pipeline_narratio_notarius_pipeline_id\":\"archive-session\",\"dnd_pipeline_notarius_cache_chunk_plans_mode\":\"bypass\"}"
|
|
alternate_log="$test_tmp/alternate.log"
|
|
run_converge "$alternate_log" "${baseline_args[@]}" -e "$alternate_config"
|
|
/usr/local/bin/notarius config validate --config /usr/local/etc/notarius/config.yml \
|
|
--pipeline archive-session
|
|
alternate_narratio_config="$test_tmp/alternate-narratio.yml"
|
|
/usr/bin/sudo -u narratio /usr/local/bin/narratio config show \
|
|
--config /usr/local/etc/narratio/pipeline.yml --campaign centaurs \
|
|
> "$alternate_narratio_config"
|
|
grep -Fq 'pipeline_id: archive-session' "$alternate_narratio_config"
|
|
alternate_effective_config="$test_tmp/alternate-effective.yml"
|
|
/usr/local/bin/notarius config effective --config /usr/local/etc/notarius/config.yml \
|
|
> "$alternate_effective_config"
|
|
grep -Fq 'archive-session:' "$alternate_effective_config"
|
|
grep -Fq 'research-session:' "$alternate_effective_config"
|
|
grep -Fq 'mode: bypass' "$alternate_effective_config"
|
|
alternate_repeat_log="$test_tmp/alternate-repeat.log"
|
|
run_converge "$alternate_repeat_log" "${baseline_args[@]}" -e "$alternate_config"
|
|
require_clean_recap "$alternate_repeat_log"
|