From 718030e631289974756c284f02a235526816b189 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 8 Sep 2026 01:12:59 +0000 Subject: [PATCH] Focus Seriatim integration coverage --- README.md | 9 +++- tests/integration | 130 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 129 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 5ed17c1..a5009ef 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,13 @@ regular file owned by `root:root` with mode `0755`. ## Validation The integration driver installs pinned collection dependencies, runs syntax -checking and ansible-lint, converges and verifies the default release, then -requires an immediate repeat with `changed=0` and `failed=0`: +checking and ansible-lint, and verifies the default release and idempotence. It +also covers missing and wrong-version binaries, metadata drift without a +rebuild, selection of the previous release, restoration of the default +release, and platform failures before application state is created. The suite +mutates only `/usr/local/src/seriatim`, `/usr/local/bin/seriatim`, and paths +inside its private temporary directory, and leaves the default release +installed: ```sh tests/integration diff --git a/tests/integration b/tests/integration index 7254bee..8d95640 100755 --- a/tests/integration +++ b/tests/integration @@ -4,7 +4,7 @@ set -euo pipefail repository_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) workspace_venv="$repository_root/../.venv" test_tmp=$(mktemp -d) -trap 'rm -rf -- "$test_tmp"' EXIT +chmod 0700 "$test_tmp" export ANSIBLE_COLLECTIONS_PATH="$test_tmp/collections" export ANSIBLE_ROLES_PATH="$(dirname "$repository_root")" @@ -12,16 +12,130 @@ export ANSIBLE_ROLES_PATH="$(dirname "$repository_root")" inventory="$repository_root/tests/inventory" converge="$repository_root/tests/converge.yml" verify="$repository_root/tests/verify.yml" +unsupported="$repository_root/tests/unsupported.yml" +ansible_playbook="$workspace_venv/bin/ansible-playbook" +default_commit=0fc92f364316c874d366b00cdc884a33e1900b25 +previous_commit=9202ccddb962ab499f0772d3c01efdf2ff703077 +source_dir=/usr/local/src/seriatim +binary_path=/usr/local/bin/seriatim + +cleanup() { + exit_status=$? + trap - EXIT + "$ansible_playbook" -i "$inventory" "$converge" >/dev/null 2>&1 || true + rm -rf -- "$test_tmp" + exit "$exit_status" +} +trap cleanup EXIT + +require_clean_recap() { + grep -Eq 'changed=0 .*failed=0' "$1" +} + +require_changed_recap() { + grep -Eq 'changed=[1-9][0-9]* .*failed=0' "$1" +} + +run_converge() { + output_file=$1 + shift + "$ansible_playbook" -i "$inventory" "$@" "$converge" | tee "$output_file" +} + +verify_release() { + expected_version=$1 + expected_commit=$2 + "$ansible_playbook" -i "$inventory" \ + -e "seriatim_expected_version=$expected_version" \ + -e "seriatim_expected_commit=$expected_commit" \ + "$verify" +} cd "$repository_root" "$workspace_venv/bin/ansible-galaxy" collection install \ -r collections/requirements.yml \ -p "$ANSIBLE_COLLECTIONS_PATH" -"$workspace_venv/bin/ansible-playbook" -i "$inventory" --syntax-check "$converge" -"$workspace_venv/bin/ansible-lint" "$converge" "$verify" -"$workspace_venv/bin/ansible-playbook" -i "$inventory" "$converge" -"$workspace_venv/bin/ansible-playbook" -i "$inventory" "$verify" +"$ansible_playbook" -i "$inventory" --syntax-check "$converge" +"$workspace_venv/bin/ansible-lint" "$converge" "$verify" "$unsupported" -repeat_output="$test_tmp/repeat.log" -"$workspace_venv/bin/ansible-playbook" -i "$inventory" "$converge" | tee "$repeat_output" -grep -Eq 'changed=0 .*failed=0' "$repeat_output" +baseline_log="$test_tmp/baseline.log" +run_converge "$baseline_log" +verify_release v1.5.0 "$default_commit" +baseline_repeat_log="$test_tmp/baseline-repeat.log" +run_converge "$baseline_repeat_log" +require_clean_recap "$baseline_repeat_log" + +sudo rm -f -- "$binary_path" +missing_binary_log="$test_tmp/missing-binary.log" +run_converge "$missing_binary_log" +require_changed_recap "$missing_binary_log" +grep -A1 -E 'TASK .*Build Seriatim' "$missing_binary_log" | grep -q '^changed:' +missing_binary_repeat_log="$test_tmp/missing-binary-repeat.log" +run_converge "$missing_binary_repeat_log" +require_clean_recap "$missing_binary_repeat_log" + +sudo cp -- /bin/false "$binary_path" +wrong_version_log="$test_tmp/wrong-version.log" +run_converge "$wrong_version_log" +require_changed_recap "$wrong_version_log" +grep -A1 -E 'TASK .*Build Seriatim' "$wrong_version_log" | grep -q '^changed:' +wrong_version_repeat_log="$test_tmp/wrong-version-repeat.log" +run_converge "$wrong_version_repeat_log" +require_clean_recap "$wrong_version_repeat_log" + +sudo chmod 0700 "$binary_path" +metadata_log="$test_tmp/metadata.log" +run_converge "$metadata_log" +require_changed_recap "$metadata_log" +grep -A1 -E 'TASK .*Build Seriatim' "$metadata_log" | grep -q '^skipping:' +grep -A3 -E 'TASK .*Converge installed metadata' "$metadata_log" | \ + grep -q "^changed:.*'path': '$binary_path'" +metadata_repeat_log="$test_tmp/metadata-repeat.log" +run_converge "$metadata_repeat_log" +require_clean_recap "$metadata_repeat_log" + +previous_release_log="$test_tmp/previous-release.log" +run_converge "$previous_release_log" -e seriatim_version=v1.4.0 +require_changed_recap "$previous_release_log" +verify_release v1.4.0 "$previous_commit" +previous_repeat_log="$test_tmp/previous-repeat.log" +run_converge "$previous_repeat_log" -e seriatim_version=v1.4.0 +require_clean_recap "$previous_repeat_log" + +restored_release_log="$test_tmp/restored-release.log" +run_converge "$restored_release_log" +require_changed_recap "$restored_release_log" +verify_release v1.5.0 "$default_commit" +restored_repeat_log="$test_tmp/restored-repeat.log" +run_converge "$restored_repeat_log" +require_clean_recap "$restored_repeat_log" + +platform_cases=( + 'Debian 26.04 x86_64 distribution' + 'Ubuntu 24.04 x86_64 release' + 'Ubuntu 26.04 riscv64 architecture' +) +for platform_case in "${platform_cases[@]}"; do + read -r distribution release architecture label <<<"$platform_case" + case_root="$test_tmp/unsupported-$label" + case_source="$case_root/source" + case_binary="$case_root/bin/seriatim" + case_log="$test_tmp/unsupported-$label.log" + if "$ansible_playbook" -i "$inventory" \ + -e "test_distribution=$distribution" \ + -e "test_distribution_version=$release" \ + -e "test_architecture=$architecture" \ + -e "seriatim_source_dir=$case_source" \ + -e "seriatim_binary_path=$case_binary" \ + "$unsupported" | tee "$case_log"; then + echo "Unsupported $label case unexpectedly succeeded" >&2 + exit 1 + fi + grep -Eq 'failed=1' "$case_log" + [[ ! -e "$case_source" ]] + [[ ! -e "$case_binary" ]] +done + +[[ "$("$binary_path" --version)" == 'seriatim version v1.5.0' ]] +[[ "$(sudo git -C "$source_dir" rev-parse HEAD)" == "$default_commit" ]] +[[ "$(sudo stat -c '%U:%G %a' "$binary_path")" == 'root:root 755' ]]