17 Commits

Author SHA1 Message Date
44eba766d6 Prepare version 0.8.0 release 2026-09-26 01:36:32 +00:00
f1566d557c Verify retry accounting and execution summary propagation 2026-09-26 01:07:31 +00:00
1d29b321dd Record remaining kit upgrade verification work 2026-09-25 23:59:43 +00:00
7ac9d94f10 Verify ConfigKit and PromptKit upgrades 2026-09-25 22:04:55 +00:00
227c05b154 Document retry composition and preserve error summaries 2026-09-25 22:03:50 +00:00
b09ce47d4a Record execution summaries in debug artifacts 2026-09-25 22:02:56 +00:00
6b8a4408c1 Add LLM execution summaries 2026-09-25 22:01:16 +00:00
bd83255a8e Preserve LLM generation failure metadata 2026-09-25 21:59:15 +00:00
10ac29d149 Configure PromptKit transient retries 2026-09-25 21:56:29 +00:00
d29a5cecd3 Upgrade PromptKit runtime baseline 2026-09-25 21:49:44 +00:00
31b48d2d5e Rename configuration selection to overlays 2026-09-25 21:46:35 +00:00
8c0e93143f Plan the ConfigKit and PromptKit upgrade implementation 2026-09-25 21:05:02 +00:00
f0abc5d220 Plan ConfigKit and PromptKit upgrades 2026-09-25 20:56:46 +00:00
acec6dc15e Simplify source releases with one guarded publication command 2026-09-25 20:33:19 +00:00
cee254651f Align architecture documentation and testing policies with current standards 2026-09-25 20:26:34 +00:00
cc51b8470a Establish a dedicated agent policy 2026-09-25 20:25:36 +00:00
53d9284f0b Upgrade PromptKit and prepare version 0.7.1 2026-09-22 14:31:55 +00:00
62 changed files with 2859 additions and 593 deletions

View File

@@ -1,3 +1,4 @@
# Advisory validation after source publication, never a release-completion gate.
when:
- event: tag
@@ -5,29 +6,4 @@ steps:
- name: validate-release
image: golang:1.25.5
commands:
- |
set -eu
version="$CI_COMMIT_TAG"
release_note="docs/releases/$version.md"
if ! printf '%s\n' "$version" | grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null; then
printf '%s\n' "invalid release tag: $version" >&2
exit 1
fi
if [ ! -s "$release_note" ]; then
printf '%s\n' "missing release note: $release_note" >&2
exit 1
fi
if ! grep -F -x "# Notarius $version" "$release_note" >/dev/null; then
printf '%s\n' "release note heading does not match $version" >&2
exit 1
fi
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
if ! grep -F -x "$heading" "$release_note" >/dev/null; then
printf '%s\n' "release note is missing heading: $heading" >&2
exit 1
fi
done
./scripts/check-release-source.sh "$version"
- ./scripts/check-release-source.sh "$CI_COMMIT_TAG"

View File

@@ -1,7 +1,5 @@
Please review `docs/development.md` for initial orientation in this repository
and follow its task-specific reading guide.
For every planning and implementation task, follow the
[`Implementation Discipline`](docs/policy/architecture.md#implementation-discipline):
keep machinery proportionate, do not invent restrictions, and do not elevate
incidental implementation details into repository policy.
Review and follow `docs/policy/agents.md` before recommending, planning, or
implementing any changes to this repository.

View File

@@ -12,8 +12,8 @@ defined in [Operations](operations.md).
notarius help
notarius --version
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate [--config path/to/config.yml] [--config-profile name] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list [--config path/to/config.yml] [--config-profile name] [--json]
notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json]
~~~
Running Notarius without arguments, or with **help**, **--help**, or **-h**,
@@ -42,7 +42,7 @@ pipeline ID and **--input** are required.
| Flag | Meaning |
| --- | --- |
| **--config path** | Use this configuration file. When omitted, configuration discovery applies; see [Configuration](config.md). |
| **--config-profile name** | Select a ConfigKit configuration overlay. Empty and **default** select the composed base configuration. |
| **--config-overlay name** | Select a ConfigKit configuration overlay. Empty and **default** select the composed base configuration. |
| **--input path** | Source input file to process. Required. |
| **--output-dir path** | Override the configured output root for this run. |
| **--json** | Write the successful run-result receipt as JSON to standard output. |
@@ -155,7 +155,7 @@ OPENROUTER_API_KEY=your-api-key \
## config validate
~~~
notarius config validate [--config path/to/config.yml] [--config-profile name] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius config validate [--config path/to/config.yml] [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
~~~
This command loads and validates the selected composed configuration. With **--pipeline**, it also
@@ -184,7 +184,7 @@ validation; it cannot run a provider-backed pipeline.
## config effective
~~~
notarius config effective [--config path/to/config.yml] [--config-profile name]
notarius config effective [--config path/to/config.yml] [--config-overlay name]
~~~
This command writes the selected composed configuration as deterministic,
@@ -194,19 +194,19 @@ pipeline, prepare modules, materialize references, or contact a provider.
## config explain
~~~
notarius config explain [--config path/to/config.yml] [--config-profile name] <json-pointer>
notarius config explain [--config path/to/config.yml] [--config-overlay name] <json-pointer>
~~~
This command writes ConfigKit's redacted, human-readable explanation for one
canonical JSON Pointer. Supply `""` explicitly to explain the root. A malformed
pointer is a syntax error; a well-formed unknown path is a command failure.
The explanation can describe a value displaced by a selected profile. Standard
The explanation can describe a value displaced by a selected overlay. Standard
output contains only the explanation.
## pipelines list
~~~
notarius pipelines list [--config path/to/config.yml] [--config-profile name] [--json]
notarius pipelines list [--config path/to/config.yml] [--config-overlay name] [--json]
~~~
This command lists configured pipeline IDs in sorted order. By default, it
@@ -225,7 +225,7 @@ errors are written to standard error.
Configuration load errors include ConfigKit's field diagnostics and source
coordinates, followed by a source-ID legend identifying the acquired files or
source labels. This includes imported files and the selected configuration profile.
source labels. This includes imported files and the selected configuration overlay.
For **run --json**, actionable process warnings remain on standard error and standard output is a
machine-readable success result only. Syntax and runtime diagnostics remain on

View File

@@ -20,12 +20,13 @@ For configuration values, precedence is:
1. built-in defaults;
2. composed YAML sources, including imports;
3. the selected ConfigKit profile; and
3. the selected ConfigKit overlay; and
4. explicitly supplied configuration-valued run flags.
`--config-profile` selects a ConfigKit overlay from `profiles/<name>.yml`
`--config-overlay` selects a ConfigKit overlay from `overlays/<name>.yml`
beneath the selected configuration root. Empty and `default` select no overlay.
Configuration profiles are distinct from PromptKit LLM profiles selected by
`profiles/<name>.yml` is not a fallback location, and `--config-profile` is no
longer accepted. Configuration overlays are distinct from PromptKit LLM profiles selected by
`llm_profile` or `--llm-profile`. The retired operational environment variables
have no effect; `NOTARIUS_CONFIG` remains a source-discovery input.
@@ -79,14 +80,14 @@ umask, setgid inheritance, and default ACLs; see [Operations](operations.md#cach
Every nonempty relative configured destination or reference path resolves from
the canonical directory of the file that supplies its surviving declaration.
This includes imported fragments and selected configuration profiles. Built-in
This includes imported fragments and selected configuration overlays. Built-in
relative destinations use the selected main configuration file's directory;
explicit `--output-dir` and `--debug-dir` values use the invocation working
directory. Absolute values remain unchanged.
The root file can use ConfigKit imports to compose disjoint definitions. Imports
and selected profiles remain under the configuration root; they do not merge or
append a pipeline's `steps` sequence. A profile replaces sequences as defined
and selected overlays remain under the configuration root; they do not merge or
append a pipeline's `steps` sequence. An overlay replaces sequences as defined
by ConfigKit, so changing an individual ordered binding requires supplying the
whole replacement sequence.
@@ -101,6 +102,7 @@ version: 5
promptkit:
profile_dir: ./profiles
# profile_file: ./profiles.yml
transient_retries: 2
local_backend:
endpoint: http://localhost:8000/v1
concurrency_limit: 2
@@ -110,6 +112,7 @@ promptkit:
| --- | --- | --- |
| **profile_dir** | string | Non-empty directory containing profile files. |
| **profile_file** | string | Non-empty profile file. |
| **transient_retries** | integer | Non-negative additional built-in HTTP attempts for eligible transient provider failures; defaults to 2. Set 0 to disable these retries. |
| **local_backend** | object | Optional registration for the conventional PromptKit backend ID **local**. |
| **local_backend.endpoint** | string | Required when **local_backend** is present; absolute HTTP or HTTPS URL with a host. |
| **local_backend.concurrency_limit** | integer | Optional non-negative limit; defaults to 0. |
@@ -201,7 +204,7 @@ debug:
**concurrency.total_llm** must be greater than zero. The only supported
**concurrency.stage_workers** key is **extract**; its value must be from 1
through **total_llm**. When omitted, it is recalculated from the effective
**total_llm** after imports, profile selection, and explicit ConfigKit
**total_llm** after imports, overlay selection, and explicit ConfigKit
overrides.
**cache.chunk_plans.mode** accepts **auto**, **bypass**, or **refresh**.
@@ -440,7 +443,7 @@ pipeline, lane, and binding scopes and precedence are defined by the
replace a configured generated artifact handoff. Required slots must be bound
after configuration and CLI reference actions are applied. A YAML reference
path resolves from the file that supplies its surviving declaration, including
an imported fragment or selected profile. CLI reference paths resolve relative
an imported fragment or selected overlay. CLI reference paths resolve relative
to the process working directory.
### D&D Reference Slots
@@ -601,5 +604,5 @@ Use [config effective](cli.md#config-effective) to write the selected composed
configuration as deterministic, redacted YAML. Use
[config explain](cli.md#config-explain) with a canonical JSON Pointer to inspect
the effective or historical provenance for one path, including a value displaced
by a selected profile. Neither command resolves a pipeline, prepares modules,
by a selected overlay. Neither command resolves a pipeline, prepares modules,
materializes references, or requires provider credentials.

View File

@@ -15,10 +15,10 @@ configured lane inventory, and downstream acceptance checklist, see
Optionally preflight a selected configuration and pipeline before work starts:
```sh
notarius config validate --config /path/to/notarius.yml --config-profile production --pipeline pipeline-id
notarius config validate --config /path/to/notarius.yml --config-overlay production --pipeline pipeline-id
```
Omit `--config-profile` when the composed base configuration is the intended
Omit `--config-overlay` when the composed base configuration is the intended
selection; it is independent from any `--llm-profile` run override.
Invoke the run with explicit paths and machine-readable output. Capture
@@ -28,7 +28,7 @@ processing the result.
```sh
notarius run pipeline-id \
--config /path/to/notarius.yml \
--config-profile production \
--config-overlay production \
--input /path/to/source.json \
--output-dir /path/to/output-root \
--json

View File

@@ -9,6 +9,9 @@ the [README](../README.md) for product context, [Architecture](policy/architectu
for system boundaries, and [Internal Overview](internal/overview.md) for the
implemented component map.
Read [Agent Policy](policy/agents.md) before recommending, planning, or
implementing changes. It owns implementation discipline and local commit rules.
## What To Read
| When working on | Read | Why |
@@ -25,7 +28,7 @@ implemented component map.
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and PromptKit integration. |
| Output, cache, resume, or debug artifacts | [Run State Internals](internal/state.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
| Release preparation, tagging, publication, or verification | [Source Releases](release.md) and [Documentation Policy](policy/documentation.md) | The release procedure owns maintainer guards and immutable-tag recovery; the policy assigns release-note ownership. |
| Release preparation, tagging, publication, or optional diagnostics | [Release Policy](policy/release.md) and [Source Releases](release.md) | Policy owns release requirements and completion; the runbook supplies the publication command and recovery procedure. |
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
For an existing subsystem, also inspect its focused tests and the package-local

View File

@@ -2,7 +2,7 @@
Notarius uses the public root package
[`gitea.maximumdirect.net/eric/ConfigKit`](https://gitea.maximumdirect.net/eric/ConfigKit)
at v0.3.0 for version 5 configuration acquisition, imports, profiles, explicit
at v0.4.0 for version 5 configuration acquisition, imports, overlays, explicit
overrides, diagnostics, and safe inspection rendering.
Notarius owns the typed document, domain defaults and validation, runtime

View File

@@ -1,11 +1,11 @@
# PromptKit Integration
Notarius pins
[`gitea.maximumdirect.net/eric/promptkit` v0.9.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0)
[`gitea.maximumdirect.net/eric/promptkit` v0.11.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0)
as its in-process prompt engine. The upstream
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/consumers/pkg-promptkit.md)
[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/consumers/pkg-promptkit.md)
owns the public engine API, and the upstream
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md)
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/formats.md)
owns prompt, profile, and schema file contracts.
## Supported Boundary
@@ -27,7 +27,7 @@ Notarius relies on the root `promptkit` package to:
admission exhaustion through `ErrCapacityExceeded`.
The pinned
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/backends.go)
[`BackendLocal`, `LocalBackend`, and `WithBackend` API](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/backends.go)
owns the registration and backend-capacity contract.
For one completion, the adapter calls `PrepareExecution`, takes a
@@ -53,7 +53,7 @@ Notarius sends one stable effective session through PromptKit's direct session
field, which is authoritative for provider session behavior. It also retains
the same value as the `session_id` prompt variable for maintained prompt
compatibility. The generated identifier is 76 ASCII characters, within
PromptKit v0.9.0's 256-code-point session limit. Session IDs are non-secret
PromptKit's 256-code-point session limit. Session IDs are non-secret
correlation identifiers and may be exposed to providers and provider
observability. The CLI contract owns generation and override behavior.
@@ -71,7 +71,15 @@ endpoint-only profiles have no such backend limit. The adapter translates
PromptKit capacity rejection into the provider-neutral Notarius
`ErrLLMCapacityExceeded` contract. It may include the normalized selected
backend ID in safe diagnostic context, without exposing PromptKit's capacity
error type, and leaves retries to the calling pipeline stage.
error type. PromptKit's built-in client retries eligible transient HTTP failures
with its v0.11.0 default of two additional attempts per generation; Notarius
keeps pipeline-stage retries outside that completion boundary.
The adapter projects built-in generation failures as the application-owned
`LLMGenerationError`, retaining a safe failure kind, HTTP status when present,
and retry-exhaustion attempt count. Transport, malformed successful envelopes,
provider statuses, and generic generation failures remain distinct without
exposing PromptKit error values to Notarius callers.
## Profile Sources And Compatibility
@@ -86,16 +94,16 @@ configuration and deployment workflow are defined in
[Operations](../operations.md#promptkit-profile-deployment).
PromptKit owns `base_profile` resolution under its
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.9.0/docs/formats.md).
[pinned format rules](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/formats.md).
Notarius records the selected leaf identity and resolved target without parsing
or merging inheritance. An unset filesystem `api_key_env` is optional and may
reach the provider without authorization, which can result in a 401 or 403.
PromptKit v0.9.0 accepts only the `developer`, `system`, `user`, and
PromptKit accepts only the `developer`, `system`, `user`, and
`assistant` text-chat roles after normalizing case and surrounding whitespace.
Maintained Notarius prompt definitions use only `system` and `user`.
For application-owned semantic correction, Notarius uses PromptKit v0.9.0's
For application-owned semantic correction, Notarius uses PromptKit's
`RunRequest.AppendedMessages` after the ordinary rendered prompt. It supplies
exactly two messages in order: the latest validated producer response with
role `assistant`, then deterministic validation guidance with role `user`.
@@ -112,12 +120,28 @@ and terminal summaries retain only safe counts, digests, identities, and usage;
complete appended messages remain limited to the explicitly requested detailed
debug trace.
PromptKit now obtains its maintained OpenRouter and Rakestrawhome backend and
PromptKit obtains its maintained OpenRouter and Rakestrawhome backend and
profile catalogs from independently versioned transitive modules. Notarius
does not import or register either catalog; PromptKit retains catalog source,
identity, precedence, credential, and capacity ownership.
Notarius supports this boundary against PromptKit v0.9.0. Its fallback source,
External catalog profile names and settings can change between dependency
releases, including removal without aliases. Review directly selected IDs and
inherited `base_profile` values when upgrading. An application-owned alias that
inherits an external profile does not freeze that profile's settings.
PromptKit's opt-in `promptkit-openrouter-standard` and
`promptkit-openrouter-auto` names provide name continuity within its
selected-and-tested dependency scope, not a fixed model or provider availability
guarantee. See the upstream
[catalog compatibility policy](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/policy/release.md#catalog-profile-compatibility).
Use a complete operator-owned profile with explicit backend or endpoint, model,
and settings when those choices must remain operator-controlled. The embedded
`dnd-extraction` fallback already follows this model and does not inherit an
external catalog profile. An upgrade updates Notarius's compiled-in catalog
checkpoint identity so prior profile-source checkpoints are not reused.
Notarius supports this boundary against PromptKit v0.11.0. Its fallback source,
prepared-execution, inspection, and typed capacity APIs are used as public
upstream contracts; other PromptKit APIs or file-format behavior are not
implicitly supported. A dependency upgrade requires reviewing the adapter,
@@ -146,3 +170,9 @@ profile/configuration contracts against the upstream documentation. Maintained
production prompts use PromptKit's bounded structural-repair contract; their
current declaration is one additional repair attempt. Notarius retains the
transport-neutral boundary and does not expose PromptKit types to modules.
PromptKit v0.11.0 is licensed under
[LGPL-3.0-or-later](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/LICENSE).
Its accompanying [GPLv3 notice](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/COPYING)
is incorporated by that license. This dependency change does not alter
Notarius's source license or distribution workflow.

View File

@@ -35,7 +35,7 @@ options before dispatch so that a missing production dependency fails as a
command error rather than reaching execution.
Commands that need configuration use one shared ConfigKit-backed loader. The
CLI discovers the file, passes the requested profile and only explicit
CLI discovers the file, passes the requested overlay and only explicit
configuration-valued run overrides, and receives a **LoadedConfig**. The
configured discovery and precedence contract is in
[Configuration](../config.md), while the loading and resolution mechanics are

View File

@@ -28,7 +28,7 @@ at their respective [CLI](cli.md), [pipeline](pipeline.md), and
## Loading And Validation
ConfigKit acquires the selected root, imports, optional selected profile, and
ConfigKit acquires the selected root, imports, optional selected overlay, and
explicit CLI overrides in precedence order. Its document definition supplies
typed defaults, source-aware path normalization, and structural validation
before conversion to the independently owned runtime **Config**. Only
@@ -44,7 +44,9 @@ concurrency settings, malformed bindings and references, invalid retries, and
invalid pipeline, step, or lane structure. PromptKit local-backend validation
accepts only an absolute HTTP or HTTPS endpoint with a host and no user
information, query, or fragment, and rejects a negative local concurrency
limit. Its errors retain the closest known pipeline, lane, and binding context.
limit. It also rejects a negative `promptkit.transient_retries` value; decoded
omission resolves to two, while explicit zero remains a distinct opt-out. Its
errors retain the closest known pipeline, lane, and binding context.
It deliberately does not require modules to be registered: that requires a
catalog and belongs to resolution.
@@ -112,13 +114,15 @@ recording arbitrary environment values or provider requests.
## Invariants To Preserve
- ConfigKit composition applies defaults, imports, an optional profile, and
- ConfigKit composition applies defaults, imports, an optional overlay, and
explicit overrides in one direction with retained origin metadata.
- A configuration is structurally valid before it is resolved, and a resolved
pipeline is compatible with the supplied catalog before preparation begins.
- Whitespace-normalized identifiers are unique wherever they identify a
pipeline, step, lane, worker, or reference slot.
- Runtime conversion and resolved-pipeline publication return detached data.
- The resolved PromptKit transient-retry budget is always passed to the shared
built-in client; document presence preserves an explicit zero.
- The resolved digest changes when resolved composition changes and never
includes itself.
@@ -126,7 +130,7 @@ recording arbitrary environment values or provider requests.
- **internal/core/config/document_test.go** and
**operational_document_test.go** cover ConfigKit document loading,
normalization, imports, profiles, and overrides.
normalization, imports, overlays, and overrides.
- **internal/core/config/validation_contract_test.go** covers configuration
invariants and contextual failures.
- **internal/core/config/effective_config_contract_test.go** covers defaults,

View File

@@ -216,6 +216,19 @@ than output-validation failures. Apart from documented context, capacity, and
invalid-output categories, provider error values and types do not cross the
adapter error chain; callers receive only a credential-redacted diagnostic.
`LLMGenerationError` preserves the provider-neutral failure kind (`transport`,
`provider_status`, `malformed_response`, or generic `generation`) and any HTTP
status. When PromptKit exhausts an eligible transient-status retry budget, it
also records that exhaustion and the attempts in that generation. Those fields
do not include repair generations, and zero attempts means unavailable rather
than proof that no request was sent.
Every execution that reaches PromptKit returns an optional request-local
`execution` summary. It contains only terminal aggregate attempts, retry and
repair counts, durations, outcome and failure classification, status, and
presence-aware token usage. Preparation failures have no summary; later local
decode failures retain the upstream terminal summary without changing it.
When PromptKit rejects backend admission before generation, the adapter maps
`promptkit.ErrCapacityExceeded` to
`contracts.ErrLLMCapacityExceeded`, retaining prompt context and a redacted
@@ -258,14 +271,18 @@ diagnostics; see
## Timeout Ownership
The caller context remains the outer cancellation authority. PromptKit applies
a positive effective generation timeout as an inner request deadline; an
explicit zero disables only that generation deadline. The HTTP client timeout
is a separate transport-wide cap. Notarius forwards the caller context and
does not install another timeout wrapper around PromptKit.
a positive effective generation timeout as an inner request deadline for each
built-in generation attempt; an explicit zero disables only that generation
deadline. The HTTP client timeout is a separate transport-wide cap. Notarius
forwards the caller context and does not install another timeout wrapper around
PromptKit.
The selected PromptKit profile owns generation settings. Notarius binding
retries remain outside the adapter and repeat the complete module operation
and validation chain. PromptKit does not add a provider retry loop.
The selected PromptKit profile owns generation settings. The resolved global
`promptkit.transient_retries` configuration is passed to the shared PromptKit
engine and controls additional built-in attempts for eligible transient HTTP
failures in one generation. It defaults to two; explicit zero disables those
attempts. Notarius binding retries remain outside the adapter and repeat the
complete module operation and validation chain.
Operator-facing behavior is summarized in
[Operations](../operations.md#operational-limits), and the pinned upstream
contract is identified in

View File

@@ -52,7 +52,7 @@ consumers do not repeat profile inheritance or configuration resolution.
Each LLM request receives its own copy of that resolved value. PromptKit spends
it only for structural correction inside one completion; the runner's binding
retry policy remains the separate outer budget for complete stage attempts.
Configuration resolution supplies the selected profile and catalog; see
Configuration resolution supplies the selected overlay and catalog; see
[Configuration Internals](configuration.md).
External reference materialization happens before preparation. The materializer

View File

@@ -1,5 +1,9 @@
# Run State Internals
LLM debug response artifacts and their safe per-call references retain an
optional terminal execution summary when one is available. This metadata is
diagnostic only and does not change receipts, checkpoints, or aggregate usage.
This document describes the implementation collaborators behind output, cache,
and debug state. User-visible fields belong in [Configuration](../config.md),
and physical layout, retention, recovery, reason codes, and cleanup belong in

View File

@@ -147,13 +147,32 @@ their exit classes are defined in the [CLI reference](cli.md#output-streams-and-
## Validation Retries And Terminal Outcomes
When debug recording is enabled, per-call response artifacts include the
terminal execution summary for retry, repair, timing, and presence-aware usage
diagnosis. Normal output, receipts, and warnings do not aggregate or publish
this diagnostic metadata.
Each producer binding has one outer **retries** budget. It covers complete
producer attempts for operational failures, invalid structured output,
normalizer fallback retry, and semantic correction. It is independent from
PromptKit's structural-repair calls inside one completion and from an
LLM-backed validator's own retry budget. A semantic correction rebuilds the
ordinary producer request and supplies only the latest rejected model response
plus aggregated validator guidance; it is not a conversation replay.
PromptKit's structural-repair calls inside one completion, its configured
`promptkit.transient_retries` built-in HTTP attempts, and an LLM-backed
validator's own retry budget. The transient setting defaults to two additional
attempts for eligible provider statuses; set it explicitly to zero when an
operator needs one HTTP attempt per generation. A semantic correction rebuilds
the ordinary producer request and supplies only the latest rejected model
response plus aggregated validator guidance; it is not a conversation replay.
These budgets multiply only by their declared scopes. With two outer producer
retries, two transient retries, and one structural repair, a single producer
can issue up to 18 HTTP attempts: three producer attempts, each with an initial
generation and one repair generation, each allowing three HTTP attempts. This
is an illustration, not a total-deadline or exactly-once guarantee. Every
built-in HTTP attempt has its own generation timeout; caller cancellation
remains the overall authority and interrupts Retry-After waiting. The shared
Notarius scheduler keeps its permit through those waits, so retry budgets can
increase latency and provider cost. Observed usage can remain incomplete when
providers omit accounting fields.
After the applicable budgets are exhausted, the resolved
[`validation_policy`](config.md#pipelines) determines the result. Structural
@@ -319,7 +338,7 @@ content. Neither surface is a cache input. Do not treat a debug bundle as safe
to share merely because its configuration summary is redacted. Invocation
metadata omits reasoning effort when it is inherited, records the replacement
value when one is supplied, and records an empty value when inherited reasoning
was explicitly cleared. It records `config_profile` only when a non-base
was explicitly cleared. It records `config_overlay` only when a non-base
configuration overlay was selected.
Notarius never creates debug state without an explicit request and never
@@ -352,7 +371,9 @@ Provider execution settings and the generation timeout come from the selected
PromptKit profile. The invocation-only **--reasoning-effort** and
**--clear-reasoning-effort** controls may replace or clear that profile setting
for all LLM-backed calls in one run without changing the profile. PromptKit
structural output repair happens within one structured-completion call. Its
v0.11.0 also retries eligible transient HTTP failures with its built-in default
of two additional attempts per generation. This retry work is separate from
structural output repair, which happens within one structured-completion call. Its
effective `structured_output_repair_attempts` limit is resolved from the
selected binding, then the pipeline, then the prompt declaration; see
[module bindings](config.md#module-bindings-and-validators). This is distinct
@@ -360,19 +381,20 @@ from Notarius binding **retries**, which rerun the complete module operation
and validation chain and do not consume or replenish the structural-repair
limit. The maintained production prompts declare one repair attempt, paid only
after a structural failure. One structured completion with repair budget **R**
makes at most **R + 1** serial provider calls. If one stage attempt performs
**C** structured completions, a binding with **retries: N** has a maximum of
**(N + 1) * C * (R + 1)** provider calls; LLM-backed validators have their own
corresponding invocation counts and budgets. This is an upper bound, not a
promise that every call reaches a provider.
contains at most **R + 1** serial generations. Each generation can now contain
more than one provider call when PromptKit retries an eligible transient
failure. LLM-backed validators have their own corresponding invocation and
repair budgets. These are bounds on attempted work, not a promise that every
call reaches a provider.
Timeouts are layered. Caller cancellation is the outer authority. A positive
effective generation timeout adds an inner request deadline, while zero
disables only that generation deadline. The HTTP client timeout remains a
transport-wide cap. Notarius does not add another timeout around PromptKit.
Repairs are serial within the same caller context, so their worst-case latency
and cost follow the provider-call bound above; provision run deadlines and
provider budgets accordingly. Credentials remain optional unless the selected
effective generation timeout adds an inner request deadline to each built-in
generation attempt, while zero disables only that generation deadline. The HTTP
client timeout remains a transport-wide cap. Notarius does not add another
timeout around PromptKit. Retries and repairs are serial within the same caller
context, so their worst-case latency and cost can exceed the structural-repair
bound above; provision run deadlines and provider budgets accordingly.
Credentials remain optional unless the selected
PromptKit profile requires one, in which case preparation fails before a
provider call when its configured credential is unavailable.
The pinned upstream boundary and profile-format links are in

90
docs/policy/agents.md Normal file
View File

@@ -0,0 +1,90 @@
# Agent Policy
**Base Policy Version:** 1.1
**Last Updated**: 2026-09-25
**Upstream Source:** `policy` commit `45591a79004c6811950f14dca554f6ff69b7f58e`.
Notarius adaptations preserve operator choice, existing explicit contracts, and
the deployment-permissions example below.
## Purpose
Agents working in this repository must read and adhere to the following house rules.
## Implementation Discipline
Simplify the mechanism, not the user's choices.
**KEY PRINCIPLES:** Write the simplest idiomatic code that satisfies the user's
request. Do not implicitly strengthen requirements, restrictions, or guarantees
beyond what the user actually requested.
**Keep implementation machinery proportionate.** Use straightforward,
idiomatic code and introduce only the machinery needed to support the requested
behavior and flexibility. Optimize for conceptual simplicity and
maintainability, not for the fewest lines, fewest options, or narrowest product
surface.
**Do not invent restrictions.** Do not reject, prohibit, constrain, cap,
harden, or otherwise narrow behavior unless required by the user, an explicit
repository contract, or an unavoidable correctness constraint. Preserve
operator and caller choice where policy has not been established. If an
additional restriction appears advisable, present it as a decision rather than
silently implementing it.
**Do not invent guarantees.** Do not strengthen public contracts, guarantees,
or invariants beyond those expressly requested by the user.
**Use the standard library.** For example: in almost every case, the best
approach to copying files is to use the standard library. Writing custom code to
implement power-loss protection at the application level, or to guard against
bad actors with privileged and concurrent access to the host's filesystem,
is wasteful, unnecessary, and not idiomatic. Stick to the simple, standard, and
idiomatic approach unless another path is expressly requested by the user.
**An implementation detail is not repository policy.** Tests and documentation
must not elevate incidental behavior into a durable guarantee or restriction
unless that behavior was expressly selected as part of the contract. Existing
explicit contracts remain authoritative; changing or weakening one requires a
deliberate decision rather than being inferred from these principles.
[ADR-0017](../adr/0017-defer-cache-access-policy-to-deployment.md) illustrates
why Unix deployments own cache access policy through permissions, umask, setgid,
and default ACLs. Notarius retains its separately adopted correctness boundaries.
## Git Commit Policy
By default, create a commit after completing each coherent unit of work, using a
concise, plain-English message describing what changed. This policy is standing
authorization to create those local commits; a separate user request is not
required. An explicit user instruction to leave work uncommitted takes precedence.
A unit of work is a focused, reviewable outcome, including its relevant tests and
documentation. Do not commit after every intermediate edit or create empty commits.
Complete the appropriate validation before committing. If work is incomplete or
validation is blocked, report that state rather than presenting it as completed.
Before committing:
1. Run `git status --short` and review the relevant diff, including any staged
changes. Identify changes that predate the current unit of work.
2. Exclude secrets, local environment files, generated junk, and unrelated changes.
3. Stage only the files or hunks belonging to the completed unit. Use `git add .`
only when every included change belongs to that unit. Do not disturb unrelated
staged or unstaged work.
4. Review `git diff --staged` and run appropriate checks for the change. Do not
repeat checks already passed for unchanged content merely to create a commit.
5. Review `git log --oneline -n 10` and use
`git commit -m "<concise plain-English commit message>"`.
6. Report the commit identifier and any remaining uncommitted work.
If the unit cannot be safely separated from pre-existing work, leave it uncommitted
and explain the overlap rather than silently including that work. If a commit
fails, preserve the changes and report the failure; do not bypass hooks.
Default commit authorization does not authorize pushing, publishing, amending,
resetting, reverting, or rewriting history. Those actions require a separate user
instruction. Do not use `git commit -a` or `git commit --no-verify` to bypass the
staging and validation discipline above.

View File

@@ -1,11 +1,25 @@
# Architecture
**Base Policy Version:** 1.1
**Last Updated**: 2026-09-25
**Upstream Source:** `policy` commit `45591a79004c6811950f14dca554f6ff69b7f58e`.
Notarius adapts the base template to its fixed pipeline, artifact families,
external adapters, state boundaries, and source-only distribution.
This document defines the intended high-level architecture of Notarius and the
invariants that changes must preserve. Implemented component details belong in
[Internal Overview](../internal/overview.md) and its linked documents. The
reasoning behind significant architectural choices belongs in
[ADRs](../adr/).
An adopted constraint can govern future changes without implying that every
component it could govern exists. Descriptions of implemented structure must
remain accurate; proposals and transition status belong in roadmaps and ADRs.
Constraints reflect selected requirements, not incidental implementation choices.
## System Shape
Notarius is a small, dependency-light Go application for extracting structured
@@ -38,35 +52,7 @@ Notarius is contract-first without being abstraction-heavy. Interfaces and
extension points should protect demonstrated boundaries. New abstraction is not
itself an architectural goal.
## Implementation Discipline
Simplify the mechanism, not the user's choices.
**Keep implementation machinery proportionate.** Use straightforward,
idiomatic code and introduce only the machinery needed to support the requested
behavior and flexibility. Optimize for conceptual simplicity and
maintainability, not for the fewest lines, fewest options, or narrowest product
surface.
**Do not invent restrictions.** Do not reject, prohibit, constrain, cap,
harden, or otherwise narrow behavior unless required by the user, an explicit
repository contract, or an unavoidable correctness constraint. Preserve
operator and caller choice where policy has not been established. If an
additional restriction appears advisable, present it as a decision rather than
silently implementing it.
**An implementation detail is not repository policy.** Tests and documentation
must not elevate incidental behavior into a durable guarantee or restriction
unless that behavior was expressly selected as part of the contract. Existing
explicit contracts remain authoritative; changing or weakening one requires a
deliberate decision rather than being inferred from this principle.
[ADR-0017](../adr/0017-defer-cache-access-policy-to-deployment.md) illustrates
these principles. Notarius previously elevated private cache creation modes
into application policy even though Unix deployments already express access
policy through directory permissions, process umask, setgid inheritance, and
default ACLs. The revised design retains application-owned correctness
boundaries while leaving deployment security policy to the operator.
Implementation discipline is owned by [Agent Policy](agents.md).
## Layers And Dependency Direction
@@ -309,6 +295,13 @@ model, correction, or large payload content.
## State, Output, And Safety
Source material and model responses cross application validation boundaries.
Host access policy belongs to the deployment; hypothetical privileged actors
concurrently changing the filesystem do not justify new application defenses
without an explicit decision. This scope statement does not remove the adopted
correctness boundaries below or claim excluded scenarios cannot occur. Follow
[Agent Policy](agents.md) when a new threat or guarantee needs a decision.
Notarius exposes three filesystem surfaces with independent roots and
lifecycle:
@@ -364,8 +357,9 @@ best-effort development and compilation environment, while Windows is
unsupported. Notarius distributes source releases only: an immutable source
tag and its checked-in release note identify a release. The project does not
publish executable binaries, archives, installers, container images,
checksums, signatures, or package-manager entries. Maintainer release commands
and tag guards belong to [Source Releases](../release.md).
checksums, signatures, or package-manager entries. [Release Policy](release.md)
owns publication requirements and completion; maintainer commands and guards
belong to [Source Releases](../release.md).
## Architectural Non-Goals

View File

@@ -1,5 +1,14 @@
# Documentation Policy
**Base Policy Version:** 1.3
**Last Updated**: 2026-09-25
**Upstream Source:** `policy` commit `45591a79004c6811950f14dca554f6ff69b7f58e`.
Notarius retains subprocess-consumer ownership, the distinction between
descriptive behavior and policy, and no placeholders for absent interfaces.
## Purpose
This policy assigns each documentation topic to one canonical owner. Its goal is
@@ -24,8 +33,12 @@ contract, maintain it only in the owning document.
Outside `docs/roadmap/`, documentation describes implemented behavior only.
Partial features may be described only to their implemented boundary.
ADRs are the narrow exception: an ADR may record an accepted architectural
decision before implementation, but acceptance must not be presented as proof
Architectural constraints may govern future changes without implying that the
components or capabilities they would govern already exist. This does not permit
describing a proposed system structure as implemented.
ADRs may record an accepted architectural decision before implementation, but
acceptance must not be presented as proof
that the behavior exists. The roadmap owns implementation status and sequencing
until the decision is implemented. Current architecture, user, operator,
integration, and internal documentation are updated when the behavior lands.
@@ -38,7 +51,7 @@ policy, default, or limit as authoritative merely because the current code
implements it. Documentation may accurately describe current behavior without
making it a permanent requirement; distinguish descriptive behavior from
normative policy. The
[implementation-discipline rules](architecture.md#implementation-discipline)
[implementation-discipline rules](agents.md#implementation-discipline)
own the criteria for adopting product restrictions and for elevating behavior
into repository policy.
@@ -71,16 +84,18 @@ secret values.
| --- | --- | --- | --- |
| Product orientation and minimal end-to-end quickstart | `README.md` | What Notarius is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
| Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
| Application architecture policy | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
| Agent policy | `docs/policy/agents.md` | Implementation discipline, rules against invented restrictions or guarantees, and local commit authorization. | Application contracts, architecture, testing policy, and contributor procedures. |
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
| Source release procedure | `docs/release.md` | Maintainer release selection, candidate validation, tagging, publication guards, verification, and immutable-tag recovery. | Product installation summary, CLI version semantics, historical release summaries, CI implementation detail. |
| Release policy | `docs/policy/release.md` | Versioning, compatibility, integrity, immutability, proportionate gate selection, publication controls, and release completion. | Concrete commands, product installation, historical notes, and CI implementation detail. |
| Repository-specific release procedure | `docs/release.md` | Prerequisites, candidate preparation, publication command, automated guards, optional diagnostics, and failure recovery. | General release requirements, product installation guidance, historical summaries, and current compatibility contracts. |
| Release-note history | `docs/releases/` | One checked-in historical summary for each source release made under the procedure. The note at the immutable tag is that release's record. | Current commands, behavior, contracts, and compatibility definitions. |
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
| Consumer guidance | `docs/consumers/` | Task-oriented subprocess use, minimal client examples, consumer responsibilities, and guidance for any future public package or API. | Wire semantics, external protocol contracts, internal implementation detail. |
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
@@ -111,11 +126,13 @@ other contracts.
### Releases
`docs/release.md` owns the source-release procedure. Release notes are
The [release policy](release.md) owns requirements and invariants; the
[repository procedure](../release.md) owns the commands that satisfy them.
Release notes are
historical summaries, not current-state contract owners: the checked-in note at
an immutable tag records that release, while current canonical documentation
must change with the behavior it describes. Do not use a release note to defer
or replace current documentation updates.
must change with the behavior it describes. Do not use a release procedure or
note to weaken policy, redefine current contracts, or defer documentation updates.
### Contracts And Implementation

204
docs/policy/release.md Normal file
View File

@@ -0,0 +1,204 @@
# Release Policy
**Base Policy Version:** 1.1
**Last Updated**: 2026-09-25
**Upstream Source:** `policy` commit `45591a79004c6811950f14dca554f6ff69b7f58e`.
Notarius adaptations: source-only delivery, lightweight tags, branch-before-tag
publication, no prereleases, and the justified native command check below.
## Purpose
This policy defines a proportionate workflow for validating and publishing an
intended source revision under a fresh version. Releases should require little
routine intervention, avoid repeated work, and preserve published contents.
The repository-specific commands belong in `docs/release.md`. Historical release
summaries belong in `docs/releases/`. The procedure must agree with the adopted
policy and link to it rather than introduce additional release requirements.
## Release Model And Completion
A source release identifies one exact committed source revision through an
immutable version tag and a version-matched, checked-in release note. The
primary release branch is `main` and the authoritative remote is `origin`.
Tags are lightweight. Publish the candidate to `main` before pushing its tag.
A successful push of the intended tag, with its source commit available on the
authoritative primary branch, completes the release. Notarius uses
branch-before-tag publication; it does not require an atomic push.
Release commands must finish at successful publication. They must not wait for
or poll CI, require successful CI status, inspect downloadable artifacts, or
check consumer installation or Go module availability as completion gates.
A failed or ambiguous push requires resolving the authoritative remote state
before declaring success or retrying; successful pushes do not require a
separate remote read-back gate.
Notarius publishes source only: no binaries, archives, images, packages,
checksums, signatures, or Gitea release objects. Tag CI may provide advisory
validation but publishes no deliverables and is not a completion gate.
## Versioning And Compatibility
Stable releases use Semantic Versioning in the form `vMAJOR.MINOR.PATCH`, with
no leading zeroes. Notarius does not support prereleases.
The selected version must be unused locally and on the authoritative remote.
Do not require queries to every configured remote or publication service.
A version known to have already been published must not be reused, even if the
normal guards do not inspect that channel. Adding a distribution channel
requires a deliberate update to the policy and procedure where needed.
For releases at or after `v1.0.0`:
- a major release may make intentional incompatible changes;
- a minor release may add backward-compatible behavior; and
- a patch release may make backward-compatible fixes.
Before `v1.0.0`, a minor release may change a documented public, operational,
configuration, data, integration, or compatibility contract when the release
note explains the impact and required consumer action. A patch release must
not intentionally break those contracts within its minor line.
Additional compatibility promises and deliberate exceptions belong in the
adopted policy or canonical contract documentation. Release notes explain
changes; they do not redefine current compatibility contracts.
## Release Documentation And Preparation
Every new release must include `docs/releases/<version>.md` in the tagged
source tree. Use the complete version tag as the filename and begin with a
title containing the product name and that exact version.
Include a short summary and any relevant compatibility impact or upgrade
instructions. Keep the note proportionate to the change. Fixed sections and
explicit no-action boilerplate are not required. Existing historical notes
need not be rewritten to adopt this format.
Keep exact current contracts in their canonical documents and update affected
documentation with the behavior change. Link to those documents from the note
where useful. A release note must not substitute for current documentation.
Do not create retrospective notes for the pre-procedure v0.1.0, v0.2.0, or
v0.3.0 tags.
Version selection, release-note content, and retirement of completed roadmaps
are preparation responsibilities, not separate human approval gates. Preserve
planned and deferred work and repair affected links when retiring roadmaps.
Ordinary change review covers unintended files, credentials, and private data;
do not repeat it as a release-specific sign-off. Ignored local files are not
prohibited merely because they exist: they are not included in the tagged source.
## Candidate Validation
Provide one publication command that validates the final committed candidate,
checks publication state, and publishes the intended refs. A validation-only
mode or command may be provided for diagnosis, but must not be a required step
before publication repeats the same checks.
Run the complete deterministic, offline default test suite once for the final
candidate, following the [testing policy](testing.md). For repositories that
publish builds, verify the required build targets. Select those targets for
the application's actual support commitments; native testing on each platform
and access to particular hardware are not default release prerequisites.
Additional gates must address a concrete risk that the existing checks do not
adequately cover. Do not automatically require linting, race testing, module
hygiene audits, documentation audits, credential scans, example validation, or
expanded platform matrices solely because they are possible. Retain such
checks when their distinct protection justifies their cost. A one-time migration
check must not silently become a permanent release requirement.
Release tests must not require real credentials, mutable application services,
paid APIs, provider probes, or inference calls. The procedure may describe dependency
preparation separately from offline validation.
Avoid duplicate validation across release phases. Successful checks for the
same final candidate need not be repeated solely to satisfy another phase;
any reuse must clearly apply to that exact candidate. Persistent validation
caches are not required. If the candidate changes, validate the new candidate.
Cheap publication-state checks may be repeated immediately before pushing
because local or remote state may have changed during validation.
Automate mechanical guards. Human review gates are strongly disfavored: add one
only through an explicit project-specific policy decision explaining the
concrete risk and why ordinary change review or automation is insufficient.
Do not add routine confirmations for checks the command can perform itself.
### Notarius Gate Selection
Run the default offline Go suite once and build the native `cmd/notarius` with
release version injection, checking its `--version` output. The command check
protects the actual executable entry point and linker-supplied version, which
package tests alone do not verify. Build products are temporary, not published.
Use `GOWORK=off` so a sibling checkout cannot substitute for declared dependencies.
Race tests, standalone vet, formatting/whitespace checks, module tidy audits,
standalone example validation, and Linux/Darwin cross-builds remain useful
change-specific development checks, not mandatory release gates. Default tests
already cover maintained examples. Source-only publication does not require a
binary distribution matrix. No new platform or hardware prerequisites are added.
## Publication Guards
Automate the checks needed to publish the intended source under a fresh version:
- the version has the adopted format and is unused locally and on the
authoritative remote;
- the worktree and index are clean on the primary release branch;
- the source being published is the exact candidate that passed validation,
including its matching release note;
- the branch push advances authoritative `main` to the candidate without
discarding remote history, before the tag is published; and
- the tag identifies the validated candidate and only the intended refs are
published.
Fetch the authoritative branch and tag state as needed for these checks.
Do not force branch or tag updates to bypass a failed guard. Publish explicit
refs rather than all local tags. Do not require CI success, publication-service
availability checks, or a separate manual confirmation for each guard.
## Immutability And Failure Recovery
Published version tags and published contents are immutable. Do not move,
overwrite, recreate, or delete a published tag, or replace a published artifact,
checksum, signature, provenance statement, or release note under that version.
The checked-in note at the immutable tag is that release's historical record.
If validation fails before publication, correct and commit the candidate and
rerun the publication command. If a push fails or its outcome is ambiguous,
inspect the authoritative remote before retrying. Remove an unpublished local
tag only after confirming it was never published or used by a release channel.
An unavailable remote leaves publication status unknown; do not overwrite a tag
to resolve uncertainty.
Transient network, authentication, CI, or consumer-availability failures do not
by themselves require a new version. Advisory CI or optional installation checks
may be retried against the same immutable tag. Notarius has no asynchronous
artifact delivery to recover.
A defect requiring changes to published source or contents requires a corrected
commit, a new semantically appropriate version, a matching release note, and
the release procedure again. Leave the original release intact. A channel's
yank, deprecation, or advisory mechanism may warn consumers away from a defective
release without replacing its contents or reusing its version.
## Repository Release Procedure
Maintain `docs/release.md` as a short, executable maintainer runbook. It must
identify:
1. prerequisites, primary branch, and authoritative remote;
2. version and release-note preparation;
3. one publication command with its selected local checks and automated guards;
4. declared deliverables, required build targets, and automation responsibilities;
5. the successful-push completion point and separately reported delivery status;
and
6. failure handling and ambiguous-push recovery.
Use copyable commands where practical. Keep optional diagnostics and consumer
verification clearly optional. The procedure must not turn preparation work,
ordinary change review, or post-publication investigation into additional
release gates.

View File

@@ -1,5 +1,14 @@
# Testing Policy
**Base Policy Version:** 1.1
**Last Updated**: 2026-09-25
**Upstream Source:** `policy` commit `45591a79004c6811950f14dca554f6ff69b7f58e`.
Adopted without substantive deviations; Notarius's established contracts remain
subject to [Agent Policy](agents.md).
## Purpose
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
@@ -8,29 +17,16 @@ We do not optimize for test count, line coverage, exhaustive isolation, or the f
## Every test has a cost
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
A test must be:
- written and reviewed;
- understood by future maintainers and coding agents;
- executed in local and CI workflows;
- diagnosed when it fails;
- updated when legitimate behavior changes;
- maintained as fixtures, APIs, and dependencies evolve; and
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
Every test has an immediate and continuing cost: writing, review, execution,
diagnosis, fixture maintenance, and updates when legitimate behavior changes.
Tests can also constrain refactoring, duplicate policy, and add failure noise.
A test is warranted only when the confidence it provides justifies these costs.
Apply this cost-benefit analysis at two levels:
Evaluate that tradeoff at two levels:
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
@@ -55,7 +51,7 @@ Control clocks, randomness, environment variables, and other process-global or m
## What deserves tests
Prioritize tests for:
To the extent the following features are present in the application, prioritize tests for:
1. Public and package-level contracts.
2. Domain rules and important invariants.
@@ -67,7 +63,18 @@ Prioritize tests for:
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
Choose tests for requested behavior and established contracts. Tests must not
introduce new restrictions, guarantees, or threat scenarios merely because they
could be asserted. Follow the [agent policy](agents.md) when deciding whether a
behavior belongs to the contract; current implementation behavior alone is not
sufficient authority.
For established behavior involving **data integrity, destructive operations,
compatibility, security, concurrency, idempotency, or recovery**, favor durable
tests for realistic failures whose consequences warrant protection. Identify
the failure and the existing contract it would violate. These categories call
for attention, not an automatic test inventory or permission to strengthen the
application's guarantees. Account for protection already provided at other layers.
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
@@ -85,18 +92,7 @@ Do not force all behavior through oversized end-to-end tests. Do not test every
## Test behavior, not implementation
A test should protect a decision, contract, or invariant—not memorialize the current implementation.
Tests must not create a stronger product contract than the requirement they
protect. In particular, do not turn an incidental limit, restriction,
permission choice, validation rule, or implementation default into policy by
asserting it without an expressly selected contract. Follow the
[implementation-discipline rules](architecture.md#implementation-discipline)
when deciding what behavior deserves durable protection.
Before adding or retaining a test, ask:
> What realistic defect would this test catch?
A test should protect a decision, contract, or invariant. A test must not memorialize the current implementation.
A test is suspect when its main purpose is to detect that someone:
@@ -109,8 +105,6 @@ A test is suspect when its main purpose is to detect that someone:
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
A test can be factually correct and still have negative value. Accurately describing current behavior is not enough; the protected behavior must be important enough to justify the future friction.
## Expected effects of different changes
Use the following expectations when evaluating test failures and test maintenance:
@@ -128,10 +122,11 @@ A test failing is not the same as a test needing to be edited. Many tests may co
Configurable thresholds and defaults must not be duplicated throughout the test suite.
For example, do not encode an internal concurrency limit indirectly:
For this example, assume the selected contract rejects a request to start more
processes than a configured limit. Do not encode its production default indirectly:
```go
// Production policy:
// Production default:
const maxConcurrency = 4
// Brittle test:
@@ -149,25 +144,30 @@ require.NoError(t, runner.Start(limit))
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
```
The test should prove:
For this selected contract, the test should prove:
- the configured limit is accepted; and
- one beyond the configured limit is rejected.
Rejection is not a general requirement for concurrency control. If the selected
contract queues excess work or limits active workers while accepting larger
batches, test that behavior instead. Do not add rejection or a configurable
limit solely to reproduce this example.
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
For concurrency limits, test both kinds of behavior when relevant:
For established concurrency limits, distinguish these behaviors where the contract requires them:
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
1. **Configuration handling:** values are accepted or rejected according to the selected contract.
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
## Avoid semantic duplication across layers
Each behavior should have a clear test owner.
Each behavior should have a clear test owner. Where these layers exist:
- Parser tests own parsing cases.
- Validator tests own validation rules.
@@ -179,8 +179,6 @@ Each behavior should have a clear test owner.
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
## Use test doubles deliberately
Choose the least elaborate test double that provides the required control or observation.
@@ -204,15 +202,16 @@ Do not use mocks merely to isolate every object or reproduce the implementation'
## Go-specific guidance
Use:
Select techniques according to the behavior and risk being tested. The following
are options, not a checklist that every package or application must satisfy:
- table-driven tests for meaningful behavioral categories and boundaries;
- `t.TempDir()` for real filesystem behavior;
- `httptest.Server` for realistic HTTP interactions;
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
- fuzz tests when exploring broad input spaces, such as parsers or path handling, adds useful protection beyond representative cases;
- golden files only when the complete output is intentionally stable;
- integration tests where correctness depends on component interaction; and
- a small number of representative end-to-end tests.
- a small number of representative end-to-end tests when assembled workflows expose meaningful risks not adequately covered at narrower boundaries.
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
@@ -230,8 +229,6 @@ Use it to find untested critical branches and unexpectedly weak packages. Do not
Pure domain logic will often warrant higher coverage than CLI wiring or external adapters. Uneven coverage is acceptable when it reflects risk.
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
## Regression tests
A bug fix should normally include a regression test that fails before the fix and passes afterward.
@@ -260,44 +257,26 @@ Strong candidates include tests that:
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
## Reviewing a proposed test
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
1. What realistic defect would it catch?
2. How likely is that defect?
3. How consequential would it be?
4. Is the behavior already protected elsewhere?
5. At which layer should this behavior be owned?
6. Does the test assert a durable contract or an incidental implementation detail?
7. Could the implementation be refactored without changing the behavior and without editing this test?
8. What should cause this test to fail?
9. What legitimate changes should not cause this test to fail?
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
11. Is there a smaller or more direct test that protects the same risk?
Do not add the test when its expected lifetime cost exceeds its expected protective value.
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
1. Which requested behavior or established contract does it protect, and what realistic defect would violate it?
2. How likely and consequential is that defect, and is it already protected elsewhere?
3. What is the narrowest stable boundary that can test it clearly?
4. Would a correct internal refactor or a change to an incidental default require test edits?
5. Does its confidence justify its maintenance, execution, and diagnostic cost, or would a smaller test suffice?
## Definition of sufficient
A test suite is sufficient when:
- important contracts and invariants are protected;
- meaningful boundaries and failure modes are exercised;
- realistic and consequential regressions are credibly protected against silent recurrence;
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
- important external boundaries have realistic integration coverage;
- representative complete workflows are tested;
- important requested behavior and established contracts are protected against realistic, consequential failures;
- boundaries and component interactions have coverage where they introduce meaningful risks not protected elsewhere;
- failures provide useful signal rather than redundant noise;
- legitimate internal changes usually do not require test edits; and
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
The governing rule is:
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
Sufficiency is a risk judgment, not a coverage percentage, test count, or required
mix of testing techniques. Reassess it as the application, its users, and the
consequences of failure evolve.

View File

@@ -1,164 +1,84 @@
# Source Releases
This procedure is for maintainers publishing Notarius source releases. A
release is an immutable lightweight `vMAJOR.MINOR.PATCH` tag on `main` together
with its checked-in `docs/releases/<tag>.md` note. Tag CI validates that source
candidate after publication; it does not publish or repair a release.
This is the executable runbook for [Release Policy](policy/release.md).
Notarius publishes an immutable lightweight tag and its checked-in release note,
not binaries or Gitea release objects. Linux is supported; macOS is best-effort
for development, and Windows is unsupported. `main` on `origin` is authoritative.
Notarius publishes no binaries, archives, checksums, signatures, containers,
package-manager entries, or Gitea release objects. Windows is not supported.
Do not create retrospective notes for the pre-procedure `v0.1.0`, `v0.2.0`, or
`v0.3.0` tags.
## Prepare
## Select And Describe The Release
Choose an unused stable semantic version in the form `vMAJOR.MINOR.PATCH`.
Prereleases are not supported. Before `v1.0.0`, a minor release may change a
documented CLI, configuration, durable artifact, integration, or operating
contract when its note explains the impact and required operator action. A
patch release must not intentionally break those documented contracts within
its minor line.
Create the version-matched note as part of the candidate. Every new note uses
this structure, with concise, truthful content in each section:
```markdown
# Notarius vMAJOR.MINOR.PATCH
This release ...
## Summary
## Compatibility
## Upgrade
## Changes
```
The note is a historical summary. Link to current canonical documentation for
exact behavior, and update that documentation in the candidate rather than
using the note as a substitute.
## Prepare The Candidate
Set the selected release version and disable Go workspace use for every
candidate command:
Use Git, a POSIX shell, and Go 1.25.5 or newer. Configure ordinary Git credentials
for `origin` and private-module access without putting secrets in the repository.
Prepare dependencies before offline validation where necessary:
```sh
RELEASE_VERSION=vMAJOR.MINOR.PATCH
export RELEASE_VERSION GOWORK=off
GOWORK=off GOPRIVATE=gitea.maximumdirect.net go mod download
```
Run the shared source-candidate checks from the repository. They cover module
hygiene, tests, race tests, vet, builds, formatting, whitespace, maintained
configuration validation, and the Linux and Darwin command-build matrix:
Choose an unused stable version under the policy's compatibility rules. Add
`docs/releases/<version>.md`, starting with `# Notarius <version>`, with a concise
summary and relevant compatibility/upgrade information. No fixed section list is
required. Update current documentation with behavior changes; retire only
completed roadmaps and preserve planned/deferred work. Commit the candidate on
`main` after ordinary change review. There is no separate release sign-off.
## Publish
Run one command from the repository with the actual selected version:
```sh
./scripts/check-release-source.sh "$RELEASE_VERSION"
./scripts/release.sh vMAJOR.MINOR.PATCH
```
Before committing, manually follow every changed local Markdown link and
review the candidate for unintended files, generated output, credentials, or
other unrelated changes. Commit the release note and all affected current
documentation, then run the shared checker against that exact candidate. Push
the candidate commit to `main` only after it succeeds. Record the exact commit
only after that push:
The command checks version availability on `origin`, branch and worktree state,
and the matching note. It runs the complete offline default test suite once and
a native CLI build/version check with `GOWORK=off`. The
[gate rationale](policy/release.md#notarius-gate-selection) explains the selected
checks. It then rechecks candidate and publication state, pushes the exact
candidate to `main` without force, creates a lightweight tag, and pushes only
that tag. It neither rewrites history nor publishes other local tags.
A successful tag push completes the source release. The command does not wait
for CI or verify consumer installation/module availability. Tag CI is advisory
validation only; there are no asynchronous deliverables to await.
## Failure And Recovery
Before publication, correct failed checks, commit any changes, and rerun the
command. If the candidate changes during validation, validation must run again
on the new candidate. A failed branch push publishes no tag.
On a failed or ambiguous push, the command queries the affected remote ref. If
a tag push reports failure but the intended lightweight tag is confirmed on the
remote, publication succeeded. Otherwise it stops and reports failure or unknown
status, leaving any local tag intact for inspection. Do not blindly rerun, force
push, or delete a possibly published tag.
Inspect authoritative state before retrying:
```sh
RELEASE_COMMIT=$(git rev-parse 'HEAD^{commit}')
export RELEASE_COMMIT
git ls-remote origin refs/heads/main refs/tags/vMAJOR.MINOR.PATCH
git rev-parse vMAJOR.MINOR.PATCH
```
For private-module installation, configure standard `GOPRIVATE` matching this
module and ordinary Git authentication for the hosting service before running
the verification below. The exact authentication mechanism belongs to the
maintainer environment; never record credentials or environment dumps in a
release note, command history, or repository file.
If the intended tag exists remotely, leave it immutable. If it is absent and
you have confirmed that the local tag was never published or used by a release
channel, remove only that unpublished local tag, then rerun the command. A
failed remote query is not proof of absence. Network/authentication failures
alone do not require a new version. A defect requiring changes to published
source requires a new semantically appropriate version and matching note.
## Guard And Publish The Tag
## Optional Diagnostics
Fetch current remote references, then run this guard without editing the
candidate. It requires `main`, a clean worktree and index, disabled workspace
use, a stable release version, the recorded and pushed commit, a matching note,
and unused local and remote tags:
`./scripts/check-release-source.sh <version>` runs validation without publication.
It is useful for diagnosis, but is not a required preliminary step that must
repeat the publication command's checks.
```sh
git fetch origin main --tags
Race tests, vet, module hygiene, formatting, example checks, and cross-builds may
be run when relevant to a change; they are not additional release gates. The
development guide and testing policy govern change-specific verification.
if ! printf '%s\n' "$RELEASE_VERSION" |
grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null
then
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
exit 1
fi
test "$GOWORK" = off
test "$(git branch --show-current)" = main
test -z "$(git status --porcelain)"
test "$RELEASE_COMMIT" = "$(git rev-parse 'HEAD^{commit}')"
test "$RELEASE_COMMIT" = "$(git rev-parse 'origin/main^{commit}')"
test -s "docs/releases/$RELEASE_VERSION.md"
grep -F -x "# Notarius $RELEASE_VERSION" "docs/releases/$RELEASE_VERSION.md"
for heading in '## Summary' '## Compatibility' '## Upgrade' '## Changes'; do
grep -F -x "$heading" "docs/releases/$RELEASE_VERSION.md"
done
if git rev-parse -q --verify "refs/tags/$RELEASE_VERSION" >/dev/null; then
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
exit 1
fi
if git ls-remote --exit-code --tags origin "refs/tags/$RELEASE_VERSION" >/dev/null 2>&1; then
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
exit 1
fi
```
Create an explicitly lightweight tag against the guarded commit, verify its
target, and push only that tag ref:
```sh
git -c tag.gpgSign=false tag "$RELEASE_VERSION" "$RELEASE_COMMIT"
test "$(git cat-file -t "$RELEASE_VERSION")" = commit
test "$(git rev-parse "$RELEASE_VERSION^{commit}")" = "$RELEASE_COMMIT"
git push origin "refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
```
Never use `git push --tags`, move a published tag, or delete a published tag.
## Verify The Published Release
Confirm that the remote tag still points at the guarded commit and that the
note is available from the tagged tree:
```sh
REMOTE_TAG_COMMIT=$(git ls-remote origin "refs/tags/$RELEASE_VERSION" | awk '{print $1}')
test "$REMOTE_TAG_COMMIT" = "$RELEASE_COMMIT"
git show "$RELEASE_VERSION:docs/releases/$RELEASE_VERSION.md" >/dev/null
```
Verify a fresh source installation and its diagnostic version. The temporary
directory confines the installed command to this check:
```sh
release_verification_dir=$(mktemp -d)
trap 'rm -rf "$release_verification_dir"' 0 HUP INT TERM
mkdir -p "$release_verification_dir/bin"
GOWORK=off GOBIN="$release_verification_dir/bin" go install \
"gitea.maximumdirect.net/eric/notarius/cmd/notarius@$RELEASE_VERSION"
test "$("$release_verification_dir/bin/notarius" --version)" = "notarius $RELEASE_VERSION"
```
An exact fresh checkout and `GOWORK=off go build ./cmd/notarius` is an
equivalent source verification when local installation policy requires it.
`notarius --version` is diagnostic only; downstream compatibility remains
defined by the published receipt and artifact contracts.
## Failure And Correction Policy
If candidate validation fails before publication, fix the candidate on `main`,
rerun the shared checker, and repeat the guards. An unpublished local tag may
be deleted after inspection.
If the remote tag or tag CI reveals a defect, leave the published tag intact.
Fix the defect on `main`, choose a new patch version, write a new matching
note, and repeat this procedure. Do not weaken tag immutability or add release
assets as a workaround.
A fresh source install or exact-tag checkout/build may be used to investigate
consumer availability after release. Such checks are optional and do not delay
or reverse release completion. `notarius --version` is diagnostic; subprocess
compatibility is defined by the receipt and artifact contracts.

44
docs/releases/v0.7.1.md Normal file
View File

@@ -0,0 +1,44 @@
# Notarius v0.7.1
This release updates the prompt engine and its provider catalogs while retaining
Notarius's application-owned D&D defaults and subprocess contracts.
## Summary
- Upgrade PromptKit from v0.9.0 to v0.10.0 and adopt its selected provider catalogs.
- Refresh the profile-source checkpoint identity for the updated catalog.
- Document profile ownership and upstream catalog compatibility.
## Compatibility
Notarius's CLI, configuration schema, durable artifact formats, and subprocess
receipt are unchanged. The `dnd-extraction` fallback retains its explicit model
and settings; the upgrade does not switch it to an upstream default profile.
Upstream catalog selections require review: `deepseek-3-2`, `deepseek-4-flash`,
and `deepseek-4-pro` are no longer present. Operators selecting or inheriting
those profiles must supply a complete operator-owned definition or select an
available profile. `deepseek-flash-latest` and `deepseek-pro-latest` are not
equivalent aliases and need not preserve previous settings. See the
[PromptKit integration contract](../integrations/pkg-promptkit.md#profile-sources-and-compatibility).
Existing checkpoints with the previous profile-source identity are not reused.
Source-keyed chunk plans and existing output artifacts are unaffected.
## Upgrade
Review selected LLM profile IDs and inherited bases before rebuilding from this
tag. No Notarius configuration migration is required for the maintained examples
or application-owned D&D defaults. For controlled model settings, use a complete
[operator profile](../config.md#promptkit-profiles), rather than inheriting a
volatile catalog entry. Follow the [source release procedure](../release.md)
for source installation and verification.
## Changes
- PromptKit v0.10.0 preserves the public engine API and adds two opt-in maintained
profile names; Notarius continues using its existing adapter and defaults.
- The dependency graph selects OpenRouter catalog v2.20260922.1 and Rakestrawhome
catalog v2.20260922.0.
- Profile checkpoint fingerprinting and current integration documentation now
identify the updated engine.

31
docs/releases/v0.8.0.md Normal file
View File

@@ -0,0 +1,31 @@
# Notarius v0.8.0
This release upgrades ConfigKit and PromptKit and makes provider retry behavior
explicit, inspectable, and available in debug diagnostics.
## Compatibility
Configuration selection now uses `--config-overlay` and
`overlays/<name>.yml`. The retired `--config-profile` flag and
`profiles/<name>.yml` lookup are no longer accepted. PromptKit LLM profiles are
unchanged.
`promptkit.transient_retries` defaults to two additional eligible transient
HTTP attempts per generation. Set it to `0` to disable built-in transient
retries. Existing pipeline and validator retry settings remain separate.
## Upgrade
Move selected configuration files from `profiles/` to `overlays/` and replace
every `--config-profile` invocation with `--config-overlay`. Review retry and
timeout budgets before deployment, especially where providers charge per
attempt. See [Configuration](../config.md), [Operations](../operations.md), and
the [PromptKit integration contract](../integrations/pkg-promptkit.md).
## Changes
- Upgrade ConfigKit to v0.4.0 and PromptKit to v0.11.0.
- Add application-owned generation failure and request-local execution-summary
metadata, including retry exhaustion details.
- Record execution summaries in existing debug response artifacts and safe
per-call references.

View File

@@ -0,0 +1,403 @@
# ConfigKit And PromptKit Upgrade Implementation Plan
## Purpose And Scope
Implement [ConfigKit And PromptKit Upgrades](upgrade-kit.md). That document
owns the desired behavior and operator migration; this document owns execution
order. Each numbered stage is one bounded implementation prompt for a
gpt-5.6-terra coding agent. Complete and validate stages in order.
The inspected baseline pins ConfigKit v0.3.0 and PromptKit v0.10.0. Adopt
exactly ConfigKit v0.4.0 and PromptKit v0.11.0, not whatever release happens to
be newest during implementation. The release guides and public APIs are
available in the sibling checkouts `../ConfigKit` and `../promptkit`; consult
the target tags or downloaded modules if those checkouts have advanced.
Build and test Notarius with `GOWORK=off`, without local dependency replacements.
Read `docs/development.md` and all five policies in `docs/policy/` before
implementation. In particular, simplify machinery without narrowing operator
choices or elevating incidental details into requirements. Preserve existing
contracts unless this roadmap expressly changes them. Use graph discovery for
code, then inspect the relevant source and tests; locations below identify the
reviewed baseline, not a requirement to preserve private file structure.
## Decisions And Boundaries
- Configuration selection becomes `--config-overlay` and `overlays/<name>.yml`,
with no legacy alias or fallback. Keep empty, omitted, and `default` selection
behavior. Do not rename PromptKit profiles or pipeline/runtime profile types.
- Keep configuration version 5, source-file-relative declarations, CLI override
precedence, strict loading, inspection provenance, and redaction.
- Add only the global `promptkit.transient_retries` setting: nonnegative integer,
default **2 additional attempts**, explicit zero disables. No arbitrary cap,
per-binding setting, new CLI retry flag, or new total-deadline setting.
- PromptKit owns transient HTTP retries and structural repairs; Notarius keeps
stage/validator retry budgets and disposition unchanged. The existing shared
scheduler retains its completion-scoped permit through upstream waiting.
- Add provider-neutral failure metadata and request-local execution summaries.
Do not change domain artifacts, result receipts, prompt content, validators,
acceptance policy, or introduce a new telemetry service/artifact.
- Successful retry recovery is ordinary success, not a warning. Debug remains
opt-in. Existing token fields keep their meaning; observed usage is separate.
- No release, tag, push, source-license change, or binary packaging is part of
these stages. Record upgrade implications in current integration/operator
documentation; the eventual release note is a separate release task.
For each stage, inspect the starting worktree, implement code with its focused
tests and current-behavior documentation, run the listed relevant checks, and
make one coherent local commit under agent policy. Report checks and any gaps.
Do not repeat passed checks for unchanged content merely to make a commit.
## Stage 1 — Cut Over Configuration Selection To Overlays
**Status:** Completed 2026-09-25.
**Goal:** Adopt ConfigKit v0.4.0 and finish its narrow operator-facing rename in
one coherent change. This is one prompt: it is a naming migration, not a new
configuration engine.
**Inspect:** `internal/core/config/load.go`, `loaded_config.go`, configuration
document/path/operational tests; `internal/cli/run.go`,
`configkit_commands.go`, CLI configuration and example contract tests;
`examples/profiles/development.yml`; upstream v0.4.0 migration guide.
**Work:**
1. Upgrade ConfigKit to v0.4.0 in `go.mod`/`go.sum`. Change `LoadOptions.Profile`
to `Overlay` and pass it into `configkit.Options[Document].Overlay`. Rename
only helpers, fields, locals, fixtures, and descriptions representing the
selected configuration layer.
2. Replace `--config-profile` with `--config-overlay` for `run`, `config validate`,
`config effective`, `config explain`, and `pipelines list`. Update usage,
flag parsing, argument classification, explicit-empty handling, and error
paths together. Retired flags use normal unknown-flag behavior.
3. Move `examples/profiles/development.yml` to
`examples/overlays/development.yml`; update maintained-example discovery and
selected-layer fixtures. Check file-relative paths after moving. Leave
`examples/llm-profiles/`, PromptKit profile sources, and ordinary explicit
imports alone unless an import actually points at the moved example.
4. Consume upstream overlay provenance and diagnostic names directly. Do not
rebuild composition, implement directory fallback, or translate metadata
back to profile vocabulary.
5. Update `docs/config.md`, `docs/cli.md`, `docs/integrations/pkg-configkit.md`,
`docs/internal/configuration.md`, relevant portions of `docs/internal/cli.md`,
and `docs/consumers/subprocess.md`. Review README, operations, and other
consumer examples for affected invocations. Explain the clean migration and
distinction from LLM profiles. Preserve accepted ADR-0018 and release history.
**Acceptance:** All five commands select overlays consistently. Representative
existing tests cover selected/base/empty/default behavior, overlay and explicit
override precedence, declaration-relative paths, redacted effective/explain
output, and source-aware failure labels. Include a selected legacy-only file
case proving there is no silent fallback. Retain ordinary-import behavior and
LLM profile coverage; do not duplicate every upstream parser case.
**Validate:** `GOWORK=off go test ./internal/core/config ./internal/cli`,
`GOWORK=off go mod tidy -diff`, `git diff --check`. Review occurrences of
`config-profile` and `examples/profiles` in current docs/code semantically;
historical docs and explicit legacy-rejection tests are not defects.
## Stage 2 — Upgrade The PromptKit Runtime Baseline
**Status:** Completed 2026-09-25.
**Goal:** Adopt the compatible upstream dependency and document its changed
default before adding Notarius controls. One small prompt.
**Inspect:** `internal/framework/llm/promptkit_client.go`, asset/profile identity
helpers and tests, `go.mod`, and upstream v0.11.0 release/consumer documentation.
**Work:**
1. Pin PromptKit v0.11.0 and tidy. Keep the Go version and provider catalog
selections unless the actual dependency graph requires a change; the target
release does not require either. Do not alter embedded D&D model settings.
2. Update `promptKitBuiltinProfileCatalogID` to the v0.11.0 identity in accordance
with the existing checkpoint fingerprint convention. No other cache policy
change or blanket invalidation is part of this work.
3. Update the pinned integration links in `docs/integrations/pkg-promptkit.md`,
including the relocated upstream `docs/policy/release.md`. Record the
LGPL-3.0-or-later dependency license and reference its license/notices; do not
relicense Notarius or invent a new distribution workflow.
4. Remove the obsolete no-provider-retry-loop statement in `docs/internal/llm.md`.
Describe the now-active upstream default and per-attempt timeouts in the
appropriate integration/operations sections. Do not yet describe the setting
or observations from later stages as implemented.
**Acceptance:** The prepared-execution adapter builds and its existing tests
pass without prompt/catalog changes. Current documentation acknowledges the
dependency's default retry behavior and license accurately.
**Validate:** `GOWORK=off go test ./internal/framework/llm ./internal/cli`,
`GOWORK=off go mod tidy -diff`, `git diff --check`.
## Stage 3 — Wire The Explicit Transient Retry Setting
**Status:** Completed 2026-09-25.
**Goal:** Make retry policy inspectable and configurable through the one shared
engine. One prompt covering a single vertical configuration slice.
**Inspect:** `internal/core/config/{document,config,load,operational_document,
loaded_config,validation}.go`, effective-config tests;
`internal/cli/catalog.go:buildProductionLLMClient`;
`internal/framework/llm/promptkit_client.go:NewPromptKitClient` and HTTP tests.
**Work:**
1. Add `TransientRetries *int` to `PromptKitDocument` and a resolved integer to
`PromptKitConfig`. Supply 2 through both the document default path and the
runtime `Default()` path. Preserve explicit zero; never use zero as omission.
Include it in document-to-runtime conversion and effective configuration.
2. Reject negative values through both existing document and direct runtime
validation entry points. Follow current source-aware diagnostics; do not
impose a new maximum. Keep ConfigKit's existing decoding behavior.
3. Add `TransientRetries *int` to `PromptKitClientConfig`: nil follows upstream
default, non-nil includes explicit zero. Production construction always
supplies the resolved configuration value. Pass it into `promptkit.Config`,
not per-request options. All calls through the injected shared client thus
share the setting without module changes.
4. Add the setting to `examples/dnd-complete.config.yml` and `docs/config.md`;
keep the minimal example minimal. Update configuration/LLM internal guides
for the new plumbing and operations for the zero-value opt-out.
**Acceptance/tests:** Demonstrate omitted default, explicit zero, a positive
override, negative rejection, and effective/explain inspection. Use an existing
CLI factory test to prove the resolved value reaches construction. Exercise
built-in generation with a local HTTP server: eligible failure then success,
disabled retries, exhausted configured count, and a noneligible 400 response.
An injected `LLMClient` stub does not test built-in HTTP retry wiring. Avoid
testing the upstream jitter formula or every equivalent HTTP status.
**Validate:** `GOWORK=off go test ./internal/core/config ./internal/cli ./internal/framework/llm`
and `git diff --check`.
## Stage 4 — Preserve Typed Failure And Exhaustion Metadata
**Status:** Completed 2026-09-25.
**Goal:** Distinguish provider failures without coupling callers to PromptKit.
One prompt; defer execution observations to the next stage.
**Inspect:** `internal/framework/contracts/errors.go`, adapter error conversion
in `promptkit_client.go`, redaction helpers, and existing pipeline handling of
`ErrLLMGeneration`, `ErrLLMCapacityExceeded`, and invalid structured output.
**Work:**
1. Extend the existing application-owned `LLMGenerationError` rather than
creating a parallel provider-error hierarchy. Retain `StatusCode()` and
`errors.Is(err, ErrLLMGeneration)`. Add application-owned failure-kind,
exhaustion, and exhausted-generation attempt metadata/accessors; use uint64
for upstream attempt counts. Preserve the existing constructor's callers
with sensible unspecified/generation metadata, adding a metadata constructor
or private adapter helper as needed.
2. Define kinds for `transport`, `provider_status`, `malformed_response`, and
generic `generation`. Exhaustion is an independent boolean and attempt count,
not a replacement kind. Zero attempts means unavailable in this error
projection, not proof no request happened. The count comes from
`RetryExhaustedError.Attempts()` and covers the exhausted generation, not all
repair generations combined.
3. Use `errors.Is`/`errors.As` on upstream sentinels/types. Keep cancellation and
deadline identity first, capacity handling intact, and HTTP status available
through an exhaustion wrapper. Do not unwrap application errors to upstream
provider objects or classify by prose. Preserve sanitized diagnostic text.
4. A malformed provider envelope is a generation failure; schema-invalid model
content stays `ErrInvalidStructuredOutput`. Do not change outer retry or
validator disposition logic. Document classification at the integration and
internal LLM boundaries.
**Acceptance/tests:** Adapter tests cover transport, malformed response,
noneligible status, exhausted eligible status with count/status preserved, and
cancellation/deadline/capacity regression behavior. Check typed fields and
identities, not full messages. Existing credential-redaction cases remain valid
for nested exhaustion diagnostics; use fake secrets only.
**Validate:** `GOWORK=off go test ./internal/framework/contracts ./internal/framework/llm ./internal/framework/pipeline`
and `git diff --check`.
## Stage 5 — Project Request-Local Execution Observations
**Status:** Completed 2026-09-25.
**Goal:** Return a lightweight terminal summary even when generation fails.
One prompt focused on contracts and adapter mapping, without persistence changes.
**Inspect:** `internal/framework/contracts/contracts.go`, `CompleteStructured`,
`responseFromResult`, prepared execution tests, upstream `observations.go` and
prepared-execution observer documentation.
**Work:**
1. Add an optional `Execution *LLMExecutionSummary` field to
`StructuredCompletionResponse` (`execution,omitempty`). Own these DTOs in
Notarius contracts; do not alias or serialize upstream event types.
2. The summary contains uint64 `attempts`, `transient_retries`, `repair_calls`;
integer millisecond `capacity_wait_ms`, `generation_ms`, `retry_delay_ms`,
`duration_ms`; `outcome`, `failure_kind`, `status_code`, `retries_exhausted`;
and `usage`. Usage contains prompt/completion/total/cached/cache-write counts,
each with `value`, `known`, `complete` fields. Do not omit these booleans or
known-zero values. The summary needs no prompts, response text, endpoints,
credentials, raw errors, or new copies of run/profile identifiers.
3. Map terminal outcome/failure values explicitly to application-owned strings.
Preserve the distinctions exposed by v0.11.0 (including canceled, deadline,
capacity, invalid output, configuration/load/render failures); unknown future
failures map to `other`. Reuse relevant application failure vocabulary from
Stage 4, without forcing every execution error into a generation error.
4. Attach a local observer to the `RunRequest` before `PrepareExecution` freezes
it. Retain the `ExecutionFinished` projection only: upstream supplies aggregate
counts, durations, and usage. Do not sum attempt events and terminal totals,
retain an event log, or reconstruct a second accounting system. Synchronous
request-local collection needs no global state or asynchronous queue.
5. Attach that summary to every subsequent return from `RunPrepared`, including
context/capacity/provider failure and post-result decode/validation failure.
Preparation failure has no execution event and returns no summary. Preserve
upstream execution outcome as such: a later Notarius decode error remains a
separate returned error, not a fabricated upstream terminal event.
6. Keep existing successful-result token and repair fields unchanged. Observed
`repair_calls` represents execution activity and is not a renamed legacy
`RepairAttempts`. No observation field is a billable-cost guarantee.
**Acceptance/tests:** Success, failed execution, cancellation, and no-observation
preparation failure retain the right presence. Test known zero versus unknown
and partial aggregate usage with controlled provider responses. One concurrent
completion case proves no summary cross-contamination. Check relationships and
nonnegative durations, not exact timing. Document the DTO and mapping in
`docs/internal/llm.md`; public dependency semantics stay in pkg-promptkit.
**Validate:** `GOWORK=off go test ./internal/framework/contracts ./internal/framework/llm`
and `GOWORK=off go test -race ./internal/framework/llm`.
## Stage 6 — Carry Summaries Through Existing Debug Paths
**Status:** Completed 2026-09-25.
**Goal:** Make observations useful to operators without a new output channel.
One prompt, limited to debug propagation and terminal allowlists.
**Inspect:** `internal/framework/pipeline/debug.go`, its response-content/fallback
helpers, terminal recording tests, and `debugLLMScope.references`. Trace
`debugLLMCallReference` through the existing timed stage/terminal envelopes;
updating only the detailed response file is insufficient. The CLI's separate
run-summary files do not need a new run-wide accounting mechanism.
**Work:**
1. Project the optional execution summary into existing debug response artifacts
and per-call references used by stage/terminal records. Explicitly preserve it
when `Debug.Response` or `RunResult` is absent and through content splitting
and fallback construction. Keep copies detached if existing clone paths
could otherwise share mutable pointers.
2. Use explicit typed fields as the allowlist for this metadata-only terminal
projection. Do not carry raw upstream events, newly copied provider diagnostics, or model
content into safe terminal summaries. Preserve established redaction and
detailed-trace content boundaries.
3. Leave warnings, normal console output, receipts, domain artifacts, checkpoint
payloads, and no-debug behavior unchanged. No new telemetry file or aggregation
of observed token totals into legacy totals.
4. Update `docs/internal/state.md`/relevant debug internals and `docs/operations.md`
to identify where this diagnostic information is available. Link to the
canonical explanation rather than repeating every field in consumer docs.
**Acceptance/tests:** Through the real debug wrapper and existing recorder seam,
verify success and failed/canceled completions retain summaries in response and
safe terminal records. Verify content splitting/fallback does not discard them,
and debug disabled writes nothing. Reuse existing redaction tests, adding only
the new projection assertions. No new warnings result from recovered retries.
**Validate:** `GOWORK=off go test ./internal/framework/pipeline ./internal/cli`,
`GOWORK=off go test -race ./internal/framework/pipeline`, `git diff --check`.
## Stage 7 — Verify Retry Composition And Finish Operator Guidance
**Status:** Completed 2026-09-25.
**Goal:** Validate the cross-boundary behavior that individual stages cannot
prove, then reconcile current documentation. One prompt; no generalized retry
test framework or runtime redesign.
**Inspect:** Existing adapter HTTP fixtures, scheduler cancellation tests,
pipeline stage retry/debug tests, CLI production tests; current config, LLM,
operations, and consumer documentation.
**Work and acceptance:**
- Add one representative local-HTTP scenario combining a recoverable transient
status with structurally invalid generated content followed by a successful
repair. Prove transient budgets apply independently per generation and
terminal attempt/repair/usage accounting is not doubled. Use small explicit
budgets rather than executing the illustrative 18-request worst case.
- Exercise cancellation during provider-directed retry waiting without a long
real sleep: signal that the request was received, cancel a context while a
long Retry-After is pending, and wait with a bounded test timeout. Preserve
cancellation identity and the terminal observation. Do not build a second
backoff implementation or test exact random delays.
- Review shared scheduler tests for completion-scoped permit ownership and
cancellation/release. Add a focused integration case only where existing
coverage does not protect that boundary across the new retry path. Keep
upstream backend capacity algorithms upstream.
- Confirm a recovered generation passes through ordinary success without
producing process warnings. Preserve existing outer stage retries and semantic
feedback. Do not alter limits to hide budget multiplication.
- Finish operations guidance: distinguish transient, structural, and stage/
validator budgets; give the 18-attempt single-producer illustration with its
assumptions; explain per-attempt timeout versus overall caller cancellation,
Retry-After waits, cost/latency implications, incomplete usage accounting, and
the outer scheduler holding its permit during waits. No new total deadline or
exactly-once guarantee is implied.
- Review all canonical docs changed earlier for consistency. Keep exact fields
and defaults in config, flags in CLI, dependency semantics in integrations,
mechanism in internals, and operating implications in operations. Keep the
migration note about overlays distinct from unchanged LLM profiles. Retain
historical ADR/release content; no new ADR is needed for these upgrades.
**Validate:** Run the focused adapter/pipeline/CLI tests and race tests covering
modified concurrency paths. Review tests for unnecessary upstream duplication
or exact-prose/change-detector assertions; consolidate rather than multiply them.
## Stage 8 — Final Integration Verification And Handoff
**Status:** Completed 2026-09-25.
**Goal:** Verify the complete upgraded application and documentation against the
feature roadmap. One final verification prompt; fix only defects within scope.
**Work:** Review all feature acceptance items and staged diffs. Confirm no
upstream types escaped the LLM boundary, no references to the retired config
flag remain in current examples/help, no unintended LLM profile rename occurred,
and no source-only/license/release scope expanded. Check edited relative links.
Leave the roadmap available for the user's completion review; do not publish a
release or delete planning documents automatically.
**Validation commands:**
```sh
GOWORK=off go test ./...
GOWORK=off go vet ./...
GOWORK=off go build ./...
GOWORK=off go mod tidy -diff
GOWORK=off go test -race ./internal/framework/llm ./internal/framework/pipeline
GOWORK=off go run ./cmd/notarius config validate --config examples/dnd-minimal.config.yml
GOWORK=off go run ./cmd/notarius config validate --config examples/dnd-complete.config.yml
GOWORK=off go run ./cmd/notarius config validate --config examples/dnd-complete.config.yml --config-overlay development
git diff --check
```
Reuse already-passed checks for the same unchanged candidate where appropriate.
The example/CLI tests own maintained examples; the commands above are a one-time
migration smoke check, not new permanent release gates. No live model calls or
real API keys are required. Report environment/dependency-access failures
separately from implementation failures; do not claim blocked checks passed.
**Completion criteria:** Both exact dependency pins are active; overlay migration,
retry setting, typed errors, observations, debug propagation, tests, examples,
and current documentation satisfy the feature roadmap. Report commits and
checks, plus any residual limitation. Future release preparation should explain
the breaking overlay cutover, retry default/opt-out, and upstream license change
in a minor-release note, but is not part of this implementation.
## Open Questions
None. The feature roadmap supplies the policy choices needed for implementation.
The DTO naming and stage boundaries above settle remaining implementation choices
without adding new product restrictions or compatibility promises.

123
docs/roadmap/remediation.md Normal file
View File

@@ -0,0 +1,123 @@
# Kit Upgrade Review Remediation
## Resolution
Completed 2026-09-26. No production changes were needed.
- `internal/framework/llm/promptkit_observation_test.go` now covers retries in
both initial and repair generations, aggregate versus legacy accounting,
unknown/known-zero/partial usage, retry-wait cancellation, exhausted execution,
preparation failure, and overlapping request isolation.
- `internal/framework/pipeline/debug_execution_test.go` exercises the real
scheduled client and debug wrapper together. It verifies returned and persisted
summaries for provider errors and cancellation without a body, split debug and
fallback content, terminal references, and explicit usage-presence JSON fields.
- Temporary regression probes confirmed that discarding failed responses in the
scheduler and collapsing usage presence into known zero cause the new tests
to fail. Those probes were reverted.
- Focused LLM/pipeline tests, their race-enabled suites, and
`GOWORK=off go test ./...` passed. Existing no-debug and redaction coverage was
retained. Runtime budgets, scheduler policy, and output contracts are unchanged.
The original finding below is retained as review context.
## Review Scope
Reviewed implementation commits `31b48d2` through `7ac9d94` against
[the feature roadmap](upgrade-kit.md) and [the implementation plan](implementation.md).
The review covered configuration selection/defaults, production client wiring,
error translation, execution observation, scheduler/debug propagation, current
documentation, and tests. No production model calls were made.
The implemented architecture is consistent with the intended design: ConfigKit
v0.4.0 owns composition and overlay provenance behind the configuration boundary;
PromptKit v0.11.0 owns transient retries and structural repairs behind the LLM
adapter. Notarius retains stage retry/disposition policy, completion-scoped
scheduling, application-owned error/observation types, and opt-in debug output.
No new runtime defect or unrequested restriction was established in this review.
## Finding 1 — Complete The Missing Retry And Observation Verification
**Priority:** Medium. This is a verification gap, not a demonstrated runtime
failure. Stages 5–7 are marked complete, but their consequential cross-boundary
acceptance tests were not implemented.
### Evidence And Risk
- `internal/framework/llm/promptkit_client_test.go:58` adds only a successful,
single-attempt observation assertion with positive prompt-token usage. It
does not exercise absent versus partial/known-zero usage, failed/canceled
execution summaries, preparation failure, or concurrent request isolation.
- `TestPromptKitClientAppliesConfiguredBuiltInTransientRetries` covers basic
status recovery, explicit zero, exhaustion, and a noneligible status. It
discards the response, so does not protect terminal accounting. There is no
combined transient-retry/structural-repair scenario or cancellation while
waiting on Retry-After.
- `internal/framework/llm/scheduled_client.go:40` now preserves a response when
the inner call fails. No corresponding test protects that behavior. Returning
an empty response again would silently remove precisely the failed-call
observations this feature is intended to retain.
- `internal/framework/pipeline/debug.go:267` and `:277` add execution summaries
to response artifacts and call references, plus fallback/content-splitting
propagation. Neither the debug nor terminal-debug tests were extended to
exercise these fields. Passing existing tests does not prove that failed-call
metadata reaches persisted records.
These are Notarius-owned integration risks, not a request to re-test PromptKit's
backoff algorithm. Losing failed-call observations or misrepresenting incomplete
usage would be difficult to detect in ordinary successful runs. A focused set
of behavioral tests is justified under the testing policy.
### Required Remediation
Use existing test assets, local HTTP fixtures, scheduler wrappers, and debug
recorders. Combine related assertions; no fixed test count or new test framework
is required.
1. Add an adapter scenario with a transient failure, a structurally invalid
completion, and a successful repair. Exercise retries in both generations
with small explicit budgets, and assert terminal attempts/retries/repair
counts without double-counting. Use controlled usage fields to distinguish
known zero, unknown, and partial aggregate usage. Preserve legacy usage
semantics independently.
2. Cover failed and canceled completions, including cancellation during a
Retry-After wait. Verify caller cancellation identity, bounded completion,
and retained execution metadata. Use signaling and a generous bounded test
timeout, not exact sleep durations or jitter assertions. Cover a preparation
failure with no execution summary. Add a small concurrent-call check for
request-local summary isolation, which race tests alone do not prove.
3. Exercise an error response through the real scheduled client and existing
debug wrapper/recorder seams. Assert that the summary reaches response
artifacts and terminal call references even without a provider response
body. Cover the content-splitting/fallback path with a representative response.
Reuse existing no-debug and redaction tests rather than duplicating their
whole inventories. Keep new summary assertions metadata-only.
Retain existing scheduler policy, stage/validator budgets, cancellation
disposition, warning semantics, and debug opt-in behavior. Do not introduce a
total deadline, new retry caps, telemetry artifacts, or live API requirements
to make these tests easier. Fix production code only if these checks reveal a
violation of the already selected behavior.
**Acceptance:** The added tests fail if failed responses lose their execution
summary at an adapter/wrapper boundary, usage presence is collapsed into zero,
or retry/repair totals are double-counted. They pass with the intended behavior
and provide evidence for the previously unverified stage acceptance criteria.
**Validation:** Run focused LLM/pipeline tests, race-enabled tests for those
packages, and `GOWORK=off go test ./...`. No production credentials are needed.
## Checks Completed During Review
- `GOWORK=off go test ./...`
- `GOWORK=off go vet ./...`
- `GOWORK=off go build ./...`
- `GOWORK=off go mod tidy -diff`
- `GOWORK=off go test -race ./internal/framework/llm ./internal/framework/pipeline`
- CLI configuration validation for the minimal and complete examples, including
the complete example with `--config-overlay development`.
- `git diff --check`
All passed. Existing green tests do not substitute for the missing behavioral
coverage above. Temporary diagnostic test material used during review was
removed; no application code was changed.

337
docs/roadmap/upgrade-kit.md Normal file
View File

@@ -0,0 +1,337 @@
# ConfigKit And PromptKit Upgrades
## Purpose
Adopt ConfigKit v0.4.0 and PromptKit v0.11.0 in one coordinated work set:
- Consistently call ConfigKit's selected configuration layer an **overlay**,
distinct from PromptKit LLM profiles.
- Adopt configurable transient HTTP retries, provider-neutral failure
classifications, and execution observations from PromptKit.
Preserve application/framework boundaries and keep the two upgrades as distinct
implementation workstreams. Neither requires a redesign of pipeline execution.
This feature roadmap owns the target state and scope. The ordered work and
stage-specific verification belong to the [implementation plan](implementation.md).
Neither document authorizes release publication.
## Current Baseline
The reviewed baseline is Notarius v0.7.1 (`53d9284`):
- `go.mod` pins ConfigKit v0.3.0 and PromptKit v0.10.0.
- `internal/core/config/load.go` passes `LoadOptions.Profile` into
`configkit.Options[Document].Profile`.
- `--config-profile` selects `profiles/<name>.yml` beneath the configuration
root for `run`, `config validate`, `config effective`, `config explain`, and
`pipelines list`.
- The maintained selected-layer example is
`examples/profiles/development.yml`; configuration and CLI tests create
equivalent directories and exercise selection and inspection.
- ConfigKit owns composition and inspection metadata. Notarius owns its
version 5 document, defaults, validation, source-relative path resolution,
runtime conversion, and CLI discovery.
A dependency-only trial against v0.4.0 fails to compile at `load.go:33` because
`configkit.Options.Profile` no longer exists. The review found no direct
Notarius references to `LayerProfile`, `CodeProfile*`, or `LoadSecretsDir`.
An isolated dependency-only trial against PromptKit v0.11.0 passes the complete
Notarius test suite and `go vet ./...`. That proves baseline compatibility, not
adoption of the new controls or observations. The current adapter does not
configure transient retries or attach an execution observer. It classifies
generation errors broadly and records successful-result usage and structural
repair counts. The outer scheduler holds a permit for the whole completion.
## Upstream Changes
### ConfigKit v0.4.0
The [ConfigKit v0.4.0 release guide](https://gitea.maximumdirect.net/eric/ConfigKit/src/tag/v0.4.0/docs/releases/v0.4.0.md)
defines four breaking naming changes:
| Surface | Previous | Target |
| --- | --- | --- |
| Selection API | `Options.Profile` | `Options.Overlay` |
| Selected file | `profiles/<name>.yml` | `overlays/<name>.yml` |
| Provenance layer | `LayerProfile` / `profile` | `LayerOverlay` / `overlay` |
| Diagnostic identifiers | `CodeProfile*` / `profile.*` | `CodeOverlay*` / `overlay.*` |
There are no legacy aliases or directory fallback. Explicit imports from a
directory named `profiles` remain valid ordinary imports. Merge rules,
precedence, lifecycle, source-origin APIs, minimum Go version, and YAML parser
dependency are unchanged.
The intervening v0.3.1 release makes the opt-in `LoadSecretsDir` API ignore
dot-prefixed entries. Notarius does not use this API; adding secrets-directory
bootstrap is outside this work.
### PromptKit v0.11.0
The [PromptKit v0.11.0 release guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.11.0/docs/releases/v0.11.0.md)
introduces additive runtime APIs and one changed default:
- Built-in generation defaults to two additional transient HTTP attempts for
statuses 429, 500, 502, 503, and 504. Engine and request controls can override
the count, including zero. Injected `LLMClient` implementations are not
automatically replayed; a custom HTTP client on the built-in path still is.
- Backoff is cancellable and honors valid provider `Retry-After` timing.
Generation and HTTP timeouts apply per attempt; the caller context bounds
the whole operation. Transport failures, malformed responses, other HTTP
statuses, and invalid generated content are not transient-retry candidates.
- `RunRequest.Observer` reports lifecycle metadata, attempts, waits, outcomes,
and presence-aware usage, including work before a failed execution. It works
with prepared execution; preparation itself emits no execution events.
- `ErrTransport`, `ErrMalformedResponse`, `ErrRetriesExhausted`, and
`RetryExhaustedError` support typed classification without parsing error text.
- The license changes to LGPL-3.0-or-later. Catalog selections and the minimum
Go version are unchanged from v0.10.0.
## Target End State
### Dependency And Ownership
Pin ConfigKit v0.4.0 and PromptKit v0.11.0 and tidy the dependency graph.
Continue using only ConfigKit's
public root API behind `internal/core/config`. No compatibility parser,
application-owned composition engine, or new configuration abstraction is
needed. PromptKit remains behind `internal/framework/llm`; its types must not
leak into domain modules or pipeline contracts. Retain the existing prepared
execution path, profile choices, and D&D model settings.
### Consistent Overlay Selection
- Replace the configuration-selection CLI flag with `--config-overlay` on all
commands that currently accept `--config-profile`.
- Rename application-owned fields, variables, helpers, help text, and test
descriptions that specifically represent this selection to overlay terms.
- Pass the selection through `configkit.Options.Overlay`.
- Select files only from `overlays/<name>.yml`, following ConfigKit's contract.
- Preserve empty and `default` selection as the composed base configuration
without a selected overlay.
- Use a clean cutover: no old CLI alias, directory fallback, automatic file
moves, or custom migration machinery. Normal unknown-flag handling is
sufficient for the retired flag; explain migration in documentation.
This rename is deliberately narrow. Preserve `llm_profile`, `--llm-profile`,
`promptkit.profile_dir`, `promptkit.profile_file`, `base_profile`, embedded
`dnd-extraction` profiles, and `examples/llm-profiles/`. Other application
concepts named `Profile`, including pipeline/runtime types, must not be renamed
merely because they share the word.
### Preserved Behavior
Retain configuration version 5, with the additive retry setting described below.
Preserve existing defaults other than the explicitly adopted transient retry policy,
imports, overlay merge behavior (including sequence replacement), explicit CLI
override precedence, strict decoding, domain validation, and secret redaction.
Do not add new restrictions or relax existing explicit contracts as part of the
rename.
Relative paths continue to resolve from the file supplying the surviving
declaration, including overlay files. Invocation-relative CLI paths and
main-file-relative built-in defaults retain their current behavior.
Effective configuration, explanations, and configuration-load diagnostics use
the new upstream overlay provenance and diagnostic vocabulary. Preserve source
file labels and actionable error context; do not translate the new metadata
back into legacy profile terminology.
Pipeline stage order, semantic acceptance, LLM selection, domain artifact
schemas, and subprocess result receipts remain unchanged. No blanket checkpoint
invalidation is added merely for the ConfigKit rename. Update the existing
PromptKit-version identity used by checkpoint fingerprinting with its dependency
pin, following the current adapter convention; no cache redesign is needed.
### Transient Retry Policy
Expose `promptkit.transient_retries` as a nonnegative integer setting with a
Notarius default of **2 additional attempts**. Explicit zero disables transient
retries. Apply the resolved setting to the shared PromptKit engine so modules,
normalizers, validators, and semantic correction calls receive the same policy.
Include the effective value in configuration inspection. Do not impose an
arbitrary application maximum or add per-binding controls in this work set.
Maintain three separate budgets and owners:
| Budget | Owner | Repeated work |
| --- | --- | --- |
| Transient retries | PromptKit | The same generation request after eligible HTTP failures |
| Structural repairs | PromptKit | A corrective generation for invalid structured output |
| Producer/validator retries | Notarius | The configured stage or validator operation, with semantic feedback where applicable |
Do not silently reduce, repurpose, or remove existing stage/validator budgets.
Each structural repair generation receives its own transient budget. For a
single-completion producer, two additional stage attempts, one structural
repair, and two transient retries permit up to 18 HTTP attempts, excluding any
separate validator calls. Document this multiplication and the possibility of
additional cost; retries do not imply exactly-once provider execution.
Preserve caller cancellation throughout generation and waiting. Do not describe
a profile's generation timeout as a total completion deadline: provider
`Retry-After` can extend waits beyond local backoff, and each HTTP attempt has
its own timeout. A new total-deadline configuration surface is outside this
upgrade; accurately document the current controls and limitations.
Retain Notarius's completion-scoped scheduler permit during upstream backoff.
PromptKit releases its active backend permit during that wait, but this does
not release the outer Notarius permit. This remains bounded and correct; do
not add an attempt-level scheduler redesign.
### Provider-Neutral Failure Diagnostics
Translate the new typed errors into application-owned classifications for
transport failures and malformed provider responses. Represent transient retry
exhaustion separately from its underlying failure/status, retaining the actual
attempt count where available. Preserve existing HTTP-status reporting,
capacity classification, cancellation, deadline handling, and redaction.
Do not turn a malformed provider envelope into a semantic rejection or confuse
it with a structurally invalid model response. Do not expose PromptKit error
types across the adapter boundary or parse prose to recover classifications.
Existing stage disposition policies continue to decide what happens after a
completion fails; this work does not redesign those policies.
### Execution Observations
Attach a request-local observer through the existing prepared-execution path.
Translate selected metadata into Notarius-owned completion/debug summaries:
- dispatched attempt, transient retry, and structural repair-call counts;
- capacity-wait, generation, retry-delay, and total execution durations;
- terminal outcome, failure classification, HTTP status, and exhaustion state;
- observed token counts with explicit known/complete indicators.
Retain available summaries on failed completions, including cancellation, rather
than discarding them with the absent `RunResult`. Preparation failures have no
execution observation; do not fabricate one. Keep existing successful-result
usage fields and their meanings unchanged. Observation totals are separate
presence-aware metadata, not a second set of tokens to add to existing usage.
Unknown counts are not known zero; partial totals are not a billing ledger.
Use lightweight per-request accumulation with no unbounded event history or
asynchronous queue. Callbacks are synchronous: avoid blocking I/O or acquisition
of scheduler resources inside them, and isolate concurrent requests. Persist
the projection through the existing opt-in debug path, including safe terminal
summaries where applicable. Do not serialize upstream event structs directly;
their JSON layout is not a public contract. Allowlist metadata and preserve
existing identifier/content handling and redaction boundaries.
Successful transient recovery is ordinary success, not a warning. This work
does not add routine console messages, change the warning policy, introduce a
new durable telemetry artifact, or add complete cost accounting.
## Required Changes
### Code And Examples
- Update `go.mod` and `go.sum` and the configuration loader/options boundary.
- Update CLI selection plumbing in `internal/cli/run.go` and
`configkit_commands.go`, including usage, flag parsing, argument classification,
explicit empty-value handling, and error paths.
- Move the maintained example to `examples/overlays/development.yml` and update
example discovery and related fixtures. Inspect relative paths when moving
files; do not assume all occurrences of `profiles` are configuration overlays.
- Update affected configuration and CLI tests, particularly document paths,
operational defaults, safe inspection, command selection, diagnostic source
labels, run contracts, and maintained-example coverage.
- Add the retry setting through the typed configuration document, defaults,
validation, runtime conversion, and CLI client-construction boundary. Update
the complete example to demonstrate it without adding unnecessary settings
to the minimal example.
- Update the PromptKit adapter's engine construction, request observation,
error conversion, and checkpoint version identity. Extend transport-neutral
response/debug contracts only with the application-owned metadata needed.
- Carry observations through the pipeline debug wrapper and safe terminal
reporting without double-counting usage or losing failed-call summaries.
### Documentation
Update current documentation with the implementation, keeping canonical
ownership under [Documentation Policy](../policy/documentation.md):
- `docs/cli.md`: new flag on every affected command and matching help behavior.
- `docs/config.md`: overlay terminology, directory layout, precedence, and the
explicit distinction from LLM profiles; transient retry setting and default.
- `docs/integrations/pkg-configkit.md`: pinned release and supported upstream
naming/inspection contract.
- `docs/internal/configuration.md` and relevant CLI internals: new selection
terminology and unchanged boundaries.
- `docs/consumers/subprocess.md` and any other current callers/examples: updated
invocation syntax. Review operations and README references where applicable.
- `docs/integrations/pkg-promptkit.md`: updated pin, typed errors, observer
support, and retry contract. Refresh pinned upstream links, including its
relocated release-policy document.
- `docs/internal/llm.md` and relevant pipeline/debug internals: observation
projection, failure handling, scheduler interaction, and separate retry
budgets. Remove the obsolete claim that PromptKit has no provider retry loop.
- `docs/operations.md`: latency/cost implications, timeout limits, opt-in debug
evidence, and how to disable transient retries.
Preserve historical release notes and the accepted text of ADR-0018. Its
ConfigKit ownership decision remains valid; a terminology migration alone does
not require a new ADR or supersession. Current documentation must not portray
the old dependency pin as the current supported version.
Record PromptKit's LGPL-3.0-or-later change in the integration documentation and
eventual release note. Review its notices and distribution implications within
Notarius's source-only release model. Do not silently relicense Notarius, copy
upstream release policy into this repository, or introduce binary packaging.
Any future binary distribution needs its own license-compliance review.
## Operator Migration And Release Impact
This work belongs in the next **minor** Notarius release, not a patch, because
both the documented selected-file layout and CLI syntax change.
Operators move selected configuration files from `profiles/` to `overlays/`
and replace `--config-profile` with `--config-overlay` in scripts, service
definitions, and subprocess callers. Preserve unrelated PromptKit profile
directories and explicit imports. Review relative paths and explicit imports
that refer to moved files. No YAML body version migration is required.
The eventual release note must explain this cutover, the newly enabled transient
retries and zero-value opt-out, and the dependency license change. PromptKit's
catalogs are unchanged from v0.10.0; this upgrade adds no further profile-name
migration or change to the D&D model defaults. Release preparation,
tagging, and publication remain a separately requested task under
[Source Releases](../release.md).
## Acceptance And Verification
- Notarius builds against ConfigKit v0.4.0 and PromptKit v0.11.0 without
compatibility shims.
- All five affected commands use the new flag; non-default overlay selection,
omitted selection, empty selection, and `default` retain their intended
behavior. The old flag is not accepted.
- Selected overlays load from the new directory. A legacy-only selected file
does not silently load; ordinary explicit imports remain supported.
- Representative tests demonstrate overlay precedence, CLI overrides,
declaration-relative paths, redacted inspection, and source-aware failures.
- Maintained examples validate, including the development overlay. LLM profile
loading and defaults remain unaffected.
- Tests prove the default transient retry policy, explicit zero, configured
counts, and negative-value rejection through the relevant configuration and
adapter boundaries. Use local HTTP test servers for eligible/noneligible
statuses, exhaustion, cancellation, and interaction with structural repair;
an injected LLM fake alone does not exercise built-in HTTP retries.
- Provider-neutral errors preserve underlying status and cancellation identity
without leaking credentials or upstream types.
- Observations are available for success and failure, preserve unknown versus
known-zero usage, avoid aggregate double-counting, and stay isolated under
concurrent completions. Successful retry recovery emits no warning.
- Representative scheduler/repair/retry checks preserve bounded execution and
cancellation; use existing coverage where it already proves the contract.
- Current documentation, help text, examples, and tests agree. Searches for old
vocabulary are reviewed semantically, excluding intentional historical and
LLM-profile references rather than enforcing a blanket text ban.
- Run focused `go test ./internal/core/config ./internal/cli ./internal/framework/llm ./internal/framework/pipeline`, followed by
`go test ./...`, `go vet ./...`, `go build ./...`, `go mod tidy -diff`, and
`git diff --check`, with `GOWORK=off`. Run race-enabled tests for the modified
adapter, scheduler, and observation integration. Validate both maintained configurations
and the complete configuration with `--config-overlay development`.
Use existing behavioral tests where possible, following
[Testing Policy](../policy/testing.md). No live model calls or production API
credentials are needed to validate this migration.

View File

@@ -4,6 +4,7 @@ configkit:
- path: pipelines/dnd-session.yml
promptkit:
profile_file: ./llm-profiles/dnd-extraction.yml
transient_retries: 2
concurrency:
total_llm: 2
stage_workers:

8
go.mod
View File

@@ -3,8 +3,8 @@ module gitea.maximumdirect.net/eric/notarius
go 1.25.5
require (
gitea.maximumdirect.net/eric/ConfigKit v0.3.0
gitea.maximumdirect.net/eric/promptkit v0.9.0
gitea.maximumdirect.net/eric/ConfigKit v0.4.0
gitea.maximumdirect.net/eric/promptkit v0.11.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1 // indirect
)
@@ -12,7 +12,7 @@ require (
require golang.org/x/text v0.40.0
require (
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 // indirect
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 // indirect
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260922.1 // indirect
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260922.0 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
)

16
go.sum
View File

@@ -1,11 +1,11 @@
gitea.maximumdirect.net/eric/ConfigKit v0.3.0 h1:LgXEUjR0KsgwGBznOWa8FJTQ+iElctHtI1via5vKdu4=
gitea.maximumdirect.net/eric/ConfigKit v0.3.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
gitea.maximumdirect.net/eric/promptkit v0.9.0 h1:IpvDRC8L6xRxQ9hpuyKOmMc5b6MeLTKYyx+h1YAjy08=
gitea.maximumdirect.net/eric/promptkit v0.9.0/go.mod h1:oMJ/WUJImUtwJ5e+6MAGECPYAErAkOaKel0G+3T/b4E=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0 h1:lc062euk2qseO//D762i3JaFyulDNML3eQQX7DkYTho=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter v1.0.0/go.mod h1:AIa7kAu2mfrRQgcspe4L+DW51WqgnALQT60lqkEywJI=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0 h1:j9YY7wsTVjzke2kHH4YAzpU0oUpM+x+nXwl1IeS+2eg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome v1.0.0/go.mod h1:4RNS+LILDg4JbS4Ts9Lwy1C92wauXJIbeQaalps4Koo=
gitea.maximumdirect.net/eric/ConfigKit v0.4.0 h1:fo+vKff3zTGeWqJIA38NtT84b6OUP//9ZmBH3lFuKgk=
gitea.maximumdirect.net/eric/ConfigKit v0.4.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
gitea.maximumdirect.net/eric/promptkit v0.11.0 h1:x7JguOUiInPVXK0GEitjyoRFqqvcuSTjYUvqJ+whtik=
gitea.maximumdirect.net/eric/promptkit v0.11.0/go.mod h1:0zEBVlJJC+ldDdXazAgybYBXNSe74vwFx5dyOuSos/w=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260922.1 h1:S2/Q/JDE+m8Ca3pDhAjVbfYrdlEhs9blj0STgNopHo0=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260922.1/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260922.0 h1:Y0EXc3eh12eKW2NrOviVqdk3oBo6UUjopt1YAa1B3Oc=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260922.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=

View File

@@ -157,13 +157,15 @@ func buildProductionLLMClient(ctx context.Context, cfg config.Config, profileID
return nil, nil, fmt.Errorf("production asset registry must not be nil")
}
recorder := llm.NewLLMProfileRecorder()
transientRetries := cfg.PromptKit.TransientRetries
client, err := llm.NewPromptKitClient(llm.PromptKitClientConfig{
ProfileDir: cfg.PromptKit.ProfileDir,
ProfileFile: cfg.PromptKit.ProfileFile,
LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend),
Assets: assets,
Recorder: recorder,
ReasoningEffort: overrides.ReasoningEffort,
ProfileDir: cfg.PromptKit.ProfileDir,
ProfileFile: cfg.PromptKit.ProfileFile,
TransientRetries: &transientRetries,
LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend),
Assets: assets,
Recorder: recorder,
ReasoningEffort: overrides.ReasoningEffort,
})
if err != nil {
return nil, nil, fmt.Errorf("create PromptKit-backed LLM client: %w", err)

View File

@@ -16,7 +16,7 @@ func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) i
fs := flag.NewFlagSet("config effective", flag.ContinueOnError)
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
profile := fs.String("config-profile", "", "configuration profile")
overlay := fs.String("config-overlay", "", "configuration overlay")
if err := fs.Parse(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
@@ -25,7 +25,7 @@ func runConfigEffective(args []string, stdout, stderr io.Writer, opts Options) i
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
return 2
}
loaded, _, err := loadConfigKit(*configPath, *profile, nil, opts)
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
@@ -41,7 +41,7 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
fs := flag.NewFlagSet("config explain", flag.ContinueOnError)
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
profile := fs.String("config-profile", "", "configuration profile")
overlay := fs.String("config-overlay", "", "configuration overlay")
if err := fs.Parse(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
@@ -56,7 +56,7 @@ func runConfigExplain(args []string, stdout, stderr io.Writer, opts Options) int
fmt.Fprintf(stderr, "notarius: %q is not a canonical JSON Pointer\n", pointer)
return 2
}
loaded, _, err := loadConfigKit(*configPath, *profile, nil, opts)
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
@@ -72,7 +72,7 @@ func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) in
fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
profile := fs.String("config-profile", "", "configuration profile")
overlay := fs.String("config-overlay", "", "configuration overlay")
pipelineID := fs.String("pipeline", "", "pipeline ID")
onlyRaw := fs.String("only", "", "comma-separated artifact lanes")
if err := fs.Parse(args); err != nil {
@@ -92,7 +92,7 @@ func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) in
fmt.Fprintf(stderr, "notarius: %v\n", err)
return 2
}
loaded, path, err := loadConfigKit(*configPath, *profile, nil, opts)
loaded, path, err := loadConfigKit(*configPath, *overlay, nil, opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
@@ -123,7 +123,7 @@ func runPipelinesList(args []string, stdout, stderr io.Writer, opts Options) int
fs := flag.NewFlagSet("pipelines list", flag.ContinueOnError)
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
profile := fs.String("config-profile", "", "configuration profile")
overlay := fs.String("config-overlay", "", "configuration overlay")
jsonOutput := fs.Bool("json", false, "write JSON output")
if err := fs.Parse(args); err != nil {
fmt.Fprintf(stderr, "notarius: %v\n", err)
@@ -133,7 +133,7 @@ func runPipelinesList(args []string, stdout, stderr io.Writer, opts Options) int
fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
return 2
}
loaded, _, err := loadConfigKit(*configPath, *profile, nil, opts)
loaded, _, err := loadConfigKit(*configPath, *overlay, nil, opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1

View File

@@ -14,22 +14,49 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestConfigKitReadOnlyCommandsLoadImportsAndProfiles(t *testing.T) {
func TestConfigKitReadOnlyCommandsLoadImportsAndOverlays(t *testing.T) {
t.Parallel()
root := t.TempDir()
path := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, filepath.Join(root, "fragment.yml"), "pipelines:\n imported:\n input: {module: input}\n")
writeConfigKitFile(t, filepath.Join(root, "profiles", "development.yml"), "pipelines:\n profiled:\n input: {module: input}\n")
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n")
writeConfigKitFile(t, path, "version: 5\nconfigkit:\n imports:\n - path: fragment.yml\npipelines:\n base:\n input: {module: input}\n")
var stdout, stderr bytes.Buffer
if code := RunWithOptions([]string{"pipelines", "list", "--config", path, "--config-profile", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\nprofiled\n" {
if code := RunWithOptions([]string{"pipelines", "list", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\noverlaid\n" {
t.Fatalf("list = %d, %q, %q", code, stdout.String(), stderr.String())
}
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--config-profile", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || !strings.Contains(stdout.String(), "is valid") {
if code := RunWithOptions([]string{"config", "validate", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, commandContractOptions(t)); code != 0 || !strings.Contains(stdout.String(), "is valid") {
t.Fatalf("validate = %d, %q, %q", code, stdout.String(), stderr.String())
}
for _, selection := range [][]string{nil, {"--config-overlay", ""}, {"--config-overlay", "default"}} {
stdout.Reset()
stderr.Reset()
args := append([]string{"pipelines", "list", "--config", path}, selection...)
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 0 || stdout.String() != "base\nimported\n" || stderr.Len() != 0 {
t.Fatalf("base list %q = %d, %q, %q", selection, code, stdout.String(), stderr.String())
}
}
}
func TestRetiredConfigProfileFlagUsesNormalUnknownFlagBehavior(t *testing.T) {
t.Parallel()
for _, args := range [][]string{
{"run", "sample", "--config-profile=development"},
{"config", "validate", "--config-profile=development"},
{"config", "effective", "--config-profile=development"},
{"config", "explain", "--config-profile=development", ""},
{"pipelines", "list", "--config-profile=development"},
} {
t.Run(strings.Join(args[:2], " "), func(t *testing.T) {
var stdout, stderr bytes.Buffer
if code := RunWithOptions(args, &stdout, &stderr, commandContractOptions(t)); code != 2 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "not defined") {
t.Fatalf("args=%q code=%d stdout=%q stderr=%q", args, code, stdout.String(), stderr.String())
}
})
}
}
func TestConfigKitReadOnlyCommandsRejectVersionFour(t *testing.T) {
@@ -53,8 +80,8 @@ pipelines:
"0":
input: {module: input}
`)
writeConfigKitFile(t, filepath.Join(root, "profiles", "development.yml"), "concurrency:\n total_llm: 3\n")
writeConfigKitFile(t, filepath.Join(root, "profiles", "retired.yml"), "pipelines: null\n")
writeConfigKitFile(t, filepath.Join(root, "overlays", "development.yml"), "concurrency:\n total_llm: 3\n")
writeConfigKitFile(t, filepath.Join(root, "overlays", "retired.yml"), "pipelines: null\n")
providerCalled := false
opts := commandContractOptions(t)
@@ -64,7 +91,7 @@ pipelines:
}
var stdout, stderr bytes.Buffer
if code := RunWithOptions([]string{"config", "effective", "--config", path, "--config-profile", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "total_llm: 3") || stderr.Len() != 0 {
if code := RunWithOptions([]string{"config", "effective", "--config", path, "--config-overlay", "development"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), "total_llm: 3") || stderr.Len() != 0 {
t.Fatalf("effective = %d, %q, %q", code, stdout.String(), stderr.String())
}
if providerCalled {
@@ -73,7 +100,7 @@ pipelines:
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-profile", "development", "/concurrency/total_llm"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/concurrency/total_llm"`) || stderr.Len() != 0 {
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "development", "/concurrency/total_llm"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/concurrency/total_llm"`) || stderr.Len() != 0 {
t.Fatalf("explain effective = %d, %q, %q", code, stdout.String(), stderr.String())
}
@@ -85,7 +112,7 @@ pipelines:
stdout.Reset()
stderr.Reset()
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-profile", "retired", "/pipelines/0"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/0"`) || stderr.Len() != 0 {
if code := RunWithOptions([]string{"config", "explain", "--config", path, "--config-overlay", "retired", "/pipelines/0"}, &stdout, &stderr, opts); code != 0 || !strings.Contains(stdout.String(), `path: "/pipelines/0"`) || stderr.Len() != 0 {
t.Fatalf("explain removed mapping key = %d, %q, %q", code, stdout.String(), stderr.String())
}
}

View File

@@ -10,7 +10,7 @@ import (
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
)
func loadConfigKit(configPath, profile string, overrides []configkit.Override, opts Options) (*config.LoadedConfig, string, error) {
func loadConfigKit(configPath, overlay string, overrides []configkit.Override, opts Options) (*config.LoadedConfig, string, error) {
path, err := discoverConfigPath(configPath, opts)
if err != nil {
return nil, "", err
@@ -19,7 +19,7 @@ func loadConfigKit(configPath, profile string, overrides []configkit.Override, o
if err != nil {
return nil, "", fmt.Errorf("determine invocation directory: %w", err)
}
loaded, err := config.Load(path, config.LoadOptions{Profile: profile, Overrides: overrides, InvocationDirectory: workingDirectory})
loaded, err := config.Load(path, config.LoadOptions{Overlay: overlay, Overrides: overrides, InvocationDirectory: workingDirectory})
if err != nil {
return nil, "", err
}
@@ -31,7 +31,7 @@ func writeConfigLoadError(output io.Writer, err error) {
if errors.As(err, &loadError) {
if writeErr := configkit.WriteDiagnostics(output, loadError.Diagnostics()); writeErr == nil {
// ConfigKit's locations contain source IDs. Keep its renderer and
// supply the acquired-source legend so imported/profile errors can
// supply the acquired-source legend so imported/overlay errors can
// be traced to a file. Quoting also escapes terminal controls.
for _, source := range loadError.Sources() {
label := source.Path

View File

@@ -13,16 +13,16 @@ import (
func TestConfigLoadDiagnosticsIdentifyComposedSources(t *testing.T) {
t.Parallel()
for _, profile := range []bool{false, true} {
t.Run(strconv.FormatBool(profile), func(t *testing.T) {
for _, overlay := range []bool{false, true} {
t.Run(strconv.FormatBool(overlay), func(t *testing.T) {
root := t.TempDir()
main := filepath.Join(root, "config.yml")
fragment := filepath.Join(root, "pipeline\n\x1b.yml")
args := []string{"config", "validate", "--config", main}
if profile {
fragment = filepath.Join(root, "profiles", "development.yml")
if overlay {
fragment = filepath.Join(root, "overlays", "development.yml")
writeConfigKitFile(t, main, "version: 5\n")
args = append(args, "--config-profile", "development")
args = append(args, "--config-overlay", "development")
} else {
writeConfigKitFile(t, main, "version: 5\nconfigkit:\n imports:\n - path: "+strconv.Quote(filepath.Base(fragment))+"\n")
}
@@ -43,18 +43,18 @@ func TestConfigLoadDiagnosticsIdentifyComposedSources(t *testing.T) {
}
}
func TestLoadConfigKitUsesDiscoveryProfileAndOverrides(t *testing.T) {
func TestLoadConfigKitUsesDiscoveryOverlayAndOverrides(t *testing.T) {
t.Parallel()
root := t.TempDir()
path := filepath.Join(root, "notarius.yml")
writeConfigKitFile(t, path, "version: 5\npipelines:\n base:\n input: {module: input}\n")
profiles := filepath.Join(root, "profiles")
writeConfigKitFile(t, filepath.Join(profiles, "development.yml"), "pipelines:\n profiled:\n input: {module: input}\n")
overlays := filepath.Join(root, "overlays")
writeConfigKitFile(t, filepath.Join(overlays, "development.yml"), "pipelines:\n overlaid:\n input: {module: input}\n")
loaded, gotPath, err := loadConfigKit(path, "development", []configkit.Override{{Path: configkit.Keys("output", "directory"), Value: "generated", Flag: "--output"}}, commandContractOptions(t))
if err != nil {
t.Fatalf("loadConfigKit() error = %v", err)
}
if gotPath != path || loaded.Config.Pipelines["profiled"].Input.Module != "input" || loaded.Config.Output.Directory == "generated" {
if gotPath != path || loaded.Config.Pipelines["overlaid"].Input.Module != "input" || loaded.Config.Output.Directory == "generated" {
t.Fatalf("loaded = %#v, path=%q", loaded.Config, gotPath)
}
}

View File

@@ -136,19 +136,19 @@ func TestMaintainedConfigurationExampleSet(t *testing.T) {
t.Fatalf("maintained configuration examples = %q, want only the minimal and complete D&D examples", got)
}
profileEntries, err := os.ReadDir(repositoryPath("examples", "profiles"))
overlayEntries, err := os.ReadDir(repositoryPath("examples", "overlays"))
if err != nil {
t.Fatal(err)
}
names = names[:0]
for _, entry := range profileEntries {
for _, entry := range overlayEntries {
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".yml") {
names = append(names, entry.Name())
}
}
sort.Strings(names)
if got := strings.Join(names, ","); got != "development.yml" {
t.Fatalf("maintained configuration profiles = %q, want development.yml", got)
t.Fatalf("maintained configuration overlays = %q, want development.yml", got)
}
llmProfiles, err := os.ReadDir(repositoryPath("examples", "llm-profiles"))
if err != nil {

View File

@@ -557,11 +557,16 @@ func TestNormalizeOptionsSharesProductionProfileAssetsWithDefaultRuntime(t *test
func TestProductionLLMClientFactoryUsesConfiguredLocalBackend(t *testing.T) {
var providerCalls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
providerCalls.Add(1)
call := providerCalls.Add(1)
if r.URL.Path != "/v1/chat/completions" {
t.Errorf("provider path = %q, want /v1/chat/completions", r.URL.Path)
}
w.Header().Set("Content-Type", "application/json")
if call == 1 {
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = w.Write([]byte(`{"error":{"message":"transient"}}`))
return
}
_, _ = w.Write([]byte(`{
"choices": [{"message": {"role": "assistant", "content": "{\"ok\":true}"}}],
"usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}
@@ -596,6 +601,7 @@ output:
cfg := config.Default()
cfg.PromptKit.ProfileFile = profilePath
cfg.PromptKit.TransientRetries = 1
cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
Endpoint: server.URL + "/v1",
ConcurrencyLimit: 2,
@@ -624,8 +630,8 @@ output:
if err != nil {
t.Fatalf("CompleteStructured() error = %v, want nil", err)
}
if providerCalls.Load() != 1 {
t.Fatalf("provider calls = %d, want 1", providerCalls.Load())
if providerCalls.Load() != 2 {
t.Fatalf("provider calls = %d, want 2", providerCalls.Load())
}
provider, ok := client.(contracts.LLMProfileManifestProvider)
if !ok {

View File

@@ -34,10 +34,10 @@ const usage = `Usage:
notarius help
notarius --version
notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
notarius config validate --config path/to/config.yml [--config-profile name] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius config effective --config path/to/config.yml [--config-profile name]
notarius config explain --config path/to/config.yml [--config-profile name] <json-pointer>
notarius pipelines list --config path/to/config.yml [--config-profile name] [--json]
notarius config validate --config path/to/config.yml [--config-overlay name] [--pipeline pipeline-id] [--only lane-a,lane-b]
notarius config effective --config path/to/config.yml [--config-overlay name]
notarius config explain --config path/to/config.yml [--config-overlay name] <json-pointer>
notarius pipelines list --config path/to/config.yml [--config-overlay name] [--json]
`
type Options struct {
@@ -160,7 +160,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
fs := flag.NewFlagSet("run", flag.ContinueOnError)
fs.SetOutput(io.Discard)
configPath := fs.String("config", "", "config file path")
configProfile := fs.String("config-profile", "", "configuration profile")
configOverlay := fs.String("config-overlay", "", "configuration overlay")
inputPath := fs.String("input", "", "source input file path")
onlyRaw := fs.String("only", "", "comma-separated artifact lanes")
outputDir := fs.String("output-dir", "", "output directory")
@@ -268,7 +268,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
return 2
}
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configProfile, runConfigurationOverrides(chunkCache, *outputDir, *debugDir), opts)
loaded, loadedConfigPath, err := loadConfigKit(*configPath, *configOverlay, runConfigurationOverrides(chunkCache, *outputDir, *debugDir), opts)
if err != nil {
writeConfigLoadError(stderr, err)
return 1
@@ -331,7 +331,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
InputPath: strings.TrimSpace(*inputPath),
ConfigPath: loadedConfigPath,
ConfigSource: configSource(*configPath),
ConfigProfile: effectiveConfigProfile(*configProfile),
ConfigOverlay: effectiveConfigOverlay(*configOverlay),
OnlyLanes: append([]string(nil), only...),
ChunkCacheOverride: chunkCache.explicitValue(),
ReasoningEffortOverride: runtimeOverrides.ReasoningEffort,
@@ -877,7 +877,7 @@ func reorderRunArgs(args []string) []string {
func runFlagTakesValue(arg string) bool {
switch arg {
case "--config", "--config-profile", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
case "--config", "--config-overlay", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
return true
default:
return false
@@ -963,7 +963,7 @@ func chunkPlanStoreForRun(cfg config.ChunkPlanCacheConfig, opts Options) (pipeli
func validateRunFlagValues(args []string) error {
for i, arg := range args {
if arg != "--config-profile" && arg != "--session-id" && arg != "--reasoning-effort" {
if arg != "--config-overlay" && arg != "--session-id" && arg != "--reasoning-effort" {
continue
}
if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") {
@@ -973,11 +973,11 @@ func validateRunFlagValues(args []string) error {
return nil
}
func effectiveConfigProfile(profile string) string {
if profile == "" || profile == "default" {
func effectiveConfigOverlay(overlay string) string {
if overlay == "" || overlay == "default" {
return ""
}
return profile
return overlay
}
func flagWasProvided(args []string, name string) bool {

View File

@@ -38,8 +38,8 @@ func TestRunControlsRejectSyntaxWithoutAllocatingState(t *testing.T) {
{name: "blank debug directory", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", ""}
}},
{name: "missing configuration profile", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--config-profile"}
{name: "missing configuration overlay", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--config-overlay"}
}},
{name: "debug directory without debug", args: func(roots stateTestRoots) []string {
return []string{"run", "sample", "--config", roots.config, "--input", roots.input, "--debug-dir", filepath.Join(filepath.Dir(roots.debug), "requested-debug")}
@@ -247,38 +247,38 @@ func TestRunStateRootsHonorEnvironmentFlagsAndDefaults(t *testing.T) {
})
}
func TestRunAppliesConfigurationProfileAndRecordsSelection(t *testing.T) {
func TestRunAppliesConfigurationOverlayAndRecordsSelection(t *testing.T) {
roots := newStateTestRoots(t)
profileDirectory := filepath.Join(filepath.Dir(roots.config), "profiles")
profileOutput := filepath.Join(profileDirectory, "profile-output")
profileDebug := filepath.Join(profileDirectory, "profile-debug")
writeConfigKitFile(t, filepath.Join(profileDirectory, "development.yml"), "output:\n directory: ./profile-output\ndebug:\n directory: ./profile-debug\n")
overlayDirectory := filepath.Join(filepath.Dir(roots.config), "overlays")
overlayOutput := filepath.Join(overlayDirectory, "overlay-output")
overlayDebug := filepath.Join(overlayDirectory, "overlay-debug")
writeConfigKitFile(t, filepath.Join(overlayDirectory, "development.yml"), "output:\n directory: ./overlay-output\ndebug:\n directory: ./overlay-debug\n")
var stdout, stderr bytes.Buffer
code := RunWithOptions([]string{
"run", "sample", "--config", roots.config, "--config-profile", "development",
"run", "sample", "--config", roots.config, "--config-overlay", "development",
"--input", roots.input, "--chunk_cache", "bypass", "--debug",
}, &stdout, &stderr, newStateTestHarness().options())
if code != 0 || stderr.Len() != 0 {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
}
assertFile(t, filepath.Join(profileOutput, filepath.Base(onlyChildDir(t, profileOutput)), "result.json"))
bundle := onlyChildDir(t, profileDebug)
assertFile(t, filepath.Join(overlayOutput, filepath.Base(onlyChildDir(t, overlayOutput)), "result.json"))
bundle := onlyChildDir(t, overlayDebug)
var invocation debugbundle.Invocation
readStateTestSummaryJSON(t, bundle, "invocation.json", &invocation)
if invocation.ConfigProfile != "development" {
t.Fatalf("config profile = %q, want development", invocation.ConfigProfile)
if invocation.ConfigOverlay != "development" {
t.Fatalf("config overlay = %q, want development", invocation.ConfigOverlay)
}
}
func TestEffectiveConfigProfileOmitsBaseSelection(t *testing.T) {
for _, profile := range []string{"", "default"} {
if got := effectiveConfigProfile(profile); got != "" {
t.Fatalf("effectiveConfigProfile(%q) = %q, want empty", profile, got)
func TestEffectiveConfigOverlayOmitsBaseSelection(t *testing.T) {
for _, overlay := range []string{"", "default"} {
if got := effectiveConfigOverlay(overlay); got != "" {
t.Fatalf("effectiveConfigOverlay(%q) = %q, want empty", overlay, got)
}
}
if got := effectiveConfigProfile("development"); got != "development" {
t.Fatalf("effectiveConfigProfile(development) = %q", got)
if got := effectiveConfigOverlay("development"); got != "development" {
t.Fatalf("effectiveConfigOverlay(development) = %q", got)
}
}

View File

@@ -14,9 +14,10 @@ type Config struct {
}
type PromptKitConfig struct {
ProfileDir string `json:"profile_dir,omitempty"`
ProfileFile string `json:"profile_file,omitempty"`
LocalBackend *PromptKitLocalBackendConfig `json:"local_backend,omitempty"`
ProfileDir string `json:"profile_dir,omitempty"`
ProfileFile string `json:"profile_file,omitempty"`
TransientRetries int `json:"transient_retries"`
LocalBackend *PromptKitLocalBackendConfig `json:"local_backend,omitempty"`
}
type PromptKitLocalBackendConfig struct {
@@ -29,7 +30,10 @@ type ConcurrencyConfig struct {
StageWorkers map[string]int `json:"stage_workers"`
}
const defaultLLMConcurrency = 16
const (
defaultLLMConcurrency = 16
defaultTransientRetries = 2
)
type OutputConfig struct {
Directory string `json:"directory"`
@@ -55,6 +59,7 @@ type DebugConfig struct {
func Default() Config {
return Config{
PromptKit: PromptKitConfig{TransientRetries: defaultTransientRetries},
Pipelines: map[string]pipeline.PipelineProfile{},
Concurrency: ConcurrencyConfig{
TotalLLM: defaultLLMConcurrency,

View File

@@ -16,9 +16,10 @@ type Document struct {
}
type PromptKitDocument struct {
ProfileDir *string `yaml:"profile_dir,omitempty"`
ProfileFile *string `yaml:"profile_file,omitempty"`
LocalBackend *PromptKitLocalBackendDocument `yaml:"local_backend,omitempty"`
ProfileDir *string `yaml:"profile_dir,omitempty"`
ProfileFile *string `yaml:"profile_file,omitempty"`
TransientRetries *int `yaml:"transient_retries,omitempty"`
LocalBackend *PromptKitLocalBackendDocument `yaml:"local_backend,omitempty"`
}
type PromptKitLocalBackendDocument struct {

View File

@@ -77,27 +77,48 @@ func TestDocumentPathsUseInvocationDirectoryForOverrides(t *testing.T) {
}
}
func TestDocumentPathsUseProfileDeclarationDirectory(t *testing.T) {
func TestDocumentPathsUseOverlayDeclarationDirectory(t *testing.T) {
t.Parallel()
root := t.TempDir()
profiles := filepath.Join(root, "profiles")
if err := os.MkdirAll(profiles, 0o700); err != nil {
overlays := filepath.Join(root, "overlays")
if err := os.MkdirAll(overlays, 0o700); err != nil {
t.Fatal(err)
}
main := filepath.Join(root, "notarius.yml")
if err := os.WriteFile(main, []byte("version: 5\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(profiles, "development.yml"), []byte("output: {directory: ./profile-output}\n"), 0o600); err != nil {
if err := os.WriteFile(filepath.Join(overlays, "development.yml"), []byte("output: {directory: ./overlay-output}\n"), 0o600); err != nil {
t.Fatal(err)
}
loaded, err := loadConfig(main, LoadOptions{Profile: "development"})
loaded, err := loadConfig(main, LoadOptions{Overlay: "development"})
if err != nil {
t.Fatalf("loadConfig() error = %v", err)
}
if got := loaded.Config.Output.Directory; got != filepath.Join(profiles, "profile-output") {
t.Fatalf("profile directory = %q", got)
if got := loaded.Config.Output.Directory; got != filepath.Join(overlays, "overlay-output") {
t.Fatalf("overlay directory = %q", got)
}
}
func TestDocumentLoadDoesNotFallBackToLegacyProfileDirectory(t *testing.T) {
t.Parallel()
root := t.TempDir()
main := filepath.Join(root, "notarius.yml")
if err := os.WriteFile(main, []byte("version: 5\n"), 0o600); err != nil {
t.Fatal(err)
}
legacy := filepath.Join(root, "profiles", "development.yml")
if err := os.MkdirAll(filepath.Dir(legacy), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(legacy, []byte("output: {directory: ./legacy-output}\n"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := loadConfig(main, LoadOptions{Overlay: "development"}); err == nil {
t.Fatal("loadConfig() error = nil, want missing overlay")
}
}

View File

@@ -10,7 +10,7 @@ import (
// boundary. It is intentionally narrow: configuration discovery and CLI flag
// parsing remain outside this package.
type LoadOptions struct {
Profile string
Overlay string
Overrides []configkit.Override
InvocationDirectory string
}
@@ -30,7 +30,7 @@ func loadDocumentWithSource(source configkit.Source, options LoadOptions, import
return configkit.Load(source, configkit.Options[Document]{
Definition: documentDefinition(state),
Imports: imports,
Profile: options.Profile,
Overlay: options.Overlay,
Overrides: options.Overrides,
})
}
@@ -63,6 +63,9 @@ func documentDefinition(state *documentLoadState) configkit.Definition[Document]
func documentDefaults() Document {
return Document{
PromptKit: &PromptKitDocument{
TransientRetries: documentValue(defaultTransientRetries),
},
Pipelines: map[string]PipelineDocument{},
Concurrency: &ConcurrencyDocument{
TotalLLM: documentValue(defaultLLMConcurrency),

View File

@@ -94,6 +94,9 @@ func documentToConfig(document Document) (Config, error) {
if document.PromptKit.ProfileFile != nil {
config.PromptKit.ProfileFile = *document.PromptKit.ProfileFile
}
if document.PromptKit.TransientRetries != nil {
config.PromptKit.TransientRetries = *document.PromptKit.TransientRetries
}
if document.PromptKit.LocalBackend != nil {
backend := PromptKitLocalBackendConfig{}
if document.PromptKit.LocalBackend.Endpoint != nil {

View File

@@ -2,6 +2,7 @@ package config
import (
"bytes"
"strconv"
"strings"
"testing"
@@ -58,6 +59,44 @@ pipelines:
}
}
func TestLoadedConfigResolvesPromptKitTransientRetriesAndRetainsInspection(t *testing.T) {
t.Parallel()
for _, test := range []struct {
name string
document string
want int
}{
{name: "omitted", document: "version: 5\n", want: defaultTransientRetries},
{name: "disabled", document: "version: 5\npromptkit:\n transient_retries: 0\n", want: 0},
{name: "configured", document: "version: 5\npromptkit:\n transient_retries: 5\n", want: 5},
} {
t.Run(test.name, func(t *testing.T) {
loaded, err := loadConfigSource(configkit.Bytes("document.yml", []byte(test.document)), LoadOptions{})
if err != nil {
t.Fatal(err)
}
if got := loaded.Config.PromptKit.TransientRetries; got != test.want {
t.Fatalf("transient retries = %d, want %d", got, test.want)
}
var effective bytes.Buffer
if err := loaded.WriteEffective(&effective); err != nil {
t.Fatal(err)
}
if !strings.Contains(effective.String(), "transient_retries: "+strconv.Itoa(test.want)) {
t.Fatalf("effective configuration = %q", effective.String())
}
var explanation bytes.Buffer
if err := loaded.WriteExplanation(&explanation, "/promptkit/transient_retries"); err != nil {
t.Fatal(err)
}
if !strings.Contains(explanation.String(), strconv.Itoa(test.want)) {
t.Fatalf("explanation = %q", explanation.String())
}
})
}
}
func TestLoadedConfigOwnsRuntimeValuesAndKeepsFrozenInspection(t *testing.T) {
t.Parallel()

View File

@@ -167,6 +167,11 @@ func validateDocumentPromptKit(promptKit *PromptKitDocument, validation *configk
if err := validatePromptKitProfileSources(profileDir, profileFile); err != nil {
validation.Error(configkit.Keys("promptkit"), err.Error())
}
if promptKit.TransientRetries == nil {
validation.Error(configkit.Keys("promptkit", "transient_retries"), "must be an integer")
} else if err := validatePromptKitTransientRetries(*promptKit.TransientRetries); err != nil {
validation.Error(configkit.Keys("promptkit", "transient_retries"), err.Error())
}
if promptKit.LocalBackend == nil {
return
}
@@ -283,6 +288,13 @@ func validatePromptKitLocalBackendConcurrencyLimit(concurrencyLimit int) error {
return nil
}
func validatePromptKitTransientRetries(transientRetries int) error {
if transientRetries < 0 {
return fmt.Errorf("promptkit.transient_retries must not be negative")
}
return nil
}
func validateRequiredPath(name, value string) error {
if strings.TrimSpace(value) == "" {
return fmt.Errorf("%s must not be empty", name)

View File

@@ -105,28 +105,28 @@ debug:
}
}
func TestDocumentLoadDerivesWorkersAfterProfileAndCLIOverrides(t *testing.T) {
func TestDocumentLoadDerivesWorkersAfterOverlayAndCLIOverrides(t *testing.T) {
t.Parallel()
directory := t.TempDir()
main := filepath.Join(directory, "config.yml")
profiles := filepath.Join(directory, "profiles")
if err := os.Mkdir(profiles, 0o755); err != nil {
overlays := filepath.Join(directory, "overlays")
if err := os.Mkdir(overlays, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(main, []byte("version: 5\nconcurrency:\n total_llm: 4\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(profiles, "development.yml"), []byte("concurrency:\n total_llm: 3\n"), 0o600); err != nil {
if err := os.WriteFile(filepath.Join(overlays, "development.yml"), []byte("concurrency:\n total_llm: 3\n"), 0o600); err != nil {
t.Fatal(err)
}
profiled, err := loadDocument(main, LoadOptions{Profile: "development"})
overlaid, err := loadDocument(main, LoadOptions{Overlay: "development"})
if err != nil {
t.Fatal(err)
}
if profiled.Config.Concurrency.TotalLLM == nil || *profiled.Config.Concurrency.TotalLLM != 3 || profiled.Config.Concurrency.StageWorkers["extract"] != 3 {
t.Fatalf("profiled concurrency = %#v", profiled.Config.Concurrency)
if overlaid.Config.Concurrency.TotalLLM == nil || *overlaid.Config.Concurrency.TotalLLM != 3 || overlaid.Config.Concurrency.StageWorkers["extract"] != 3 {
t.Fatalf("overlaid concurrency = %#v", overlaid.Config.Concurrency)
}
overridden, err := loadDocument(main, LoadOptions{Overrides: []configkit.Override{{
@@ -138,6 +138,16 @@ func TestDocumentLoadDerivesWorkersAfterProfileAndCLIOverrides(t *testing.T) {
if overridden.Config.Concurrency.TotalLLM == nil || *overridden.Config.Concurrency.TotalLLM != 2 || overridden.Config.Concurrency.StageWorkers["extract"] != 2 {
t.Fatalf("overridden concurrency = %#v", overridden.Config.Concurrency)
}
overlaidAndOverridden, err := loadDocument(main, LoadOptions{Overlay: "development", Overrides: []configkit.Override{{
Path: configkit.Keys("concurrency", "total_llm"), Value: 2, Flag: "--total-llm",
}}})
if err != nil {
t.Fatal(err)
}
if overlaidAndOverridden.Config.Concurrency.TotalLLM == nil || *overlaidAndOverridden.Config.Concurrency.TotalLLM != 2 || overlaidAndOverridden.Config.Concurrency.StageWorkers["extract"] != 2 {
t.Fatalf("overlaid and overridden concurrency = %#v", overlaidAndOverridden.Config.Concurrency)
}
}
func TestDocumentLoadRetainsExplicitWorkerAndRejectsInvalidOperationalValues(t *testing.T) {
@@ -213,6 +223,13 @@ cache:
assertDocumentDiagnostic(t, err, configkit.CodeValidateError, "/promptkit/local_backend/endpoint")
}
func TestDocumentLoadRejectsNegativePromptKitTransientRetries(t *testing.T) {
t.Parallel()
_, err := loadDocumentSource(configkit.Bytes("promptkit.yml", []byte("version: 5\npromptkit:\n transient_retries: -1\n")), LoadOptions{})
assertDocumentDiagnostic(t, err, configkit.CodeValidateError, "/promptkit/transient_retries")
}
func assertDocumentDiagnostic(t *testing.T, err error, code configkit.Code, path string) {
t.Helper()
var loadError *configkit.LoadError

View File

@@ -46,6 +46,9 @@ func validatePromptKit(cfg PromptKitConfig) error {
if err := validatePromptKitProfileSources(cfg.ProfileDir, cfg.ProfileFile); err != nil {
return err
}
if err := validatePromptKitTransientRetries(cfg.TransientRetries); err != nil {
return err
}
if cfg.LocalBackend == nil {
return nil
}

View File

@@ -89,6 +89,22 @@ func TestValidatePromptKitSourcesAreMutuallyExclusive(t *testing.T) {
assertValidationContains(t, cfg, "promptkit profile_dir and profile_file are mutually exclusive")
}
func TestValidatePromptKitTransientRetries(t *testing.T) {
if got := Default().PromptKit.TransientRetries; got != defaultTransientRetries {
t.Fatalf("default transient retries = %d, want %d", got, defaultTransientRetries)
}
for _, retries := range []int{0, 4} {
cfg := Default()
cfg.PromptKit.TransientRetries = retries
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() with transient retries %d error = %v", retries, err)
}
}
negative := Default()
negative.PromptKit.TransientRetries = -1
assertValidationContains(t, negative, "promptkit.transient_retries must not be negative")
}
func TestValidatePromptKitLocalBackendEndpoints(t *testing.T) {
tests := []struct {
name string

View File

@@ -39,7 +39,7 @@ type Invocation struct {
InputPath string `json:"input_path,omitempty"`
ConfigPath string `json:"config_path,omitempty"`
ConfigSource string `json:"config_source,omitempty"`
ConfigProfile string `json:"config_profile,omitempty"`
ConfigOverlay string `json:"config_overlay,omitempty"`
OnlyLanes []string `json:"only_lanes,omitempty"`
ChunkCacheOverride string `json:"chunk_cache_override,omitempty"`
ReasoningEffortOverride *string `json:"reasoning_effort_override,omitempty"`

View File

@@ -21,15 +21,45 @@ type StructuredCompletionRequest struct {
}
type StructuredCompletionResponse struct {
Content json.RawMessage `json:"content"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Debug *LLMDebugMaterial `json:"debug,omitempty"`
Content json.RawMessage `json:"content"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Debug *LLMDebugMaterial `json:"debug,omitempty"`
Execution *LLMExecutionSummary `json:"execution,omitempty"`
}
type LLMObservedTokenCount struct {
Value int `json:"value"`
Known bool `json:"known"`
Complete bool `json:"complete"`
}
type LLMObservedTokenUsage struct {
PromptTokens LLMObservedTokenCount `json:"prompt_tokens"`
CompletionTokens LLMObservedTokenCount `json:"completion_tokens"`
TotalTokens LLMObservedTokenCount `json:"total_tokens"`
CachedTokens LLMObservedTokenCount `json:"cached_tokens"`
CacheWriteTokens LLMObservedTokenCount `json:"cache_write_tokens"`
}
type LLMExecutionSummary struct {
Attempts uint64 `json:"attempts"`
TransientRetries uint64 `json:"transient_retries"`
RepairCalls uint64 `json:"repair_calls"`
CapacityWaitMS int64 `json:"capacity_wait_ms"`
GenerationMS int64 `json:"generation_ms"`
RetryDelayMS int64 `json:"retry_delay_ms"`
DurationMS int64 `json:"duration_ms"`
Outcome string `json:"outcome"`
FailureKind string `json:"failure_kind"`
StatusCode int `json:"status_code"`
RetriesExhausted bool `json:"retries_exhausted"`
Usage LLMObservedTokenUsage `json:"usage"`
}
type StructuredLLMClient interface {

View File

@@ -13,16 +13,42 @@ var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded")
// ErrLLMGeneration identifies a provider generation failure.
var ErrLLMGeneration = errors.New("LLM generation failed")
type LLMGenerationFailureKind string
const (
LLMGenerationFailureGeneration LLMGenerationFailureKind = "generation"
LLMGenerationFailureTransport LLMGenerationFailureKind = "transport"
LLMGenerationFailureProviderStatus LLMGenerationFailureKind = "provider_status"
LLMGenerationFailureMalformedResponse LLMGenerationFailureKind = "malformed_response"
)
type LLMGenerationMetadata struct {
FailureKind LLMGenerationFailureKind
RetriesExhausted bool
Attempts uint64
}
type LLMGenerationError struct {
status int
diagnostic string
status int
diagnostic string
failureKind LLMGenerationFailureKind
exhausted bool
attempts uint64
}
func NewLLMGenerationError(status int, diagnostic string) *LLMGenerationError {
return NewLLMGenerationErrorWithMetadata(status, diagnostic, LLMGenerationMetadata{})
}
func NewLLMGenerationErrorWithMetadata(status int, diagnostic string, metadata LLMGenerationMetadata) *LLMGenerationError {
if status < 0 {
status = 0
}
return &LLMGenerationError{status: status, diagnostic: diagnostic}
kind := metadata.FailureKind
if kind == "" {
kind = LLMGenerationFailureGeneration
}
return &LLMGenerationError{status: status, diagnostic: diagnostic, failureKind: kind, exhausted: metadata.RetriesExhausted, attempts: metadata.Attempts}
}
func (e *LLMGenerationError) Error() string {
if e == nil || e.diagnostic == "" {
@@ -37,3 +63,21 @@ func (e *LLMGenerationError) StatusCode() int {
}
return e.status
}
func (e *LLMGenerationError) FailureKind() LLMGenerationFailureKind {
if e == nil || e.failureKind == "" {
return LLMGenerationFailureGeneration
}
return e.failureKind
}
func (e *LLMGenerationError) RetriesExhausted() bool {
return e != nil && e.exhausted
}
func (e *LLMGenerationError) Attempts() uint64 {
if e == nil {
return 0
}
return e.attempts
}

View File

@@ -0,0 +1,22 @@
package contracts
import (
"errors"
"testing"
)
func TestLLMGenerationErrorPreservesApplicationMetadata(t *testing.T) {
legacy := NewLLMGenerationError(503, "provider failure")
if !errors.Is(legacy, ErrLLMGeneration) || legacy.StatusCode() != 503 || legacy.FailureKind() != LLMGenerationFailureGeneration || legacy.RetriesExhausted() || legacy.Attempts() != 0 {
t.Fatalf("legacy generation error = %#v", legacy)
}
err := NewLLMGenerationErrorWithMetadata(429, "provider failure", LLMGenerationMetadata{
FailureKind: LLMGenerationFailureProviderStatus,
RetriesExhausted: true,
Attempts: 3,
})
if !errors.Is(err, ErrLLMGeneration) || err.StatusCode() != 429 || err.FailureKind() != LLMGenerationFailureProviderStatus || !err.RetriesExhausted() || err.Attempts() != 3 {
t.Fatalf("generation error = %#v", err)
}
}

View File

@@ -26,15 +26,16 @@ type PromptKitLocalBackendConfig struct {
}
type PromptKitClientConfig struct {
ProfileDir string
ProfileFile string
LocalBackend *PromptKitLocalBackendConfig
Assets *AssetRegistry
Timeout time.Duration
HTTPClient *http.Client
EngineOptions []promptkit.Option
Recorder *LLMProfileRecorder
ReasoningEffort *string
ProfileDir string
ProfileFile string
TransientRetries *int
LocalBackend *PromptKitLocalBackendConfig
Assets *AssetRegistry
Timeout time.Duration
HTTPClient *http.Client
EngineOptions []promptkit.Option
Recorder *LLMProfileRecorder
ReasoningEffort *string
}
type PromptKitClient struct {
@@ -79,9 +80,10 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
options = append(options, cfg.EngineOptions...)
engine, err := promptkit.NewEngine(promptkit.Config{
ProfileDir: profileSource.ProfileDir,
Timeout: cfg.Timeout,
HTTPClient: cfg.HTTPClient,
ProfileDir: profileSource.ProfileDir,
Timeout: cfg.Timeout,
HTTPClient: cfg.HTTPClient,
TransientRetries: cfg.TransientRetries,
}, options...)
if err != nil {
return nil, fmt.Errorf("create PromptKit engine: %w", err)
@@ -128,10 +130,10 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
}
sessionID := strings.TrimSpace(req.SessionID)
var execution *promptkit.ExecutionTargetOverride
var targetOverride *promptkit.ExecutionTargetOverride
if c.reasoningEffort != nil {
reasoningEffort := *c.reasoningEffort
execution = &promptkit.ExecutionTargetOverride{
targetOverride = &promptkit.ExecutionTargetOverride{
ReasoningEffort: &reasoningEffort,
}
}
@@ -143,9 +145,15 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
SessionID: sessionID,
Inputs: promptKitInputs(req.Inputs),
Vars: promptKitVars(req, sessionID),
Execution: execution,
Execution: targetOverride,
AppendedMessages: appendedMessages,
}
var executionSummary *contracts.LLMExecutionSummary
runReq.Observer = func(event promptkit.ExecutionEvent) {
if event.Kind == promptkit.ExecutionFinished {
executionSummary = promptKitExecutionSummary(event)
}
}
if req.StructuredOutputRepairAttempts != nil {
inspection, err := c.engine.InspectPrompt(ctx, promptID, strings.TrimSpace(req.PromptVersion))
if err != nil {
@@ -170,12 +178,12 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
result, err := c.engine.RunPrepared(ctx, prepared)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return contracts.StructuredCompletionResponse{}, ctxErr
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), ctxErr
}
if errors.Is(err, promptkit.ErrCapacityExceeded) {
var capacityErr *promptkit.CapacityError
if errors.As(err, &capacityErr) && strings.TrimSpace(capacityErr.BackendID) != "" {
return contracts.StructuredCompletionResponse{}, fmt.Errorf(
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf(
"run PromptKit prompt %q on backend %q: %w: %v",
promptID,
strings.TrimSpace(capacityErr.BackendID),
@@ -183,7 +191,7 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
redactPromptKitError(err),
)
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf(
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf(
"run PromptKit prompt %q: %w: %v",
promptID,
contracts.ErrLLMCapacityExceeded,
@@ -192,20 +200,26 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
}
if errors.Is(err, promptkit.ErrLLMGenerate) {
var generationErr *promptkit.GenerationError
var exhaustedErr *promptkit.RetryExhaustedError
status := 0
metadata := contracts.LLMGenerationMetadata{FailureKind: promptKitGenerationFailureKind(err)}
if errors.As(err, &exhaustedErr) {
metadata.RetriesExhausted = true
metadata.Attempts = exhaustedErr.Attempts()
}
response := contracts.StructuredCompletionResponse{Debug: &contracts.LLMDebugMaterial{Prompt: promptKitDebugPrompt(&preparedDetails)}}
if errors.As(err, &generationErr) {
status = generationErr.StatusCode()
response.Debug.Response = promptKitDebugGenerationError(&preparedDetails, generationErr)
}
return response, contracts.NewLLMGenerationError(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)))
return promptKitResponseWithExecution(response, executionSummary), contracts.NewLLMGenerationErrorWithMetadata(status, fmt.Sprintf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err)), metadata)
}
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf("run PromptKit prompt %q: %v", promptID, redactPromptKitError(err))
}
if result == nil {
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
return promptKitResponseWithExecution(contracts.StructuredCompletionResponse{}, executionSummary), fmt.Errorf("run PromptKit prompt %q: %w: empty result", promptID, contracts.ErrInvalidStructuredOutput)
}
response := c.responseFromResult(result, &preparedDetails)
response := promptKitResponseWithExecution(c.responseFromResult(result, &preparedDetails), executionSummary)
if result.Validation.Status == promptkit.ValidationFailed || !result.Validation.IsValid {
return response, fmt.Errorf("run PromptKit prompt %q: %w: validation failed: %s", promptID, contracts.ErrInvalidStructuredOutput, strings.Join(result.Validation.Errors, "; "))
}
@@ -218,6 +232,61 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts.
return response, nil
}
func promptKitResponseWithExecution(response contracts.StructuredCompletionResponse, execution *contracts.LLMExecutionSummary) contracts.StructuredCompletionResponse {
response.Execution = execution
return response
}
func promptKitExecutionSummary(event promptkit.ExecutionEvent) *contracts.LLMExecutionSummary {
count := func(value promptkit.ObservedTokenCount) contracts.LLMObservedTokenCount {
return contracts.LLMObservedTokenCount{Value: value.Value, Known: value.Known, Complete: value.Complete}
}
return &contracts.LLMExecutionSummary{
Attempts: event.Attempts, TransientRetries: event.TransientRetries, RepairCalls: event.RepairCalls,
CapacityWaitMS: event.CapacityWait.Milliseconds(), GenerationMS: event.GenerationDuration.Milliseconds(), RetryDelayMS: event.RetryDelay.Milliseconds(), DurationMS: event.Duration.Milliseconds(),
Outcome: promptKitExecutionOutcome(event.Outcome), FailureKind: promptKitExecutionFailure(event.Failure), StatusCode: event.StatusCode, RetriesExhausted: event.RetriesExhausted,
Usage: contracts.LLMObservedTokenUsage{PromptTokens: count(event.Usage.PromptTokens), CompletionTokens: count(event.Usage.CompletionTokens), TotalTokens: count(event.Usage.TotalTokens), CachedTokens: count(event.Usage.CachedTokens), CacheWriteTokens: count(event.Usage.CacheWriteTokens)},
}
}
func promptKitExecutionOutcome(value promptkit.ExecutionOutcome) string {
switch value {
case promptkit.ExecutionOutcomeSuccess:
return "success"
case promptkit.ExecutionOutcomeInvalidOutput:
return "invalid_output"
case promptkit.ExecutionOutcomeError:
return "error"
default:
return "other"
}
}
func promptKitExecutionFailure(value promptkit.ExecutionFailureKind) string {
switch value {
case promptkit.ExecutionFailureNone:
return ""
case promptkit.ExecutionFailureCanceled, promptkit.ExecutionFailureDeadline, promptkit.ExecutionFailureTransport, promptkit.ExecutionFailureProviderStatus, promptkit.ExecutionFailureMalformedResponse, promptkit.ExecutionFailureCapacity, promptkit.ExecutionFailureInvalidConfig, promptkit.ExecutionFailureInvalidRequest, promptkit.ExecutionFailurePromptNotFound, promptkit.ExecutionFailureProfileNotFound, promptkit.ExecutionFailurePromptLoad, promptkit.ExecutionFailureProfileLoad, promptkit.ExecutionFailureArtifactLoad, promptkit.ExecutionFailurePromptRender, promptkit.ExecutionFailureValidation, promptkit.ExecutionFailureGeneration:
return string(value)
default:
return "other"
}
}
func promptKitGenerationFailureKind(err error) contracts.LLMGenerationFailureKind {
switch {
case errors.Is(err, promptkit.ErrTransport):
return contracts.LLMGenerationFailureTransport
case errors.Is(err, promptkit.ErrMalformedResponse):
return contracts.LLMGenerationFailureMalformedResponse
}
var generationErr *promptkit.GenerationError
if errors.As(err, &generationErr) && generationErr.StatusCode() != 0 {
return contracts.LLMGenerationFailureProviderStatus
}
return contracts.LLMGenerationFailureGeneration
}
func promptKitCorrectionMessages(correction *contracts.SemanticCorrection) ([]promptkit.RenderedMessage, error) {
owned, err := contracts.CloneSemanticCorrection(correction)
if err != nil {

View File

@@ -55,6 +55,10 @@ func TestPromptKitClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
t.Fatalf("usage = %#v, want mapped token counts", resp)
}
if resp.Execution == nil || resp.Execution.Outcome != "success" || resp.Execution.Attempts != 1 ||
!resp.Execution.Usage.PromptTokens.Known || resp.Execution.Usage.PromptTokens.Value != 11 {
t.Fatalf("execution summary = %#v", resp.Execution)
}
if resp.Debug == nil || resp.Debug.Prompt == nil {
t.Fatalf("debug prompt = nil, want prepared prompt material")
}
@@ -971,6 +975,83 @@ model: local-model
}
}
func TestPromptKitClientAppliesConfiguredBuiltInTransientRetries(t *testing.T) {
for _, test := range []struct {
name string
retries int
statuses []int
wantCalls int32
wantErr bool
}{
{name: "eligible failure then success", retries: 1, statuses: []int{http.StatusServiceUnavailable, http.StatusOK}, wantCalls: 2},
{name: "disabled", retries: 0, statuses: []int{http.StatusServiceUnavailable}, wantCalls: 1, wantErr: true},
{name: "exhausted", retries: 1, statuses: []int{http.StatusServiceUnavailable, http.StatusServiceUnavailable}, wantCalls: 2, wantErr: true},
{name: "noneligible client error", retries: 2, statuses: []int{http.StatusBadRequest}, wantCalls: 1, wantErr: true},
} {
t.Run(test.name, func(t *testing.T) {
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
index := calls.Add(1) - 1
if int(index) >= len(test.statuses) {
t.Errorf("unexpected HTTP attempt %d", index+1)
w.WriteHeader(http.StatusInternalServerError)
return
}
status := test.statuses[index]
w.Header().Set("Content-Type", "application/json")
if status != http.StatusOK {
w.WriteHeader(status)
_, _ = w.Write([]byte(`{"error":{"message":"provider failure"}}`))
return
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"ok\":true}"}}]}`))
}))
defer server.Close()
profilePath := filepath.Join(t.TempDir(), "profile.yml")
if err := os.WriteFile(profilePath, []byte("id: retry-profile\nendpoint: "+server.URL+"/v1\nmodel: retry-model\n"), 0o600); err != nil {
t.Fatal(err)
}
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
ProfileFile: profilePath,
TransientRetries: &test.retries,
})
if err != nil {
t.Fatal(err)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test",
ProfileID: "retry-profile",
SessionID: "retry-test",
Inputs: contracts.LLMInputSet{
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
},
}, &out)
if (err != nil) != test.wantErr {
t.Fatalf("CompleteStructured() error = %v, want error=%t", err, test.wantErr)
}
if test.wantErr {
var generation *contracts.LLMGenerationError
if !errors.As(err, &generation) || generation.FailureKind() != contracts.LLMGenerationFailureProviderStatus {
t.Fatalf("generation error = %#v, want provider-status classification", generation)
}
wantExhausted := test.name == "exhausted"
if generation.RetriesExhausted() != wantExhausted {
t.Fatalf("retries exhausted = %t, want %t", generation.RetriesExhausted(), wantExhausted)
}
if wantExhausted && (generation.Attempts() != 2 || generation.StatusCode() != http.StatusServiceUnavailable) {
t.Fatalf("exhausted generation = %#v", generation)
}
}
if got := calls.Load(); got != test.wantCalls {
t.Fatalf("HTTP calls = %d, want %d", got, test.wantCalls)
}
})
}
}
func TestLLMProfileRecorderDistinguishesEffectiveTargets(t *testing.T) {
recorder := NewLLMProfileRecorder()
for _, profile := range []artifacts.LLMProfileManifest{
@@ -1309,6 +1390,51 @@ func TestPromptKitClientForwardsConfiguredTransportTimeout(t *testing.T) {
}
}
func TestPromptKitClientClassifiesBuiltInTransportAndMalformedResponses(t *testing.T) {
for _, test := range []struct {
name string
kind contracts.LLMGenerationFailureKind
roundTrip func(*http.Request) (*http.Response, error)
}{
{
name: "transport",
kind: contracts.LLMGenerationFailureTransport,
roundTrip: func(*http.Request) (*http.Response, error) {
return nil, errors.New("transport unavailable")
},
},
{
name: "malformed response",
kind: contracts.LLMGenerationFailureMalformedResponse,
roundTrip: func(request *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("not-json")), Request: request}, nil
},
},
} {
t.Run(test.name, func(t *testing.T) {
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t),
HTTPClient: &http.Client{Transport: roundTripFunc(test.roundTrip)},
EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile", Endpoint: "http://promptkit.test/v1", Model: "default-model",
}))},
})
if err != nil {
t.Fatal(err)
}
var out map[string]any
_, err = client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
PromptID: "adapter.test", SessionID: "failure-kind-test",
Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", "")},
}, &out)
var generation *contracts.LLMGenerationError
if !errors.As(err, &generation) || generation.FailureKind() != test.kind || generation.RetriesExhausted() || generation.Attempts() != 0 {
t.Fatalf("generation error = %#v, want kind %q", generation, test.kind)
}
})
}
}
func TestPromptKitClientClassifiesEmptyStructuredCompletion(t *testing.T) {
client := newTestPromptKitClient(t, &fakePromptKitLLM{allowEmpty: true})

View File

@@ -0,0 +1,245 @@
package llm
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/promptkit"
)
func newObservedHTTPClient(t *testing.T, handler http.HandlerFunc, retries int, transport http.RoundTripper) *PromptKitClient {
t.Helper()
server := httptest.NewServer(handler)
t.Cleanup(server.Close)
client, err := NewPromptKitClient(PromptKitClientConfig{
Assets: newTestPromptKitAssets(t), TransientRetries: &retries,
HTTPClient: &http.Client{Transport: transport},
EngineOptions: []promptkit.Option{promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "default-profile", Endpoint: server.URL + "/v1", Model: "test-model",
}))},
})
if err != nil {
t.Fatal(err)
}
return client
}
func observedRequest() contracts.StructuredCompletionRequest {
return contracts.StructuredCompletionRequest{
PromptID: "adapter.test", SessionID: "observation-test",
Inputs: contracts.LLMInputSet{"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{}`), "", "")},
}
}
func TestPromptKitObservationsComposeTransientRetriesAndRepair(t *testing.T) {
var calls atomic.Int32
client := newObservedHTTPClient(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch calls.Add(1) {
case 1, 3:
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = io.WriteString(w, `{"error":{"message":"try again"}}`)
case 2:
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"bad\":true}"}}],"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5,"prompt_tokens_details":{"cached_tokens":0}}}`)
case 4:
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"ok\":true}"}}],"usage":{"prompt_tokens":7,"completion_tokens":0,"total_tokens":7}}`)
default:
t.Error("unexpected extra HTTP attempt")
w.WriteHeader(http.StatusBadRequest)
}
}, 1, nil)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
req := observedRequest()
repairs := 1
req.StructuredOutputRepairAttempts = &repairs
var out struct{ OK bool }
response, err := client.CompleteStructured(ctx, req, &out)
if err != nil || !out.OK {
t.Fatalf("completion = (%+v, %v), output = %+v", response, err, out)
}
summary := response.Execution
if calls.Load() != 4 || summary == nil || summary.Attempts != 4 || summary.TransientRetries != 2 || summary.RepairCalls != 1 || summary.Outcome != "success" || summary.RetriesExhausted {
t.Fatalf("calls = %d, execution = %+v", calls.Load(), summary)
}
wantUsage := contracts.LLMObservedTokenUsage{
PromptTokens: contracts.LLMObservedTokenCount{Value: 10, Known: true},
CompletionTokens: contracts.LLMObservedTokenCount{Value: 2, Known: true},
TotalTokens: contracts.LLMObservedTokenCount{Value: 12, Known: true},
CachedTokens: contracts.LLMObservedTokenCount{Known: true},
}
// Failed HTTP attempts contribute unknown usage, not known zero. Neither
// terminal totals nor legacy result totals should be added a second time.
if summary.Usage != wantUsage {
t.Fatalf("observed usage = %+v, want %+v", summary.Usage, wantUsage)
}
if response.PromptTokens != 10 || response.CompletionTokens != 2 || response.TotalTokens != 12 || response.RepairAttempts != 1 {
t.Fatalf("legacy result accounting changed: %+v", response)
}
}
type closeSignalingBody struct {
io.ReadCloser
closed chan<- struct{}
}
func (b closeSignalingBody) Close() error {
err := b.ReadCloser.Close()
b.closed <- struct{}{}
return err
}
func TestPromptKitObservationSurvivesCanceledRetryWait(t *testing.T) {
consumed := make(chan struct{}, 2)
transport := http.DefaultTransport.(*http.Transport).Clone()
t.Cleanup(transport.CloseIdleConnections)
var calls atomic.Int32
client := newObservedHTTPClient(t, func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
w.Header().Set("Retry-After", "3600")
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = io.WriteString(w, `{"error":{"message":"busy"}}`)
}, 1, roundTripFunc(func(req *http.Request) (*http.Response, error) {
response, err := transport.RoundTrip(req)
if err == nil {
response.Body = closeSignalingBody{ReadCloser: response.Body, closed: consumed}
}
return response, err
}))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
type result struct {
response contracts.StructuredCompletionResponse
err error
}
done := make(chan result, 1)
go func() {
var out map[string]any
response, err := client.CompleteStructured(ctx, observedRequest(), &out)
done <- result{response, err}
}()
// The failure has been consumed, so cancellation must interrupt the pending
// retry wait rather than depend on another provider response or its timeout.
select {
case <-consumed:
case <-time.After(5 * time.Second):
t.Fatal("provider response was not consumed")
}
cancel()
select {
case got := <-done:
if !errors.Is(got.err, context.Canceled) || got.response.Execution == nil {
t.Fatalf("canceled completion = (%+v, %v)", got.response, got.err)
}
summary := got.response.Execution
if summary.Outcome != "error" || summary.FailureKind != "canceled" || summary.Attempts != 1 || summary.RetriesExhausted || calls.Load() != 1 {
t.Fatalf("canceled execution = %+v, HTTP calls = %d", summary, calls.Load())
}
case <-time.After(5 * time.Second):
t.Fatal("cancellation did not interrupt retry waiting")
}
}
func TestPromptKitObservationsSeparatePreparationAndExecutionFailure(t *testing.T) {
var calls atomic.Int32
client := newObservedHTTPClient(t, func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = io.WriteString(w, `{"error":{"message":"busy"}}`)
}, 1, nil)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
req := observedRequest()
req.PromptID = "missing-prompt"
var out map[string]any
response, err := client.CompleteStructured(ctx, req, &out)
if err == nil || response.Execution != nil || calls.Load() != 0 {
t.Fatalf("preparation failure = (%+v, %v), calls = %d", response, err, calls.Load())
}
response, err = client.CompleteStructured(ctx, observedRequest(), &out)
var generation *contracts.LLMGenerationError
if !errors.As(err, &generation) || response.Execution == nil {
t.Fatalf("execution failure = (%+v, %v)", response, err)
}
summary := response.Execution
if summary.Outcome != "error" || summary.FailureKind != "provider_status" || summary.StatusCode != http.StatusServiceUnavailable || !summary.RetriesExhausted || summary.Attempts != 2 || summary.TransientRetries != 1 || calls.Load() != 2 {
t.Fatalf("failed execution = %+v, calls = %d", summary, calls.Load())
}
if summary.Usage != (contracts.LLMObservedTokenUsage{}) {
t.Fatalf("failed attempts should have unknown usage: %+v", summary.Usage)
}
}
func TestPromptKitObservationsAreIsolatedAcrossConcurrentCalls(t *testing.T) {
arrived := make(chan struct{}, 2)
release := make(chan struct{})
client := newObservedHTTPClient(t, func(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
if err != nil {
t.Error(err)
return
}
tokens := 11
if strings.Contains(string(body), "second-call") {
tokens = 22
}
arrived <- struct{}{}
select {
case <-release:
case <-r.Context().Done():
return
}
_ = json.NewEncoder(w).Encode(map[string]any{
"choices": []any{map[string]any{"message": map[string]string{"content": `{"ok":true}`}}},
"usage": map[string]int{"prompt_tokens": tokens, "completion_tokens": 0, "total_tokens": tokens},
})
}, 0, nil)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
type result struct {
want int
response contracts.StructuredCompletionResponse
err error
}
done := make(chan result, 2)
for name, tokens := range map[string]int{"first-call": 11, "second-call": 22} {
go func() {
req := observedRequest()
req.Vars = map[string]any{"custom": name}
var out map[string]any
response, err := client.CompleteStructured(ctx, req, &out)
done <- result{tokens, response, err}
}()
}
for i := 0; i < 2; i++ {
select {
case <-arrived:
case <-ctx.Done():
t.Fatal("requests did not overlap")
}
}
close(release)
for i := 0; i < 2; i++ {
select {
case got := <-done:
summary := got.response.Execution
if got.err != nil || summary == nil {
t.Fatalf("concurrent completion = (%+v, %v)", got.response, got.err)
}
if summary.Attempts != 1 || summary.Usage.PromptTokens != (contracts.LLMObservedTokenCount{Value: got.want, Known: true, Complete: true}) || summary.Usage.CompletionTokens != (contracts.LLMObservedTokenCount{Known: true, Complete: true}) || summary.Usage.CachedTokens != (contracts.LLMObservedTokenCount{}) {
t.Fatalf("execution for %d tokens = %+v", got.want, summary)
}
case <-ctx.Done():
t.Fatal("concurrent completions did not return")
}
}
}

View File

@@ -17,7 +17,7 @@ const (
promptKitLocalBackendMarker = "notarius:promptkit-local-backend:v1"
// The built-in profile catalog is compiled into this pinned PromptKit
// release. Update this identity when the dependency is upgraded.
promptKitBuiltinProfileCatalogID = "promptkit:v0.9.0:builtin-profiles"
promptKitBuiltinProfileCatalogID = "promptkit:v0.11.0:builtin-profiles"
)
func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest string) (CheckpointFingerprint, error) {

View File

@@ -38,7 +38,7 @@ func (c *scheduledClient) CompleteStructured(ctx context.Context, req contracts.
return callErr
})
if err != nil {
return contracts.StructuredCompletionResponse{}, err
return response, err
}
return response, nil
}

View File

@@ -113,13 +113,14 @@ type debugSerializedOutput struct {
type debugStructuredCompletionRequest = contracts.DebugStructuredCompletionRequest
type debugStructuredCompletionResponse struct {
Content string `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Content string `json:"content,omitempty"`
Provider string `json:"provider,omitempty"`
Model string `json:"model,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugLLMPromptArtifact struct {
@@ -133,21 +134,23 @@ type debugLLMResponseArtifact struct {
Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"`
ContentPath string `json:"content_path,omitempty"`
Error string `json:"error,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugLLMCallReference struct {
CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"`
ResponseContentPath string `json:"response_content_path,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Error bool `json:"error,omitempty"`
CallID string `json:"call_id"`
PromptPath string `json:"prompt_path,omitempty"`
ResponsePath string `json:"response_path"`
ResponseContentPath string `json:"response_content_path,omitempty"`
PromptID string `json:"prompt_id,omitempty"`
ProfileID string `json:"profile_id,omitempty"`
Model string `json:"model,omitempty"`
PromptTokens int `json:"prompt_tokens,omitempty"`
CompletionTokens int `json:"completion_tokens,omitempty"`
TotalTokens int `json:"total_tokens,omitempty"`
RepairAttempts int `json:"repair_attempts,omitempty"`
Error bool `json:"error,omitempty"`
Execution *contracts.LLMExecutionSummary `json:"execution,omitempty"`
}
type debugValidationOutcome struct {
@@ -267,6 +270,7 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
Fallback: fallbackForArtifact,
ContentPath: responseContentPath,
Error: errorText,
Execution: cloneLLMExecutionSummary(response.Execution),
},
Error: errorText,
}))
@@ -283,6 +287,7 @@ func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contra
TotalTokens: response.TotalTokens,
RepairAttempts: response.RepairAttempts,
Error: err != nil,
Execution: cloneLLMExecutionSummary(response.Execution),
}
if scope := debugLLMScopeFromContext(ctx); scope != nil {
scope.record(callRef)
@@ -637,6 +642,7 @@ func debugCompletionResponse(response contracts.StructuredCompletionResponse) de
PromptTokens: response.PromptTokens,
CompletionTokens: response.CompletionTokens,
TotalTokens: response.TotalTokens,
Execution: cloneLLMExecutionSummary(response.Execution),
}
}
@@ -651,7 +657,8 @@ func debugCompletionFallback(response contracts.StructuredCompletionResponse) *d
fallback.ProfileID == "" &&
fallback.PromptTokens == 0 &&
fallback.CompletionTokens == 0 &&
fallback.TotalTokens == 0 {
fallback.TotalTokens == 0 &&
fallback.Execution == nil {
return nil
}
return &fallback
@@ -696,6 +703,15 @@ func cloneDebugFallbackWithoutContent(fallback *debugStructuredCompletionRespons
}
clone := *fallback
clone.Content = ""
clone.Execution = cloneLLMExecutionSummary(fallback.Execution)
return &clone
}
func cloneLLMExecutionSummary(summary *contracts.LLMExecutionSummary) *contracts.LLMExecutionSummary {
if summary == nil {
return nil
}
clone := *summary
return &clone
}

View File

@@ -0,0 +1,122 @@
package pipeline
import (
"bytes"
"context"
"encoding/json"
"errors"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)
type executionDebugClient struct {
response contracts.StructuredCompletionResponse
err error
}
func (c executionDebugClient) CompleteStructured(context.Context, contracts.StructuredCompletionRequest, any) (contracts.StructuredCompletionResponse, error) {
return c.response, c.err
}
func TestScheduledDebugClientPreservesExecutionThroughTerminalRecords(t *testing.T) {
providerErr := contracts.NewLLMGenerationError(503, "provider unavailable")
const content = `{"ok":true}`
for _, test := range []struct {
name string
err error
content string
debugResponse bool
}{
{name: "provider failure without body", err: providerErr},
{name: "canceled without body", err: context.Canceled},
{name: "split debug response", content: content, debugResponse: true},
{name: "split fallback response", content: content},
} {
t.Run(test.name, func(t *testing.T) {
want := contracts.LLMExecutionSummary{
Attempts: 2, TransientRetries: 1, Outcome: "success",
CapacityWaitMS: 1, GenerationMS: 3, RetryDelayMS: 5, DurationMS: 9,
Usage: contracts.LLMObservedTokenUsage{
PromptTokens: contracts.LLMObservedTokenCount{Value: 7, Known: true},
CompletionTokens: contracts.LLMObservedTokenCount{Known: true},
},
}
if test.err != nil {
want.Outcome = "error"
want.FailureKind = "provider_status"
want.StatusCode = 503
want.RetriesExhausted = true
if errors.Is(test.err, context.Canceled) {
want.FailureKind = "canceled"
want.StatusCode = 0
want.RetriesExhausted = false
}
}
execution := want
response := contracts.StructuredCompletionResponse{Execution: &execution}
if test.debugResponse {
response.Debug = &contracts.LLMDebugMaterial{Response: &contracts.LLMDebugResponse{Content: test.content}}
} else {
response.Content = json.RawMessage(test.content)
}
scheduler, err := llm.NewScheduler(1)
if err != nil {
t.Fatal(err)
}
recorder := newCapturedDebugRecorder()
client := WithDebugLLMRecording(llm.NewScheduledClient(executionDebugClient{response, test.err}, scheduler), recorder)
ctx, scope := withDebugLLMScope(context.Background(), "attempt")
got, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{StageName: "extract"}, &struct{}{})
if !errors.Is(err, test.err) || got.Execution == nil || *got.Execution != want {
t.Fatalf("scheduled debug completion = (%+v, %v), want summary and original error", got, err)
}
terminal := newAttemptTerminalRecorder(recorder, "attempt/terminal", "extract", scope, debugTimedEnvelope{Stage: "extract"})
if err := terminal.record(nil, test.err); !errors.Is(err, test.err) {
t.Fatalf("terminal recording error = %v", err)
}
envelope := recorder.envelope(t, "attempt/response-0001.json")
encoded, err := json.Marshal(envelope.Payload)
if err != nil {
t.Fatal(err)
}
var artifact debugLLMResponseArtifact
if err := json.Unmarshal(encoded, &artifact); err != nil {
t.Fatal(err)
}
if artifact.Execution == nil || *artifact.Execution != want {
t.Fatalf("persisted response lost execution summary: %+v", artifact)
}
if !test.debugResponse && (artifact.Fallback == nil || artifact.Fallback.Execution == nil || *artifact.Fallback.Execution != want) {
t.Fatalf("persisted fallback lost execution summary: %+v", artifact.Fallback)
}
if test.content == "" {
if artifact.ContentPath != "" {
t.Fatalf("bodyless failure has content path %q", artifact.ContentPath)
}
} else {
var compact bytes.Buffer
if artifact.ContentPath == "" || json.Compact(&compact, recorder.bytes[artifact.ContentPath]) != nil || compact.String() != test.content {
t.Fatalf("split content not preserved at %q", artifact.ContentPath)
}
if (artifact.Fallback != nil && artifact.Fallback.Content != "") || (artifact.Response != nil && artifact.Response.Content != "") {
t.Fatal("split content remains in response metadata")
}
}
refs := recorder.envelope(t, "attempt/terminal.json").LLMCalls
if len(refs) != 1 || refs[0].Execution == nil || *refs[0].Execution != want || refs[0].Error != (test.err != nil) {
t.Fatalf("terminal references = %+v", refs)
}
// Decode the persisted JSON, not just Go defaults: omitted false and
// zero fields would erase the presence-aware usage contract.
usage := envelope.Payload.(map[string]any)["execution"].(map[string]any)["usage"].(map[string]any)
unknown := usage["cached_tokens"].(map[string]any)
for _, key := range []string{"value", "known", "complete"} {
if _, ok := unknown[key]; !ok {
t.Fatalf("unknown token usage omits %q: %+v", key, unknown)
}
}
})
}
}

View File

@@ -2,85 +2,23 @@
set -eu
fail() {
printf '%s\n' "check-release-source: $*" >&2
exit 1
}
if [ "$#" -ne 1 ]; then
fail "usage: $0 vMAJOR.MINOR.PATCH"
fi
release_version=$1
if ! printf '%s\n' "$release_version" | grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null; then
fail "version must be a stable semantic version tag"
fi
fail() { printf '%s\n' "check-release-source: $*" >&2; exit 1; }
[ "$#" -eq 1 ] || fail "usage: $0 vMAJOR.MINOR.PATCH"
version=$1
printf '%s\n' "$version" | grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null || fail "invalid stable release version"
script_dir=$(CDPATH= cd "$(dirname "$0")" && pwd)
repo_root=$(CDPATH= cd "$script_dir/.." && pwd)
cd "$repo_root"
cd "$script_dir/.."
note="docs/releases/$version.md"
[ -s "$note" ] || fail "missing release note: $note"
[ "$(head -n 1 "$note")" = "# Notarius $version" ] || fail "release note title does not match version"
temporary_dir=$(mktemp -d "${TMPDIR:-/tmp}/notarius-release-source.XXXXXX") || fail "create temporary directory"
trap 'rm -rf "$temporary_dir"' 0 HUP INT TERM
export GOWORK=off
go test -count=1 ./...
if ! grep -E '^module[[:space:]]+gitea\.maximumdirect\.net/eric/notarius([[:space:]]|$)' go.mod >/dev/null; then
fail "go.mod does not declare the expected module path"
fi
if git ls-files --error-unmatch go.work >/dev/null 2>&1 || git ls-files --error-unmatch go.work.sum >/dev/null 2>&1; then
fail "tracked Go workspace files are not allowed"
fi
if [ -d vendor ]; then
fail "vendor directory is not allowed"
fi
if grep -E '^[[:space:]]*replace([[:space:]]|$)' go.mod >/dev/null; then
fail "go.mod replace directives are not allowed"
fi
GOWORK=off go test -count=1 ./...
GOWORK=off go test -race -count=1 ./...
GOWORK=off go vet ./...
GOWORK=off go build ./...
GOWORK=off go mod tidy -diff
unformatted=$(git ls-files '*.go' | while IFS= read -r go_file; do
fmt_output=$(gofmt -l "$go_file")
if [ -n "$fmt_output" ]; then
printf '%s\n' "$fmt_output"
fi
done)
if [ -n "$unformatted" ]; then
printf '%s\n' "$unformatted" >&2
fail "tracked Go files are not gofmt formatted"
fi
git diff --check
git diff --cached --check
GOWORK=off go run ./cmd/notarius config validate --config examples/dnd-minimal.config.yml --pipeline dnd-session
GOWORK=off OPENROUTER_API_KEY=validation-placeholder go run ./cmd/notarius config validate --config examples/dnd-complete.config.yml --pipeline dnd-session
host_os=$(GOWORK=off go env GOOS)
host_arch=$(GOWORK=off go env GOARCH)
version_symbol='gitea.maximumdirect.net/eric/notarius/internal/buildinfo.Override'
build_target() {
target_os=$1
target_arch=$2
target_path="$temporary_dir/notarius-$target_os-$target_arch"
CGO_ENABLED=0 GOOS="$target_os" GOARCH="$target_arch" GOWORK=off go build -trimpath \
-ldflags "-X $version_symbol=$release_version" \
-o "$target_path" ./cmd/notarius
if [ "$host_os" = "$target_os" ] && [ "$host_arch" = "$target_arch" ]; then
"$target_path" --version > "$temporary_dir/version-output"
printf 'notarius %s\n' "$release_version" > "$temporary_dir/expected-version-output"
if ! cmp -s "$temporary_dir/expected-version-output" "$temporary_dir/version-output"; then
fail "version output is incorrect for $target_os/$target_arch"
fi
fi
}
build_target linux amd64
build_target linux arm64
build_target darwin amd64
build_target darwin arm64
# Exercise the actual command and its linker-supplied diagnostic version.
verification_dir=$(mktemp -d "${TMPDIR:-/tmp}/notarius-release-check.XXXXXX")
trap 'rm -rf "$verification_dir"' 0 HUP INT TERM
go build -trimpath -ldflags "-X gitea.maximumdirect.net/eric/notarius/internal/buildinfo.Override=$version" \
-o "$verification_dir/notarius" ./cmd/notarius
[ "$("$verification_dir/notarius" --version)" = "notarius $version" ] || fail "incorrect diagnostic version"

56
scripts/release.sh Executable file
View File

@@ -0,0 +1,56 @@
#!/bin/sh
set -eu
fail() { printf '%s\n' "release: $*" >&2; exit 1; }
[ "$#" -eq 1 ] || fail "usage: $0 vMAJOR.MINOR.PATCH"
version=$1
printf '%s\n' "$version" | grep -E -x 'v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)' >/dev/null || fail "invalid stable release version"
script_dir=$(CDPATH= cd "$(dirname "$0")" && pwd)
cd "$script_dir/.."
export GOWORK=off
candidate=$(git rev-parse 'HEAD^{commit}')
check_candidate() {
[ "$(git branch --show-current)" = main ] || fail "release branch must be main"
[ -z "$(git status --porcelain)" ] || fail "worktree and index must be clean"
[ "$(git rev-parse 'HEAD^{commit}')" = "$candidate" ] || fail "candidate changed during validation"
git cat-file -e "$candidate:docs/releases/$version.md" || fail "release note must be committed"
}
check_publication() {
# Query failure is not proof that the tag is unused.
remote_tag=$(git ls-remote --tags origin "refs/tags/$version" "refs/tags/$version^{}") || fail "cannot determine remote tag availability"
[ -z "$remote_tag" ] || fail "remote tag already exists: $version"
if git show-ref --verify --quiet "refs/tags/$version"; then
fail "local tag already exists: $version"
fi
git fetch --no-tags origin refs/heads/main || fail "cannot fetch authoritative main"
remote_main=$(git rev-parse 'FETCH_HEAD^{commit}')
git merge-base --is-ancestor "$remote_main" "$candidate" || fail "candidate does not contain authoritative main"
}
check_candidate
check_publication
"$script_dir/check-release-source.sh" "$version"
check_candidate
check_publication
if ! git -c push.followTags=false push origin "$candidate:refs/heads/main"; then
# Do not publish a tag after a failed branch push, even if the branch arrived.
git ls-remote origin refs/heads/main >&2 || fail "branch publication status unknown; inspect origin before retrying"
fail "branch push failed; inspect the reported remote state before retrying"
fi
git -c tag.gpgSign=false tag "$version" "$candidate"
[ "$(git cat-file -t "$version")" = commit ] || fail "tag is not lightweight"
[ "$(git rev-parse "$version^{commit}")" = "$candidate" ] || fail "tag target changed"
if ! git -c push.followTags=false push origin "refs/tags/$version:refs/tags/$version"; then
# A connection can fail after the server accepted the tag. Preserve it locally
# until authoritative state proves whether publication succeeded.
remote_tag=$(git ls-remote --tags origin "refs/tags/$version" "refs/tags/$version^{}") || fail "tag publication status unknown; preserve local tag and inspect origin"
expected=$(printf '%s\t%s' "$candidate" "refs/tags/$version")
[ "$remote_tag" = "$expected" ] || fail "tag push failed; preserve local tag and inspect origin before retrying"
fi
printf 'Released %s at %s\n' "$version" "$candidate"

243
scripts/release_test.go Normal file
View File

@@ -0,0 +1,243 @@
package scripts
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
// Keep publication tests offline: real Git repositories exercise ref safety,
// while a fake Go command represents the expensive candidate validation boundary.
type releaseFixture struct {
t *testing.T
root string
remote string
bin string
log string
initial string
}
func writeFile(t *testing.T, path, content string, mode os.FileMode) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), mode); err != nil {
t.Fatal(err)
}
}
func command(t *testing.T, dir string, args ...string) string {
t.Helper()
cmd := exec.Command(args[0], args[1:]...)
cmd.Dir = dir
output, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("%v: %v\n%s", args, err, output)
}
return strings.TrimSpace(string(output))
}
func newReleaseFixture(t *testing.T) *releaseFixture {
t.Helper()
dir := t.TempDir()
f := &releaseFixture{t: t, root: filepath.Join(dir, "work"), remote: filepath.Join(dir, "origin.git"), bin: filepath.Join(dir, "bin"), log: filepath.Join(dir, "validation.log")}
command(t, dir, "git", "init", "--bare", f.remote)
command(t, dir, "git", "init", "-b", "main", f.root)
f.git("config", "user.name", "Release Test")
f.git("config", "user.email", "release@example.invalid")
f.git("config", "commit.gpgSign", "false")
f.git("config", "tag.gpgSign", "false")
for _, name := range []string{"release.sh", "check-release-source.sh"} {
content, err := os.ReadFile(name)
if err != nil {
t.Fatal(err)
}
writeFile(t, filepath.Join(f.root, "scripts", name), string(content), 0755)
}
writeFile(t, filepath.Join(f.root, ".gitignore"), "vendor/\n", 0644)
f.git("add", ".")
f.git("commit", "-m", "Initial source")
f.git("remote", "add", "origin", f.remote)
f.git("push", "origin", "main")
f.initial = f.git("rev-parse", "HEAD")
writeFile(t, filepath.Join(f.root, "docs/releases/v0.8.0.md"), "# Notarius v0.8.0\n\nA concise note without fixed sections.\n", 0644)
f.git("add", ".")
f.git("commit", "-m", "Prepare release")
writeFile(t, filepath.Join(f.bin, "go"), `#!/bin/sh
set -eu
[ "$GOWORK" = off ]
printf '%s\n' "$1" >> "$RELEASE_TEST_LOG"
case "$1" in
test)
[ "${RELEASE_TEST_FAIL:-}" != test ] || exit 1
if [ "${RELEASE_TEST_CHANGE:-}" = yes ]; then
printf 'changed\n' >> docs/releases/v0.8.0.md
fi
;;
build)
while [ "$1" != -o ]; do shift; done
shift
printf '#!/bin/sh\nprintf "notarius v0.8.0\\n"\n' > "$1"
chmod +x "$1"
;;
*) exit 1 ;;
esac
`, 0755)
return f
}
func (f *releaseFixture) git(args ...string) string {
f.t.Helper()
return command(f.t, f.root, append([]string{"git"}, args...)...)
}
func (f *releaseFixture) run(version string, env ...string) (string, error) {
f.t.Helper()
cmd := exec.Command("sh", "scripts/release.sh", version)
cmd.Dir = f.root
cmd.Env = append(os.Environ(), "PATH="+f.bin+string(os.PathListSeparator)+os.Getenv("PATH"), "RELEASE_TEST_LOG="+f.log)
cmd.Env = append(cmd.Env, env...)
output, err := cmd.CombinedOutput()
return string(output), err
}
func TestReleasePublishesExactCandidateOnce(t *testing.T) {
f := newReleaseFixture(t)
// Neither ignored local files nor unrelated local tags are release inputs.
writeFile(t, filepath.Join(f.root, "vendor/local.txt"), "ignored", 0644)
f.git("tag", "unrelated")
f.git("tag", "-a", "unrelated-annotated", "-m", "Do not publish")
f.git("config", "push.followTags", "true")
candidate := f.git("rev-parse", "HEAD")
output, err := f.run("v0.8.0")
if err != nil {
t.Fatalf("release: %v\n%s", err, output)
}
for _, ref := range []string{"refs/heads/main", "refs/tags/v0.8.0"} {
if got := command(t, f.remote, "git", "rev-parse", ref); got != candidate {
t.Fatalf("%s = %s, want %s", ref, got, candidate)
}
}
if got := command(t, f.remote, "git", "cat-file", "-t", "v0.8.0"); got != "commit" {
t.Fatalf("tag type = %s", got)
}
if got := f.git("ls-remote", "--tags", "origin", "refs/tags/unrelated*"); got != "" {
t.Fatal("published unrelated tag")
}
log, err := os.ReadFile(f.log)
if err != nil || string(log) != "test\nbuild\n" {
t.Fatalf("validation was not performed once: %q, %v", log, err)
}
}
func TestReleaseGuardsPreventPublication(t *testing.T) {
for _, name := range []string{"invalid_version", "missing_note", "wrong_title", "dirty", "staged", "wrong_branch", "local_tag", "remote_tag", "remote_ahead", "remote_unavailable", "validation_failure", "candidate_changed"} {
t.Run(name, func(t *testing.T) {
f := newReleaseFixture(t)
version := "v0.8.0"
var env []string
switch name {
case "invalid_version":
version = "v00.8.0"
case "missing_note":
version = "v0.8.1"
case "wrong_title":
writeFile(t, filepath.Join(f.root, "docs/releases/v0.8.0.md"), "# Wrong version\n", 0644)
f.git("add", ".")
f.git("commit", "-m", "Wrong title")
case "dirty", "staged":
writeFile(t, filepath.Join(f.root, "dirty.txt"), "dirty", 0644)
if name == "staged" {
f.git("add", ".")
}
case "wrong_branch":
f.git("checkout", "-b", "other")
case "local_tag":
f.git("tag", version)
case "remote_tag":
f.git("push", "origin", f.initial+":refs/tags/"+version)
case "remote_ahead":
// Publish an independent descendant of the old remote tip.
f.git("checkout", "-b", "other", f.initial)
f.git("commit", "--allow-empty", "-m", "Other work")
f.git("push", "origin", "HEAD:main")
f.initial = f.git("rev-parse", "HEAD")
f.git("checkout", "main")
case "remote_unavailable":
f.git("remote", "set-url", "origin", filepath.Join(t.TempDir(), "missing.git"))
case "validation_failure":
env = []string{"RELEASE_TEST_FAIL=test"}
case "candidate_changed":
env = []string{"RELEASE_TEST_CHANGE=yes"}
}
if output, err := f.run(version, env...); err == nil {
t.Fatalf("guard allowed publication: %s", output)
}
if got := command(t, f.remote, "git", "rev-parse", "main"); got != f.initial {
t.Fatal("guard advanced remote branch")
}
if name != "remote_tag" {
if got := command(t, f.remote, "git", "tag", "--list"); got != "" {
t.Fatalf("guard published tag: %s", got)
}
}
})
}
}
func TestReleasePushFailureRecovery(t *testing.T) {
for _, mode := range []string{"branch_rejected", "tag_rejected", "tag_accepted_disconnect", "tag_unknown"} {
t.Run(mode, func(t *testing.T) {
f := newReleaseFixture(t)
if mode == "branch_rejected" || mode == "tag_rejected" {
pattern := "refs/heads/main"
if mode == "tag_rejected" {
pattern = "refs/tags/*"
}
writeFile(t, filepath.Join(f.remote, "hooks/update"), "#!/bin/sh\ncase \"$1\" in "+pattern+") exit 1;; esac\n", 0755)
} else {
realGit, err := exec.LookPath("git")
if err != nil {
t.Fatal(err)
}
// Simulate a lost acknowledgement, optionally followed by an outage.
writeFile(t, filepath.Join(f.bin, "git"), `#!/bin/sh
set -eu
if [ "$1" = -c ] && [ "$3" = push ] && [ "$5" = refs/tags/v0.8.0:refs/tags/v0.8.0 ]; then
"$RELEASE_TEST_GIT" "$@"
touch "$RELEASE_TEST_MARKER"
exit 1
fi
if [ "$1" = ls-remote ] && [ -e "$RELEASE_TEST_MARKER" ] && [ "$RELEASE_TEST_MODE" = tag_unknown ]; then
exit 1
fi
exec "$RELEASE_TEST_GIT" "$@"
`, 0755)
output, err := f.run("v0.8.0", "RELEASE_TEST_GIT="+realGit, "RELEASE_TEST_MARKER="+filepath.Join(f.bin, "marker"), "RELEASE_TEST_MODE="+mode)
if (err == nil) != (mode == "tag_accepted_disconnect") {
t.Fatalf("unexpected outcome: %v\n%s", err, output)
}
if got := command(t, f.remote, "git", "rev-parse", "v0.8.0"); got != f.git("rev-parse", "HEAD") {
t.Fatal("wrong remote tag")
}
return
}
if output, err := f.run("v0.8.0"); err == nil {
t.Fatalf("rejected push reported success: %s", output)
}
if got := command(t, f.remote, "git", "tag", "--list"); got != "" {
t.Fatal("rejected push published tag")
}
if mode == "tag_rejected" {
if got := f.git("rev-parse", "v0.8.0"); got != f.git("rev-parse", "HEAD") {
t.Fatal("unpublished local tag not preserved")
}
} else if got := f.git("tag", "--list"); got != "" {
t.Fatal("created tag after failed branch push")
}
})
}
}