From 2a9db9a9571b5e5559af35c693ebbb1adfa8cd6e Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 30 Jul 2026 01:54:47 +0000 Subject: [PATCH] Upgrade PromptKit to version 0.2.0 --- docs/integrations/pkg-promptkit.md | 8 +- docs/roadmap/implementation.md | 817 ++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- internal/framework/llm/promptkit_client.go | 12 - .../framework/llm/promptkit_client_test.go | 4 + .../llm/promptkit_profile_fingerprint.go | 2 +- 7 files changed, 829 insertions(+), 20 deletions(-) create mode 100644 docs/roadmap/implementation.md diff --git a/docs/integrations/pkg-promptkit.md b/docs/integrations/pkg-promptkit.md index 582f26c..67bd59b 100644 --- a/docs/integrations/pkg-promptkit.md +++ b/docs/integrations/pkg-promptkit.md @@ -1,11 +1,11 @@ # PromptKit Integration Notarius pins -[`gitea.maximumdirect.net/eric/promptkit` v0.1.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0) +[`gitea.maximumdirect.net/eric/promptkit` v0.2.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0) as its in-process prompt engine. The upstream -[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md) +[Go package consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/docs/consumers/pkg-promptkit.md) owns the public engine API, and the upstream -[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/formats.md) +[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/docs/formats.md) owns prompt, profile, and schema file contracts. ## Supported Boundary @@ -15,7 +15,7 @@ Notarius relies on the root `promptkit` package to: - construct an `Engine` with filesystem-backed prompt, schema, and optional profile sources; - prepare and run a `RunRequest` with named inline artifacts, variables, - metadata, prompt identity, and profile selection; + prompt identity, and profile selection; - return rendered debug material, validated structured output, selected profile and model metadata, and token usage; - distinguish structured-output validation failure from execution failure; and diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md new file mode 100644 index 0000000..b70e0f4 --- /dev/null +++ b/docs/roadmap/implementation.md @@ -0,0 +1,817 @@ +# PromptKit v0.2.0 Upgrade Implementation Plan + +## Objective + +Upgrade Notarius from `gitea.maximumdirect.net/eric/promptkit` v0.1.0 to +v0.2.0 and adopt the release capabilities that fit the current application: + +- native request-level session propagation; +- tri-state, run-wide reasoning-effort overrides; +- backend-aware LLM provenance; +- provider-neutral handling of PromptKit backend-capacity exhaustion; and +- accurate documentation of the layered Notarius and PromptKit concurrency + model. + +Each numbered stage below is intended to be implemented by one LLM coding-agent +prompt, in order. A stage is complete only when its focused tests, documentation +updates, and acceptance criteria pass. Do not implement later-stage behavior +early unless a small compile-preserving type addition is explicitly required by +the current stage. + +## Repository Baseline + +The plan is grounded in the following current implementation: + +- `go.mod` pins PromptKit v0.1.0. +- `internal/framework/llm/promptkit_client.go` is the sole production adapter + from Notarius's provider-neutral `StructuredLLMClient` contract to PromptKit. +- `PromptKitClient.CompleteStructured` currently assigns + `promptkit.RunRequest.Metadata`; v0.2.0 removes that field. A temporary + v0.2.0 modfile build fails at this assignment. +- The same adapter currently supplies + `StructuredCompletionRequest.SessionID` only through a `session_id` prompt + variable. It does not set a direct PromptKit request session. +- The CLI already supplies one run-wide `--session-id`, passes it to every + prompt-facing stage, records it in checkpoint runtime identity, and exposes + it in debug/run metadata. +- PromptKit public values are not persisted wholesale. The adapter maps them + into Notarius-owned contracts, debug values, and manifests. PromptKit + v0.2.0's public JSON changes therefore do not require conversion of existing + Notarius output or checkpoint payloads. +- `concurrency.total_llm` is enforced by Notarius's application-wide scheduled + client. PromptKit v0.2.0 adds an independent engine-local scheduler keyed by + backend ID. +- Current production profiles are either PromptKit built-ins or user-supplied + endpoint profiles. Notarius has no configuration contract for registering + custom PromptKit backends. +- PromptKit composite literals in the repository are keyed; no positional + literal migration is expected. +- The pre-upgrade `go test ./...` baseline passes. + +Use the PromptKit v0.2.0 tag, not the moving `main` branch, as the dependency +and API authority. The release guide and exact tagged contracts are: + +- `https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/docs/releases/v0.2.0.md` +- `https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/docs/consumers/pkg-promptkit.md` +- `https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/docs/formats.md` +- `https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/types.go` +- `https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/backends.go` +- `https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/engine.go` + +## Decisions + +### Dependency and compatibility + +- Pin exactly PromptKit v0.2.0. +- Keep all PromptKit struct literals keyed. +- Delete the obsolete `RunRequest.Metadata` mapping. Do not invent a + replacement upstream metadata channel. +- Keep `StructuredCompletionRequest.StageName`; it remains useful in + Notarius-owned stage and debug behavior even though it is no longer sent to + PromptKit. +- Update the built-in profile-catalog checkpoint identity from v0.1.0 to + v0.2.0. +- Do not add a migration for PromptKit public JSON because Notarius does not + persist those values directly. + +### Session propagation + +- `--session-id` remains the only user-facing session surface. +- Set `promptkit.RunRequest.SessionID` to the trimmed Notarius request session. +- Continue populating `Vars["session_id"]` with the same value for prompt + compatibility. The direct field is authoritative for provider session + behavior; the variable exists only so a maintained prompt that renders + `.session_id` does not break silently. +- Keep the existing checkpoint session fingerprint unchanged. +- Do not add a profile or configuration session setting. + +### Reasoning override + +Expose PromptKit's three states through two CLI flags: + +- no flag: inherit the selected PromptKit profile (`nil`); +- `--reasoning-effort VALUE`: replace the profile value with trimmed, + non-empty `VALUE`; and +- `--clear-reasoning-effort`: pass a pointer to an empty string, explicitly + clearing inherited reasoning. + +The two flags are mutually exclusive. An explicitly empty +`--reasoning-effort` is a CLI usage error with exit status 2. + +The override is invocation-wide and applies to every PromptKit call made by +that command. Persistent reasoning policy remains in PromptKit profiles; do +not add a Notarius configuration field or per-lane reasoning option. + +Use a dedicated value object, named `LLMRuntimeOverrides`, at the CLI factory +boundary. It must contain `ReasoningEffort *string`, where: + +- `nil` means inherit; +- a non-empty pointed-to string means replace; and +- an empty pointed-to string means clear. + +The PromptKit client must defensively copy the pointed-to value during +construction. It must create a keyed +`promptkit.ExecutionTargetOverride{ReasoningEffort: ...}` for each run request +when the override is non-nil and leave `RunRequest.Execution` nil otherwise. + +### Checkpoint and debug identity + +- Add the reasoning override to checkpoint runtime identity because it can + change generated output. +- Omit the fingerprint when the override is inherited. +- Use fingerprint name `reasoning_effort_override`. +- Use the trimmed value for replacement and the stable literal `` for + explicit clearing. Do not use an empty fingerprint value. +- Record the tri-state value in debug invocation metadata as + `reasoning_effort_override` using `*string` with `omitempty`; a non-nil empty + string must remain distinguishable from omission. +- Reasoning effort is non-secret provider configuration, but it must still pass + through the existing redaction-safe debug path. + +### Backend identity and concurrency + +- Keep the Notarius `total_llm` scheduler. It is an application-wide limit + across every provider and backend. +- Allow PromptKit's v0.2.0 backend scheduler to impose a narrower + backend-specific limit. Do not attempt to merge, disable, or reimplement the + two schedulers. +- Document the effective active generation bound as the intersection of the + Notarius global limit and the selected PromptKit backend limit. +- Endpoint-only profiles remain limited only by Notarius. +- Built-in OpenRouter profiles automatically select PromptKit's reserved + `openrouter` backend and its upstream capacity policy. +- Add optional `backend_id` and `reasoning_effort` fields to Notarius + `LLMProfileManifest` values and published manifest JSON. +- Populate those fields from `RunResult.SelectedBackendID` and + `RunResult.EffectiveModelParams.ReasoningEffort`. +- Include both fields in recorder identity and sorting so distinct effective + executions are not collapsed. +- Extend Notarius debug prompt metadata with `selected_backend_id`; retain the + existing serialized effective-model parameters. + +### Capacity failure + +- Add a provider-neutral sentinel + `contracts.ErrLLMCapacityExceeded`. +- When `PromptKitClient` receives an error matching + `promptkit.ErrCapacityExceeded`, return an error matching the Notarius + sentinel and include redacted diagnostic context. +- Do not require callers outside `internal/framework/llm` to import PromptKit + or match a PromptKit error. +- Capacity exhaustion is an operational stage error, not invalid structured + output and not a validator rejection. +- Do not retry inside the adapter. Existing configured stage retry behavior + applies, and the run fails if those retries are exhausted. + +### Configuration and versioning + +- Keep Notarius configuration version 4. +- Do not add `promptkit.backends`, backend registration fields, or a + configuration-level reasoning override. +- Existing PromptKit profile files remain valid because v0.2.0 continues to + support endpoint-only profiles. + +### Documentation and decision records + +- Update canonical documentation in the same stage that changes its behavior. +- No ADR is required: this work advances the existing LLM boundary and + scheduler architecture without changing its direction or replacing a prior + durable decision. +- Remove the completed “Native Session Propagation” item from + `docs/roadmap/future.md` only after native session behavior is implemented + and documented. +- Do not add custom backend registration to this implementation plan. A concise + future-roadmap item may be added during final cleanup if it records the + demonstrated boundary and avoids implying a commitment. + +## Non-Goals + +This implementation must not: + +- register user-defined PromptKit backends; +- expose backend endpoint, API-key environment, extra-parameter, concurrency, + or queue-capacity fields in Notarius configuration; +- remove or weaken `concurrency.total_llm`; +- add module-specific or lane-specific reasoning overrides; +- add another session flag or configuration field; +- remove the compatibility `session_id` prompt variable; +- persist or expose raw PromptKit `RunResult`/`PreparedRun` JSON as a Notarius + contract; +- translate old PromptKit v0.1.0 JSON; +- change D&D prompt text, response schemas, or artifact contracts; +- change pipeline retry semantics; or +- call a live model provider in tests. + +## Stage 1 — Upgrade The Dependency And Restore Compilation + +### Goal + +Move to PromptKit v0.2.0 with no intentional user-visible behavior change +beyond using the new upstream release. + +### Files and areas + +- `go.mod` +- `go.sum` +- `internal/framework/llm/promptkit_client.go` +- `internal/framework/llm/promptkit_profile_fingerprint.go` +- corresponding LLM adapter and fingerprint tests +- `docs/integrations/pkg-promptkit.md` + +### Implementation + +1. Run: + + ```sh + go get gitea.maximumdirect.net/eric/promptkit@v0.2.0 + go mod tidy + ``` + +2. Remove the `Metadata` field from the constructed + `promptkit.RunRequest`. +3. Delete `promptKitMetadata` and tests that exist only for the removed + upstream metadata mapping. Do not delete `StageName` from Notarius + contracts. +4. Change `promptKitBuiltinProfileCatalogID` to + `promptkit:v0.2.0:builtin-profiles`. +5. Update fingerprint expectations so v0.2.0 has a distinct deterministic + checkpoint identity. +6. Review every `promptkit.*{...}` literal touched by compilation and keep it + keyed. +7. Update the PromptKit integration document to identify v0.2.0 and link to + the tagged v0.2.0 consumer and format documentation. Remove metadata from + the stated supported boundary. Do not yet claim that later-stage native + session, reasoning CLI, or Notarius capacity classification is implemented. + +### Tests + +- Existing PromptKit asset, profile, adapter, scheduler, and D&D prompt tests + must compile and pass. +- Do not add a test whose only purpose is to assert the dependency-version + string. The fingerprint behavior is the meaningful compatibility boundary. + +### Acceptance criteria + +- `go.mod` and `go.sum` contain v0.2.0 and no v0.1.0 PromptKit entry. +- No production use of `RunRequest.Metadata` remains. +- The built-in profile checkpoint fingerprint changes deterministically. +- Existing endpoint-only profiles continue to load. +- No Notarius durable schema or configuration version changes. +- Stage documentation describes the actual post-stage boundary. + +### Validation + +```sh +go mod tidy +git diff --check +go test ./internal/framework/llm/... ./internal/cli/... +go test ./internal/modules/dnd/... +go test ./... +go vet ./... +go build ./cmd/notarius +``` + +This stage is small enough for one implementation prompt. + +## Stage 2 — Adopt Native Session Propagation + +### Goal + +Make the existing run-wide Notarius session ID reach PromptKit's native direct +session field without breaking prompt-variable consumers. + +### Files and areas + +- `internal/framework/llm/promptkit_client.go` +- `internal/framework/llm/promptkit_client_test.go` +- PromptKit adapter test prompt fixtures +- existing CLI assembled-run session tests +- `docs/internal/llm.md` +- `docs/integrations/pkg-promptkit.md` +- `docs/roadmap/future.md` + +### Implementation + +1. In `PromptKitClient.CompleteStructured`, trim + `StructuredCompletionRequest.SessionID` once and assign it to + `promptkit.RunRequest.SessionID`. +2. Continue passing that same trimmed value as `Vars["session_id"]`. + Preserve caller-supplied variables, with the explicit Notarius session + taking precedence over a caller-supplied variable of the same name. +3. Leave an empty Notarius session as an empty direct value so PromptKit's + prompt-defined session template behavior remains available. +4. Keep session checkpoint identity, CLI behavior, module request propagation, + and debug/run metadata unchanged except that debug material should now show + PromptKit's native effective session. +5. Update LLM and integration documentation to state: + - the direct PromptKit session is authoritative for provider behavior; + - the prompt variable is retained for compatibility; + - session IDs are stable, non-secret correlation identifiers and may be + exposed to providers and provider observability. +6. Remove the completed `Native Session Propagation` section from + `docs/roadmap/future.md`. Do not replace it with implementation history. + +### Tests + +Add or adapt behavior-focused coverage proving: + +- a nonblank Notarius session reaches + `GenerateRequest.Prompt.SessionID` through a prompt definition that has no + `session_id` template; +- the effective session appears in prepared/debug prompt material; +- a prompt that renders `.session_id` still receives the compatibility + variable; +- an empty request does not invent a direct session; and +- the existing assembled-run test still observes one session across every + prompt-facing module. + +Rely on PromptKit for exhaustive Unicode-length and internal normalization +tests. Notarius needs only representative adapter-boundary coverage. Do not add +tests for PromptKit's private implementation sequence. + +### Acceptance criteria + +- Provider-facing PromptKit requests use the native session field. +- Existing `--session-id` syntax and exit behavior are unchanged. +- Checkpoints remain session-separated. +- Prompt-variable compatibility remains intact. +- No second session configuration surface exists. +- The future roadmap no longer describes native session support as blocked. + +### Validation + +```sh +go test ./internal/framework/llm/... -run 'Session|PromptKit' +go test ./internal/cli/... -run 'Session|Checkpoint' +go test ./internal/modules/dnd/... +go test ./... +git diff --check +``` + +This stage is small enough for one implementation prompt. + +## Stage 3 — Add Internal Tri-State Reasoning Support + +### Goal + +Teach the PromptKit adapter and CLI factory boundary to represent and apply +inherit, replace, and clear states before exposing new command flags. + +### Files and areas + +- `internal/framework/llm/promptkit_client.go` +- `internal/framework/llm/promptkit_client_test.go` +- `internal/cli/run.go` +- `internal/cli/catalog.go` +- CLI option/factory test helpers and fakes +- `docs/internal/llm.md` +- `docs/internal/cli.md` + +### Implementation + +1. Define `LLMRuntimeOverrides` at the CLI's LLM-client factory boundary with + `ReasoningEffort *string`. +2. Change `LLMClientFactory` and all production/test implementations to accept + `LLMRuntimeOverrides`. At this stage, ordinary CLI execution passes the zero + value, preserving inheritance. +3. Add `ReasoningEffort *string` to `PromptKitClientConfig` or an equivalently + focused run-override field. +4. Defensively copy the pointer and its string in + `NewPromptKitClient`; do not retain caller-owned mutable pointer state. +5. When non-nil, construct a keyed + `promptkit.ExecutionTargetOverride` for every adapter request and set its + `ReasoningEffort` pointer. When nil, keep `RunRequest.Execution` nil. +6. Preserve an empty pointed-to value exactly; do not trim it into a missing + override. Trim a non-empty replacement once at the CLI boundary in the next + stage. +7. Update internal docs to describe the tri-state adapter capability and + factory boundary without documenting CLI flags that do not yet exist. + +### Tests + +Use an injected PromptKit `LLMClient` and table-driven adapter test for: + +- nil pointer inherits the selected profile value; +- pointer to a nonblank value replaces it; +- pointer to an empty string clears it; and +- mutating the caller's original pointer after client construction cannot + change subsequent requests. + +Assert the effective value received in +`promptkit.GenerateRequest.Target.ReasoningEffort`, not internal PromptKit call +order. + +### Acceptance criteria + +- The factory and adapter preserve all three states. +- Existing production construction passes nil and retains current profile + behavior. +- No CLI or config surface has been added yet. +- Tests use offline injected clients only. + +### Validation + +```sh +go test ./internal/framework/llm/... -run 'Reasoning|PromptKit' +go test ./internal/cli/... +go test ./... +git diff --check +``` + +This stage is small enough for one implementation prompt. + +## Stage 4 — Expose The Run-Wide Reasoning CLI Contract + +### Goal + +Make the internal tri-state reasoning override available to users while +preserving checkpoint and debug correctness. + +### Files and areas + +- `internal/cli/run.go` +- `internal/cli/catalog.go` +- CLI parsing, run-contract, checkpoint, and debug-summary tests +- `internal/core/debugbundle/summary.go` +- related debugbundle tests +- `docs/cli.md` +- `docs/operations.md` +- `docs/internal/cli.md` +- `docs/internal/llm.md` + +### Implementation + +1. Add `--reasoning-effort VALUE` and + `--clear-reasoning-effort` to `notarius run`. +2. Use a flag value that tracks presence separately from content, following + the established `sessionIDFlag`/single-value patterns where appropriate. +3. Validate before run construction: + - both flags together are a usage error; + - an explicitly empty or whitespace-only reasoning value is a usage error; + - repeated single-value flags follow the CLI's existing duplicate-flag + policy. +4. Construct one `LLMRuntimeOverrides` value: + - neither flag -> nil; + - replacement -> pointer to the trimmed value; + - clear -> pointer to `""`. +5. Pass it through `LLMClientFactory` to production PromptKit client + construction. The same client is shared across the invocation, so the + override applies to chunking, extraction, normalization, validators, and + retries without adding fields to module request types. +6. Extend `runtimeOverrideFingerprints` and every relevant checkpoint-identity + call to include the reasoning fingerprint defined in “Decisions.” +7. Add `ReasoningEffortOverride *string` with JSON name + `reasoning_effort_override` and `omitempty` to + `debugbundle.Invocation`. Copy the invocation value defensively if any + writer retains it. +8. Update command, operations, and internal documentation. Persistent + reasoning remains a PromptKit profile concern. + +### Tests + +Add focused behavior coverage for: + +- no flags, replacement, and clear; +- the mutual-exclusion error; +- empty replacement rejection; +- forwarding of all three states to the test LLM factory; +- different reasoning states produce different checkpoint identities; +- inherit omits the reasoning fingerprint; +- explicit clear uses ``; +- debug invocation JSON distinguishes omission from `""`; and +- existing CLI commands and examples remain unaffected. + +Do not replicate the same tri-state cases at every module layer. The CLI, +checkpoint, debugbundle, and adapter boundaries each own one distinct risk. + +### Acceptance criteria + +- The public CLI exposes exactly the two agreed flags. +- The override applies run-wide and does not modify resolved pipeline + structure or PromptKit profile files. +- Checkpoint reuse cannot cross reasoning states. +- Debug invocation metadata preserves the tri-state distinction. +- CLI reference and operations documentation are current. +- Notarius configuration remains version 4 with no reasoning field. + +### Validation + +```sh +go test ./internal/cli/... -run 'Reasoning|Checkpoint|Run' +go test ./internal/core/debugbundle/... +go test ./internal/framework/llm/... -run 'Reasoning' +go test ./... +go run ./cmd/notarius run --help +git diff --check +``` + +`run --help` is expected to follow the repository's documented flag-parser +usage behavior and may exit with status 2. Verify the displayed result and +status against `docs/cli.md`. + +This stage is substantial but coherent and small enough for one focused +implementation prompt. Do not combine it with backend provenance. + +## Stage 5 — Publish Backend And Effective Reasoning Provenance + +### Goal + +Expose the new PromptKit backend identity and effective reasoning setting in +Notarius-owned debug and manifest contracts. + +### Files and areas + +- `internal/core/artifacts/artifacts.go` +- artifact cloning, sorting, equality, and JSON tests +- `internal/framework/llm/promptkit_client.go` +- profile recorder and adapter tests +- `internal/framework/pipeline/runner.go` and profile-manifest merge logic +- pipeline tests involving LLM profile manifests +- `internal/framework/contracts/contracts.go` debug types if required +- `docs/integrations/json-output.md` +- `docs/integrations/pkg-promptkit.md` +- `docs/internal/llm.md` + +### Implementation + +1. Add optional JSON fields to `artifacts.LLMProfileManifest`: + + ```go + BackendID string `json:"backend_id,omitempty"` + ReasoningEffort string `json:"reasoning_effort,omitempty"` + ``` + + Use the repository's normal Go alignment/formatting. + +2. Populate: + - `BackendID` from `promptkit.RunResult.SelectedBackendID`; and + - `ReasoningEffort` from + `RunResult.EffectiveModelParams.ReasoningEffort`. +3. Trim both strings at the Notarius boundary. +4. Include both fields in `LLMProfileRecorder`'s deduplication key. +5. Update deterministic merge/sort keys everywhere LLM profile manifests are + combined so entries differing in backend or effective reasoning are not + collapsed and output order remains stable. +6. Add `SelectedBackendID` to the Notarius debug prompt representation and + populate it from `promptkit.PreparedRun.SelectedBackendID`. +7. Keep `EffectiveModelParams` in debug output; it now uses PromptKit v0.2.0's + stable lower-case JSON representation and can include `backend_id`. +8. Update the JSON output integration contract to define the optional + `backend_id` and `reasoning_effort` members of each `llm_profiles` entry. + Do not claim that Notarius supports user-defined backend registration. +9. Do not bump a D&D artifact schema or configuration version. These are + additive optional run-manifest provenance fields, and the JSON bundle + contract already requires consumers to tolerate omitted optional manifest + fields. + +### Tests + +Add or update tests proving: + +- the adapter records backend ID and effective reasoning from a PromptKit + result; +- endpoint-only profiles omit `backend_id`; +- built-in or test registered backend profiles record a non-empty ID; +- profile entries with otherwise equal ID/provider/model but different backend + or reasoning remain distinct; +- ordering is deterministic; and +- published manifest JSON uses the documented optional field names. + +Do not snapshot an entire debug bundle or published manifest when focused +semantic assertions protect the same contract. + +### Acceptance criteria + +- Backend identity does not leak PromptKit types beyond the adapter. +- Published provenance distinguishes effective backend and reasoning. +- Endpoint-only profiles remain valid and omit backend identity. +- Existing JSON consumers remain compatible with additive optional fields. +- Documentation owns the field semantics in the JSON output contract. + +### Validation + +```sh +go test ./internal/core/artifacts/... +go test ./internal/framework/llm/... -run 'Profile|Backend|Reasoning' +go test ./internal/framework/pipeline/... -run 'Profile|Manifest' +go test ./internal/modules/generic/output/json/... -run 'Manifest|Profile' +go test ./... +git diff --check +``` + +This stage is small enough for one implementation prompt. + +## Stage 6 — Translate PromptKit Capacity Exhaustion + +### Goal + +Give the application a provider-neutral overload classification while leaving +scheduling and retry ownership in their existing layers. + +### Files and areas + +- `internal/framework/contracts/errors.go` +- `internal/framework/llm/promptkit_client.go` +- PromptKit adapter tests +- scheduler/scheduled-client tests for regression only +- `docs/internal/llm.md` +- `docs/operations.md` +- `docs/integrations/pkg-promptkit.md` +- `docs/policy/architecture.md` + +### Implementation + +1. Add: + + ```go + var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded") + ``` + + with a comment that identifies backend admission exhaustion before + generation. +2. In the PromptKit `Run` error path, check + `errors.Is(err, promptkit.ErrCapacityExceeded)` before generic wrapping. +3. Return an error matching `contracts.ErrLLMCapacityExceeded`. Include the + prompt context and sanitized upstream diagnostic, but do not require an + external caller to match the PromptKit sentinel. +4. Preserve context cancellation precedence: when `ctx.Err()` is non-nil, + return the context error as today. +5. Do not classify capacity as `ErrInvalidStructuredOutput`. +6. Do not add an adapter retry. Allow the calling module/pipeline's existing + configured attempt policy to see an operational error. +7. Keep the Notarius scheduled client around the PromptKit adapter. +8. Document: + - the two independent concurrency layers; + - built-in OpenRouter's upstream backend limit; + - endpoint-only profiles having no PromptKit backend limit; + - capacity failure and stage retry behavior; and + - the possible effective limit being narrower than `total_llm`. +9. Add one sentence to architecture policy permitting provider runtime + adapters to apply a narrower backend-specific limit beneath the mandatory + application-wide scheduler. Do not copy numeric upstream defaults into + architecture policy. + +### Tests + +At the adapter boundary, use: + +- a test `promptkit.Backend` with `ConcurrencyLimit: 1`; +- an explicit zero `QueueCapacity`; +- a profile selecting that backend; and +- a blocking injected `promptkit.LLMClient`. + +Prove relationally that: + +- the first run occupies the one active slot; +- one additional direct adapter run receives an error matching + `contracts.ErrLLMCapacityExceeded`; +- the injected client is not called for the rejected request; +- the error is not `ErrInvalidStructuredOutput`; and +- provider diagnostic redaction still applies. + +Do not duplicate PromptKit's queue fairness, default-capacity, or complete +backend scheduler tests. Existing Notarius scheduler tests continue to own the +global limit. + +### Acceptance criteria + +- Capacity exhaustion has a stable Notarius error identity. +- PromptKit error types do not become a framework-wide contract. +- The global scheduler remains in production composition. +- No hidden retry or HTTP/status policy is introduced. +- Architecture, operations, integration, and internal docs agree on + concurrency ownership. + +### Validation + +```sh +go test ./internal/framework/llm/... -run 'Capacity|Scheduler|PromptKit' +go test -race ./internal/framework/llm/... +go test ./internal/cli/... -run 'Concurrency|Retry|Production' +go test ./... +git diff --check +``` + +This stage is small enough for one implementation prompt. + +## Stage 7 — Final Documentation And Release Verification + +### Goal + +Verify the completed upgrade as one coherent current contract, remove stale +v0.1.0/native-session planning language, and run the release-recommended test +matrix. + +### Files and areas + +- `docs/integrations/pkg-promptkit.md` +- `docs/internal/llm.md` +- `docs/internal/cli.md` +- `docs/cli.md` +- `docs/operations.md` +- `docs/integrations/json-output.md` +- `docs/policy/architecture.md` +- `docs/config.md` only if it currently states upstream version or concurrency + behavior that became inaccurate +- `docs/roadmap/future.md` +- `go.mod` +- `go.sum` +- all tests + +### Implementation + +1. Review all PromptKit references outside `docs/roadmap/` against the v0.2.0 + tag. +2. Ensure canonical ownership: + - CLI owns reasoning flag syntax and errors; + - configuration owns profile-source and `total_llm` fields; + - operations owns effective limits and failure handling; + - internal LLM docs own adapter/session/reasoning/scheduler mechanics; + - PromptKit integration owns the upstream version and supported dependency + boundary; + - JSON output owns manifest field semantics; and + - architecture owns only the invariant that backend limits may narrow the + application-wide bound. +3. Remove all claims that PromptKit v0.1.0 is pinned or that native session + propagation is blocked. +4. Ensure `future.md` contains only unimplemented work. If adding a deferred + custom-backend item, state that it requires a demonstrated reusable custom + endpoint or per-backend policy and must define configuration, redaction, + fingerprint, validation, and layered-queue behavior. Do not add + implementation staging or imply commitment. +5. Verify that no documentation says Notarius registers custom backends. +6. Confirm examples remain version 4 and unchanged unless a factual + documentation link requires adjustment. +7. Run `go mod tidy` and confirm it produces no additional diff. + +### Final validation + +```sh +go mod tidy +go test ./... +go test -race ./... +go vet ./... +go build ./cmd/notarius +git diff --check + +go run ./cmd/notarius config validate \ + --config examples/dnd-minimal.config.yml \ + --pipeline dnd-session + +OPENROUTER_API_KEY=validation-placeholder \ + go run ./cmd/notarius config validate \ + --config examples/dnd-complete.config.yml \ + --pipeline dnd-session + +go run ./cmd/notarius pipelines list \ + --config examples/dnd-minimal.config.yml + +OPENROUTER_API_KEY=validation-placeholder \ + go run ./cmd/notarius pipelines list \ + --config examples/dnd-complete.config.yml + +rg -n 'promptkit v0\\.1\\.0|PromptKit v0\\.1\\.0|native session.*blocked|Metadata:' \ + README.md docs internal go.mod go.sum +``` + +Review search results rather than deleting legitimate historical text blindly. +No v0.1.0 PromptKit ADR history currently exists, so the expected final result +is no stale version claim. + +### Acceptance criteria + +- PromptKit v0.2.0 is the only selected PromptKit module version. +- Full ordinary and race-enabled test suites pass offline. +- Vet, build, example validation, pipeline listing, and whitespace checks pass. +- Documentation describes only implemented behavior outside the roadmap. +- Native session, reasoning tri-state, backend provenance, and capacity + behavior each have one canonical documentation owner. +- Custom backend registration remains unimplemented and is not implied. +- The working tree contains only intentional upgrade changes. + +This stage is small enough for one implementation prompt because it is a +verification and documentation-consistency pass, not a feature stage. + +## Completion Checklist + +Before declaring the plan complete, verify all of the following: + +- [ ] PromptKit is pinned to v0.2.0. +- [ ] `RunRequest.Metadata` and `promptKitMetadata` are gone. +- [ ] Built-in profile checkpoint identity names v0.2.0. +- [ ] Native direct sessions and the compatibility variable are both present. +- [ ] Existing session checkpoint isolation remains intact. +- [ ] Reasoning inherit/replace/clear states reach PromptKit exactly. +- [ ] CLI reasoning states are mutually exclusive and checkpoint-safe. +- [ ] Backend ID and effective reasoning are published as optional provenance. +- [ ] Endpoint-only profiles remain supported. +- [ ] PromptKit capacity maps to `ErrLLMCapacityExceeded`. +- [ ] Notarius's global scheduler remains active. +- [ ] No custom backend configuration was added. +- [ ] No PromptKit raw-JSON migration was added. +- [ ] Canonical documentation and `future.md` are current. +- [ ] All focused, full, race, vet, build, example, and diff checks pass. + +## Open Questions + +None. The implementation choices required for this scope are resolved above. diff --git a/go.mod b/go.mod index a19ea04..73afcb4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module gitea.maximumdirect.net/eric/notarius go 1.25.5 require ( - gitea.maximumdirect.net/eric/promptkit v0.1.0 + gitea.maximumdirect.net/eric/promptkit v0.2.0 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 47ac713..433ab7f 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -gitea.maximumdirect.net/eric/promptkit v0.1.0 h1:vuKeBxkiY8E54LRFbLQFjlJJCiOfMvB1++DYBCrD/ug= -gitea.maximumdirect.net/eric/promptkit v0.1.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ= +gitea.maximumdirect.net/eric/promptkit v0.2.0 h1:6bUATKnVgLlHSbKjGb8EnCt2r7jV5Xw6t+pX7G9S/lU= +gitea.maximumdirect.net/eric/promptkit v0.2.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= diff --git a/internal/framework/llm/promptkit_client.go b/internal/framework/llm/promptkit_client.go index 053eabe..d7c386c 100644 --- a/internal/framework/llm/promptkit_client.go +++ b/internal/framework/llm/promptkit_client.go @@ -100,7 +100,6 @@ func (c *PromptKitClient) CompleteStructured(ctx context.Context, req contracts. ProfileID: strings.TrimSpace(req.ProfileID), Inputs: promptKitInputs(req.Inputs), Vars: promptKitVars(req), - Metadata: promptKitMetadata(req), } prepared, err := c.engine.Prepare(ctx, runReq) if err != nil { @@ -361,17 +360,6 @@ func promptKitVars(req contracts.StructuredCompletionRequest) map[string]string return vars } -func promptKitMetadata(req contracts.StructuredCompletionRequest) map[string]string { - metadata := map[string]string{} - if stageName := strings.TrimSpace(req.StageName); stageName != "" { - metadata["stage_name"] = stageName - } - if len(metadata) == 0 { - return nil - } - return metadata -} - var bearerTokenPattern = regexp.MustCompile(`(?i)Bearer\s+[A-Za-z0-9._~+/=-]+`) func redactPromptKitError(err error) error { diff --git a/internal/framework/llm/promptkit_client_test.go b/internal/framework/llm/promptkit_client_test.go index 67af1dc..6d5a1bd 100644 --- a/internal/framework/llm/promptkit_client_test.go +++ b/internal/framework/llm/promptkit_client_test.go @@ -176,6 +176,10 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) { if fresh[0].Value == "mutated" { t.Fatal("LLMCheckpointFingerprints exposed mutable backing storage") } + const wantBuiltinFingerprint = "sha256:4b48cd1ef010b587cd8d56bc13b73a0cdd84da1414a8de9f9a2adc695dfcf0a7" + if fresh[0].Value != wantBuiltinFingerprint { + t.Fatalf("built-in profile fingerprint = %q, want %q", fresh[0].Value, wantBuiltinFingerprint) + } } func TestPromptKitClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) { diff --git a/internal/framework/llm/promptkit_profile_fingerprint.go b/internal/framework/llm/promptkit_profile_fingerprint.go index 9570626..15e3a9d 100644 --- a/internal/framework/llm/promptkit_profile_fingerprint.go +++ b/internal/framework/llm/promptkit_profile_fingerprint.go @@ -15,7 +15,7 @@ const ( promptKitProfileFingerprintName = "promptkit_profile_source" // The built-in profile catalog is compiled into this pinned PromptKit // release. Update this identity when the dependency is upgraded. - promptKitBuiltinProfileCatalogID = "promptkit:v0.1.0:builtin-profiles" + promptKitBuiltinProfileCatalogID = "promptkit:v0.2.0:builtin-profiles" ) func promptKitProfileFingerprint(profileDir, profileFile string) (CheckpointFingerprint, error) {