diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md deleted file mode 100644 index b70e0f4..0000000 --- a/docs/roadmap/implementation.md +++ /dev/null @@ -1,817 +0,0 @@ -# PromptKit v0.2.0 Upgrade Implementation Plan - -## Objective - -Upgrade Notarius from `gitea.maximumdirect.net/eric/promptkit` v0.1.0 to -v0.2.0 and adopt the release capabilities that fit the current application: - -- native request-level session propagation; -- tri-state, run-wide reasoning-effort overrides; -- backend-aware LLM provenance; -- provider-neutral handling of PromptKit backend-capacity exhaustion; and -- accurate documentation of the layered Notarius and PromptKit concurrency - model. - -Each numbered stage below is intended to be implemented by one LLM coding-agent -prompt, in order. A stage is complete only when its focused tests, documentation -updates, and acceptance criteria pass. Do not implement later-stage behavior -early unless a small compile-preserving type addition is explicitly required by -the current stage. - -## Repository Baseline - -The plan is grounded in the following current implementation: - -- `go.mod` pins PromptKit v0.1.0. -- `internal/framework/llm/promptkit_client.go` is the sole production adapter - from Notarius's provider-neutral `StructuredLLMClient` contract to PromptKit. -- `PromptKitClient.CompleteStructured` currently assigns - `promptkit.RunRequest.Metadata`; v0.2.0 removes that field. A temporary - v0.2.0 modfile build fails at this assignment. -- The same adapter currently supplies - `StructuredCompletionRequest.SessionID` only through a `session_id` prompt - variable. It does not set a direct PromptKit request session. -- The CLI already supplies one run-wide `--session-id`, passes it to every - prompt-facing stage, records it in checkpoint runtime identity, and exposes - it in debug/run metadata. -- PromptKit public values are not persisted wholesale. The adapter maps them - into Notarius-owned contracts, debug values, and manifests. PromptKit - v0.2.0's public JSON changes therefore do not require conversion of existing - Notarius output or checkpoint payloads. -- `concurrency.total_llm` is enforced by Notarius's application-wide scheduled - client. PromptKit v0.2.0 adds an independent engine-local scheduler keyed by - backend ID. -- Current production profiles are either PromptKit built-ins or user-supplied - endpoint profiles. Notarius has no configuration contract for registering - custom PromptKit backends. -- PromptKit composite literals in the repository are keyed; no positional - literal migration is expected. -- The pre-upgrade `go test ./...` baseline passes. - -Use the PromptKit v0.2.0 tag, not the moving `main` branch, as the dependency -and API authority. The release guide and exact tagged contracts are: - -- `https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/docs/releases/v0.2.0.md` -- `https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/docs/consumers/pkg-promptkit.md` -- `https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/docs/formats.md` -- `https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/types.go` -- `https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/backends.go` -- `https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.2.0/engine.go` - -## Decisions - -### Dependency and compatibility - -- Pin exactly PromptKit v0.2.0. -- Keep all PromptKit struct literals keyed. -- Delete the obsolete `RunRequest.Metadata` mapping. Do not invent a - replacement upstream metadata channel. -- Keep `StructuredCompletionRequest.StageName`; it remains useful in - Notarius-owned stage and debug behavior even though it is no longer sent to - PromptKit. -- Update the built-in profile-catalog checkpoint identity from v0.1.0 to - v0.2.0. -- Do not add a migration for PromptKit public JSON because Notarius does not - persist those values directly. - -### Session propagation - -- `--session-id` remains the only user-facing session surface. -- Set `promptkit.RunRequest.SessionID` to the trimmed Notarius request session. -- Continue populating `Vars["session_id"]` with the same value for prompt - compatibility. The direct field is authoritative for provider session - behavior; the variable exists only so a maintained prompt that renders - `.session_id` does not break silently. -- Keep the existing checkpoint session fingerprint unchanged. -- Do not add a profile or configuration session setting. - -### Reasoning override - -Expose PromptKit's three states through two CLI flags: - -- no flag: inherit the selected PromptKit profile (`nil`); -- `--reasoning-effort VALUE`: replace the profile value with trimmed, - non-empty `VALUE`; and -- `--clear-reasoning-effort`: pass a pointer to an empty string, explicitly - clearing inherited reasoning. - -The two flags are mutually exclusive. An explicitly empty -`--reasoning-effort` is a CLI usage error with exit status 2. - -The override is invocation-wide and applies to every PromptKit call made by -that command. Persistent reasoning policy remains in PromptKit profiles; do -not add a Notarius configuration field or per-lane reasoning option. - -Use a dedicated value object, named `LLMRuntimeOverrides`, at the CLI factory -boundary. It must contain `ReasoningEffort *string`, where: - -- `nil` means inherit; -- a non-empty pointed-to string means replace; and -- an empty pointed-to string means clear. - -The PromptKit client must defensively copy the pointed-to value during -construction. It must create a keyed -`promptkit.ExecutionTargetOverride{ReasoningEffort: ...}` for each run request -when the override is non-nil and leave `RunRequest.Execution` nil otherwise. - -### Checkpoint and debug identity - -- Add the reasoning override to checkpoint runtime identity because it can - change generated output. -- Omit the fingerprint when the override is inherited. -- Use fingerprint name `reasoning_effort_override`. -- Use the trimmed value for replacement and the stable literal `` for - explicit clearing. Do not use an empty fingerprint value. -- Record the tri-state value in debug invocation metadata as - `reasoning_effort_override` using `*string` with `omitempty`; a non-nil empty - string must remain distinguishable from omission. -- Reasoning effort is non-secret provider configuration, but it must still pass - through the existing redaction-safe debug path. - -### Backend identity and concurrency - -- Keep the Notarius `total_llm` scheduler. It is an application-wide limit - across every provider and backend. -- Allow PromptKit's v0.2.0 backend scheduler to impose a narrower - backend-specific limit. Do not attempt to merge, disable, or reimplement the - two schedulers. -- Document the effective active generation bound as the intersection of the - Notarius global limit and the selected PromptKit backend limit. -- Endpoint-only profiles remain limited only by Notarius. -- Built-in OpenRouter profiles automatically select PromptKit's reserved - `openrouter` backend and its upstream capacity policy. -- Add optional `backend_id` and `reasoning_effort` fields to Notarius - `LLMProfileManifest` values and published manifest JSON. -- Populate those fields from `RunResult.SelectedBackendID` and - `RunResult.EffectiveModelParams.ReasoningEffort`. -- Include both fields in recorder identity and sorting so distinct effective - executions are not collapsed. -- Extend Notarius debug prompt metadata with `selected_backend_id`; retain the - existing serialized effective-model parameters. - -### Capacity failure - -- Add a provider-neutral sentinel - `contracts.ErrLLMCapacityExceeded`. -- When `PromptKitClient` receives an error matching - `promptkit.ErrCapacityExceeded`, return an error matching the Notarius - sentinel and include redacted diagnostic context. -- Do not require callers outside `internal/framework/llm` to import PromptKit - or match a PromptKit error. -- Capacity exhaustion is an operational stage error, not invalid structured - output and not a validator rejection. -- Do not retry inside the adapter. Existing configured stage retry behavior - applies, and the run fails if those retries are exhausted. - -### Configuration and versioning - -- Keep Notarius configuration version 4. -- Do not add `promptkit.backends`, backend registration fields, or a - configuration-level reasoning override. -- Existing PromptKit profile files remain valid because v0.2.0 continues to - support endpoint-only profiles. - -### Documentation and decision records - -- Update canonical documentation in the same stage that changes its behavior. -- No ADR is required: this work advances the existing LLM boundary and - scheduler architecture without changing its direction or replacing a prior - durable decision. -- Remove the completed “Native Session Propagation” item from - `docs/roadmap/future.md` only after native session behavior is implemented - and documented. -- Do not add custom backend registration to this implementation plan. A concise - future-roadmap item may be added during final cleanup if it records the - demonstrated boundary and avoids implying a commitment. - -## Non-Goals - -This implementation must not: - -- register user-defined PromptKit backends; -- expose backend endpoint, API-key environment, extra-parameter, concurrency, - or queue-capacity fields in Notarius configuration; -- remove or weaken `concurrency.total_llm`; -- add module-specific or lane-specific reasoning overrides; -- add another session flag or configuration field; -- remove the compatibility `session_id` prompt variable; -- persist or expose raw PromptKit `RunResult`/`PreparedRun` JSON as a Notarius - contract; -- translate old PromptKit v0.1.0 JSON; -- change D&D prompt text, response schemas, or artifact contracts; -- change pipeline retry semantics; or -- call a live model provider in tests. - -## Stage 1 — Upgrade The Dependency And Restore Compilation - -### Goal - -Move to PromptKit v0.2.0 with no intentional user-visible behavior change -beyond using the new upstream release. - -### Files and areas - -- `go.mod` -- `go.sum` -- `internal/framework/llm/promptkit_client.go` -- `internal/framework/llm/promptkit_profile_fingerprint.go` -- corresponding LLM adapter and fingerprint tests -- `docs/integrations/pkg-promptkit.md` - -### Implementation - -1. Run: - - ```sh - go get gitea.maximumdirect.net/eric/promptkit@v0.2.0 - go mod tidy - ``` - -2. Remove the `Metadata` field from the constructed - `promptkit.RunRequest`. -3. Delete `promptKitMetadata` and tests that exist only for the removed - upstream metadata mapping. Do not delete `StageName` from Notarius - contracts. -4. Change `promptKitBuiltinProfileCatalogID` to - `promptkit:v0.2.0:builtin-profiles`. -5. Update fingerprint expectations so v0.2.0 has a distinct deterministic - checkpoint identity. -6. Review every `promptkit.*{...}` literal touched by compilation and keep it - keyed. -7. Update the PromptKit integration document to identify v0.2.0 and link to - the tagged v0.2.0 consumer and format documentation. Remove metadata from - the stated supported boundary. Do not yet claim that later-stage native - session, reasoning CLI, or Notarius capacity classification is implemented. - -### Tests - -- Existing PromptKit asset, profile, adapter, scheduler, and D&D prompt tests - must compile and pass. -- Do not add a test whose only purpose is to assert the dependency-version - string. The fingerprint behavior is the meaningful compatibility boundary. - -### Acceptance criteria - -- `go.mod` and `go.sum` contain v0.2.0 and no v0.1.0 PromptKit entry. -- No production use of `RunRequest.Metadata` remains. -- The built-in profile checkpoint fingerprint changes deterministically. -- Existing endpoint-only profiles continue to load. -- No Notarius durable schema or configuration version changes. -- Stage documentation describes the actual post-stage boundary. - -### Validation - -```sh -go mod tidy -git diff --check -go test ./internal/framework/llm/... ./internal/cli/... -go test ./internal/modules/dnd/... -go test ./... -go vet ./... -go build ./cmd/notarius -``` - -This stage is small enough for one implementation prompt. - -## Stage 2 — Adopt Native Session Propagation - -### Goal - -Make the existing run-wide Notarius session ID reach PromptKit's native direct -session field without breaking prompt-variable consumers. - -### Files and areas - -- `internal/framework/llm/promptkit_client.go` -- `internal/framework/llm/promptkit_client_test.go` -- PromptKit adapter test prompt fixtures -- existing CLI assembled-run session tests -- `docs/internal/llm.md` -- `docs/integrations/pkg-promptkit.md` -- `docs/roadmap/future.md` - -### Implementation - -1. In `PromptKitClient.CompleteStructured`, trim - `StructuredCompletionRequest.SessionID` once and assign it to - `promptkit.RunRequest.SessionID`. -2. Continue passing that same trimmed value as `Vars["session_id"]`. - Preserve caller-supplied variables, with the explicit Notarius session - taking precedence over a caller-supplied variable of the same name. -3. Leave an empty Notarius session as an empty direct value so PromptKit's - prompt-defined session template behavior remains available. -4. Keep session checkpoint identity, CLI behavior, module request propagation, - and debug/run metadata unchanged except that debug material should now show - PromptKit's native effective session. -5. Update LLM and integration documentation to state: - - the direct PromptKit session is authoritative for provider behavior; - - the prompt variable is retained for compatibility; - - session IDs are stable, non-secret correlation identifiers and may be - exposed to providers and provider observability. -6. Remove the completed `Native Session Propagation` section from - `docs/roadmap/future.md`. Do not replace it with implementation history. - -### Tests - -Add or adapt behavior-focused coverage proving: - -- a nonblank Notarius session reaches - `GenerateRequest.Prompt.SessionID` through a prompt definition that has no - `session_id` template; -- the effective session appears in prepared/debug prompt material; -- a prompt that renders `.session_id` still receives the compatibility - variable; -- an empty request does not invent a direct session; and -- the existing assembled-run test still observes one session across every - prompt-facing module. - -Rely on PromptKit for exhaustive Unicode-length and internal normalization -tests. Notarius needs only representative adapter-boundary coverage. Do not add -tests for PromptKit's private implementation sequence. - -### Acceptance criteria - -- Provider-facing PromptKit requests use the native session field. -- Existing `--session-id` syntax and exit behavior are unchanged. -- Checkpoints remain session-separated. -- Prompt-variable compatibility remains intact. -- No second session configuration surface exists. -- The future roadmap no longer describes native session support as blocked. - -### Validation - -```sh -go test ./internal/framework/llm/... -run 'Session|PromptKit' -go test ./internal/cli/... -run 'Session|Checkpoint' -go test ./internal/modules/dnd/... -go test ./... -git diff --check -``` - -This stage is small enough for one implementation prompt. - -## Stage 3 — Add Internal Tri-State Reasoning Support - -### Goal - -Teach the PromptKit adapter and CLI factory boundary to represent and apply -inherit, replace, and clear states before exposing new command flags. - -### Files and areas - -- `internal/framework/llm/promptkit_client.go` -- `internal/framework/llm/promptkit_client_test.go` -- `internal/cli/run.go` -- `internal/cli/catalog.go` -- CLI option/factory test helpers and fakes -- `docs/internal/llm.md` -- `docs/internal/cli.md` - -### Implementation - -1. Define `LLMRuntimeOverrides` at the CLI's LLM-client factory boundary with - `ReasoningEffort *string`. -2. Change `LLMClientFactory` and all production/test implementations to accept - `LLMRuntimeOverrides`. At this stage, ordinary CLI execution passes the zero - value, preserving inheritance. -3. Add `ReasoningEffort *string` to `PromptKitClientConfig` or an equivalently - focused run-override field. -4. Defensively copy the pointer and its string in - `NewPromptKitClient`; do not retain caller-owned mutable pointer state. -5. When non-nil, construct a keyed - `promptkit.ExecutionTargetOverride` for every adapter request and set its - `ReasoningEffort` pointer. When nil, keep `RunRequest.Execution` nil. -6. Preserve an empty pointed-to value exactly; do not trim it into a missing - override. Trim a non-empty replacement once at the CLI boundary in the next - stage. -7. Update internal docs to describe the tri-state adapter capability and - factory boundary without documenting CLI flags that do not yet exist. - -### Tests - -Use an injected PromptKit `LLMClient` and table-driven adapter test for: - -- nil pointer inherits the selected profile value; -- pointer to a nonblank value replaces it; -- pointer to an empty string clears it; and -- mutating the caller's original pointer after client construction cannot - change subsequent requests. - -Assert the effective value received in -`promptkit.GenerateRequest.Target.ReasoningEffort`, not internal PromptKit call -order. - -### Acceptance criteria - -- The factory and adapter preserve all three states. -- Existing production construction passes nil and retains current profile - behavior. -- No CLI or config surface has been added yet. -- Tests use offline injected clients only. - -### Validation - -```sh -go test ./internal/framework/llm/... -run 'Reasoning|PromptKit' -go test ./internal/cli/... -go test ./... -git diff --check -``` - -This stage is small enough for one implementation prompt. - -## Stage 4 — Expose The Run-Wide Reasoning CLI Contract - -### Goal - -Make the internal tri-state reasoning override available to users while -preserving checkpoint and debug correctness. - -### Files and areas - -- `internal/cli/run.go` -- `internal/cli/catalog.go` -- CLI parsing, run-contract, checkpoint, and debug-summary tests -- `internal/core/debugbundle/summary.go` -- related debugbundle tests -- `docs/cli.md` -- `docs/operations.md` -- `docs/internal/cli.md` -- `docs/internal/llm.md` - -### Implementation - -1. Add `--reasoning-effort VALUE` and - `--clear-reasoning-effort` to `notarius run`. -2. Use a flag value that tracks presence separately from content, following - the established `sessionIDFlag`/single-value patterns where appropriate. -3. Validate before run construction: - - both flags together are a usage error; - - an explicitly empty or whitespace-only reasoning value is a usage error; - - repeated single-value flags follow the CLI's existing duplicate-flag - policy. -4. Construct one `LLMRuntimeOverrides` value: - - neither flag -> nil; - - replacement -> pointer to the trimmed value; - - clear -> pointer to `""`. -5. Pass it through `LLMClientFactory` to production PromptKit client - construction. The same client is shared across the invocation, so the - override applies to chunking, extraction, normalization, validators, and - retries without adding fields to module request types. -6. Extend `runtimeOverrideFingerprints` and every relevant checkpoint-identity - call to include the reasoning fingerprint defined in “Decisions.” -7. Add `ReasoningEffortOverride *string` with JSON name - `reasoning_effort_override` and `omitempty` to - `debugbundle.Invocation`. Copy the invocation value defensively if any - writer retains it. -8. Update command, operations, and internal documentation. Persistent - reasoning remains a PromptKit profile concern. - -### Tests - -Add focused behavior coverage for: - -- no flags, replacement, and clear; -- the mutual-exclusion error; -- empty replacement rejection; -- forwarding of all three states to the test LLM factory; -- different reasoning states produce different checkpoint identities; -- inherit omits the reasoning fingerprint; -- explicit clear uses ``; -- debug invocation JSON distinguishes omission from `""`; and -- existing CLI commands and examples remain unaffected. - -Do not replicate the same tri-state cases at every module layer. The CLI, -checkpoint, debugbundle, and adapter boundaries each own one distinct risk. - -### Acceptance criteria - -- The public CLI exposes exactly the two agreed flags. -- The override applies run-wide and does not modify resolved pipeline - structure or PromptKit profile files. -- Checkpoint reuse cannot cross reasoning states. -- Debug invocation metadata preserves the tri-state distinction. -- CLI reference and operations documentation are current. -- Notarius configuration remains version 4 with no reasoning field. - -### Validation - -```sh -go test ./internal/cli/... -run 'Reasoning|Checkpoint|Run' -go test ./internal/core/debugbundle/... -go test ./internal/framework/llm/... -run 'Reasoning' -go test ./... -go run ./cmd/notarius run --help -git diff --check -``` - -`run --help` is expected to follow the repository's documented flag-parser -usage behavior and may exit with status 2. Verify the displayed result and -status against `docs/cli.md`. - -This stage is substantial but coherent and small enough for one focused -implementation prompt. Do not combine it with backend provenance. - -## Stage 5 — Publish Backend And Effective Reasoning Provenance - -### Goal - -Expose the new PromptKit backend identity and effective reasoning setting in -Notarius-owned debug and manifest contracts. - -### Files and areas - -- `internal/core/artifacts/artifacts.go` -- artifact cloning, sorting, equality, and JSON tests -- `internal/framework/llm/promptkit_client.go` -- profile recorder and adapter tests -- `internal/framework/pipeline/runner.go` and profile-manifest merge logic -- pipeline tests involving LLM profile manifests -- `internal/framework/contracts/contracts.go` debug types if required -- `docs/integrations/json-output.md` -- `docs/integrations/pkg-promptkit.md` -- `docs/internal/llm.md` - -### Implementation - -1. Add optional JSON fields to `artifacts.LLMProfileManifest`: - - ```go - BackendID string `json:"backend_id,omitempty"` - ReasoningEffort string `json:"reasoning_effort,omitempty"` - ``` - - Use the repository's normal Go alignment/formatting. - -2. Populate: - - `BackendID` from `promptkit.RunResult.SelectedBackendID`; and - - `ReasoningEffort` from - `RunResult.EffectiveModelParams.ReasoningEffort`. -3. Trim both strings at the Notarius boundary. -4. Include both fields in `LLMProfileRecorder`'s deduplication key. -5. Update deterministic merge/sort keys everywhere LLM profile manifests are - combined so entries differing in backend or effective reasoning are not - collapsed and output order remains stable. -6. Add `SelectedBackendID` to the Notarius debug prompt representation and - populate it from `promptkit.PreparedRun.SelectedBackendID`. -7. Keep `EffectiveModelParams` in debug output; it now uses PromptKit v0.2.0's - stable lower-case JSON representation and can include `backend_id`. -8. Update the JSON output integration contract to define the optional - `backend_id` and `reasoning_effort` members of each `llm_profiles` entry. - Do not claim that Notarius supports user-defined backend registration. -9. Do not bump a D&D artifact schema or configuration version. These are - additive optional run-manifest provenance fields, and the JSON bundle - contract already requires consumers to tolerate omitted optional manifest - fields. - -### Tests - -Add or update tests proving: - -- the adapter records backend ID and effective reasoning from a PromptKit - result; -- endpoint-only profiles omit `backend_id`; -- built-in or test registered backend profiles record a non-empty ID; -- profile entries with otherwise equal ID/provider/model but different backend - or reasoning remain distinct; -- ordering is deterministic; and -- published manifest JSON uses the documented optional field names. - -Do not snapshot an entire debug bundle or published manifest when focused -semantic assertions protect the same contract. - -### Acceptance criteria - -- Backend identity does not leak PromptKit types beyond the adapter. -- Published provenance distinguishes effective backend and reasoning. -- Endpoint-only profiles remain valid and omit backend identity. -- Existing JSON consumers remain compatible with additive optional fields. -- Documentation owns the field semantics in the JSON output contract. - -### Validation - -```sh -go test ./internal/core/artifacts/... -go test ./internal/framework/llm/... -run 'Profile|Backend|Reasoning' -go test ./internal/framework/pipeline/... -run 'Profile|Manifest' -go test ./internal/modules/generic/output/json/... -run 'Manifest|Profile' -go test ./... -git diff --check -``` - -This stage is small enough for one implementation prompt. - -## Stage 6 — Translate PromptKit Capacity Exhaustion - -### Goal - -Give the application a provider-neutral overload classification while leaving -scheduling and retry ownership in their existing layers. - -### Files and areas - -- `internal/framework/contracts/errors.go` -- `internal/framework/llm/promptkit_client.go` -- PromptKit adapter tests -- scheduler/scheduled-client tests for regression only -- `docs/internal/llm.md` -- `docs/operations.md` -- `docs/integrations/pkg-promptkit.md` -- `docs/policy/architecture.md` - -### Implementation - -1. Add: - - ```go - var ErrLLMCapacityExceeded = errors.New("LLM capacity exceeded") - ``` - - with a comment that identifies backend admission exhaustion before - generation. -2. In the PromptKit `Run` error path, check - `errors.Is(err, promptkit.ErrCapacityExceeded)` before generic wrapping. -3. Return an error matching `contracts.ErrLLMCapacityExceeded`. Include the - prompt context and sanitized upstream diagnostic, but do not require an - external caller to match the PromptKit sentinel. -4. Preserve context cancellation precedence: when `ctx.Err()` is non-nil, - return the context error as today. -5. Do not classify capacity as `ErrInvalidStructuredOutput`. -6. Do not add an adapter retry. Allow the calling module/pipeline's existing - configured attempt policy to see an operational error. -7. Keep the Notarius scheduled client around the PromptKit adapter. -8. Document: - - the two independent concurrency layers; - - built-in OpenRouter's upstream backend limit; - - endpoint-only profiles having no PromptKit backend limit; - - capacity failure and stage retry behavior; and - - the possible effective limit being narrower than `total_llm`. -9. Add one sentence to architecture policy permitting provider runtime - adapters to apply a narrower backend-specific limit beneath the mandatory - application-wide scheduler. Do not copy numeric upstream defaults into - architecture policy. - -### Tests - -At the adapter boundary, use: - -- a test `promptkit.Backend` with `ConcurrencyLimit: 1`; -- an explicit zero `QueueCapacity`; -- a profile selecting that backend; and -- a blocking injected `promptkit.LLMClient`. - -Prove relationally that: - -- the first run occupies the one active slot; -- one additional direct adapter run receives an error matching - `contracts.ErrLLMCapacityExceeded`; -- the injected client is not called for the rejected request; -- the error is not `ErrInvalidStructuredOutput`; and -- provider diagnostic redaction still applies. - -Do not duplicate PromptKit's queue fairness, default-capacity, or complete -backend scheduler tests. Existing Notarius scheduler tests continue to own the -global limit. - -### Acceptance criteria - -- Capacity exhaustion has a stable Notarius error identity. -- PromptKit error types do not become a framework-wide contract. -- The global scheduler remains in production composition. -- No hidden retry or HTTP/status policy is introduced. -- Architecture, operations, integration, and internal docs agree on - concurrency ownership. - -### Validation - -```sh -go test ./internal/framework/llm/... -run 'Capacity|Scheduler|PromptKit' -go test -race ./internal/framework/llm/... -go test ./internal/cli/... -run 'Concurrency|Retry|Production' -go test ./... -git diff --check -``` - -This stage is small enough for one implementation prompt. - -## Stage 7 — Final Documentation And Release Verification - -### Goal - -Verify the completed upgrade as one coherent current contract, remove stale -v0.1.0/native-session planning language, and run the release-recommended test -matrix. - -### Files and areas - -- `docs/integrations/pkg-promptkit.md` -- `docs/internal/llm.md` -- `docs/internal/cli.md` -- `docs/cli.md` -- `docs/operations.md` -- `docs/integrations/json-output.md` -- `docs/policy/architecture.md` -- `docs/config.md` only if it currently states upstream version or concurrency - behavior that became inaccurate -- `docs/roadmap/future.md` -- `go.mod` -- `go.sum` -- all tests - -### Implementation - -1. Review all PromptKit references outside `docs/roadmap/` against the v0.2.0 - tag. -2. Ensure canonical ownership: - - CLI owns reasoning flag syntax and errors; - - configuration owns profile-source and `total_llm` fields; - - operations owns effective limits and failure handling; - - internal LLM docs own adapter/session/reasoning/scheduler mechanics; - - PromptKit integration owns the upstream version and supported dependency - boundary; - - JSON output owns manifest field semantics; and - - architecture owns only the invariant that backend limits may narrow the - application-wide bound. -3. Remove all claims that PromptKit v0.1.0 is pinned or that native session - propagation is blocked. -4. Ensure `future.md` contains only unimplemented work. If adding a deferred - custom-backend item, state that it requires a demonstrated reusable custom - endpoint or per-backend policy and must define configuration, redaction, - fingerprint, validation, and layered-queue behavior. Do not add - implementation staging or imply commitment. -5. Verify that no documentation says Notarius registers custom backends. -6. Confirm examples remain version 4 and unchanged unless a factual - documentation link requires adjustment. -7. Run `go mod tidy` and confirm it produces no additional diff. - -### Final validation - -```sh -go mod tidy -go test ./... -go test -race ./... -go vet ./... -go build ./cmd/notarius -git diff --check - -go run ./cmd/notarius config validate \ - --config examples/dnd-minimal.config.yml \ - --pipeline dnd-session - -OPENROUTER_API_KEY=validation-placeholder \ - go run ./cmd/notarius config validate \ - --config examples/dnd-complete.config.yml \ - --pipeline dnd-session - -go run ./cmd/notarius pipelines list \ - --config examples/dnd-minimal.config.yml - -OPENROUTER_API_KEY=validation-placeholder \ - go run ./cmd/notarius pipelines list \ - --config examples/dnd-complete.config.yml - -rg -n 'promptkit v0\\.1\\.0|PromptKit v0\\.1\\.0|native session.*blocked|Metadata:' \ - README.md docs internal go.mod go.sum -``` - -Review search results rather than deleting legitimate historical text blindly. -No v0.1.0 PromptKit ADR history currently exists, so the expected final result -is no stale version claim. - -### Acceptance criteria - -- PromptKit v0.2.0 is the only selected PromptKit module version. -- Full ordinary and race-enabled test suites pass offline. -- Vet, build, example validation, pipeline listing, and whitespace checks pass. -- Documentation describes only implemented behavior outside the roadmap. -- Native session, reasoning tri-state, backend provenance, and capacity - behavior each have one canonical documentation owner. -- Custom backend registration remains unimplemented and is not implied. -- The working tree contains only intentional upgrade changes. - -This stage is small enough for one implementation prompt because it is a -verification and documentation-consistency pass, not a feature stage. - -## Completion Checklist - -Before declaring the plan complete, verify all of the following: - -- [ ] PromptKit is pinned to v0.2.0. -- [ ] `RunRequest.Metadata` and `promptKitMetadata` are gone. -- [ ] Built-in profile checkpoint identity names v0.2.0. -- [ ] Native direct sessions and the compatibility variable are both present. -- [ ] Existing session checkpoint isolation remains intact. -- [ ] Reasoning inherit/replace/clear states reach PromptKit exactly. -- [ ] CLI reasoning states are mutually exclusive and checkpoint-safe. -- [ ] Backend ID and effective reasoning are published as optional provenance. -- [ ] Endpoint-only profiles remain supported. -- [ ] PromptKit capacity maps to `ErrLLMCapacityExceeded`. -- [ ] Notarius's global scheduler remains active. -- [ ] No custom backend configuration was added. -- [ ] No PromptKit raw-JSON migration was added. -- [ ] Canonical documentation and `future.md` are current. -- [ ] All focused, full, race, vet, build, example, and diff checks pass. - -## Open Questions - -None. The implementation choices required for this scope are resolved above. diff --git a/internal/cli/run.go b/internal/cli/run.go index 07e70ba..bcee8af 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -30,7 +30,7 @@ import ( const defaultConfigPath = "/usr/local/etc/notarius/config.yml" const usage = `Usage: notarius help - notarius run --input path/to/source.json [--config path/to/config.yml] [--output-dir path] [--json] [--chunk_cache auto|bypass|refresh] [--resume] [--recompute-step step-id] [--debug [--debug-dir path]] [--only lane-a,lane-b] [--session-id id] [--reference selector=path] [--without-reference selector] + notarius run --input path/to/source.json [--json] [flags] notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b] notarius pipelines list --config path/to/config.yml [--json] ` diff --git a/internal/core/artifacts/artifacts.go b/internal/core/artifacts/artifacts.go index 8abe914..f96d0b5 100644 --- a/internal/core/artifacts/artifacts.go +++ b/internal/core/artifacts/artifacts.go @@ -1,6 +1,7 @@ package artifacts import ( + "strings" "time" ) @@ -33,6 +34,27 @@ type LLMProfileManifest struct { ReasoningEffort string `json:"reasoning_effort,omitempty"` } +// Normalized returns the canonical representation used for manifest identity +// and publication. +func (profile LLMProfileManifest) Normalized() LLMProfileManifest { + profile.ID = strings.TrimSpace(profile.ID) + profile.Provider = strings.TrimSpace(profile.Provider) + profile.Model = strings.TrimSpace(profile.Model) + profile.BackendID = strings.TrimSpace(profile.BackendID) + profile.ReasoningEffort = strings.TrimSpace(profile.ReasoningEffort) + return profile +} + +// IdentityKey returns an opaque, deterministic key for the effective profile. +func (profile LLMProfileManifest) IdentityKey() string { + profile = profile.Normalized() + return profile.ID + "\x00" + + profile.Provider + "\x00" + + profile.Model + "\x00" + + profile.BackendID + "\x00" + + profile.ReasoningEffort +} + type ReferenceProvenance struct { Stage string `json:"stage,omitempty"` StepID string `json:"step_id,omitempty"` diff --git a/internal/framework/llm/promptkit_client.go b/internal/framework/llm/promptkit_client.go index 33c935f..c336aca 100644 --- a/internal/framework/llm/promptkit_client.go +++ b/internal/framework/llm/promptkit_client.go @@ -313,12 +313,8 @@ func (r *LLMProfileRecorder) Record(profile artifacts.LLMProfileManifest) { if r == nil { return } - profile.ID = strings.TrimSpace(profile.ID) - profile.Provider = strings.TrimSpace(profile.Provider) - profile.Model = strings.TrimSpace(profile.Model) - profile.BackendID = strings.TrimSpace(profile.BackendID) - profile.ReasoningEffort = strings.TrimSpace(profile.ReasoningEffort) - key := profile.ID + "\x00" + profile.Provider + "\x00" + profile.Model + "\x00" + profile.BackendID + "\x00" + profile.ReasoningEffort + profile = profile.Normalized() + key := profile.IdentityKey() r.mu.Lock() defer r.mu.Unlock() if r.profiles == nil { diff --git a/internal/framework/pipeline/runner.go b/internal/framework/pipeline/runner.go index 3e14be2..0faead7 100644 --- a/internal/framework/pipeline/runner.go +++ b/internal/framework/pipeline/runner.go @@ -829,22 +829,12 @@ func mergeLLMProfileManifests(sources ...[]artifacts.LLMProfileManifest) []artif merged := make(map[string]artifacts.LLMProfileManifest) for _, source := range sources { for _, profile := range source { - id := strings.TrimSpace(profile.ID) - provider := strings.TrimSpace(profile.Provider) - model := strings.TrimSpace(profile.Model) - backendID := strings.TrimSpace(profile.BackendID) - reasoningEffort := strings.TrimSpace(profile.ReasoningEffort) - key := id + "\x00" + provider + "\x00" + model + "\x00" + backendID + "\x00" + reasoningEffort + profile = profile.Normalized() + key := profile.IdentityKey() if _, exists := merged[key]; exists { continue } - merged[key] = artifacts.LLMProfileManifest{ - ID: id, - Provider: provider, - Model: model, - BackendID: backendID, - ReasoningEffort: reasoningEffort, - } + merged[key] = profile } } if len(merged) == 0 {