diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md deleted file mode 100644 index 772c2f8..0000000 --- a/docs/roadmap/implementation.md +++ /dev/null @@ -1,586 +0,0 @@ -# PromptKit v0.5 Implementation Plan - -## Objective - -Implement the target state in -[PromptKit v0.5 Integration And LLM Profile Policy](promptkit.md). Each numbered -stage is intended to be one implementation prompt for a GPT-5.6-Terra coding -agent. Complete stages in order and leave the repository buildable, tested, and -internally coherent after every stage. - -Follow [Architecture](../policy/architecture.md), -[Testing Policy](../policy/testing.md), and -[Documentation Policy](../policy/documentation.md) throughout. Preserve -unrelated user changes. Use `apply_patch` for source and documentation edits, -run `gofmt` on changed Go files, and add only tests that protect the behaviors -and risks assigned to that stage. - -Do not implement the separate deterministic session-ID or default-concurrency -roadmap items as part of this plan. Do not perform paid or credentialed LLM -calls. - -## Background Summary - -Notarius currently pins PromptKit v0.3.0, calls `Prepare` and then `Run` for one -completion, validates profiles through a synthetic prompt, has no application -fallback profile source, and accepts LLM profiles only at individual bindings -or through the run-wide CLI override. PromptKit v0.5.0 is source-compatible -with the current tree; a temporary v0.5.0 module override has already passed -`go test ./...`. - -The implementation must nevertheless treat the upstream optional-parameter -change as intentional: unset `temperature`, `max_tokens`, and `top_p` remain -unset and are omitted from compatible provider requests. Do not restore the old -implicit `top_p: 1` default. - -## Stage 1: Upgrade The PromptKit Dependency - -### Goal - -Establish a clean PromptKit v0.5.0 baseline before adopting its new APIs. - -### Work - -- Update `go.mod` and `go.sum` from PromptKit v0.3.0 to v0.5.0 and run - `go mod tidy`. -- Change the PromptKit built-in profile-catalog marker in - `internal/framework/llm/promptkit_profile_fingerprint.go` to identify - v0.5.0. This deliberately invalidates LLM checkpoints tied to the prior - catalog identity. -- Review PromptKit-facing compile errors or test failures against the v0.4.0 - and v0.5.0 release guides. Do not adopt prepared execution, inspection, or - fallback profiles in this stage. -- Replace the existing test assertion for one exact built-in fingerprint hash - with durable assertions that the fingerprint is deterministic, non-empty, - non-secret, and changes when a semantic profile source changes. Do not add a - new version-constant or exact-hash change detector. -- Update `docs/integrations/pkg-promptkit.md` to pin and link v0.5.0 and state - the implemented dependency-level behavior: unset optional sampling controls - are provider defaults. Do not document later stages as implemented. -- Update any other canonical text that explicitly claims the dependency is - v0.3.0, but defer descriptions of unimplemented v0.5 APIs. - -### Tests And Validation - -- `go test ./internal/framework/llm ./internal/cli` -- `go test ./...` -- `go vet ./...` -- `go build ./cmd/notarius` -- `rg -n 'promptkit v0\.3\.0|promptkit@v0\.3\.0|PromptKit v0\.3\.0' .` -- `git diff --check` - -### Completion Criteria - -- The repository directly pins v0.5.0 and all default offline checks pass. -- The profile-source fingerprint identifies the new upstream catalog without a - brittle literal-hash test. -- Current documentation no longer identifies v0.3.0 as the supported version. - -## Stage 2: Execute One Frozen Prepared Snapshot - -### Goal - -Make Notarius debug details and generation use one exact PromptKit preparation. - -### Work - -- Refactor `PromptKitClient.CompleteStructured` to call - `PrepareExecution`, immediately defer `Discard`, obtain a caller-owned - `Details` value, and execute with `RunPrepared`. -- Preserve the existing Notarius request mapping, cancellation precedence, - validation classification, raw structured bytes, response decoding, - profile recording, usage reporting, and credential redaction. -- Ensure every preparation, execution, validation, empty-result, and decode - error retains useful Notarius prompt context without exposing prepared handle - state or secrets. -- Use `errors.As` to obtain `*promptkit.CapacityError` on admission rejection. - Preserve `contracts.ErrLLMCapacityExceeded` as the stable classification and - add a nonblank backend ID only to safe application-owned diagnostic context. - Do not expose `promptkit.CapacityError` outside the LLM adapter. -- Update `docs/internal/llm.md` and the implemented-mechanics portion of - `docs/integrations/pkg-promptkit.md` to describe the single frozen execution - snapshot and structured capacity adaptation. - -### Tests And Validation - -- Adapt existing PromptKit client tests to the prepared-execution path. -- Retain or add one behavioral test proving that the debug prompt details match - the request actually passed to generation when a backing prompt source could - otherwise change between independent preparations. Test the resulting - snapshot consistency, not a private helper call count. -- Retain capacity tests proving `errors.Is` reaches - `contracts.ErrLLMCapacityExceeded`, the selected backend can appear in safe - diagnostic context, and provider calls are not made after rejected - admission. -- Run `go test ./internal/framework/llm` and - `go test -race ./internal/framework/llm`. -- Run `go test ./...` and `git diff --check`. - -### Completion Criteria - -- `CompleteStructured` no longer calls independent `Prepare` and `Run` - operations for one request. -- Debug prompt material and generation result originate from the same frozen - PromptKit snapshot. -- Capacity remains a provider-neutral Notarius error classification. - -## Stage 3: Replace Synthetic Profile Validation With Inspection - -### Goal - -Validate profiles through PromptKit's exact profile-inspection boundary and -centralize engine profile-source construction. - -### Work - -- Introduce a small provider-adapter-owned profile inspection or validation - function in `internal/framework/llm`. Its public internal signature must use - Notarius-owned configuration and result/error types rather than returning - PromptKit types to the CLI. -- Share the code that applies `profile_dir`, `profile_file`, and registered - backend options between the production PromptKit engine and the inspection - engine. Preserve the mutual-exclusion and local-backend rules. -- Change CLI explicit-profile preflight to use `Engine.InspectProfile` through - that LLM boundary. -- Remove `profileCheckPromptID`, `profileCheckPromptFS`, the `testing/fstest` - production dependency, and the synthetic `Prepare` request. -- Preserve distinct, useful errors for an absent profile, invalid profile, - unknown backend registration, cancellation, and invalid profile source. -- Do not require `api_key_env` to be populated during configuration validation. - Inspection may report credential requirements internally, but actual - preparation remains responsible for credential availability before a model - call. -- Update current-behavior sections in `docs/internal/cli.md` and - `docs/internal/llm.md`. Keep field definitions in `docs/config.md`. - -### Tests And Validation - -- Replace synthetic-prompt tests with profile inspection tests covering: - configured local backend success; missing local backend failure; absent - profile; malformed profile; and an otherwise valid profile whose credential - environment variable is intentionally unset. -- Prove validation performs no provider HTTP call and remains offline. -- Run `go test ./internal/framework/llm ./internal/cli` and `go test ./...`. -- Run `git diff --check`. - -### Completion Criteria - -- No production synthetic profile-check prompt remains. -- Profile validation uses the same ordinary profile source and backend - registrations as execution. -- Configuration validation succeeds for structurally valid profiles without - reading credential values. - -## Stage 4: Add Application Fallback Profile Asset Plumbing - -### Goal - -Allow module families to register application-owned fallback profile YAML -without placing domain policy in generic LLM code. - -### Work - -- Extend `internal/framework/llm.AssetRegistry` with a separate fallback - profile source collection, registration method, flattened filesystem, and - safe content digest. -- Reuse the existing asset-source path validation and flattening behavior where - appropriate. Reject invalid roots, unreadable assets, and duplicate flattened - paths. Do not parse PromptKit profile YAML in Notarius. -- Add `promptkit.WithFallbackProfileFS` to production engine options only when - at least one fallback profile source is registered. -- Supply the identical assembled fallback source to the profile-inspection - engine. Adjust CLI composition so pipeline-aware profile validation can use - the production LLM asset registry without exposing PromptKit types. -- Extend profile-source checkpoint identity to include the exact fallback - profile asset digest in addition to the PromptKit catalog marker and operator - source. Keep the resulting fingerprint hash-only and path/content/credential - free. -- Keep operator source precedence owned by PromptKit. Do not implement profile - merging or duplicate PromptKit source resolution in Notarius. -- Update `docs/internal/llm.md` only for the new implemented generic asset and - fingerprint mechanics. No domain fallback exists until Stage 5. - -### Tests And Validation - -- Add focused AssetRegistry tests for successful flattening, invalid roots, - duplicate paths, and hash changes when fallback bytes change. -- Add adapter-level tests showing that the fallback filesystem reaches both - execution construction and inspection construction. -- Extend checkpoint tests to prove fallback content changes profile-source - identity without exposing raw YAML or paths. Use relational comparisons, not - a fixed hash literal. -- Run `go test ./internal/framework/llm ./internal/cli` and `go test ./...`. -- Run `git diff --check`. - -### Completion Criteria - -- Generic plumbing can carry application fallback profiles while remaining - unaware of D&D IDs or model settings. -- Inspection, execution, and checkpoint identity use the same fallback asset - source. - -## Stage 5: Adopt The D&D `dnd-extraction` Fallback - -### Goal - -Give the D&D module family one stable embedded workload profile that operators -can replace. - -### Work - -- Add a D&D-owned embedded PromptKit profile asset with ID `dnd-extraction` - under `internal/modules/dnd`. Use the exact baseline defined in - `promptkit.md`: OpenRouter, `openai/gpt-5.6-luna`, no explicit reasoning - effort, a 240-second timeout, flex service tier, and no selected temperature, - token limit, or `top_p`. The omitted reasoning value intentionally allows - OpenAI's backend to apply its `medium` default. -- Register the profile filesystem from the D&D registrar through the generic - fallback profile asset boundary. Keep D&D policy out of - `internal/framework/llm` and the CLI composition root. -- Change every maintained D&D LLM prompt definition—including scene chunking, - all D&D extractors, and NPC normalization—from the model-named default to - `default_profile: dnd-extraction`. -- Add an integration-level profile-resolution test proving that: - - the fallback resolves when no operator source defines the ID; - - a valid operator profile with the same ID wins completely; and - - an invalid matching operator profile fails rather than falling through. -- Test through Notarius's assembled production assets and PromptKit boundary; - do not duplicate every upstream source-precedence case. -- Update the implemented profile ownership and prompt-default behavior in - `docs/internal/dnd.md`, `docs/internal/llm.md`, and - `docs/integrations/pkg-promptkit.md`. Defer the complete operator walkthrough - and examples to Stage 10. - -### Tests And Validation - -- Run focused D&D prompt preparation tests and the production composition - tests. -- Run `go test ./internal/modules/dnd/... ./internal/framework/llm - ./internal/cli`. -- Run `go test ./...`. -- Verify `rg -n 'default_profile: gemini-2-flash' internal/modules/dnd` - returns no matches. -- Run `git diff --check`. - -### Completion Criteria - -- All maintained D&D prompts use the application-owned logical profile ID. -- The fallback works without an operator profile and remains authoritatively - overridable by a matching valid operator definition. - -## Stage 6: Introduce Module Execution-Class Metadata - -### Goal - -Make each production module's ability to use an LLM statically discoverable -without yet changing profile inheritance. - -### Work - -- Add `ExecutionClass contracts.ExecutionClass` to `pipeline.ModuleSpec` and - preserve it through normalization, cloning, catalogs, registries, JSON/debug - views, and lookup helpers. -- In this transitional stage only, allow an omitted execution class to - normalize to deterministic so existing test-only fixtures can be migrated in - Stage 7 without breaking the repository midway. -- Explicitly classify every production module: - - D&D scene chunking, every D&D extractor, and D&D NPC normalization as - `llm_backed`; - - all other current production input, chunk, merge, normalize, and output - modules as `deterministic`. -- Update production module specification tests and production catalog tests to - assert the semantic class alongside stage, artifact kind, and capabilities. -- Add catalog lookup support needed by later resolution to retrieve a selected - module's execution class by stage and key without constructing it. -- Do not implement pipeline-level profile inheritance or reject deterministic - profiles yet. -- Update `docs/internal/modules.md` and `docs/internal/dnd.md` to identify - execution class as registered module metadata, while noting only implemented - uses. - -### Tests And Validation - -- Run module registration/spec tests across generic, Seriatim, and D&D - families. -- Run `go test ./internal/framework/pipeline ./internal/modules/...`. -- Run `go test ./...` and `git diff --check`. - -### Completion Criteria - -- Every production module has an explicit correct execution class. -- Catalog consumers can retrieve that class without a concrete module - instance. -- Test-only omitted classes remain the only temporary compatibility behavior. - -## Stage 7: Enforce Execution Metadata And Remove Runtime Probing - -### Goal - -Finish the execution-class contract so missing metadata cannot cause future -profile drift. - -### Work - -- Update every framework, CLI, and integration test module specification to - declare an explicit execution class appropriate to the fake behavior. -- Change module-spec validation so an empty or unsupported execution class is a - registration error. Remove the transitional deterministic default from - Stage 6. -- Replace the chunk runner's special `ChunkExecutionClassProvider` probe with - specification-derived behavior. Remove the now-redundant provider interface, - implementation methods, and tests when they have no remaining consumer. -- Ensure chunk producer provenance remains unchanged: it records a non-empty - effective binding profile for an LLM-backed chunker, while a deterministic - chunker records no profile. A profile selected only through the prompt - default remains represented by PromptKit's actual-profile manifest rather - than being invented as an explicit chunk binding. -- Review helper constructors and fixtures for opportunities to set execution - class once without obscuring the class under test. Do not introduce an - elaborate test-spec framework. -- Update internal documentation if the removal changes any described runtime - mechanics. - -### Tests And Validation - -- Add or retain focused registration tests for missing and invalid execution - classes. -- Retain chunk-plan provenance tests for LLM-backed and deterministic - chunkers. -- Run `go test ./internal/framework/pipeline ./internal/modules/...`. -- Run `go test ./...`, `go vet ./...`, and `git diff --check`. - -### Completion Criteria - -- No registered module specification relies on an implicit execution class. -- Pipeline metadata, not a concrete runtime type assertion, owns module - execution classification. - -## Stage 8: Resolve Programmatic Pipeline Profile Defaults - -### Goal - -Implement profile inheritance and precedence inside the pipeline resolver -before exposing the field through YAML configuration. - -### Work - -- Add an optional trimmed `LLMProfile` field to - `pipeline.PipelineProfile`. Add a non-empty runtime override field to - `pipeline.ResolveOptions` so all precedence decisions occur in the resolver - rather than through pre-resolution mutation. -- After module selection, `--only` filtering, default validator-chain - selection, and validator compatibility resolution, apply effective profiles - to every selected input, chunk, extract, merge, normalize, output, and - validator binding according to the precedence in `promptkit.md`. -- Apply profiles only when the selected module or validator execution class is - `llm_backed`. -- Reject a binding-specific `llm_profile` on any deterministic module or - validator. Do not reject or inspect an unused pipeline default when no - selected LLM-backed binding consumes it. -- Leave an LLM-backed binding empty when no CLI, binding, or pipeline profile is - selected so PromptKit can use the prompt's `default_profile`. -- Store the effective values on resolved bindings before digest construction. - Do not add a second inheritance decision to execution. -- Ensure semantically equivalent repeated binding profiles and one inherited - default produce the same resolved pipeline digest. Ensure any changed - effective profile changes the digest. -- Do not modify file configuration or CLI parsing in this stage. - -### Tests And Validation - -- Add pipeline package tests for the complete precedence matrix: - runtime override; binding-specific exception; pipeline default; prompt - fallback; and deterministic bindings. -- Cover default and explicitly configured validator chains, all relevant stage - categories, `--only` lane selection, unused defaults, deterministic-profile - rejection, and semantic digest equivalence. -- Prefer table-driven package-level tests over assertions on private traversal - helpers. -- Run `go test ./internal/framework/pipeline` and `go test ./...`. -- Run `git diff --check`. - -### Completion Criteria - -- Programmatic pipelines resolve one canonical effective profile policy. -- Only LLM-backed resolved bindings can contain a profile. -- Runtime override, binding, pipeline, and prompt precedence is unambiguous and - digest-stable. - -## Stage 9: Expose Pipeline Defaults Through Configuration And CLI - -### Goal - -Make the profile-default workflow available to operators while preserving -validation and override behavior. - -### Work - -- Add optional `pipelines..llm_profile` support to the version 4 file - configuration model. Use presence-aware decoding so an explicitly set blank - value is rejected, while omission remains valid. -- Preserve the field through file application, configuration cloning, - effective configuration, and programmatic profile copies without aliasing or - trimming drift. -- Remove `applyLLMProfileOverride`. Pass the CLI override through the resolver's - runtime-override input so deterministic bindings are never populated. -- Update effective profile-ID collection to cover every selected LLM-backed - module stage and LLM-backed validator, including future LLM-backed input and - output modules. Do not inspect deterministic or unselected profiles. -- Ensure `run`, `config validate --pipeline`, resume/checkpoint identity, and - relevant dry preflight paths all use the same resolved effective profiles. -- Preserve `--llm-profile` as the highest-precedence non-empty run-wide - override and preserve binding-specific profiles as exceptions when no CLI - override is present. -- Do not increment the configuration version. -- Update current configuration and CLI contracts in `docs/config.md` and - `docs/cli.md` in the same stage. Link to operations for the deployment - workflow rather than duplicating it prematurely. - -### Tests And Validation - -- Add file-config tests for omission, trimming, explicit blank rejection, - unknown-key behavior, cloning, and round-trip application. -- Add effective-config and CLI contract tests for precedence, LLM-only - application, inherited-profile inspection failure before factory execution, - `--only`, and digest changes. -- Retain offline operation and do not require credentials for - `config validate --pipeline`. -- Run `go test ./internal/core/config ./internal/framework/pipeline - ./internal/cli`. -- Run `go test ./...`, `go vet ./...`, and `git diff --check`. - -### Completion Criteria - -- Operators can select `dnd-extraction` once per pipeline. -- Configuration and CLI paths share the resolver's precedence policy. -- Unknown effective profiles fail preflight, while deterministic and unused - profiles do not cause spurious inspection. - -## Stage 10: Complete Operator Documentation, Examples, And Decision Record - -### Goal - -Make the implemented workflow understandable, copyable, and maintainable -without duplicating canonical facts. - -### Work - -- Create an ADR using the next sequential number for the durable decision to - use workload-oriented pipeline defaults with operator-overridable application - fallback profiles. Record context, decision, alternatives, and consequences; - do not turn the ADR into a field reference or implementation log. -- Complete `docs/config.md` as the canonical owner of profile-source fields, - `pipelines..llm_profile`, validation, and precedence. -- Complete `docs/operations.md` with an operator workflow that distinguishes - Notarius embedded prompts, Notarius fallback profiles, PromptKit built-ins, - and deployment filesystem profiles. Include production/development/local use - of the same `dnd-extraction` ID, credential handling, absolute-path guidance, - and the fact that current relative profile paths use the process working - directory rather than the configuration file's directory. -- Complete `docs/integrations/pkg-promptkit.md` with the v0.5.0 boundary, - prepared execution, inspection, fallback and ordinary source precedence, - optional provider controls, capacity adaptation, and compatibility policy. -- Update `docs/internal/configuration.md`, `docs/internal/pipeline.md`, - `docs/internal/cli.md`, `docs/internal/llm.md`, `docs/internal/modules.md`, and - `docs/internal/dnd.md` only for their owned implementation details. Link to - canonical configuration, operations, and upstream format contracts rather - than restating them. -- Keep exactly the existing two D&D configuration examples. Add - `llm_profile: dnd-extraction` to the minimal and complete pipelines and remove - the now-redundant model-named binding override from the complete example. -- Add one secret-free maintained operator profile at - `examples/profiles/dnd-extraction.yml`. It should be a complete valid profile - for the same logical ID and may mirror the embedded baseline; its purpose is - to demonstrate file ownership and format, not claim automatic environment - detection. Link it from the configuration and operations documentation. -- If the complete example selects the external profile file, use a path that - is valid for the documented repository-root invocation and explicitly note - the working-directory rule. Keep the minimal example dependent only on the - embedded fallback. -- Add or extend maintained-example validation so both configuration examples - and the profile YAML are checked without generation or credentials. -- Remove the now-implemented `Pipeline-Level LLM Profile Defaults` section from - `docs/roadmap/future.md`. Preserve the unrelated deterministic session and - concurrency items. -- Do not delete `promptkit.md` or this implementation plan during the feature - implementation; retire them only after post-implementation review. - -### Tests And Validation - -- Run maintained example/configuration tests and relevant CLI help/parser - tests. -- Run `go test ./...`. -- Run `rg -n 'gemini-2-flash' examples docs` and review every remaining match - for intentional model-policy or historical context. -- Run `rg -n 'v0\.3\.0|profileCheckPrompt|applyLLMProfileOverride' .` and resolve - stale production or current-documentation matches. -- Verify all new links and `git diff --check`. - -### Completion Criteria - -- Every current fact has one canonical documentation owner. -- Operators can distinguish and deploy all profile layers without reading Go - source. -- Both maintained configurations and the maintained external profile are valid, - secret-free, and tested offline. -- Implemented profile work no longer remains in `future.md`. - -## Stage 11: Final Verification And Quality Review - -### Goal - -Verify the complete migration as one integrated change and correct only defects -or omissions found during that review. - -### Work - -- Review the final diff against every acceptance criterion in `promptkit.md`. -- Confirm provider-specific PromptKit types remain inside the LLM integration - boundary and D&D policy remains inside the D&D module family. -- Confirm execution and inspection receive identical ordinary, fallback, and - backend configuration. -- Confirm no paths, profile YAML, endpoints, credentials, or prepared handle - state leak into fingerprints or ordinary diagnostics. -- Confirm all production module specs have explicit correct execution classes - and every resolved deterministic binding is profile-free. -- Confirm prompt default, pipeline default, binding override, and CLI override - behavior through representative assembled configurations. -- Review tests for redundancy and remove obsolete synthetic-prompt, - runtime-probe, exact-hash, or duplicated upstream-behavior tests superseded by - stronger contract tests. -- Perform an optional manual D&D quality comparison if credentials and an - evaluation transcript are deliberately supplied. Record no private input or - credential material, and do not make this comparison a completion gate. - -### Validation Commands - -```sh -gofmt -w -go test ./... -go test -race ./internal/framework/llm ./internal/core/config ./internal/framework/pipeline ./internal/cli -go vet ./... -go build ./cmd/notarius -git diff --check -``` - -Also run focused stale-contract searches: - -```sh -rg -n 'gitea.maximumdirect.net/eric/promptkit v0\.3\.0|PromptKit v0\.3\.0' . -rg -n 'default_profile: gemini-2-flash|profileCheckPrompt|applyLLMProfileOverride' internal docs examples -``` - -Review any matches rather than deleting intentional historical references -blindly. - -### Completion Criteria - -- All automated checks pass offline and without real credentials. -- The implemented behavior matches `promptkit.md` with no known architecture, - provenance, checkpoint, profile-precedence, or documentation gap. -- Any optional live evaluation is clearly separate from correctness testing. - -## Open Questions - -None. The roadmap decisions are sufficient to implement every stage without an -additional product or architecture choice. diff --git a/docs/roadmap/promptkit.md b/docs/roadmap/promptkit.md deleted file mode 100644 index 55ed4f6..0000000 --- a/docs/roadmap/promptkit.md +++ /dev/null @@ -1,318 +0,0 @@ -# PromptKit v0.5 Integration And LLM Profile Policy - -## Purpose - -This roadmap defines the target state for upgrading Notarius from PromptKit -v0.3.0 to v0.5.0 and adopting the upstream runtime and profile facilities that -directly improve Notarius. It also defines the application policy for stable, -domain-oriented LLM profile names, operator overrides, pipeline inheritance, -profile validation, provider defaults, checkpoint identity, and documentation. - -The ordered work needed to reach this state belongs in -[the implementation plan](implementation.md). Current behavior remains defined -by the canonical documentation outside `docs/roadmap/` until the corresponding -work is implemented. - -## Background - -Notarius currently pins PromptKit v0.3.0. Its adapter prepares a request once -for debug material and then independently runs the original request, causing -PromptKit to prepare the same logical call a second time. The CLI validates an -explicit profile by preparing a synthetic prompt. PromptKit profile selection -can be repeated on individual module bindings or replaced for one invocation -with `--llm-profile`, but a configured pipeline cannot yet declare one inherited -profile policy. - -PromptKit v0.4.0 and v0.5.0 add the upstream boundaries needed to improve these -areas: - -- [v0.4.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/releases/v0.4.0.md) - adds opaque prepared executions, exact profile and prompt inspection, and a - typed backend-capacity error; -- [v0.5.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/releases/v0.5.0.md) - adds application fallback profile filesystems and stops sending unset - optional sampling controls as framework-selected provider values; and -- the [v0.5.0 format contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.5.0/docs/formats.md) - defines the resulting profile-source and execution-setting precedence. - -A source-compatibility test of the current Notarius repository against -PromptKit v0.5.0 completed successfully. The work is therefore primarily an -intentional runtime and configuration migration rather than a repair for a -breaking Go API change. - -## Goals - -- Pin and document PromptKit v0.5.0 as Notarius's supported upstream contract. -- Execute the exact prepared request snapshot whose safe details are recorded - in Notarius debug material. -- Validate configured PromptKit profiles through the upstream inspection API - without synthetic prompts, provider calls, or credential-value access. -- Give Notarius an application-owned, operator-overridable - `dnd-extraction` profile fallback. -- Let a pipeline choose one default LLM profile without repeating that ID on - every LLM-backed binding. -- Apply profile inheritance and run-wide overrides only where the resolved - module or validator can use an LLM. -- Preserve accurate checkpoint invalidation, effective profile provenance, - redaction, cancellation, concurrency, and provider-neutral module contracts. -- Provide operators with one clear deployment pattern for production, - development, and local profile definitions. - -## Target End State - -### PromptKit Runtime Boundary - -Notarius depends on PromptKit v0.5.0 and uses its public APIs rather than -reimplementing source or execution resolution. - -For each structured completion, the adapter: - -1. builds one PromptKit run request from the provider-neutral Notarius request; -2. calls `PrepareExecution` once; -3. immediately arranges an idempotent `Discard` for every unexecuted handle; -4. obtains credential-redacted `Details` for debug and response metadata; and -5. calls `RunPrepared` so generation uses that exact frozen snapshot. - -The debug prompt and successful result therefore describe the same selected -profile, rendered messages, input bytes, session, output contract, and effective -settings even when a filesystem-backed source changes concurrently. PromptKit -handle types remain private to `internal/framework/llm`. - -PromptKit admission failures continue to match Notarius's provider-neutral -`ErrLLMCapacityExceeded` contract. When PromptKit supplies a `CapacityError`, -the adapter obtains the normalized backend ID through `errors.As` and may add it -to safe application-owned diagnostics without parsing upstream error wording. -The backend ID does not become a provider-specific module contract. - -### Optional Provider Controls - -Notarius accepts PromptKit v0.5.0's new behavior for `temperature`, -`max_tokens`, and `top_p`: an unset setting is omitted from compatible provider -requests and the provider chooses its own default. Notarius does not restore -PromptKit's former implicit `top_p: 1` value globally. - -An operator who requires a particular value specifies it in the selected -PromptKit profile. The application fallback described below intentionally -leaves these controls unset. A human-reviewed D&D extraction comparison should -be performed after the upgrade, but paid or nondeterministic model output is -not part of the default automated test suite. - -### Profile Inspection - -Pipeline-aware configuration validation uses `Engine.InspectProfile` for every -effective explicit profile ID. It verifies that the profile exists, parses and -validates, resolves its backend and target, and is compatible with the engine's -registered backends. It does not create a synthetic prompt, load prompt inputs, -contact a provider, or require credential values to exist in the validation -process environment. - -Credential availability is execution-time state. PromptKit preparation still -enforces the selected profile's credential contract before generation. This -keeps `notarius config validate` useful in build and deployment validation -environments where secrets are deliberately absent. - -PromptKit construction for inspection and execution uses one shared internal -profile-source and backend-option path. The CLI does not expose PromptKit public -types across the Notarius LLM boundary merely to perform inspection. - -`InspectPrompt` is not adopted merely because it exists. It remains available -for a later, separately defined module-to-prompt interface preflight if a -concrete validation requirement justifies that additional contract. - -### Application And Operator Profile Sources - -Notarius embeds one ordinary PromptKit YAML profile with the stable ID -`dnd-extraction`. It is an application fallback registered through -`WithFallbackProfileFS`, is owned by the D&D module family, and initially -preserves the current effective D&D baseline: - -- backend: PromptKit's built-in `openrouter` backend; -- model: `openai/gpt-5.6-luna`; -- reasoning effort: unset, allowing OpenAI's backend to apply its default of - `medium`; -- generation timeout: 240 seconds; -- service tier: `flex`; and -- no application-selected `temperature`, `max_tokens`, or `top_p`. - -All maintained D&D LLM prompt definitions use `dnd-extraction` as their -`default_profile`. The ID communicates workload intent rather than a provider, -model, or environment. Changing the embedded fallback is an intentional -Notarius execution-policy change and participates in checkpoint identity. - -Effective profile definitions resolve in PromptKit's order: - -1. programmatic in-memory profiles used by tests or explicit consumers; -2. the operator source configured by `promptkit.profile_file` or - `promptkit.profile_dir`; -3. the Notarius application fallback source; and -4. PromptKit's embedded built-in catalog. - -Only an absent ID falls through to the next source. A matching profile is a -complete definition: fields are not merged with a lower-precedence definition, -and a malformed matching operator profile fails rather than silently selecting -the application fallback. - -Production, development, and local deployments should normally provide -different complete definitions for the same `dnd-extraction` ID. An operator -source is optional because the application fallback keeps the maintained D&D -workflow usable, but a deployment that needs an intentional model or backend -policy should configure its own definition. - -### Domain Ownership And Asset Assembly - -The D&D fallback profile remains under `internal/modules/dnd` and is registered -by the D&D registrar, consistent with ADR-0004. Generic LLM plumbing knows how -to collect and flatten application fallback profile filesystems but contains no -D&D model or policy knowledge. - -The shared asset registry detects invalid roots, unreadable sources, and -duplicate flattened paths. PromptKit remains responsible for strict profile -YAML parsing, duplicate profile-ID detection, source precedence, and effective -target resolution. The same assembled fallback source is supplied to runtime -execution and CLI profile inspection. - -### Explicit Module Execution Metadata - -Every registered input, chunk, extract, merge, normalize, and output module -declares one required execution class: `deterministic` or `llm_backed`. -Validator registrations continue to declare the same distinction through their -validator specifications. - -The registered specification is authoritative for configuration resolution. -Current production classifications are: - -- the D&D scene chunker, all D&D extractors, and the D&D NPC normalizer are - LLM-backed; -- the Seriatim input adapter, generic chunker, all current mergers, all other - current normalizers, and the JSON output encoder are deterministic; and -- current validators retain their declared classifications. - -Missing or unsupported execution metadata is a registration error. Explicitly -assigning `llm_profile` to a deterministic module or validator is a pipeline -resolution error. The framework does not infer execution class by inspecting -domain package names or concrete implementation types at runtime. - -The module specification replaces the chunk runner's special runtime -execution-class probe. Effective resolved bindings already express the result: -only LLM-backed bindings may retain a non-empty profile. - -### Pipeline-Level Profile Default - -Configuration version 4 gains one optional non-empty pipeline field: - -```yaml -pipelines: - dnd-session: - llm_profile: dnd-extraction -``` - -No configuration-version increment is required because the field is additive -and existing files remain valid. An explicitly present blank value is invalid. - -For every selected LLM-backed module and validator, the effective profile uses -this precedence: - -1. non-empty run-wide `--llm-profile` override; -2. binding-specific `llm_profile`; -3. pipeline-level `llm_profile`; and -4. the prompt definition's `default_profile`, represented by an empty effective - Notarius binding profile. - -The run-wide override and inherited pipeline default never attach to a -deterministic binding. Binding-specific exceptions remain available when one -operation needs a different cost, latency, quality, backend, or reasoning -policy. - -Inheritance is resolved after module and validator selection, including -`--only` lane filtering, but before effective-pipeline validation, digest -construction, explicit-profile inspection, checkpoint construction, -preparation, execution, or provenance capture. Only profiles used by selected -LLM-backed bindings are inspected. An unused pipeline default in a pipeline -with no selected LLM-backed work does not require an otherwise unused profile -to exist. - -The resolved pipeline contains effective binding profiles rather than a second -runtime inheritance mechanism. Two pipelines that differ only by spelling the -same effective policy once as a pipeline default and once on every LLM-backed -binding have the same semantic resolved digest. Changing an effective profile -changes the digest and applicable checkpoint identity. - -### Provenance And Checkpoints - -The PromptKit profile-source checkpoint fingerprint covers: - -- the PromptKit v0.5.0 built-in profile catalog identity; -- exact application fallback profile asset content; and -- exact configured operator profile YAML content, when present. - -The existing local-backend target fingerprint remains separate and continues -to exclude scheduling-only concurrency limits. Fingerprints contain hashes and -stable markers, not profile contents, filesystem paths, endpoints, credentials, -or other secrets. - -Changing the PromptKit version, application fallback, operator profile, or -effective pipeline profile makes incompatible LLM checkpoints ineligible for -reuse. The dependency upgrade is expected to invalidate checkpoints produced -under v0.3.0. - -Successful run manifests continue to record only profiles actually selected by -PromptKit, including their effective model, backend, and reasoning metadata. -Debug output reports the same effective execution snapshot used for generation. - -### Operator Documentation And Examples - -Canonical documentation clearly distinguishes: - -- Notarius prompt and schema assets embedded in the application; -- Notarius application fallback profiles embedded in the application; -- PromptKit's own embedded built-in profiles; and -- operator profile files on the deployment filesystem. - -The configuration reference owns the pipeline field, profile-source fields, -validation rules, and precedence. Operations owns deployment layout, working -directory behavior, credentials, and environment-specific profile management. -The PromptKit integration document owns the pinned upstream contract and -source-precedence boundary. Internal documents describe asset registration, -resolution, inspection, prepared execution, fingerprinting, and tests without -duplicating user-facing field definitions. - -The maintained examples continue to include only the minimal and complete D&D -configurations. They use the stable `dnd-extraction` policy, and one maintained -PromptKit profile file under `examples/` demonstrates an operator override. -Examples remain secret-free and are validated without live provider calls. - -## Out Of Scope - -- Implementing the separate deterministic prompt-session identity roadmap - item. -- Changing the default `concurrency.total_llm` value; PromptKit's retained - OpenRouter capacity of 16 remains relevant to that separate item. -- Adding model evaluation as a deterministic or CI correctness gate. -- Automatically selecting production, development, or local environments. - Deployment configuration chooses the operator profile source. -- Profile inheritance, partial profile merging, or cross-profile aliases. -- Exposing PromptKit types to modules, validators, durable output contracts, or - public configuration structures. -- Adopting `InspectPrompt` without a separately justified prompt-interface - validation contract. - -## Acceptance Criteria - -- Notarius builds and its offline test suite passes with PromptKit v0.5.0. -- Every structured completion executes the exact snapshot used for safe debug - prompt details. -- Profile preflight uses profile inspection and no synthetic prompt. -- The embedded `dnd-extraction` fallback resolves without an operator source, - and a matching valid operator profile replaces it completely. -- Every production module has explicit, correct execution metadata. -- Pipeline, binding, CLI, and prompt-default precedence behaves as defined for - modules and validators, while deterministic bindings remain profile-free. -- Effective profiles participate in pipeline digests, profile inspection, - checkpoint identity, debug records, and run provenance at the appropriate - boundaries. -- The dependency and application fallback changes invalidate incompatible old - checkpoints without exposing profile or credential content. -- Canonical documentation and maintained examples accurately describe and - exercise the implemented operator workflow. -- Default tests remain deterministic, offline, credential-free, and focused on - Notarius-owned behavior rather than duplicating PromptKit's upstream suite. diff --git a/internal/cli/example_contract_test.go b/internal/cli/example_contract_test.go index d70b33d..57711e0 100644 --- a/internal/cli/example_contract_test.go +++ b/internal/cli/example_contract_test.go @@ -245,6 +245,12 @@ func TestMaintainedMalformedInputOnlyRecordsDebugFailureWhenRequested(t *testing if report.Succeeded || report.PipelineID != "dnd-session" { t.Fatalf("failure report = %#v, want failed dnd-session report", report) } + invocation := readProductionJSON[debugbundle.Invocation](t, filepath.Join(bundle, "summary", "invocation.json")) + manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(bundle, "summary", "run-manifest.json")) + manifestSession, found := manifest.Metadata["session_id"] + if invocation.SessionID == "" || !found || manifestSession != invocation.SessionID { + t.Fatalf("failed run sessions: invocation=%q manifest=%#v metadata=%#v", invocation.SessionID, manifestSession, manifest.Metadata) + } }) } } diff --git a/internal/framework/pipeline/runner.go b/internal/framework/pipeline/runner.go index fded7e4..768f36b 100644 --- a/internal/framework/pipeline/runner.go +++ b/internal/framework/pipeline/runner.go @@ -91,6 +91,11 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err input.llmClient = input.Prepared.dependencies.LLM output.Manifest = manifestFromPipeline(input) + sessionID := strings.TrimSpace(input.SessionID) + output.Manifest.Metadata, err = manifestMetadataWithSessionID(output.Manifest.Metadata, sessionID) + if err != nil { + return failOutput(output), err + } output.ChunkPlan = &artifacts.ChunkPlanSummary{ Mode: string(effectiveChunkCacheMode(input.ChunkCacheMode)), RequestedModule: input.pipeline.Chunk.Module, LookupStatus: "skipped", LookupReason: "chunk plan lookup skipped", @@ -191,11 +196,6 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err return failOutput(output), fmt.Errorf("write source debug artifact: %w", err) } sourceInput := sourceInputMaterial(input.Path, input.RawInput) - sessionID := strings.TrimSpace(input.SessionID) - output.Manifest.Metadata, err = manifestMetadataWithSessionID(output.Manifest.Metadata, sessionID) - if err != nil { - return failOutput(output), err - } output.Manifest.SourceDigests = []string{doc.Digest} chunker := input.Prepared.chunker