Clean up roadmap and fallback profile guidance
This commit is contained in:
@@ -53,14 +53,18 @@ var applicationProfiles embed.FS
|
|||||||
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
PromptDir: "prompts",
|
PromptDir: "prompts",
|
||||||
ProfileDir: "profiles",
|
ProfileDir: operatorProfileDir,
|
||||||
},
|
},
|
||||||
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
|
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
Keep application-owned profile IDs and definitions in the embedded source.
|
Keep application-owned profile IDs and definitions in the embedded source.
|
||||||
Use the ordinary configured profile source for operator overrides. The
|
Use the ordinary configured profile source for operator overrides. Leave
|
||||||
|
`operatorProfileDir` empty when the operator did not configure an override
|
||||||
|
directory; a non-empty path names an authoritative higher-precedence source,
|
||||||
|
so an unavailable or unreadable directory is an error rather than a reason to
|
||||||
|
fall back. The
|
||||||
[framework format reference](../formats.md#source-and-profile-precedence)
|
[framework format reference](../formats.md#source-and-profile-precedence)
|
||||||
owns the exact profile format and lookup order; the
|
owns the exact profile format and lookup order; the
|
||||||
[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and
|
[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and
|
||||||
|
|||||||
@@ -44,98 +44,3 @@ Start with:
|
|||||||
For cross-cutting changes, follow every applicable row. Do not create
|
For cross-cutting changes, follow every applicable row. Do not create
|
||||||
placeholder documents for packages, APIs, or integrations that do not yet
|
placeholder documents for packages, APIs, or integrations that do not yet
|
||||||
exist.
|
exist.
|
||||||
|
|
||||||
## Maintainer-Run Validation
|
|
||||||
|
|
||||||
Promptkit does not currently use hosted CI. Maintainers are responsible for
|
|
||||||
running the documented checks before accepting changes. Run the default Go
|
|
||||||
validation from the Promptkit repository root:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
go test -race ./...
|
|
||||||
go vet ./...
|
|
||||||
go build ./...
|
|
||||||
go run ./examples/go-library/prepare
|
|
||||||
```
|
|
||||||
|
|
||||||
Check formatting across every tracked Go file:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
gofmt -l $(git ls-files '*.go')
|
|
||||||
```
|
|
||||||
|
|
||||||
The formatting command must produce no paths. Follow every added or changed
|
|
||||||
Markdown link and confirm its target exists. Finally, check whitespace:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
Documentation-only work does not require unrelated new tests, but it still
|
|
||||||
requires link validation and `git diff --check`. Run the Go validation whenever
|
|
||||||
documentation changes commands, examples, generated output, or another
|
|
||||||
behavior checked by the module.
|
|
||||||
|
|
||||||
## Focused Validation
|
|
||||||
|
|
||||||
Use focused checks while iterating, then run the complete validation sequence
|
|
||||||
before accepting the change. The root package supports:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test .
|
|
||||||
go vet .
|
|
||||||
go build .
|
|
||||||
```
|
|
||||||
|
|
||||||
Filter tests by name without assuming a fixed internal package layout:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./... -run 'TestName'
|
|
||||||
```
|
|
||||||
|
|
||||||
Replace `TestName` with a useful regular expression. Target only paths that
|
|
||||||
exist, and consult the internal component overview for their owning
|
|
||||||
documentation. A filtered or package-specific run does not replace the
|
|
||||||
complete repository validation.
|
|
||||||
|
|
||||||
## Coordinated Work With Scriptorium
|
|
||||||
|
|
||||||
Promptkit and Scriptorium must remain independently valid. For temporary local
|
|
||||||
integration, use either a Go workspace outside both repositories or an
|
|
||||||
uncommitted replacement in the consuming module.
|
|
||||||
|
|
||||||
If the repositories are sibling directories, run the workspace commands from
|
|
||||||
their parent directory:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go work init ./promptkit ./scriptorium
|
|
||||||
go work sync
|
|
||||||
```
|
|
||||||
|
|
||||||
Use the workspace only for coordinated local checks. From the same parent
|
|
||||||
directory, remove it when finished:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
rm -f go.work go.work.sum
|
|
||||||
```
|
|
||||||
|
|
||||||
Alternatively, from the Scriptorium repository root, temporarily point its
|
|
||||||
Promptkit dependency at the sibling checkout:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go mod edit -replace gitea.maximumdirect.net/eric/promptkit=../promptkit
|
|
||||||
```
|
|
||||||
|
|
||||||
After coordinated checks, remove the replacement and reconcile module
|
|
||||||
metadata:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go mod edit -dropreplace gitea.maximumdirect.net/eric/promptkit
|
|
||||||
go mod tidy
|
|
||||||
```
|
|
||||||
|
|
||||||
Never commit `go.work`, `go.work.sum`, or a local filesystem `replace`
|
|
||||||
directive. Before committing in either repository, inspect its module files and
|
|
||||||
working tree independently. Published consumer versions must depend on a tagged
|
|
||||||
Promptkit version, not a workspace, local replacement, or unpublished commit.
|
|
||||||
|
|||||||
82
docs/roadmap/deferred.md
Normal file
82
docs/roadmap/deferred.md
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
# Deferred Feature Ideas
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document catalogs feature ideas that remain potentially useful but have
|
||||||
|
been deliberately postponed. These ideas are not awaiting ordinary selection
|
||||||
|
from the [future feature catalog](future.md); each has a stated reason to wait
|
||||||
|
and should be reconsidered only when its trigger becomes relevant.
|
||||||
|
|
||||||
|
Deferred entries are not commitments, schedules, active implementation plans,
|
||||||
|
or descriptions of current behavior. When an entry is reactivated, move it to
|
||||||
|
`future.md` for evaluation or directly into a focused roadmap after its open
|
||||||
|
design dependencies have been resolved.
|
||||||
|
|
||||||
|
## Deferred Ideas
|
||||||
|
|
||||||
|
### Semantic Execution-Target Fingerprints
|
||||||
|
|
||||||
|
**Reason for deferral:** A stable digest requires a deliberate semantic-
|
||||||
|
equality and versioning design. Notarius can safely use conservative source
|
||||||
|
hashes and a Promptkit release marker today, while Weatherreporter does not
|
||||||
|
currently reuse LLM-dependent checkpoints.
|
||||||
|
|
||||||
|
Promptkit could expose an opaque equality value for a resolved profile and its
|
||||||
|
effective generation target. This would let checkpointing consumers detect
|
||||||
|
generation-affecting configuration changes without hashing YAML presentation
|
||||||
|
or depending on Promptkit's built-in catalog layout.
|
||||||
|
|
||||||
|
The digest should change with semantically relevant state such as the resolved
|
||||||
|
model, endpoint, backend routing identity, request defaults, extra parameters,
|
||||||
|
profile generation settings, and selected built-in profile semantics. It
|
||||||
|
should exclude credential values, concurrency and queue policy, source paths,
|
||||||
|
comments, formatting, and other representation-only changes. Whether a
|
||||||
|
credential environment-variable name affects equality must be decided
|
||||||
|
explicitly. The encoding should remain opaque and internally versioned so
|
||||||
|
Promptkit can deliberately invalidate earlier digests when its resolution
|
||||||
|
semantics change.
|
||||||
|
|
||||||
|
Reconsider this idea when a downstream consumer needs Promptkit-owned
|
||||||
|
checkpoint equality or when a broader semantic identity design is selected.
|
||||||
|
|
||||||
|
### Eager Source Validation
|
||||||
|
|
||||||
|
**Reason for deferral:** Exact prompt and profile inspection may already
|
||||||
|
provide a sufficiently small validation surface. Experience from downstream
|
||||||
|
adoption should establish whether an engine-wide operation would add enough
|
||||||
|
value to justify its broader contract.
|
||||||
|
|
||||||
|
Promptkit could provide an explicit offline operation that discovers and
|
||||||
|
structurally validates configured prompt, profile, and schema sources without
|
||||||
|
model generation. The normal `NewEngine` path would remain lazy.
|
||||||
|
|
||||||
|
An eager operation would need coherent handling for duplicate prompt IDs and
|
||||||
|
versions, strict YAML decoding, referenced content files, profile/backend
|
||||||
|
membership, schema syntax and transitive references, context cancellation,
|
||||||
|
and source-specific public errors. Credential declarations must remain
|
||||||
|
separate from credential values; checking current environment availability,
|
||||||
|
if supported at all, should be an explicit option and must not expose secrets.
|
||||||
|
|
||||||
|
Reconsider this idea after downstream use of `InspectPrompt`,
|
||||||
|
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.
|
||||||
|
|
||||||
|
### Structured Generation Errors
|
||||||
|
|
||||||
|
**Reason for deferral:** Existing `ErrLLMGenerate` classification, preserved
|
||||||
|
injected-client errors, and prepared execution details currently provide the
|
||||||
|
necessary failure boundary. A typed error should wait for stronger downstream
|
||||||
|
demand and a transport-neutral field design.
|
||||||
|
|
||||||
|
Promptkit could expose safe structured generation context through
|
||||||
|
`errors.As` while preserving `errors.Is(err, ErrLLMGenerate)`. Potential
|
||||||
|
fields include the selected backend ID and model plus an optional HTTP status
|
||||||
|
when the built-in OpenAI-compatible transport supplies one.
|
||||||
|
|
||||||
|
The design must not expose provider response bodies, endpoints, credential
|
||||||
|
environment names or values, request content, or generated content. It should
|
||||||
|
not duplicate prompt and profile provenance already available from a prepared
|
||||||
|
execution, and it must preserve the identity of errors returned by injected
|
||||||
|
clients. Retry and backoff policy remains a consumer responsibility.
|
||||||
|
|
||||||
|
Reconsider this idea when consumers need structured generation diagnostics
|
||||||
|
beyond the existing sentinel, wrapped client error, and preparation record.
|
||||||
@@ -1,152 +0,0 @@
|
|||||||
# Application Fallback Profiles
|
|
||||||
|
|
||||||
Status: Complete.
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Promptkit will allow a consuming application to supply an embedded fallback
|
|
||||||
profile source below operator-configured profiles and above Promptkit's
|
|
||||||
built-in profile catalog.
|
|
||||||
|
|
||||||
This gives consumers stable, application-owned profile IDs with useful
|
|
||||||
packaged defaults while preserving the existing ability for an operator to
|
|
||||||
replace those definitions. Promptkit will own the reusable source layer and
|
|
||||||
lookup semantics without owning downstream profile names, model assignments,
|
|
||||||
or application configuration policy.
|
|
||||||
|
|
||||||
## Consumer Outcome
|
|
||||||
|
|
||||||
A consumer such as Weatherreporter can embed profiles named for application
|
|
||||||
workloads or execution tiers, such as `weather-light`, `weather-balanced`, and
|
|
||||||
`weather-deep`. Prompts can select those logical IDs without coupling the
|
|
||||||
application to a particular provider or model.
|
|
||||||
|
|
||||||
An operator can define the same profile ID in the application's ordinary
|
|
||||||
configured profile source to replace the packaged default. If no operator
|
|
||||||
definition exists, Promptkit resolves the application's embedded definition.
|
|
||||||
If neither source contains the ID, Promptkit retains access to its own built-in
|
|
||||||
profile catalog.
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
|
|
||||||
Promptkit will add one optional `fs.FS`-backed application fallback profile
|
|
||||||
source to engine construction. The intended public surface is:
|
|
||||||
|
|
||||||
```go
|
|
||||||
promptkit.WithFallbackProfileFS(profileFS, ".")
|
|
||||||
```
|
|
||||||
|
|
||||||
The source will use the existing profile YAML format, discovery behavior,
|
|
||||||
strict decoding, validation rules, and credential restrictions. The option
|
|
||||||
will accept a non-nil filesystem and nonblank root. Repeating the option will
|
|
||||||
follow Promptkit's same-category convention: the last valid value replaces the
|
|
||||||
earlier fallback source, while an invalid option still fails construction when
|
|
||||||
it is applied.
|
|
||||||
|
|
||||||
No programmatic fallback-profile option is included. Consumers that need the
|
|
||||||
new precedence relationship can embed YAML assets, while `WithProfiles`
|
|
||||||
continues to serve the distinct highest-precedence in-memory use case.
|
|
||||||
|
|
||||||
## Profile Selection And Source Precedence
|
|
||||||
|
|
||||||
Profile ID selection remains separate from profile definition lookup. An
|
|
||||||
explicit request profile ID continues to take precedence over a prompt's
|
|
||||||
`default_profile`. After an ID has been selected, matching definitions resolve
|
|
||||||
in this order:
|
|
||||||
|
|
||||||
1. in-memory profiles supplied through `WithProfiles`;
|
|
||||||
2. the ordinary configured profile source selected through `WithProfileFile`,
|
|
||||||
`WithProfileFS`, or `Config.ProfileDir`;
|
|
||||||
3. the application fallback profile source; and
|
|
||||||
4. Promptkit's embedded built-in profiles.
|
|
||||||
|
|
||||||
A higher-precedence source falls through only when the requested profile ID is
|
|
||||||
absent. An unreadable, malformed, duplicate, ambiguous, or otherwise invalid
|
|
||||||
matching definition is an error and does not permit lookup in a lower layer.
|
|
||||||
Profiles are selected as complete values; sources do not merge fields or
|
|
||||||
inherit from one another.
|
|
||||||
|
|
||||||
The fallback source remains lazily read and validated when a requested ID
|
|
||||||
reaches that layer. This feature does not introduce engine-wide eager source
|
|
||||||
validation, and an unrelated malformed asset does not acquire stronger
|
|
||||||
validation guarantees than it has in an ordinary profile source.
|
|
||||||
|
|
||||||
## Consistent Engine Behavior
|
|
||||||
|
|
||||||
The engine will assemble one profile repository with the complete precedence
|
|
||||||
chain. Exact profile inspection, ordinary preparation, prepared execution, and
|
|
||||||
ordinary execution will all use that same repository and therefore observe the
|
|
||||||
same definition for a given profile ID.
|
|
||||||
|
|
||||||
Exact profile inspection remains side-effect free and does not contact a model
|
|
||||||
provider. Existing public error identities remain applicable to cancellation,
|
|
||||||
profile-not-found and profile-load failures, invalid fallback definitions,
|
|
||||||
credential resolution, and unknown backends. The feature does not add a new
|
|
||||||
fallback-specific public error category.
|
|
||||||
|
|
||||||
Internally, the root facade will own composition of all profile-source layers.
|
|
||||||
The application fallback is another use of the profile repository's existing
|
|
||||||
error-preserving overlay semantics; it is not a separate profile-loading or
|
|
||||||
validation implementation.
|
|
||||||
|
|
||||||
## Compatibility
|
|
||||||
|
|
||||||
The feature is additive. An engine that does not configure an application
|
|
||||||
fallback source retains its current behavior and profile precedence. Existing
|
|
||||||
uses of `Config.ProfileDir`, `WithProfileFile`, `WithProfileFS`, and
|
|
||||||
`WithProfiles` keep their meanings.
|
|
||||||
|
|
||||||
The application fallback is deliberately below every existing
|
|
||||||
consumer-configured profile source. An ordinary configured profile therefore
|
|
||||||
continues to override any packaged definition with the same ID. A missing or
|
|
||||||
invalid configured source also retains its current behavior; the new layer
|
|
||||||
does not turn configuration failures into silent fallthrough.
|
|
||||||
|
|
||||||
## Ownership Boundaries
|
|
||||||
|
|
||||||
Promptkit owns:
|
|
||||||
|
|
||||||
- the additional source layer and its construction option;
|
|
||||||
- deterministic lookup and fallthrough semantics;
|
|
||||||
- use of the existing profile format, validation, and public error mapping;
|
|
||||||
- consistent repository use across inspection, preparation, and execution;
|
|
||||||
and
|
|
||||||
- canonical public, format, consumer, and contributor documentation for the
|
|
||||||
implemented capability.
|
|
||||||
|
|
||||||
The consuming application owns:
|
|
||||||
|
|
||||||
- whether to supply a fallback source;
|
|
||||||
- application-specific profile IDs and their domain meaning;
|
|
||||||
- embedded profile contents, backend selections, and model choices;
|
|
||||||
- application configuration discovery and operator override policy;
|
|
||||||
- assignment of profiles to reports or other workloads; and
|
|
||||||
- credential policy and operator-facing error presentation beyond Promptkit's
|
|
||||||
public contract.
|
|
||||||
|
|
||||||
## Non-Goals
|
|
||||||
|
|
||||||
This scope does not add:
|
|
||||||
|
|
||||||
- downstream-specific profiles to Promptkit's built-in catalog;
|
|
||||||
- profile inheritance, aliases, or field-level merging;
|
|
||||||
- provider failover after a selected profile or generation attempt fails;
|
|
||||||
- automatic endpoint discovery, probing, benchmarking, or tier selection;
|
|
||||||
- application configuration discovery;
|
|
||||||
- eager validation of every profile in every source;
|
|
||||||
- source-provenance fields in profile inspection or preparation results; or
|
|
||||||
- an in-memory companion to `WithFallbackProfileFS`.
|
|
||||||
|
|
||||||
## Target End State
|
|
||||||
|
|
||||||
The feature is complete when a consumer can embed an application fallback
|
|
||||||
profile source through the root facade and rely on the documented four-layer
|
|
||||||
precedence everywhere profiles are resolved. Operator definitions override
|
|
||||||
application defaults, application defaults override Promptkit built-ins,
|
|
||||||
invalid matching definitions never silently fall through, and engines that do
|
|
||||||
not use the new option remain behaviorally compatible.
|
|
||||||
|
|
||||||
The implemented public option and precedence contract will be owned by GoDoc
|
|
||||||
and the framework format reference. Consumer guidance will show the embedded
|
|
||||||
application-default workflow without making this temporary roadmap a second
|
|
||||||
current-state reference.
|
|
||||||
@@ -12,6 +12,9 @@ consumer value, and important scope boundaries. Defer API design,
|
|||||||
implementation details, sequencing, and acceptance criteria until an idea is
|
implementation details, sequencing, and acceptance criteria until an idea is
|
||||||
selected.
|
selected.
|
||||||
|
|
||||||
|
Ideas that have been deliberately postponed rather than left available for
|
||||||
|
ordinary selection belong in the [deferred catalog](deferred.md).
|
||||||
|
|
||||||
## Using This Catalog
|
## Using This Catalog
|
||||||
|
|
||||||
- Add an idea when its purpose and likely value can be stated clearly.
|
- Add an idea when its purpose and likely value can be stated clearly.
|
||||||
@@ -23,6 +26,8 @@ selected.
|
|||||||
- When an idea is selected, move its active planning to a focused roadmap or,
|
- When an idea is selected, move its active planning to a focused roadmap or,
|
||||||
when it requires a durable architectural decision, an ADR. Update
|
when it requires a durable architectural decision, an ADR. Update
|
||||||
current-state documentation only when implementation lands.
|
current-state documentation only when implementation lands.
|
||||||
|
- Move an idea to `deferred.md` when maintainers decide to retain it but wait
|
||||||
|
for a stated design dependency, demand signal, or reconsideration trigger.
|
||||||
- Remove ideas that are no longer relevant. Retain a rejected idea only when
|
- Remove ideas that are no longer relevant. Retain a rejected idea only when
|
||||||
its rationale is likely to prevent repeated reconsideration.
|
its rationale is likely to prevent repeated reconsideration.
|
||||||
|
|
||||||
|
|||||||
@@ -1,502 +0,0 @@
|
|||||||
# Application Fallback Profiles Implementation Plan
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This document is the decision-complete implementation plan for
|
|
||||||
[application fallback profiles](fallback-profiles.md). It is written for a
|
|
||||||
`gpt-5.6-terra` coding agent that will implement each stage in order.
|
|
||||||
|
|
||||||
The feature roadmap owns the motivation, policy choices, compatibility
|
|
||||||
boundary, non-goals, and target end state. This document owns the concrete
|
|
||||||
design, file-level work, test ownership, documentation updates, validation,
|
|
||||||
and completion gates.
|
|
||||||
|
|
||||||
## Implementation Rules
|
|
||||||
|
|
||||||
- Complete the stages in order. Every stage must leave the repository
|
|
||||||
buildable, tested for the behavior changed in that stage, and accurately
|
|
||||||
documented for its implemented state.
|
|
||||||
- Preserve unrelated working-tree changes. Inspect `git status --short` and
|
|
||||||
the relevant diffs before editing, and do not overwrite or reformat
|
|
||||||
pre-existing user work.
|
|
||||||
- Follow every policy under `docs/policy/`, the task-specific reading guide in
|
|
||||||
`docs/development.md`, and the accepted behavior in
|
|
||||||
`fallback-profiles.md`.
|
|
||||||
- Keep the module root as the public facade. The root package owns profile
|
|
||||||
source selection and composition; `internal/profile` owns repository,
|
|
||||||
parsing, validation, and error-preserving overlay behavior; and
|
|
||||||
`internal/profile/builtin` owns only the embedded built-in catalog.
|
|
||||||
- Add only the public `WithFallbackProfileFS` option. Do not add an exported
|
|
||||||
repository type, source-provenance value, fallback-specific error, config
|
|
||||||
field, programmatic fallback-profile option, or public package.
|
|
||||||
- Reuse `profile.NewFSRepository` and `profile.NewOverlayRepository`. Do not
|
|
||||||
add another parser, validator, repository implementation, or merge model.
|
|
||||||
- Preserve lazy loading. Engine construction validates the option arguments,
|
|
||||||
not every file in the supplied filesystem. A profile source is read only
|
|
||||||
when resolution reaches it.
|
|
||||||
- Preserve existing error identities and mappings. Only
|
|
||||||
`profile.ErrProfileNotFound` permits an overlay to consult its next layer;
|
|
||||||
every other error from a higher layer must be returned and mapped through
|
|
||||||
the existing public profile-load path.
|
|
||||||
- Keep tests classical and behavior-focused. Public source precedence and
|
|
||||||
exported option semantics belong in external-package root tests; generic
|
|
||||||
overlay behavior remains owned by `internal/profile` tests.
|
|
||||||
- Update GoDoc and current-state documentation in the same stage that exposes
|
|
||||||
the public option. Do not describe the feature as implemented before that
|
|
||||||
stage is complete.
|
|
||||||
- Do not add release notes, change a module version, commit, tag, push, or
|
|
||||||
publish a release as part of this plan.
|
|
||||||
|
|
||||||
## Fixed Design
|
|
||||||
|
|
||||||
### Public API
|
|
||||||
|
|
||||||
Add this function to `engine.go` beside the existing profile-source options:
|
|
||||||
|
|
||||||
```go
|
|
||||||
func WithFallbackProfileFS(fsys fs.FS, root string) Option
|
|
||||||
```
|
|
||||||
|
|
||||||
The option accepts the same filesystem and root forms as `WithProfileFS` and
|
|
||||||
constructs its repository with `profile.NewFSRepository(fsys, root)`. It must
|
|
||||||
return `ErrInvalidConfig` from option application when `fsys` is nil or when
|
|
||||||
`strings.TrimSpace(root)` is empty. Do not normalize or replace a valid root
|
|
||||||
before passing it to the repository.
|
|
||||||
|
|
||||||
The fallback source is its own last-value-wins option category. Add these two
|
|
||||||
private fields to `engineOptions`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
fallbackProfiles profile.Repository
|
|
||||||
fallbackProfileSource bool
|
|
||||||
```
|
|
||||||
|
|
||||||
Use the repository field for the selected source and the boolean only to
|
|
||||||
distinguish an unapplied option from an applied option. A later valid
|
|
||||||
`WithFallbackProfileFS` replaces both values. Option application remains
|
|
||||||
sequential, so an invalid option fails `NewEngine` immediately even if a later
|
|
||||||
option could otherwise replace it.
|
|
||||||
|
|
||||||
The exact GoDoc for `WithFallbackProfileFS` must state:
|
|
||||||
|
|
||||||
- that it supplies application-owned fallback profile definitions;
|
|
||||||
- the full four-layer lookup order;
|
|
||||||
- that definitions are whole profiles and are not field-merged;
|
|
||||||
- that only a missing ID falls through, while a matching read, parse,
|
|
||||||
duplicate, validation, or credential-format failure stops resolution;
|
|
||||||
- that loading and validation are lazy;
|
|
||||||
- that files use the ordinary strict profile YAML and `api_key_env` rules;
|
|
||||||
- that nil filesystems and blank roots cause `NewEngine` to match
|
|
||||||
`ErrInvalidConfig`;
|
|
||||||
- that repeated calls use the last valid fallback source; and
|
|
||||||
- that this is definition lookup, not provider or generation failover.
|
|
||||||
|
|
||||||
Update the `Option`, `Config.ProfileDir`, `WithProfileFS`, `WithProfileFile`,
|
|
||||||
and `WithProfiles` GoDoc in `engine.go` where necessary so their relative
|
|
||||||
precedence is unambiguous. Exact declarations and behavior remain owned by
|
|
||||||
GoDoc; consumer documentation should summarize the workflow and link readers
|
|
||||||
back to the API rather than reproduce every error clause.
|
|
||||||
|
|
||||||
### Repository Composition
|
|
||||||
|
|
||||||
Move all profile-source composition into one private root helper in
|
|
||||||
`engine.go`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
func newProfileRepository(profileDir string, options engineOptions) profile.Repository
|
|
||||||
```
|
|
||||||
|
|
||||||
`NewEngine` must call this helper once and pass its returned repository to the
|
|
||||||
runner. The helper must build from lowest to highest precedence:
|
|
||||||
|
|
||||||
1. begin with `builtin.NewRepository()`;
|
|
||||||
2. if `options.fallbackProfileSource` is true, overlay
|
|
||||||
`options.fallbackProfiles` over the built-in repository;
|
|
||||||
3. select exactly one ordinary configured source: use `options.profiles` when
|
|
||||||
`options.profileSource` is true; otherwise, when `profileDir` is nonblank,
|
|
||||||
use `profile.NewFilesystemRepository(profileDir)`; overlay that selected
|
|
||||||
source over the current repository;
|
|
||||||
4. if `options.memorySource` is true, overlay `options.memoryProfiles` over
|
|
||||||
the current repository; and
|
|
||||||
5. return the resulting chain.
|
|
||||||
|
|
||||||
This preserves the existing rule that `WithProfileFS` or `WithProfileFile`
|
|
||||||
replaces `Config.ProfileDir`; those sources are alternatives in one ordinary
|
|
||||||
configured-source category, not two independent layers. `WithProfiles` remains
|
|
||||||
a distinct highest-precedence category.
|
|
||||||
|
|
||||||
The final lookup order is therefore:
|
|
||||||
|
|
||||||
```text
|
|
||||||
WithProfiles
|
|
||||||
-> WithProfileFile / WithProfileFS / Config.ProfileDir
|
|
||||||
-> WithFallbackProfileFS
|
|
||||||
-> Promptkit built-ins
|
|
||||||
```
|
|
||||||
|
|
||||||
Every arrow is a whole-profile, not-found-only fallback. Do not inspect or
|
|
||||||
copy profile fields in the composition helper.
|
|
||||||
|
|
||||||
### Built-In Package Boundary
|
|
||||||
|
|
||||||
Reduce `internal/profile/builtin/repository.go` to the embedded catalog leaf:
|
|
||||||
|
|
||||||
```go
|
|
||||||
func NewRepository() profile.Repository
|
|
||||||
```
|
|
||||||
|
|
||||||
Remove `NewRepositoryWithPrimary` and `NewRepositoryWithDirectory`. Remove
|
|
||||||
their now-obsolete tests from `internal/profile/builtin/repository_test.go` and
|
|
||||||
remove imports used only by those helpers or tests. Do not move their tests to
|
|
||||||
another private helper: existing root public-behavior tests own assembled
|
|
||||||
precedence, and `internal/profile.TestOverlayRepository` owns not-found-only
|
|
||||||
overlay semantics.
|
|
||||||
|
|
||||||
Do not change built-in YAML assets, built-in validation, the `profile.Repository`
|
|
||||||
interface, `profile.NewOverlayRepository`, or filesystem repository behavior.
|
|
||||||
|
|
||||||
### Resolution And Error Semantics
|
|
||||||
|
|
||||||
The runner receives one assembled `profile.Repository`; do not add fallback
|
|
||||||
logic to `InspectProfile`, `Prepare`, `PrepareExecution`, `Run`, or
|
|
||||||
`RunPrepared`. Those paths must continue to resolve through the runner's one
|
|
||||||
repository dependency.
|
|
||||||
|
|
||||||
The existing overlay contract is authoritative:
|
|
||||||
|
|
||||||
- a successful lookup returns the complete higher-layer profile;
|
|
||||||
- `profile.ErrProfileNotFound` consults the next layer;
|
|
||||||
- cancellation, filesystem read failures, malformed YAML, duplicate matches,
|
|
||||||
raw `api_key`, invalid profiles, and all other errors stop lookup; and
|
|
||||||
- the root facade maps failures through the existing public identities such as
|
|
||||||
`ErrProfileNotFound` and `ErrProfileLoad`.
|
|
||||||
|
|
||||||
Do not add eager filesystem walking in the option or `NewEngine`. A malformed
|
|
||||||
asset unrelated to the requested ID retains the existing ordinary
|
|
||||||
`FSRepository` behavior; this plan does not strengthen that package's global
|
|
||||||
validation guarantees.
|
|
||||||
|
|
||||||
### Test Ownership
|
|
||||||
|
|
||||||
Use the following test boundaries and avoid duplicating the profile parser's
|
|
||||||
existing case matrix.
|
|
||||||
|
|
||||||
In `public_contract_test.go`:
|
|
||||||
|
|
||||||
- Add `TestFallbackProfileSourcePrecedence`. Use minimal synthetic
|
|
||||||
`fstest.MapFS` profiles and, where `Config.ProfileDir` is under test, a
|
|
||||||
`t.TempDir`. Cover these distinct relationships: an in-memory profile beats
|
|
||||||
both ordinary and fallback definitions; an ordinary `WithProfileFS` source
|
|
||||||
beats a fallback definition; `Config.ProfileDir` beats a fallback
|
|
||||||
definition when no ordinary source option replaces it; a fallback
|
|
||||||
definition beats a built-in definition with the same ID; and an ID absent
|
|
||||||
from the fallback source still resolves from the built-in catalog. Assert
|
|
||||||
the selected model or another stable complete-profile field rather than
|
|
||||||
internal repository structure.
|
|
||||||
- Extend `TestRepeatedOptionsUseLastValueInEachCategory` with a
|
|
||||||
`fallback profile source` subtest proving that the later valid fallback
|
|
||||||
filesystem is selected.
|
|
||||||
- Add `TestFallbackProfileSourcePreservesLazyLoadingAndErrors`. Prove that
|
|
||||||
engine construction succeeds without reading malformed fallback YAML, that
|
|
||||||
an unrelated malformed file does not prevent a valid requested fallback
|
|
||||||
definition from resolving under existing FS-repository semantics, that a
|
|
||||||
malformed fallback file whose stem matches a built-in profile ID yields
|
|
||||||
`ErrProfileLoad` instead of silently reaching the built-in, and that a
|
|
||||||
malformed ordinary configured definition yields `ErrProfileLoad` instead of
|
|
||||||
reaching a valid application fallback definition. Use `errors.Is`; do not
|
|
||||||
assert complete error strings.
|
|
||||||
- Add one representative workflow test that supplies a fallback-only profile
|
|
||||||
and verifies the same effective model through `InspectProfile`, `Prepare`,
|
|
||||||
a `PrepareExecution` followed by `RunPrepared`, and direct `Run`. Use the
|
|
||||||
existing deterministic injected-client style, no live provider, and no real
|
|
||||||
credential. This test owns the cross-workflow repository wiring; do not
|
|
||||||
repeat the full precedence matrix through every method.
|
|
||||||
|
|
||||||
In `engine_test.go`, extend `TestSourceOptionsRejectInvalidInputs` with nil
|
|
||||||
filesystem and blank-root cases for `WithFallbackProfileFS`. Both must make
|
|
||||||
`NewEngine` match `ErrInvalidConfig`.
|
|
||||||
|
|
||||||
Retain `internal/profile.TestOverlayRepository` unchanged unless a genuine
|
|
||||||
existing defect is found. It already owns success, not-found fallback, and
|
|
||||||
non-not-found error preservation. Do not add package-private tests for the new
|
|
||||||
root helper, snapshots, golden files, provider calls, or one test per profile
|
|
||||||
format error already covered by `internal/profile`.
|
|
||||||
|
|
||||||
### Canonical Documentation
|
|
||||||
|
|
||||||
Update current-state documentation when the option is implemented:
|
|
||||||
|
|
||||||
- In `docs/formats.md`, make the source-precedence section the canonical
|
|
||||||
four-layer definition lookup order. State that ordinary configured sources
|
|
||||||
override application fallbacks, application fallbacks override built-ins,
|
|
||||||
profiles are whole values, and only a missing ID falls through. Retain this
|
|
||||||
document's ownership of strict YAML, credentials, validation, and source
|
|
||||||
discovery details.
|
|
||||||
- In `docs/consumers/pkg-promptkit.md`, add a short task-oriented section that
|
|
||||||
shows an illustrative `embed.FS` declaration and
|
|
||||||
`WithFallbackProfileFS`. Explain that application defaults belong in the
|
|
||||||
embedded fallback and operator overrides belong in the ordinary configured
|
|
||||||
source. Link to `docs/formats.md` for exact format and precedence rules, and
|
|
||||||
do not turn the snippet into a second complete maintained application.
|
|
||||||
- In `docs/internal/sources.md`, describe the root-owned four-layer
|
|
||||||
composition and the existing not-found-only overlay mechanism. Remove any
|
|
||||||
claim that the built-in package composes caller-selected repositories.
|
|
||||||
- In `docs/internal/overview.md`, keep the root facade responsible for source
|
|
||||||
assembly, describe `internal/profile/builtin` only as the embedded catalog,
|
|
||||||
and reflect the implemented fallback layer without duplicating the exact
|
|
||||||
public API contract.
|
|
||||||
- In `engine.go`, apply the GoDoc changes under Public API. GoDoc owns the
|
|
||||||
exact option signature, validation, category, and public semantics.
|
|
||||||
|
|
||||||
The architecture policy already assigns assembly to the root facade and the
|
|
||||||
built-in catalog to `internal/profile/builtin`; do not edit it unless the
|
|
||||||
implementation reveals an actual contradiction. No integration protocol,
|
|
||||||
outbound request body, profile YAML shape, README orientation, or maintained
|
|
||||||
example changes as part of this feature.
|
|
||||||
|
|
||||||
## Stage 1: Move Existing Profile Composition To The Root Facade
|
|
||||||
|
|
||||||
### Objective
|
|
||||||
|
|
||||||
Establish the intended ownership boundary and a single root composition point
|
|
||||||
without changing public behavior or adding the fallback option.
|
|
||||||
|
|
||||||
### Implementation Prompt
|
|
||||||
|
|
||||||
Implement only Stage 1 of `docs/roadmap/implementation.md`. Read the complete
|
|
||||||
feature roadmap, implementation rules, and fixed design above before editing.
|
|
||||||
|
|
||||||
1. In `engine.go`, add `newProfileRepository(profileDir string, options
|
|
||||||
engineOptions) profile.Repository` and move the existing three-layer
|
|
||||||
assembly into it: built-ins, then the selected ordinary configured source,
|
|
||||||
then `WithProfiles`. Do not add fallback fields or the public option yet.
|
|
||||||
2. Replace the inline profile assembly in `NewEngine` with one call to the
|
|
||||||
helper. Preserve the existing replacement relationship between
|
|
||||||
`Config.ProfileDir` and `WithProfileFS`/`WithProfileFile`.
|
|
||||||
3. In `internal/profile/builtin/repository.go`, remove
|
|
||||||
`NewRepositoryWithPrimary` and `NewRepositoryWithDirectory`, leaving
|
|
||||||
`NewRepository` as the only constructor.
|
|
||||||
4. Remove the three tests dedicated to the deleted built-in composition
|
|
||||||
helpers from `internal/profile/builtin/repository_test.go`. Preserve tests
|
|
||||||
that validate the embedded catalog itself.
|
|
||||||
5. Update `docs/internal/sources.md` and `docs/internal/overview.md` so they
|
|
||||||
describe the implemented Stage 1 ownership accurately. At this boundary
|
|
||||||
the source order is still in-memory, ordinary configured source, built-ins;
|
|
||||||
do not document the application fallback as implemented yet.
|
|
||||||
6. Run the focused validation below. Fix in-scope regressions without adding
|
|
||||||
fallback behavior early.
|
|
||||||
|
|
||||||
Do not add or mention an implemented `WithFallbackProfileFS` in Stage 1. Do
|
|
||||||
not change exported declarations, profile parsing, profile assets, error
|
|
||||||
mapping, runner behavior, or consumer and format documentation.
|
|
||||||
|
|
||||||
### Focused Validation
|
|
||||||
|
|
||||||
Run from the repository root:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
gofmt -w engine.go internal/profile/builtin/repository.go \
|
|
||||||
internal/profile/builtin/repository_test.go
|
|
||||||
go test . -run \
|
|
||||||
'Test(PrepareUsesBuiltInProfileWithoutProfileDir|CustomProfileOverridesBuiltInProfile|InMemoryProfilesOverrideBuiltInsAndProfileSources)$'
|
|
||||||
go test ./internal/profile/...
|
|
||||||
go test . ./internal/profile/...
|
|
||||||
go vet . ./internal/profile/...
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
If a focused expression does not match an existing test name, inspect the
|
|
||||||
current suite and run the narrowest equivalent public precedence coverage; do
|
|
||||||
not silently skip the intended relationship.
|
|
||||||
|
|
||||||
### Completion Gate
|
|
||||||
|
|
||||||
Stage 1 is complete only when:
|
|
||||||
|
|
||||||
- the root facade assembles the unchanged three-layer profile chain in one
|
|
||||||
private helper;
|
|
||||||
- ordinary source options still replace `Config.ProfileDir` and in-memory
|
|
||||||
profiles still have highest precedence;
|
|
||||||
- built-in profiles remain available and remain lower than consumer sources;
|
|
||||||
- only `internal/profile` owns generic overlay behavior and the built-in
|
|
||||||
package owns only its embedded catalog;
|
|
||||||
- no public API or behavior changed;
|
|
||||||
- internal current-state documentation matches that boundary; and
|
|
||||||
- all focused tests, vet, formatting, and whitespace checks pass.
|
|
||||||
|
|
||||||
## Stage 2: Add Application Fallback Profiles And Public Contracts
|
|
||||||
|
|
||||||
### Objective
|
|
||||||
|
|
||||||
Add the public option, insert the application fallback into the root-owned
|
|
||||||
repository chain, prove its precedence and failure behavior through public
|
|
||||||
workflows, and publish the canonical current-state documentation.
|
|
||||||
|
|
||||||
### Implementation Prompt
|
|
||||||
|
|
||||||
Implement only Stage 2 of `docs/roadmap/implementation.md` after Stage 1
|
|
||||||
satisfies its completion gate.
|
|
||||||
|
|
||||||
1. Add `fallbackProfiles` and `fallbackProfileSource` to `engineOptions`, then
|
|
||||||
implement `WithFallbackProfileFS` exactly as specified under Public API.
|
|
||||||
2. Extend `newProfileRepository` so it constructs the fixed four-layer chain
|
|
||||||
in the prescribed low-to-high order. Do not alter generic overlay logic or
|
|
||||||
add fallback branches to runner methods.
|
|
||||||
3. Update all affected `engine.go` GoDoc, including the option category list
|
|
||||||
and the relative precedence descriptions for existing profile sources.
|
|
||||||
4. Add and extend the external-package root tests exactly as specified under
|
|
||||||
Test Ownership. Reuse small existing fakes and fixture helpers where they
|
|
||||||
remain clear; add only minimal synthetic YAML helpers needed by these tests.
|
|
||||||
5. Extend `TestSourceOptionsRejectInvalidInputs` with the two fallback option
|
|
||||||
validation cases.
|
|
||||||
6. Update `docs/formats.md`, `docs/consumers/pkg-promptkit.md`,
|
|
||||||
`docs/internal/sources.md`, and `docs/internal/overview.md` according to
|
|
||||||
Canonical Documentation.
|
|
||||||
7. Run the focused validation below. Repair in-scope failures without
|
|
||||||
weakening existing parser, error-identity, prepared-execution, or profile
|
|
||||||
precedence guarantees.
|
|
||||||
|
|
||||||
Do not add a config field, in-memory fallback API, source provenance, profile
|
|
||||||
inheritance, provider failover, eager validation, application-specific assets,
|
|
||||||
or backend/model policy. Do not edit `internal/profile/builtin/assets/`.
|
|
||||||
|
|
||||||
### Focused Validation
|
|
||||||
|
|
||||||
Run from the repository root:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
gofmt -w engine.go engine_test.go public_contract_test.go
|
|
||||||
go test . -run \
|
|
||||||
'Test(FallbackProfileSource|RepeatedOptionsUseLastValueInEachCategory|SourceOptionsRejectInvalidInputs)'
|
|
||||||
go test ./internal/profile/...
|
|
||||||
go test . ./internal/profile/... ./internal/usecase
|
|
||||||
go vet . ./internal/profile/... ./internal/usecase
|
|
||||||
git diff --check
|
|
||||||
```
|
|
||||||
|
|
||||||
The focused root expression must execute the precedence, lazy/error,
|
|
||||||
cross-workflow, repeated-option, and invalid-input coverage described above.
|
|
||||||
If the implemented names differ slightly, run explicit equivalent expressions
|
|
||||||
and record no skipped contract category.
|
|
||||||
|
|
||||||
### Completion Gate
|
|
||||||
|
|
||||||
Stage 2 is complete only when:
|
|
||||||
|
|
||||||
- `WithFallbackProfileFS` is the sole new public declaration and has complete,
|
|
||||||
accurate GoDoc;
|
|
||||||
- nil filesystem and blank root inputs fail construction with
|
|
||||||
`ErrInvalidConfig`, and the last valid repeated fallback option wins;
|
|
||||||
- the assembled order is in-memory, ordinary configured, application
|
|
||||||
fallback, built-ins;
|
|
||||||
- existing ordinary source options still replace `Config.ProfileDir`;
|
|
||||||
- lookup falls through only on a missing ID and never after a matching
|
|
||||||
higher-layer failure;
|
|
||||||
- profiles remain whole values and loading remains lazy;
|
|
||||||
- inspection, preparation, prepared execution, and ordinary execution resolve
|
|
||||||
the same fallback definition through one runner repository;
|
|
||||||
- no built-in asset, profile format, public error identity, provider request,
|
|
||||||
or existing consumer behavior changed unintentionally;
|
|
||||||
- GoDoc and all affected canonical documents describe implemented behavior
|
|
||||||
without duplicating ownership; and
|
|
||||||
- all focused tests, vet, formatting, links, and whitespace checks pass.
|
|
||||||
|
|
||||||
## Stage 3: Audit Compatibility And Validate The Repository
|
|
||||||
|
|
||||||
### Objective
|
|
||||||
|
|
||||||
Confirm that the implementation is complete, minimal, and consistent across
|
|
||||||
the public facade, internal boundaries, tests, and documentation, then mark
|
|
||||||
the temporary planning documents complete.
|
|
||||||
|
|
||||||
### Implementation Prompt
|
|
||||||
|
|
||||||
Implement only Stage 3 of `docs/roadmap/implementation.md` after Stage 2
|
|
||||||
satisfies its completion gate.
|
|
||||||
|
|
||||||
1. Search tracked Go and Markdown files for
|
|
||||||
`NewRepositoryWithPrimary`, `NewRepositoryWithDirectory`, profile source
|
|
||||||
precedence lists, and descriptions of built-in repository composition.
|
|
||||||
Remove stale references and correct only feature-owned contradictions.
|
|
||||||
2. Review `newProfileRepository` directly and confirm it has exactly four
|
|
||||||
possible layers in the required order, selects only one ordinary configured
|
|
||||||
source, and contains no profile field merging or eager I/O.
|
|
||||||
3. Review the public tests as a suite. Confirm that each distinct risk in Test
|
|
||||||
Ownership is protected once, generic parser and overlay cases remain with
|
|
||||||
`internal/profile`, and no test depends on private helper shape.
|
|
||||||
4. Confirm that `internal/profile/builtin/assets/`, external wire behavior,
|
|
||||||
backend configuration, credential resolution, public result shapes, and
|
|
||||||
stable JSON tags have no feature-related changes.
|
|
||||||
5. Follow every added or changed Markdown link and confirm that its target and
|
|
||||||
relevant heading exist. Verify that exact API details live in GoDoc, exact
|
|
||||||
profile format and precedence details live in `docs/formats.md`, consumer
|
|
||||||
guidance remains task-oriented, and internal documents describe only
|
|
||||||
implementation responsibility.
|
|
||||||
6. Run the complete validation sequence below and repair only in-scope
|
|
||||||
failures.
|
|
||||||
7. After every check passes, change the status of
|
|
||||||
`fallback-profiles.md` and this document to `Complete`. Do not delete or
|
|
||||||
retire either roadmap; retirement is a separate maintainer action.
|
|
||||||
8. Re-run `git diff --check`, inspect `git status --short`, and review the full
|
|
||||||
diff while distinguishing pre-existing user changes from this feature.
|
|
||||||
|
|
||||||
Do not add release notes, change versions, or create a commit, tag, push, or
|
|
||||||
release during this stage.
|
|
||||||
|
|
||||||
### Full Validation
|
|
||||||
|
|
||||||
Run from the repository root:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
gofmt -w engine.go engine_test.go public_contract_test.go \
|
|
||||||
internal/profile/builtin/repository.go \
|
|
||||||
internal/profile/builtin/repository_test.go
|
|
||||||
gofmt -l $(git ls-files '*.go')
|
|
||||||
go test ./...
|
|
||||||
go test -race ./...
|
|
||||||
go vet ./...
|
|
||||||
go build ./...
|
|
||||||
go run ./examples/go-library/prepare
|
|
||||||
git diff --check
|
|
||||||
git status --short
|
|
||||||
```
|
|
||||||
|
|
||||||
The `gofmt -l` command must print no paths. The maintained example must remain
|
|
||||||
offline and require no real credential or provider.
|
|
||||||
|
|
||||||
Inspect the final diff and confirm:
|
|
||||||
|
|
||||||
- only this feature's files and pre-existing user changes are present;
|
|
||||||
- no built-in asset, public value shape, stable JSON tag, provider payload,
|
|
||||||
workspace file, local module replacement, generated binary, or unrelated
|
|
||||||
formatting changed;
|
|
||||||
- deleted built-in composition helpers have no remaining references;
|
|
||||||
- the fallback option reuses the ordinary FS repository and generic overlay;
|
|
||||||
- the root constructs one repository used by every resolution workflow;
|
|
||||||
- the feature roadmap and this plan are both complete; and
|
|
||||||
- no commit, tag, push, or release was created.
|
|
||||||
|
|
||||||
### Completion Gate
|
|
||||||
|
|
||||||
The implementation is complete only when:
|
|
||||||
|
|
||||||
- every Stage 1 and Stage 2 gate remains satisfied;
|
|
||||||
- the ordinary and race-enabled suites pass;
|
|
||||||
- vet, build, formatting, the maintained offline example, Markdown links, and
|
|
||||||
whitespace checks pass;
|
|
||||||
- the four-layer precedence and not-found-only fallthrough are consistent in
|
|
||||||
code, GoDoc, public tests, format reference, consumer guidance, and internal
|
|
||||||
documentation;
|
|
||||||
- engines without `WithFallbackProfileFS` retain their previous behavior;
|
|
||||||
- the public surface contains no speculative companion API or provenance;
|
|
||||||
- both temporary roadmap statuses are `Complete`; and
|
|
||||||
- the repository is ready for maintainer review without a commit or release
|
|
||||||
having been created by this plan.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
None. The feature roadmap and fixed design above fully specify the public API,
|
|
||||||
repository composition, error and validation behavior, compatibility boundary,
|
|
||||||
documentation ownership, test strategy, and staged implementation sequence.
|
|
||||||
@@ -1,245 +0,0 @@
|
|||||||
# Notarius PromptKit Wishlist
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This document records features and interface changes that would be useful
|
|
||||||
additions to PromptKit from the perspective of the maintainers of Notarius, a
|
|
||||||
downstream application that consumes PromptKit.
|
|
||||||
|
|
||||||
PromptKit now provides the capabilities Notarius currently needs. The
|
|
||||||
remaining deferred ideas are optional opportunities to improve checkpointing
|
|
||||||
and operational observability.
|
|
||||||
|
|
||||||
The examples are API sketches intended to communicate the desired capability,
|
|
||||||
not prescriptive names or finalized Go contracts.
|
|
||||||
|
|
||||||
## Priority 1: Atomic Execution With Prepared Details
|
|
||||||
|
|
||||||
**Disposition:** Implemented through [`Engine.PrepareExecution` and
|
|
||||||
`Engine.RunPrepared`](../../engine.go). See the
|
|
||||||
[consumer guidance](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later).
|
|
||||||
A separate `RunDetailed` method is not cataloged.
|
|
||||||
|
|
||||||
### Downstream need
|
|
||||||
|
|
||||||
Notarius needs both:
|
|
||||||
|
|
||||||
- the completed `RunResult`; and
|
|
||||||
- the rendered messages, effective output contract, hashes, and other
|
|
||||||
preparation details exposed by `PreparedRun`.
|
|
||||||
|
|
||||||
Notarius uses the prepared details to construct redaction-aware debug bundles
|
|
||||||
and retain enough information to diagnose model behavior.
|
|
||||||
|
|
||||||
### Implemented behavior
|
|
||||||
|
|
||||||
Notarius can prepare one frozen execution snapshot, retain a caller-owned and
|
|
||||||
credential-redacted `Details` value for its debug bundle, and execute the same
|
|
||||||
snapshot through `RunPrepared`. The opaque handle is engine-bound and
|
|
||||||
single-use; an unused handle can be released with `Discard`. The consumer
|
|
||||||
guide and exported GoDoc own the exact lifecycle and failure contracts.
|
|
||||||
|
|
||||||
### Value to Notarius
|
|
||||||
|
|
||||||
This removes duplicate work from PromptKit-backed calls and ensures that
|
|
||||||
retained debug material corresponds atomically to the actual execution.
|
|
||||||
|
|
||||||
## Priority 2: Prompt-Independent Profile Inspection
|
|
||||||
|
|
||||||
**Disposition:** Implemented as
|
|
||||||
[`Engine.InspectProfile`](../../engine.go). See the
|
|
||||||
[consumer guidance](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work).
|
|
||||||
|
|
||||||
### Downstream need
|
|
||||||
|
|
||||||
Notarius validates configured pipeline profile IDs before beginning a run. It
|
|
||||||
needs to determine whether:
|
|
||||||
|
|
||||||
- a profile exists;
|
|
||||||
- its referenced backend is registered;
|
|
||||||
- its execution target can be resolved; and
|
|
||||||
- it declares a credential requirement that the application may need to
|
|
||||||
enforce.
|
|
||||||
|
|
||||||
This validation should not require model generation.
|
|
||||||
|
|
||||||
### Previous integration
|
|
||||||
|
|
||||||
Before profile inspection was available, Notarius constructed a synthetic
|
|
||||||
prompt using `testing/fstest.MapFS`, supplied a dummy transcript, and called
|
|
||||||
`Engine.Prepare` solely to exercise profile and backend resolution.
|
|
||||||
|
|
||||||
### Value to Notarius
|
|
||||||
|
|
||||||
The implemented interface eliminates a synthetic production-only prompt
|
|
||||||
fixture and establishes a direct, supported contract for configuration-time
|
|
||||||
profile and backend validation.
|
|
||||||
|
|
||||||
## Priority 3: Semantic Execution-Target Fingerprints
|
|
||||||
|
|
||||||
**Disposition:** Deferred pending a separate semantic-equality design for
|
|
||||||
resolved execution targets.
|
|
||||||
|
|
||||||
### Downstream need
|
|
||||||
|
|
||||||
Notarius checkpoints model-backed pipeline stages. A checkpoint must not be
|
|
||||||
reused when generation-affecting PromptKit configuration changes.
|
|
||||||
|
|
||||||
Notarius therefore needs a stable equality signal for the effective profile
|
|
||||||
and backend target used by a pipeline.
|
|
||||||
|
|
||||||
### Current integration
|
|
||||||
|
|
||||||
Notarius currently constructs this identity itself from:
|
|
||||||
|
|
||||||
- a manually maintained marker for the PromptKit release and built-in profile
|
|
||||||
catalog;
|
|
||||||
- raw hashes of configured profile files; and
|
|
||||||
- a separate hash of the configured conventional local-backend endpoint.
|
|
||||||
|
|
||||||
This is safe but conservative and coupled to PromptKit details. Raw file
|
|
||||||
hashing also invalidates checkpoints for semantically irrelevant YAML changes,
|
|
||||||
such as comments or formatting.
|
|
||||||
|
|
||||||
### Requested capability
|
|
||||||
|
|
||||||
Expose an opaque semantic digest for a resolved profile and its effective
|
|
||||||
generation target. It could be returned by the proposed profile-resolution
|
|
||||||
API:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type ResolvedProfile struct {
|
|
||||||
ProfileID string
|
|
||||||
BackendID string
|
|
||||||
EffectiveTarget ExecutionTarget
|
|
||||||
ExecutionDigest string
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Alternatively, PromptKit could expose a dedicated method such as
|
|
||||||
`ProfileExecutionDigest(profileID)`.
|
|
||||||
|
|
||||||
### Desired equality semantics
|
|
||||||
|
|
||||||
The digest should change when generation-affecting state changes, including:
|
|
||||||
|
|
||||||
- resolved model and endpoint;
|
|
||||||
- backend routing identity;
|
|
||||||
- backend request defaults and extra parameters;
|
|
||||||
- profile generation parameters; and
|
|
||||||
- the semantic identity of any selected built-in profile.
|
|
||||||
|
|
||||||
The digest should not incorporate:
|
|
||||||
|
|
||||||
- credential values;
|
|
||||||
- concurrency or queue capacity;
|
|
||||||
- filesystem source paths;
|
|
||||||
- YAML comments or formatting; or
|
|
||||||
- other settings that affect scheduling or source representation without
|
|
||||||
changing the generation target.
|
|
||||||
|
|
||||||
The credential environment-variable name may need to participate if changing
|
|
||||||
it can select a materially different provider account or target. PromptKit
|
|
||||||
should define this deliberately while continuing to exclude the resolved
|
|
||||||
secret value.
|
|
||||||
|
|
||||||
### Design considerations
|
|
||||||
|
|
||||||
- Treat the digest as an opaque equality value rather than a public encoding
|
|
||||||
of internal structures.
|
|
||||||
- Document which categories of change affect equality.
|
|
||||||
- Include a versioned semantic marker internally so PromptKit can deliberately
|
|
||||||
invalidate old digests when its resolution semantics change.
|
|
||||||
- Prefer a per-profile digest over a digest of every profile known to an
|
|
||||||
engine. Notarius generally knows which profiles a resolved pipeline uses.
|
|
||||||
- Do not require consumers to know PromptKit's built-in catalog version.
|
|
||||||
|
|
||||||
### Value to Notarius
|
|
||||||
|
|
||||||
This would let Notarius remove its PromptKit release marker and raw
|
|
||||||
profile-source fingerprinting, reduce unnecessary checkpoint invalidation, and
|
|
||||||
delegate execution-target equality to the component that owns target
|
|
||||||
resolution.
|
|
||||||
|
|
||||||
## Priority 4: Structured Capacity Errors
|
|
||||||
|
|
||||||
**Disposition:** Implemented behavior. See the consumer guide's
|
|
||||||
[Handle Errors](../consumers/pkg-promptkit.md#handle-errors) section.
|
|
||||||
|
|
||||||
### Downstream need
|
|
||||||
|
|
||||||
Notarius translates PromptKit backend-capacity rejection into a
|
|
||||||
provider-neutral application error. When multiple backends are active,
|
|
||||||
operators would benefit from knowing which backend rejected admission without
|
|
||||||
parsing an error string or exposing endpoint details.
|
|
||||||
|
|
||||||
### Implemented behavior
|
|
||||||
|
|
||||||
PromptKit now retains the broad capacity classification while allowing
|
|
||||||
Notarius to obtain the selected backend ID without parsing diagnostic text.
|
|
||||||
The consumer guide owns the application workflow, including retry and backoff
|
|
||||||
policy.
|
|
||||||
|
|
||||||
### Value to Notarius
|
|
||||||
|
|
||||||
This improves operational diagnostics and future metrics while preserving
|
|
||||||
the provider-neutral error boundary used by Notarius.
|
|
||||||
|
|
||||||
## Capabilities PromptKit Already Provides Well
|
|
||||||
|
|
||||||
The current PromptKit boundary is sufficient for Notarius's implemented
|
|
||||||
behavior. In particular, PromptKit already provides:
|
|
||||||
|
|
||||||
- filesystem, `fs.FS`, and programmatic prompt, profile, and schema sources;
|
|
||||||
- offline preparation without model execution;
|
|
||||||
- structured output and content validation;
|
|
||||||
- direct session propagation;
|
|
||||||
- tri-state per-run reasoning overrides;
|
|
||||||
- selected profile, backend, model, endpoint, effective parameters, hashes, and
|
|
||||||
token-usage provenance;
|
|
||||||
- endpoint-only profiles;
|
|
||||||
- the conventional `local` backend helper;
|
|
||||||
- arbitrary engine-scoped `Backend` registrations;
|
|
||||||
- backend authentication environment names, extra parameters, concurrency
|
|
||||||
limits, and queue-capacity policies;
|
|
||||||
- provider-client and artifact-reader extension interfaces;
|
|
||||||
- context cancellation; and
|
|
||||||
- useful public error sentinels, including profile absence and capacity
|
|
||||||
exhaustion.
|
|
||||||
|
|
||||||
The wishlist does not imply that Notarius needs PromptKit to broaden its core
|
|
||||||
responsibilities. It primarily asks for more direct access to information and
|
|
||||||
operations that PromptKit already computes internally.
|
|
||||||
|
|
||||||
## Responsibilities That Should Remain In Notarius
|
|
||||||
|
|
||||||
The following concerns belong to the downstream application and should not
|
|
||||||
move into PromptKit for the sake of Notarius:
|
|
||||||
|
|
||||||
- pipeline staging, dependencies, and generated references;
|
|
||||||
- application-wide scheduling across providers and backends;
|
|
||||||
- module and validation retry policy;
|
|
||||||
- checkpoints, resume, and recomputation;
|
|
||||||
- durable run artifacts and manifests;
|
|
||||||
- D&D prompts, schemas, extractors, validators, and normalizers;
|
|
||||||
- Notarius configuration-file parsing and precedence;
|
|
||||||
- domain-specific prompt-cache prefix policy; and
|
|
||||||
- application-specific redaction, retention, and debug-bundle policy.
|
|
||||||
|
|
||||||
PromptKit's complete `Backend` API already supports custom IDs, multiple local
|
|
||||||
endpoints, authentication, extra parameters, and explicit queue policies.
|
|
||||||
Whether Notarius exposes those capabilities in its own configuration is an
|
|
||||||
application-policy decision, not an upstream PromptKit gap.
|
|
||||||
|
|
||||||
## Suggested Upstream Sequence
|
|
||||||
|
|
||||||
For downstream adoption and any remaining upstream work, the useful order is:
|
|
||||||
|
|
||||||
1. Adopt prepared execution for atomic details and results.
|
|
||||||
2. Add a semantic execution-target digest, preferably alongside profile
|
|
||||||
inspection.
|
|
||||||
3. Use the implemented typed capacity error where backend admission diagnostics
|
|
||||||
are needed.
|
|
||||||
|
|
||||||
The first removes the concrete execution workaround. The second would improve
|
|
||||||
checkpoint correctness and reduce coupling. The third is operational polish.
|
|
||||||
@@ -1,83 +0,0 @@
|
|||||||
# Omit Unset Optional Request Parameters
|
|
||||||
|
|
||||||
Status: Complete.
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Promptkit will stop turning an unset optional provider control into an explicit
|
|
||||||
outbound value. Required protocol fields and settings needed to enforce
|
|
||||||
Promptkit's own contracts will retain defined behavior, while provider tuning
|
|
||||||
choices will be sent only when a supported configuration layer selects them.
|
|
||||||
|
|
||||||
This keeps profiles intentional, avoids overriding model- or backend-specific
|
|
||||||
defaults, and improves compatibility across OpenAI-compatible endpoints with
|
|
||||||
different supported parameter sets.
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
|
|
||||||
The framework-level `top_p` default of `1` will become unspecified. When no
|
|
||||||
profile or runtime override supplies `top_p`, the built-in OpenAI-compatible
|
|
||||||
client will omit it from the request body rather than serialize `1`.
|
|
||||||
|
|
||||||
This establishes the general outbound policy for currently supported optional
|
|
||||||
controls:
|
|
||||||
|
|
||||||
- `temperature`, `max_tokens`, and `top_p` are omitted when unspecified and
|
|
||||||
included when selected by a profile or runtime override;
|
|
||||||
- an explicit numeric zero supplied through a runtime override remains present
|
|
||||||
on the wire through the existing numeric-presence contract;
|
|
||||||
- `service_tier` and `reasoning_effort` remain omitted when their resolved
|
|
||||||
values are empty;
|
|
||||||
- backend, profile, or runtime `extra_params` remain explicit configuration and
|
|
||||||
are sent when present; and
|
|
||||||
- `session_id` remains conditional on a supplied nonempty value, while
|
|
||||||
`response_format` remains conditional on the effective output contract.
|
|
||||||
|
|
||||||
The request body will continue to require `model` and `messages`. Endpoint and
|
|
||||||
credential resolution remain transport concerns rather than body defaults.
|
|
||||||
The positive framework `timeout_seconds` default also remains in place because
|
|
||||||
it enforces a Promptkit-owned generation deadline and is not serialized as a
|
|
||||||
provider request field.
|
|
||||||
|
|
||||||
## Effective Settings And Compatibility
|
|
||||||
|
|
||||||
An unspecified numeric provider control continues to use its zero Go value in
|
|
||||||
resolved public metadata. For a `GenerateRequest` delivered to an injected
|
|
||||||
client, the existing `ExecutionTargetPresence` value distinguishes an explicit
|
|
||||||
runtime zero from an inherited unspecified zero. Prepared and inspection
|
|
||||||
metadata will continue to report the resolved numeric value without adding
|
|
||||||
source-provenance fields.
|
|
||||||
|
|
||||||
The behavior change is intentional and belongs in a minor release. Consumers
|
|
||||||
that require stable sampling behavior should declare the desired values in
|
|
||||||
their profiles or runtime overrides instead of relying on Promptkit to repeat a
|
|
||||||
provider's conventional default.
|
|
||||||
|
|
||||||
Built-in profile parameters remain explicit profile policy and will continue
|
|
||||||
to be sent. An explicit built-in setting will not be removed merely because
|
|
||||||
the framework default becomes unspecified.
|
|
||||||
|
|
||||||
## Non-Goals
|
|
||||||
|
|
||||||
This change does not add:
|
|
||||||
|
|
||||||
- pointer-valued numeric fields to file or in-memory profiles;
|
|
||||||
- a general provenance model for effective settings;
|
|
||||||
- automatic discovery of provider defaults or supported parameters;
|
|
||||||
- backend- or model-specific request-shape negotiation;
|
|
||||||
- changes to required request fields, structured-output behavior, credentials,
|
|
||||||
or timeout enforcement; or
|
|
||||||
- removal of deliberately configured settings from consumer or built-in
|
|
||||||
profiles.
|
|
||||||
|
|
||||||
## Target End State
|
|
||||||
|
|
||||||
An otherwise unset optional provider tuning parameter is absent from the
|
|
||||||
OpenAI-compatible request body. Explicit profile values and runtime overrides,
|
|
||||||
including explicit runtime zero values, retain their current precedence and
|
|
||||||
wire effect. Promptkit continues to supply only the required protocol fields
|
|
||||||
and the settings necessary to honor its own execution and output contracts.
|
|
||||||
|
|
||||||
GoDoc, the framework format reference, and the OpenAI-compatible integration
|
|
||||||
contract will own the implemented omission and metadata semantics once the
|
|
||||||
feature lands.
|
|
||||||
@@ -1,331 +0,0 @@
|
|||||||
# Weatherreporter PromptKit Wishlist
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This document records features and interface changes that would be useful
|
|
||||||
additions to PromptKit from the perspective of the maintainers of
|
|
||||||
Weatherreporter, a downstream application planning to replace its Scriptorium
|
|
||||||
CLI integration with PromptKit.
|
|
||||||
|
|
||||||
PromptKit now provides the capabilities Weatherreporter needs for the
|
|
||||||
migration. The remaining deferred ideas are optional opportunities to validate
|
|
||||||
configuration earlier and improve durable failure diagnostics.
|
|
||||||
|
|
||||||
The examples are API sketches intended to communicate the desired capability,
|
|
||||||
not prescriptive names or finalized Go contracts. The related
|
|
||||||
[Notarius PromptKit wishlist](notarius-promptkit-wishlist.md) proposes several
|
|
||||||
overlapping features from another downstream consumer's perspective.
|
|
||||||
|
|
||||||
## Priority 1: Executable Preparation Handles
|
|
||||||
|
|
||||||
**Disposition:** Implemented as [`Engine.PrepareExecution` and
|
|
||||||
`Engine.RunPrepared`](../../engine.go). See the
|
|
||||||
[consumer guidance](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later).
|
|
||||||
|
|
||||||
### Downstream need
|
|
||||||
|
|
||||||
Weatherreporter treats prompt preparation as a durable preflight boundary. It
|
|
||||||
needs to:
|
|
||||||
|
|
||||||
1. prepare the exact request that will be executed;
|
|
||||||
2. persist a safe preparation record before starting the provider call; and
|
|
||||||
3. execute without reloading or rerendering prompt, profile, schema, or input
|
|
||||||
sources.
|
|
||||||
|
|
||||||
Persisting preflight before generation leaves useful evidence when a provider
|
|
||||||
call fails or the process is interrupted during generation.
|
|
||||||
|
|
||||||
### Implemented behavior
|
|
||||||
|
|
||||||
Weatherreporter can prepare one frozen execution snapshot, persist a
|
|
||||||
caller-owned and credential-redacted `Details` value, and execute that same
|
|
||||||
snapshot through `RunPrepared`. The opaque handle is engine-bound and
|
|
||||||
single-use; an unused handle can be released with `Discard`. The consumer
|
|
||||||
guide and exported GoDoc own the exact lifecycle, credential, cancellation,
|
|
||||||
and capacity contracts.
|
|
||||||
|
|
||||||
### Value to Weatherreporter
|
|
||||||
|
|
||||||
This preserves Weatherreporter's durable preflight behavior, removes duplicate
|
|
||||||
work, eliminates the source-consistency window, and ensures that persisted
|
|
||||||
provenance describes the actual execution.
|
|
||||||
|
|
||||||
## Priority 2: Prompt-Definition Inspection
|
|
||||||
|
|
||||||
**Disposition:** Implemented as
|
|
||||||
[`Engine.InspectPrompt`](../../engine.go). See the
|
|
||||||
[consumer guidance](../consumers/pkg-promptkit.md#inspect-a-prompt-before-preparation).
|
|
||||||
|
|
||||||
### Downstream need
|
|
||||||
|
|
||||||
Weatherreporter has a fixed registry of seven report definitions. Each report
|
|
||||||
selects a prompt ID and one of two output workflows:
|
|
||||||
|
|
||||||
- direct Markdown; or
|
|
||||||
- structured generated text followed by application-owned domain validation
|
|
||||||
and Markdown template rendering.
|
|
||||||
|
|
||||||
Weatherreporter will embed the PromptKit prompt definitions and private
|
|
||||||
response schemas that implement those reports. It needs to validate that the
|
|
||||||
report registry and embedded prompt corpus agree before weather collection or
|
|
||||||
provider execution.
|
|
||||||
|
|
||||||
### Previous integration option
|
|
||||||
|
|
||||||
Before prompt inspection was available, Weatherreporter could maintain
|
|
||||||
synthetic data-package fixtures and call `Engine.Prepare` for every report
|
|
||||||
prompt during tests. Runtime validation could also occur through the ordinary
|
|
||||||
per-report preparation stage.
|
|
||||||
|
|
||||||
This required complete placeholder inputs and profile resolution when the
|
|
||||||
application primarily wanted to inspect prompt identity and declared contracts.
|
|
||||||
|
|
||||||
### Value to Weatherreporter
|
|
||||||
|
|
||||||
The implemented interface lets Weatherreporter directly verify that every
|
|
||||||
report prompt exists, requires the curated `data_package` input, and declares
|
|
||||||
the expected Markdown or JSON Schema output contract. It reduces synthetic
|
|
||||||
test setup and moves failures ahead of weather collection.
|
|
||||||
|
|
||||||
## Priority 3: Prompt-Independent Profile Inspection
|
|
||||||
|
|
||||||
**Disposition:** Implemented as
|
|
||||||
[`Engine.InspectProfile`](../../engine.go). See the
|
|
||||||
[consumer guidance](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work).
|
|
||||||
|
|
||||||
### Downstream need
|
|
||||||
|
|
||||||
Weatherreporter will allow operators to select an external PromptKit profile
|
|
||||||
source and may allow an explicit profile override. It needs to reject a missing
|
|
||||||
profile, unknown backend, or malformed execution target before collecting
|
|
||||||
weather data or writing report artifacts, and to apply its own policy to a
|
|
||||||
reported credential requirement.
|
|
||||||
|
|
||||||
### Value to Weatherreporter
|
|
||||||
|
|
||||||
The implemented interface improves fail-fast configuration validation and gives
|
|
||||||
operator-facing errors direct profile and backend context. It remains optional
|
|
||||||
for the initial migration.
|
|
||||||
|
|
||||||
## Priority 4: Eager Source Validation
|
|
||||||
|
|
||||||
**Disposition:** Deferred until prompt and profile inspection have been used
|
|
||||||
to determine whether a broader engine-wide validation operation is still
|
|
||||||
needed.
|
|
||||||
|
|
||||||
### Downstream need
|
|
||||||
|
|
||||||
PromptKit deliberately defers reading and validating filesystem and `fs.FS`
|
|
||||||
prompt, profile, and schema content until a request needs it. Weatherreporter
|
|
||||||
has a small fixed embedded prompt corpus and one optional external profile
|
|
||||||
source. It would benefit from an explicit offline validation operation for
|
|
||||||
tests, startup diagnostics, and configuration checks.
|
|
||||||
|
|
||||||
### Current integration option
|
|
||||||
|
|
||||||
Weatherreporter can prepare every report prompt with fixture inputs and inspect
|
|
||||||
any explicit profiles individually. That provides strong coverage but requires
|
|
||||||
consumer-maintained traversal and synthetic material.
|
|
||||||
|
|
||||||
### Requested capability
|
|
||||||
|
|
||||||
Consider an opt-in source-validation operation:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type SourceValidationOptions struct {
|
|
||||||
RequireCredentials bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (e *Engine) ValidateSources(
|
|
||||||
ctx context.Context,
|
|
||||||
opts SourceValidationOptions,
|
|
||||||
) error
|
|
||||||
```
|
|
||||||
|
|
||||||
The operation should eagerly discover and structurally validate the configured
|
|
||||||
prompt, profile, and schema sources without model generation.
|
|
||||||
|
|
||||||
### Design considerations
|
|
||||||
|
|
||||||
- Keep deferred validation as the normal `NewEngine` behavior.
|
|
||||||
- Make eager validation an explicit consumer choice.
|
|
||||||
- Validate duplicate IDs and versions, strict YAML decoding, referenced content
|
|
||||||
files, profile/backend membership, schema syntax, and schema references.
|
|
||||||
- Distinguish structural credential declarations from current environment
|
|
||||||
availability.
|
|
||||||
- Do not read or expose credential values when credential availability is not
|
|
||||||
requested.
|
|
||||||
- Preserve source-specific public error identities and useful path context.
|
|
||||||
- Respect context cancellation during filesystem discovery and schema work.
|
|
||||||
- Consider whether exact prompt and profile inspection APIs already provide a
|
|
||||||
smaller sufficient surface before adding an engine-wide operation.
|
|
||||||
|
|
||||||
### Value to Weatherreporter
|
|
||||||
|
|
||||||
This would simplify offline corpus checks and catch malformed operator profile
|
|
||||||
sources before report work begins. It is helpful but lower priority than exact
|
|
||||||
prompt and profile inspection.
|
|
||||||
|
|
||||||
## Priority 5: Structured Generation Errors
|
|
||||||
|
|
||||||
**Disposition:** Deferred pending stronger downstream demand and a narrower
|
|
||||||
design that does not duplicate prepared provenance or impose HTTP-specific
|
|
||||||
fields on injected model clients.
|
|
||||||
|
|
||||||
### Downstream need
|
|
||||||
|
|
||||||
Weatherreporter preserves redacted, inspectable failure receipts for report
|
|
||||||
runs. When model generation fails operationally, it needs to classify the
|
|
||||||
failure and retain safe execution context without parsing error prose.
|
|
||||||
|
|
||||||
Prompt preparation already supplies selected profile, backend, and model
|
|
||||||
identity. Provider status classification would add useful operator context,
|
|
||||||
especially when the built-in OpenAI-compatible client receives a non-success
|
|
||||||
HTTP status.
|
|
||||||
|
|
||||||
### Current integration option
|
|
||||||
|
|
||||||
PromptKit exposes `ErrLLMGenerate` and preserves injected client errors through
|
|
||||||
`errors.Is`. Weatherreporter can reliably classify generation failure and use
|
|
||||||
its preparation record for profile, backend, and model provenance. Any further
|
|
||||||
diagnostic detail remains a redacted error string.
|
|
||||||
|
|
||||||
### Requested capability
|
|
||||||
|
|
||||||
Consider a typed generation error that continues to match `ErrLLMGenerate`:
|
|
||||||
|
|
||||||
```go
|
|
||||||
type GenerationError struct {
|
|
||||||
BackendID string
|
|
||||||
Model string
|
|
||||||
StatusCode int
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The exact fields may differ. The useful contract is safe structured context
|
|
||||||
available through `errors.As`, while `errors.Is(err, ErrLLMGenerate)` remains
|
|
||||||
compatible.
|
|
||||||
|
|
||||||
### Design considerations
|
|
||||||
|
|
||||||
- Include only fields that PromptKit knows reliably and can expose safely.
|
|
||||||
- Treat an HTTP status as optional because injected model clients may not use
|
|
||||||
HTTP.
|
|
||||||
- Do not expose provider response bodies, endpoints, credential environment
|
|
||||||
names, credential values, request content, or generated content.
|
|
||||||
- Do not make a structured error a second source of prompt/profile provenance
|
|
||||||
already present in a prepared execution.
|
|
||||||
- Preserve injected client error identity.
|
|
||||||
- Keep retry and backoff policy with the consuming application.
|
|
||||||
|
|
||||||
### Value to Weatherreporter
|
|
||||||
|
|
||||||
This would improve durable failure receipts and troubleshooting, particularly
|
|
||||||
for built-in transport failures. It is not required if preparation details and
|
|
||||||
the existing sentinel remain available.
|
|
||||||
|
|
||||||
## Lower-Priority Shared Wishlist Items
|
|
||||||
|
|
||||||
### Structured Capacity Errors
|
|
||||||
|
|
||||||
**Disposition:** Implemented behavior. See the consumer guide's
|
|
||||||
[Handle Errors](../consumers/pkg-promptkit.md#handle-errors) section.
|
|
||||||
|
|
||||||
PromptKit now exposes the stable backend ID for capacity rejection without
|
|
||||||
requiring Weatherreporter to parse error text.
|
|
||||||
|
|
||||||
Weatherreporter currently generates batch reports sequentially and constructs
|
|
||||||
one engine per invocation, so engine-local capacity exhaustion is unlikely in
|
|
||||||
the initial design. The typed error becomes more valuable if report generation
|
|
||||||
later becomes concurrent or PromptKit engines become longer-lived.
|
|
||||||
It should not block adoption.
|
|
||||||
|
|
||||||
### Semantic Execution-Target Fingerprints
|
|
||||||
|
|
||||||
**Disposition:** Deferred pending a separate semantic-equality design for
|
|
||||||
resolved execution targets.
|
|
||||||
|
|
||||||
The semantic target digest proposed by the
|
|
||||||
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-3-semantic-execution-target-fingerprints)
|
|
||||||
would provide a compact equality signal for audit metadata.
|
|
||||||
|
|
||||||
Weatherreporter does not currently reuse LLM-dependent checkpoints. Its Recent
|
|
||||||
Changes behavior compares deterministic module snapshots rather than generated
|
|
||||||
reports, so the digest has no immediate cache-correctness role. Existing
|
|
||||||
PromptKit result metadata is sufficient for the initial integration. A digest
|
|
||||||
would still be useful provenance and future-proofing, but it is not a
|
|
||||||
migration priority.
|
|
||||||
|
|
||||||
## Capabilities PromptKit Already Provides Well
|
|
||||||
|
|
||||||
PromptKit already provides the essential Weatherreporter integration surface:
|
|
||||||
|
|
||||||
- importable in-process engine construction;
|
|
||||||
- filesystem, `fs.FS`, and programmatic prompt, profile, and schema sources;
|
|
||||||
- offline preparation without model execution;
|
|
||||||
- prepared execution handles for a durable preflight boundary;
|
|
||||||
- exact prompt and profile inspection;
|
|
||||||
- versioned prompt selection;
|
|
||||||
- text, Markdown, JSON, and JSON Schema output contracts;
|
|
||||||
- single-pass output validation with raw output retained after completed
|
|
||||||
validation failure;
|
|
||||||
- inline input artifacts with provenance URIs and input hashes;
|
|
||||||
- selected profile, backend, model, effective target, prompt hashes, timing,
|
|
||||||
and token-usage provenance;
|
|
||||||
- endpoint-only profiles and engine-scoped backend registration;
|
|
||||||
- injected model-client and artifact-reader interfaces;
|
|
||||||
- caller cancellation, generation timeout, and transport timeout behavior; and
|
|
||||||
- public error sentinels for configuration, prompt, profile, artifact,
|
|
||||||
validation, capacity, and generation failures; and
|
|
||||||
- structured backend identity for capacity rejection.
|
|
||||||
|
|
||||||
These capabilities are sufficient for Weatherreporter to adopt PromptKit
|
|
||||||
without waiting for new upstream work.
|
|
||||||
|
|
||||||
## Responsibilities That Should Remain In Weatherreporter
|
|
||||||
|
|
||||||
The following concerns belong to Weatherreporter and should not move into
|
|
||||||
PromptKit:
|
|
||||||
|
|
||||||
- report definitions, valid periods, batches, and output naming;
|
|
||||||
- application prompt content and private report response schemas;
|
|
||||||
- deterministic weather facts, modules, and Recent Changes;
|
|
||||||
- curated `data_package` construction and persistence;
|
|
||||||
- generated-text domain validation and Markdown template rendering;
|
|
||||||
- managed artifact paths, atomic writes, metadata, and inspection commands;
|
|
||||||
- preparation, execution, raw-output, and failure-receipt schemas;
|
|
||||||
- CLI configuration loading and precedence;
|
|
||||||
- debug enablement, redaction, placement, sensitivity, and retention;
|
|
||||||
- distributor notification;
|
|
||||||
- batch continuation and any future retry policy; and
|
|
||||||
- application-level compatibility and migration policy.
|
|
||||||
|
|
||||||
## Suggested Upstream Sequence
|
|
||||||
|
|
||||||
For downstream adoption and any remaining upstream work, the useful order is:
|
|
||||||
|
|
||||||
1. Adopt the implemented executable preparation handles.
|
|
||||||
2. Consider eager source validation after evaluating whether the two exact
|
|
||||||
inspection APIs are sufficient.
|
|
||||||
3. Add structured generation errors.
|
|
||||||
4. Use structured capacity errors and consider semantic execution-target
|
|
||||||
fingerprints as lower-priority operational improvements.
|
|
||||||
|
|
||||||
The first item removes the material integration workaround. Prompt and profile
|
|
||||||
inspection improve fail-fast validation. The remaining items are optional
|
|
||||||
ergonomic and diagnostic improvements.
|
|
||||||
|
|
||||||
## Adoption Sequencing
|
|
||||||
|
|
||||||
Weatherreporter should not wait for the deferred wishlist items. The current
|
|
||||||
PromptKit interface is sufficient when Weatherreporter:
|
|
||||||
|
|
||||||
- embeds immutable prompt and schema assets;
|
|
||||||
- supplies immutable inline data-package bytes;
|
|
||||||
- constructs one engine per CLI invocation;
|
|
||||||
- prepares an execution, persists selected `Details`, and calls
|
|
||||||
`RunPrepared`; and
|
|
||||||
- keeps PromptKit behind a weatherreporter-owned adapter contract.
|
|
||||||
|
|
||||||
Prompt inspection, profile inspection, source validation, structured errors,
|
|
||||||
capacity details, and semantic fingerprints should not gate adoption.
|
|
||||||
@@ -1146,6 +1146,14 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) {
|
|||||||
|
|
||||||
t.Run("missing fallback profile uses built-in profile", func(t *testing.T) {
|
t.Run("missing fallback profile uses built-in profile", func(t *testing.T) {
|
||||||
t.Setenv("OPENROUTER_API_KEY", "test-key")
|
t.Setenv("OPENROUTER_API_KEY", "test-key")
|
||||||
|
baseline, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct baseline engine: %v", err)
|
||||||
|
}
|
||||||
|
want := prepareModel(t, baseline, "prompt")
|
||||||
|
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
|
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
|
||||||
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
|
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
|
||||||
@@ -1153,8 +1161,8 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine: %v", err)
|
||||||
}
|
}
|
||||||
if model := prepareModel(t, engine, "prompt"); model != "mistralai/mistral-small-3.2-24b-instruct" {
|
if model := prepareModel(t, engine, "prompt"); model != want {
|
||||||
t.Fatalf("expected built-in profile, got %q", model)
|
t.Fatalf("expected built-in profile model %q, got %q", want, model)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user