Compare commits
106 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2f42bdde39 | |||
| 9a00f30c7b | |||
| f71d2bbb73 | |||
| 6f64947e42 | |||
| 7bb4cf35b9 | |||
| fb0b21c51d | |||
| 5a00ca81a2 | |||
| e13610481d | |||
| 309fe9b7ea | |||
| adfd08ffe2 | |||
| c7263ab2a8 | |||
| 532c31c09a | |||
| 68cd90c657 | |||
| 416438d80d | |||
| 9298d8ae73 | |||
| 4c7278febc | |||
| 096208532e | |||
| 50bd19b9d1 | |||
| 2fc7204bd5 | |||
| c0d4ea0d4e | |||
| 4d7e1327ad | |||
| 3c33b52b15 | |||
| 280916bf4a | |||
| 033bc93d3c | |||
| 45c2644b9d | |||
| a74c03bd9b | |||
| ad115a2259 | |||
| 3074b3165a | |||
| 8703793b0c | |||
| 4cb4943a57 | |||
| c6c747e94d | |||
| 2bbf13e739 | |||
| 5edb24a9c1 | |||
| 99d5e96316 | |||
| 144d840fbe | |||
| ed0c9f6370 | |||
| a0e905ce46 | |||
| 719243e90c | |||
| a9e1b7435c | |||
| eb6dfb19b0 | |||
| d86b65adad | |||
| 31faaf4259 | |||
| 9932153b97 | |||
| f0ca233c25 | |||
| ff31f8daf8 | |||
| c927b7819d | |||
| 6d1fb66dd7 | |||
| e0b1d6a0dc | |||
| 33698903be | |||
| 90b76ddad3 | |||
| d5b3d1e061 | |||
| 41083de46a | |||
| 07ac7e54c5 | |||
| 879cb021b2 | |||
| 574f88bd6a | |||
| d5d7a222a4 | |||
| aabd89aea7 | |||
| 9189cbfc22 | |||
| 872c166ed7 | |||
| 6742def4d3 | |||
| 1b39f82117 | |||
| f7d821067f | |||
| a16f66cbc7 | |||
| 39485d87f6 | |||
| 61e5b0fe58 | |||
| f3c21c7d9f | |||
| bc5f5d3731 | |||
| 93a76f1d36 | |||
| 5c882f26a9 | |||
| 0d45ac6e3c | |||
| f7ad756fc3 | |||
| 4fe11b1b2b | |||
| 296f9b1817 | |||
| 7a8516b0c6 | |||
| 2df2f530b3 | |||
| 8b25ca72e5 | |||
| fa02791fe9 | |||
| 32767b4eb4 | |||
| e1e5351c5d | |||
| d60ef66f53 | |||
| 4669b73d38 | |||
| 6f91603168 | |||
| 3ad247039b | |||
| 32e2433628 | |||
| 712c6b92b8 | |||
| 89cafcefec | |||
| 1d7fac0a47 | |||
| 03d4f27d2b | |||
| 4ac2038331 | |||
| 14a7e7e04c | |||
| 5e522bad8b | |||
| 23872dd742 | |||
| 7ffbf5f6ca | |||
| d0dc30fcc9 | |||
| b38f7b4dc3 | |||
| 0512995931 | |||
| 049a5feadb | |||
| 1798e9c575 | |||
| 5d4bc8c2b9 | |||
| 63fb8fc132 | |||
| 4d4bb7a121 | |||
| 5dcb3cd4fc | |||
| efe346893c | |||
| c95d6fcfec | |||
| 0badb4364d | |||
| 1f63f8afbb |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -1,6 +1,5 @@
|
|||||||
# ---> Codex
|
# ---> Codex
|
||||||
.codex
|
.codex
|
||||||
AGENTS.md
|
|
||||||
|
|
||||||
# ---> Go
|
# ---> Go
|
||||||
# If you prefer the allow list template instead of the deny list, see community template:
|
# If you prefer the allow list template instead of the deny list, see community template:
|
||||||
@@ -57,6 +56,8 @@ mono_crash.*
|
|||||||
[Dd]ebugPublic/
|
[Dd]ebugPublic/
|
||||||
[Rr]elease/
|
[Rr]elease/
|
||||||
[Rr]eleases/
|
[Rr]eleases/
|
||||||
|
!docs/releases/
|
||||||
|
!docs/releases/*.md
|
||||||
x64/
|
x64/
|
||||||
x86/
|
x86/
|
||||||
[Ww][Ii][Nn]32/
|
[Ww][Ii][Nn]32/
|
||||||
@@ -434,4 +435,3 @@ FodyWeavers.xsd
|
|||||||
|
|
||||||
# JetBrains Rider
|
# JetBrains Rider
|
||||||
*.sln.iml
|
*.sln.iml
|
||||||
|
|
||||||
|
|||||||
@@ -11,9 +11,16 @@ steps:
|
|||||||
version="$CI_COMMIT_TAG"
|
version="$CI_COMMIT_TAG"
|
||||||
dist="dist"
|
dist="dist"
|
||||||
pkg="gitea.maximumdirect.net/eric/scriptorium/cmd/scriptorium"
|
pkg="gitea.maximumdirect.net/eric/scriptorium/cmd/scriptorium"
|
||||||
|
notes="docs/releases/$version.md"
|
||||||
|
|
||||||
|
if [ ! -f "$notes" ]; then
|
||||||
|
printf 'release notes not found: %s\n' "$notes" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
rm -rf "$dist"
|
rm -rf "$dist"
|
||||||
mkdir -p "$dist"
|
mkdir -p "$dist"
|
||||||
|
cp "$notes" "$dist/RELEASE_NOTES.md"
|
||||||
|
|
||||||
build_binary() {
|
build_binary() {
|
||||||
goos="$1"
|
goos="$1"
|
||||||
@@ -22,16 +29,12 @@ steps:
|
|||||||
output="$dist/scriptorium-$version-$goos-$goarch$suffix"
|
output="$dist/scriptorium-$version-$goos-$goarch$suffix"
|
||||||
|
|
||||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" \
|
||||||
go build -trimpath -ldflags "-s -w -X gitea.maximumdirect.net/eric/scriptorium/internal/buildinfo.Version=$version" \
|
go build -trimpath -ldflags "-s -w" \
|
||||||
-o "$output" "$pkg"
|
-o "$output" "$pkg"
|
||||||
}
|
}
|
||||||
|
|
||||||
build_binary linux amd64 ""
|
build_binary linux amd64 ""
|
||||||
build_binary linux arm64 ""
|
build_binary linux arm64 ""
|
||||||
build_binary darwin amd64 ""
|
|
||||||
build_binary darwin arm64 ""
|
|
||||||
build_binary windows amd64 ".exe"
|
|
||||||
build_binary windows arm64 ".exe"
|
|
||||||
|
|
||||||
- name: publish-release
|
- name: publish-release
|
||||||
image: woodpeckerci/plugin-release
|
image: woodpeckerci/plugin-release
|
||||||
@@ -42,6 +45,7 @@ steps:
|
|||||||
from_secret: GITEA_RELEASE_TOKEN
|
from_secret: GITEA_RELEASE_TOKEN
|
||||||
files:
|
files:
|
||||||
- dist/scriptorium-*
|
- dist/scriptorium-*
|
||||||
|
note: dist/RELEASE_NOTES.md
|
||||||
checksum: sha256
|
checksum: sha256
|
||||||
checksum-file: SHA256SUMS
|
checksum-file: SHA256SUMS
|
||||||
checksum-flatten: true
|
checksum-flatten: true
|
||||||
|
|||||||
1
AGENTS.md
Normal file
1
AGENTS.md
Normal file
@@ -0,0 +1 @@
|
|||||||
|
Please review `docs/development.md` for initial orientation in this repository and follow its task-specific reading guide.
|
||||||
2
LICENSE
2
LICENSE
@@ -1,4 +1,4 @@
|
|||||||
Copyright (c) 2026 eric.
|
Copyright (c) 2026 Eric Rakestraw.
|
||||||
|
|
||||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||||
|
|
||||||
|
|||||||
35
README.md
35
README.md
@@ -1,8 +1,15 @@
|
|||||||
# scriptorium
|
# Scriptorium
|
||||||
|
|
||||||
Scriptorium is a config-driven prompt execution engine.
|
Scriptorium is a prompt-execution application with a command-line interface and
|
||||||
|
an HTTP service. It prepares prompt requests, runs them against
|
||||||
|
OpenAI-compatible model endpoints, and returns generated output with validation
|
||||||
|
metadata.
|
||||||
|
|
||||||
It separates prompt definitions (what to generate) from execution profiles (how to call an OpenAI-compatible model endpoint), then runs or renders a prepared request from named input artifacts.
|
The application uses
|
||||||
|
[Promptkit v0.1.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/)
|
||||||
|
for prompt, profile, schema, preparation, generation, and validation behavior.
|
||||||
|
Scriptorium owns executable configuration, CLI and HTTP mapping, process
|
||||||
|
behavior, output presentation, and HTTP artifact-containment policy.
|
||||||
|
|
||||||
## Quickstart
|
## Quickstart
|
||||||
|
|
||||||
@@ -17,20 +24,28 @@ go run ./cmd/scriptorium render \
|
|||||||
--format json
|
--format json
|
||||||
```
|
```
|
||||||
|
|
||||||
This command renders the prepared prompt and effective runtime settings without calling an LLM.
|
This renders the prepared prompt and effective runtime settings without calling
|
||||||
|
a model. For complete invocation and output behavior, see the
|
||||||
|
[CLI reference](docs/cli.md).
|
||||||
|
|
||||||
## Documentation
|
## Documentation
|
||||||
|
|
||||||
- [CLI reference](docs/cli.md)
|
- [CLI reference](docs/cli.md)
|
||||||
- [Configuration reference](docs/config.md)
|
- [Configuration reference](docs/config.md)
|
||||||
|
- [HTTP API reference](docs/api.md)
|
||||||
- [Operations guide](docs/operations.md)
|
- [Operations guide](docs/operations.md)
|
||||||
- [Troubleshooting](docs/troubleshooting.md)
|
- [Consumer integration overview](docs/consumers/api.md)
|
||||||
- [HTTP API integration](docs/integrations/http-api.md)
|
- [Migration from the former Go package](docs/consumers/migrating-to-promptkit.md)
|
||||||
- [OpenAI-compatible chat integration](docs/integrations/openai-compatible-chat.md)
|
- [Subprocess integration](docs/integrations/subprocess.md)
|
||||||
- [Narratio subprocess integration](docs/integrations/narratio.md)
|
|
||||||
- [Architecture policy](docs/policy/architecture.md)
|
- [Architecture policy](docs/policy/architecture.md)
|
||||||
|
- [Promptkit framework formats](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/formats.md)
|
||||||
|
- [Promptkit Go consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
- `examples/render-markdown-summary.sh`
|
- [Minimal configuration](examples/config.yml) and
|
||||||
- `examples/http-run.json`
|
[complete configuration](examples/config.full.yml)
|
||||||
|
- [Prompt definitions](examples/prompts/), [execution profiles](examples/profiles/),
|
||||||
|
[schemas](examples/schemas/), and [synthetic input fixtures](examples/fixtures/)
|
||||||
|
- [Render script](examples/render-markdown-summary.sh)
|
||||||
|
- [HTTP request](examples/http-run.json)
|
||||||
|
|||||||
48
docs/adr/0001-adopt-canonical-documentation-ownership.md
Normal file
48
docs/adr/0001-adopt-canonical-documentation-ownership.md
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
# ADR 0001: Adopt Canonical Documentation Ownership
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted
|
||||||
|
|
||||||
|
## Date
|
||||||
|
|
||||||
|
2026-07-26
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Scriptorium's documentation grew alongside its CLI, HTTP, public Go, and
|
||||||
|
integration interfaces. As a result, several documents repeated mutable
|
||||||
|
contracts such as flags, configuration fields, and status behavior. Those
|
||||||
|
parallel definitions made it unclear which document to update when behavior
|
||||||
|
changed and increased the risk of documentation drift.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Assign each documentation topic one canonical owner, as defined in
|
||||||
|
[`docs/policy/documentation.md`](../policy/documentation.md). Non-owning
|
||||||
|
documents may provide short orientation and links, but do not redefine volatile
|
||||||
|
contracts. Current behavior is documented outside `docs/roadmap/`; roadmaps own
|
||||||
|
future work, sequencing, and implementation status.
|
||||||
|
|
||||||
|
## Alternatives Considered
|
||||||
|
|
||||||
|
- Keep broad reference material in several audience-specific documents. This
|
||||||
|
would preserve local convenience but leave conflicting contract definitions
|
||||||
|
likely.
|
||||||
|
- Consolidate all documentation into one reference. This would reduce duplicate
|
||||||
|
text but would not serve the distinct needs of users, operators, consumers,
|
||||||
|
and contributors.
|
||||||
|
|
||||||
|
## Rationale
|
||||||
|
|
||||||
|
Canonical ownership retains audience-specific guidance while making the source
|
||||||
|
of truth for each contract discoverable. It also makes documentation changes
|
||||||
|
reviewable alongside the implementation change that requires them.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- Changes to behavior must update the canonical owner in the same change.
|
||||||
|
- Cross-cutting documentation links to the owner instead of copying its
|
||||||
|
details.
|
||||||
|
- Documentation restructuring followed a dedicated implementation roadmap;
|
||||||
|
repository history, not this ADR, records its completion.
|
||||||
288
docs/adr/0002-split-promptkit-from-scriptorium.md
Normal file
288
docs/adr/0002-split-promptkit-from-scriptorium.md
Normal file
@@ -0,0 +1,288 @@
|
|||||||
|
# ADR 0002: Split Promptkit From Scriptorium
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted
|
||||||
|
|
||||||
|
## Date
|
||||||
|
|
||||||
|
2026-07-26
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
Scriptorium currently combines two products in one Go module:
|
||||||
|
|
||||||
|
- a reusable prompt-execution framework with a public Go facade; and
|
||||||
|
- a runnable application with CLI and HTTP interfaces.
|
||||||
|
|
||||||
|
Downstream Go projects increasingly import the framework directly and do not
|
||||||
|
use the executable interfaces. Keeping both products in one module couples
|
||||||
|
framework releases, dependencies, documentation, and public API evolution to
|
||||||
|
application-specific transport concerns.
|
||||||
|
|
||||||
|
Promptkit will become the framework project, and Scriptorium will become a slim
|
||||||
|
application that consumes it. This ADR records that end-state boundary. It does
|
||||||
|
not assert that the split has been implemented; until then, the current
|
||||||
|
repository structure and contracts remain authoritative.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
### Projects And Module Paths
|
||||||
|
|
||||||
|
Create a repository named `promptkit` alongside Scriptorium:
|
||||||
|
|
||||||
|
| Project | Repository and Go module path | Root Go package |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Promptkit | `gitea.maximumdirect.net/eric/promptkit` | `promptkit` |
|
||||||
|
| Scriptorium | `gitea.maximumdirect.net/eric/scriptorium` | No reusable root facade after migration |
|
||||||
|
|
||||||
|
Promptkit will expose its supported public API from the module root. Its
|
||||||
|
implementation packages will remain under `internal/` unless a real consumer
|
||||||
|
extension point requires a public type or interface.
|
||||||
|
|
||||||
|
Scriptorium will import only Promptkit's supported public packages. It will not
|
||||||
|
import Promptkit implementation packages or reproduce Promptkit orchestration.
|
||||||
|
|
||||||
|
### Product Responsibilities
|
||||||
|
|
||||||
|
Promptkit owns application-neutral framework behavior:
|
||||||
|
|
||||||
|
- the engine and its `Prepare` and `Run` workflow;
|
||||||
|
- public request, result, profile, option, extension, and error APIs;
|
||||||
|
- prompt-definition loading and rendering;
|
||||||
|
- profile loading, overlays, and the embedded built-in profile registry;
|
||||||
|
- schema loading and output validation;
|
||||||
|
- provider-neutral model-client boundaries and the OpenAI-compatible client;
|
||||||
|
- artifact types, artifact-reader injection, and general-purpose inline and
|
||||||
|
caller-selected file readers;
|
||||||
|
- execution-setting resolution and framework defaults; and
|
||||||
|
- framework-level secret redaction and error classification.
|
||||||
|
|
||||||
|
Scriptorium owns executable and transport behavior:
|
||||||
|
|
||||||
|
- the `scriptorium` process and its `run`, `render`, and `serve` commands;
|
||||||
|
- CLI parsing, streams, output files, formatting, exit codes, and process
|
||||||
|
cancellation behavior;
|
||||||
|
- application-configuration discovery and CLI-over-configuration precedence;
|
||||||
|
- HTTP routing, strict request decoding, DTO mapping, response encoding,
|
||||||
|
status codes, and transport limits;
|
||||||
|
- HTTP artifact-root containment and deployment policy;
|
||||||
|
- server construction, server defaults, and process logging; and
|
||||||
|
- executable release artifacts.
|
||||||
|
|
||||||
|
The dependency direction is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Scriptorium CLI and HTTP adapters
|
||||||
|
|
|
||||||
|
v
|
||||||
|
Promptkit public API
|
||||||
|
|
|
||||||
|
v
|
||||||
|
injected sources, readers, and model clients
|
||||||
|
```
|
||||||
|
|
||||||
|
### Current Package Disposition
|
||||||
|
|
||||||
|
Implementation may reorganize files during extraction, but each current package
|
||||||
|
has this target owner:
|
||||||
|
|
||||||
|
| Current package or file group | Target owner | Disposition |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Root `scriptorium` facade files and tests | Promptkit | Move and rename the public package to `promptkit`; Scriptorium retains no compatibility facade. |
|
||||||
|
| `internal/domain`, `internal/usecase` | Promptkit | Move as internal engine implementation. |
|
||||||
|
| `internal/promptdef`, `internal/prompt` | Promptkit | Move as internal prompt loading and rendering. |
|
||||||
|
| `internal/profile`, `internal/profile/builtin` | Promptkit | Move with embedded built-in assets and registry tests. |
|
||||||
|
| `internal/filecatalog` | Promptkit | Move as source-loading support. |
|
||||||
|
| `internal/validate` | Promptkit | Move as schema and output-validation implementation. |
|
||||||
|
| `internal/llm` | Promptkit | Move with the OpenAI-compatible integration. |
|
||||||
|
| `internal/artifact` | Split | Move general inline/file reading to Promptkit; keep rooted, denied, and byte-limited HTTP file reading in Scriptorium behind a Promptkit reader interface. |
|
||||||
|
| `internal/defaults` | Split | Move framework, execution, output-artifact, content-type, and model-client defaults to Promptkit; keep CLI, HTTP, and server defaults in Scriptorium. |
|
||||||
|
| `internal/adapter/cli`, `internal/adapter/http` | Scriptorium | Keep and refactor to use Promptkit's public API. |
|
||||||
|
| `internal/config` | Scriptorium | Keep application settings, discovery, validation, and CLI precedence. |
|
||||||
|
| `internal/format` | Scriptorium | Keep prepared-run presentation, rewritten against Promptkit public values. |
|
||||||
|
| `cmd/scriptorium` | Scriptorium | Keep as the process entrypoint. |
|
||||||
|
|
||||||
|
Tests move with the behavior they protect. Cross-boundary tests will live with
|
||||||
|
the consuming side: Promptkit protects framework contracts, while Scriptorium
|
||||||
|
protects adapter mapping, HTTP containment, and executable behavior.
|
||||||
|
|
||||||
|
### Public Boundary
|
||||||
|
|
||||||
|
Promptkit's initial facade will preserve the useful shape of the current
|
||||||
|
Scriptorium Go API where that reduces extraction risk. It will expose only the
|
||||||
|
capabilities required by Promptkit consumers and by Scriptorium:
|
||||||
|
|
||||||
|
- engine construction, preparation, and execution;
|
||||||
|
- public request, result, profile, and error values;
|
||||||
|
- prompt, profile, schema, artifact-reader, validator, and model-client source
|
||||||
|
or injection options that have demonstrated consumers; and
|
||||||
|
- enough stable error identity for Scriptorium to map CLI and HTTP outcomes.
|
||||||
|
|
||||||
|
Promptkit will not export its domain package, runner implementation,
|
||||||
|
repositories, adapter DTOs, or general internal constructors merely to
|
||||||
|
simplify the move.
|
||||||
|
|
||||||
|
Scriptorium's CLI and HTTP adapters will depend on a small consumer-facing
|
||||||
|
`Prepare`/`Run` interface where test substitution is needed. That interface
|
||||||
|
belongs at the consuming boundary rather than forcing adapter concepts into
|
||||||
|
Promptkit.
|
||||||
|
|
||||||
|
### Artifact Reading And HTTP Containment
|
||||||
|
|
||||||
|
Promptkit will define the artifact-reader extension point used during
|
||||||
|
preparation. Its ordinary file reader may read a path deliberately supplied by
|
||||||
|
an in-process or CLI caller and does not claim to be a deployment sandbox.
|
||||||
|
|
||||||
|
Scriptorium will implement the HTTP-specific reader that:
|
||||||
|
|
||||||
|
- denies file references when no artifact root is configured;
|
||||||
|
- applies the configured artifact byte limit;
|
||||||
|
- enforces Scriptorium's documented lexical root-containment rule; and
|
||||||
|
- maps reader failures to Scriptorium HTTP error responses.
|
||||||
|
|
||||||
|
Scriptorium will inject that reader through Promptkit's public construction
|
||||||
|
boundary. Promptkit will not know about HTTP roots, status codes, request DTOs,
|
||||||
|
or deployment policy.
|
||||||
|
|
||||||
|
### Configuration And Default Ownership
|
||||||
|
|
||||||
|
Configuration ownership follows the behavior configured, not the current file
|
||||||
|
location:
|
||||||
|
|
||||||
|
| Configuration category | Owner |
|
||||||
|
| --- | --- |
|
||||||
|
| Application configuration discovery, configuration-file precedence, `prompt_dir`, `profile_dir`, and `schema_dir` | Scriptorium |
|
||||||
|
| CLI flags and their mapping to application settings or request overrides | Scriptorium |
|
||||||
|
| `server.*`, render-output settings, HTTP byte limits, and server defaults | Scriptorium |
|
||||||
|
| Prompt-definition, profile, and output-contract file formats | Promptkit |
|
||||||
|
| Prompt/profile source selection, overlays, schema behavior, and built-in profiles | Promptkit |
|
||||||
|
| Execution settings, presence-aware request overrides, and execution defaults | Promptkit |
|
||||||
|
| Built-in OpenAI-compatible client settings, timeout behavior, and provider wire mapping | Promptkit |
|
||||||
|
| HTTP request and response fields, including their mapping to framework values | Scriptorium |
|
||||||
|
|
||||||
|
Scriptorium will translate its application settings and external request
|
||||||
|
values into Promptkit construction options and requests. When an omitted
|
||||||
|
Scriptorium setting means “use the framework default,” Scriptorium will omit
|
||||||
|
the override rather than copy Promptkit's numeric default.
|
||||||
|
|
||||||
|
### Compatibility And Versioning
|
||||||
|
|
||||||
|
This migration is intentionally breaking:
|
||||||
|
|
||||||
|
- new Go consumers will import `gitea.maximumdirect.net/eric/promptkit`;
|
||||||
|
- Scriptorium will not provide aliases, forwarding wrappers, or deprecated
|
||||||
|
compatibility packages for its former Go facade;
|
||||||
|
- existing consumers may remain pinned to the final framework-bearing
|
||||||
|
Scriptorium tag until migrated; and
|
||||||
|
- intermediate migration phases need not preserve source compatibility, but
|
||||||
|
each merged phase must be internally buildable and tested.
|
||||||
|
|
||||||
|
Promptkit's first release will be `v0.1.0`. During the migration, incompatible
|
||||||
|
Promptkit changes may advance its minor version until a stable `v1` contract is
|
||||||
|
declared. The first slim Scriptorium release will advance the Scriptorium minor
|
||||||
|
version beyond the final framework-bearing release. Normal semantic-versioning
|
||||||
|
rules apply independently to both projects after the migration.
|
||||||
|
|
||||||
|
Promptkit must be tagged before Scriptorium or another consumer publishes a
|
||||||
|
release that depends on it. Release branches must use tagged module
|
||||||
|
dependencies, not local replacements or unpublished revisions.
|
||||||
|
|
||||||
|
### Local Development And Cross-Repository Coordination
|
||||||
|
|
||||||
|
For coordinated local work, place both repositories in a temporary Go
|
||||||
|
workspace or use an uncommitted module replacement. `go.work`,
|
||||||
|
`go.work.sum`, and local filesystem `replace` directives must not be committed
|
||||||
|
to release branches.
|
||||||
|
|
||||||
|
Cross-repository changes follow this order:
|
||||||
|
|
||||||
|
1. land and tag the required Promptkit capability;
|
||||||
|
2. update Scriptorium and other consumers to that tag;
|
||||||
|
3. run each repository's own CI and smoke checks; and
|
||||||
|
4. release consumers only after the Promptkit tag is available.
|
||||||
|
|
||||||
|
Migration coordination must confirm out-of-band repository creation, Promptkit
|
||||||
|
tags, and downstream migrations before dependent work proceeds.
|
||||||
|
Cross-repository changes are coordinated, not treated as atomic commits.
|
||||||
|
|
||||||
|
### Documentation And Maintained Assets
|
||||||
|
|
||||||
|
Each repository will maintain its own README, contributor guide, architecture,
|
||||||
|
documentation, testing, release, and operations material appropriate to that
|
||||||
|
project. Cross-project documents will link to the canonical owner rather than
|
||||||
|
copy its contract.
|
||||||
|
|
||||||
|
Existing documentation and maintained assets have these target owners:
|
||||||
|
|
||||||
|
| Current material | Target owner |
|
||||||
|
| --- | --- |
|
||||||
|
| Current README and executable quickstart | Scriptorium; Promptkit creates its own framework orientation |
|
||||||
|
| Public Go package and Go-consumer guidance | Promptkit |
|
||||||
|
| Prompt, profile, schema, execution-setting, and framework credential reference | Promptkit |
|
||||||
|
| OpenAI-compatible integration contract and framework internal documents | Promptkit |
|
||||||
|
| CLI, HTTP API, subprocess, and Scriptorium operations contracts | Scriptorium |
|
||||||
|
| Consumer interface overview | Scriptorium, revised to route Go consumers to Promptkit |
|
||||||
|
| Application-configuration discovery, server settings, and adapter internals | Scriptorium |
|
||||||
|
| Current internal overview and source documentation | Split into repository-local overviews; Promptkit owns framework sources and Scriptorium owns HTTP containment |
|
||||||
|
| This ADR and cross-project migration records | Scriptorium |
|
||||||
|
| `examples/go-library` | Promptkit |
|
||||||
|
| `examples/config*.yml`, `examples/render-markdown-summary.sh`, and `examples/http-run.json` | Scriptorium |
|
||||||
|
| Example prompts, profiles, schemas, and synthetic fixtures used by the executable examples | Scriptorium |
|
||||||
|
| Embedded built-in profile assets | Promptkit |
|
||||||
|
| Scriptorium release workflow and executable packaging | Scriptorium |
|
||||||
|
| Repository-level license, ignore rules, agent guidance, and development policies | Each repository maintains its own applicable copy |
|
||||||
|
|
||||||
|
Promptkit will create or retain its own minimal framework examples and test
|
||||||
|
fixtures rather than making either repository's tests depend on the other's
|
||||||
|
working tree. Scriptorium's framework-format documentation will become a short
|
||||||
|
version-appropriate link to Promptkit, while its maintained executable examples
|
||||||
|
remain self-contained.
|
||||||
|
|
||||||
|
## Alternatives Considered
|
||||||
|
|
||||||
|
- Keep the current combined repository and improve package naming. This avoids
|
||||||
|
migration work but retains release and ownership coupling between the
|
||||||
|
framework and executable.
|
||||||
|
- Add Promptkit as a wrapper around the Scriptorium public package. This gives
|
||||||
|
consumers a new import path but leaves framework ownership and dependency
|
||||||
|
direction inverted.
|
||||||
|
- Extract Promptkit while retaining a Scriptorium compatibility facade. This
|
||||||
|
reduces immediate consumer changes but creates a second public API surface
|
||||||
|
and prolongs duplicate maintenance.
|
||||||
|
- Move all artifact reading into Promptkit. This would place HTTP containment,
|
||||||
|
byte limits, and deployment policy in the application-neutral framework.
|
||||||
|
- Keep Promptkit and Scriptorium as separate modules in one repository. This
|
||||||
|
separates imports but not repository permissions, release workflows,
|
||||||
|
issue ownership, or independent project evolution.
|
||||||
|
|
||||||
|
## Rationale
|
||||||
|
|
||||||
|
A separate Promptkit project makes the reusable framework the direct owner of
|
||||||
|
the API that downstream Go projects already consume. Keeping Scriptorium as a
|
||||||
|
public-API consumer exercises the same boundary as other consumers and prevents
|
||||||
|
its adapters from relying on framework internals.
|
||||||
|
|
||||||
|
The selected split keeps transport and deployment policy close to the
|
||||||
|
Scriptorium interfaces that expose it, while allowing Promptkit to remain
|
||||||
|
useful to in-process consumers with different IO and security requirements.
|
||||||
|
Explicit package, configuration, documentation, and asset ownership reduces
|
||||||
|
ambiguity during extraction and after release.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- All Go consumers of the framework must change their import path.
|
||||||
|
- Promptkit and Scriptorium gain independent issue, release, CI, policy, and
|
||||||
|
documentation lifecycles.
|
||||||
|
- Scriptorium becomes a real downstream integration test of Promptkit's public
|
||||||
|
facade.
|
||||||
|
- Framework changes that affect Scriptorium require tagged, ordered
|
||||||
|
cross-repository coordination.
|
||||||
|
- Some current packages, especially artifact reading and defaults, must be
|
||||||
|
separated by responsibility rather than moved intact.
|
||||||
|
- Scriptorium's current configuration and documentation references must be
|
||||||
|
split between application and framework owners.
|
||||||
|
- Maintainers must inventory and migrate downstream consumers explicitly; no
|
||||||
|
compatibility facade will hide incomplete migration.
|
||||||
|
- Until the split is implemented, the current repository structure and
|
||||||
|
contracts remain authoritative.
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
# ADR 0003: Use Maintainer-Run Validation and Tag-Only Releases for Promptkit
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted
|
||||||
|
|
||||||
|
## Date
|
||||||
|
|
||||||
|
2026-07-28
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
[ADR 0002](0002-split-promptkit-from-scriptorium.md) established Promptkit as
|
||||||
|
an independent Go library with its own repository, version history, validation,
|
||||||
|
and release coordination. It anticipated independent hosted CI for Promptkit
|
||||||
|
alongside Scriptorium's existing executable build and CI policy.
|
||||||
|
|
||||||
|
Promptkit is presently a single-maintainer library. It does not produce a
|
||||||
|
runnable command, so executable packaging and binary-release automation do not
|
||||||
|
apply. Its validation and release model should be explicit before repository
|
||||||
|
guidance relies on it.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
Promptkit will use maintainer-run validation rather than hosted CI at this
|
||||||
|
stage. From a clean checkout, the maintainer will run the repository-documented
|
||||||
|
test, vet, build, formatting, documentation-link, and repository-hygiene checks
|
||||||
|
before changes are accepted and before a release tag is published.
|
||||||
|
|
||||||
|
Promptkit releases consist of source commits and semantic Go module tags. The
|
||||||
|
project does not release runnable binaries or maintain binary-packaging
|
||||||
|
automation.
|
||||||
|
|
||||||
|
Scriptorium's executable build, hosted CI, and binary-release policies are
|
||||||
|
unaffected. The repository boundary, independent version history, release
|
||||||
|
ordering, and other migration decisions accepted by ADR 0002 remain in force.
|
||||||
|
Where ADR 0002 anticipated independent hosted CI for Promptkit, this later ADR
|
||||||
|
controls Promptkit validation.
|
||||||
|
|
||||||
|
## Alternatives Considered
|
||||||
|
|
||||||
|
- Add hosted Promptkit CI now. This would provide automated remote enforcement,
|
||||||
|
but its setup and maintenance are not proportionate to the present
|
||||||
|
single-maintainer library and do not replace the maintainer's release
|
||||||
|
responsibility.
|
||||||
|
- Require local Git hooks. Hooks can provide fast feedback, but they are
|
||||||
|
machine-local, can be bypassed, and are not a durable substitute for the
|
||||||
|
documented clean-checkout validation procedure.
|
||||||
|
|
||||||
|
## Rationale
|
||||||
|
|
||||||
|
A documented maintainer-run procedure provides a clear acceptance and release
|
||||||
|
gate with little operational overhead for the project's current contribution
|
||||||
|
pattern. If maintenance load or contributor patterns change, a later ADR may
|
||||||
|
introduce hosted CI without changing Promptkit's library or tag-based release
|
||||||
|
model.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- Promptkit repository guidance must define the complete local validation
|
||||||
|
procedure and the checks required before accepting or tagging a change.
|
||||||
|
- Release evidence is the maintainer's successful clean-checkout validation,
|
||||||
|
not a hosted CI result.
|
||||||
|
- Promptkit releases contain source and semantic Go module tags only.
|
||||||
|
- A future move to hosted CI requires a later architectural decision.
|
||||||
|
- Scriptorium continues to validate, build, package, and release its executable
|
||||||
|
under its own policies.
|
||||||
146
docs/api.md
Normal file
146
docs/api.md
Normal file
@@ -0,0 +1,146 @@
|
|||||||
|
# HTTP API Reference
|
||||||
|
|
||||||
|
This is the canonical public HTTP contract for Scriptorium.
|
||||||
|
|
||||||
|
## Service And Route
|
||||||
|
|
||||||
|
`POST /v1/runs` runs one prompt request and returns generated output,
|
||||||
|
validation, and metadata. The service has no built-in authentication or
|
||||||
|
authorization; deploy it behind appropriate network and authentication controls.
|
||||||
|
|
||||||
|
The service address and HTTP limits are configured as described in the
|
||||||
|
[configuration reference](config.md). `serve` invocation is defined in the
|
||||||
|
[CLI reference](cli.md).
|
||||||
|
|
||||||
|
Requests and responses are JSON objects. Requests are decoded as JSON regardless
|
||||||
|
of their `Content-Type`; successful JSON responses use
|
||||||
|
`Content-Type: application/json`. There are no query parameters.
|
||||||
|
|
||||||
|
## Request Limits
|
||||||
|
|
||||||
|
The configured request-body limit includes inline artifact bodies. The artifact
|
||||||
|
limit applies to HTTP `file` inputs. The response limit applies to the encoded
|
||||||
|
response, including the artifact body and optional raw output. A limit of zero
|
||||||
|
disables that limit.
|
||||||
|
|
||||||
|
A request body over its limit returns `413 request_too_large`; an oversized
|
||||||
|
file input returns `413 artifact_too_large`; an oversized encoded response
|
||||||
|
returns `413 response_too_large`.
|
||||||
|
|
||||||
|
## `POST /v1/runs`
|
||||||
|
|
||||||
|
### Request Body
|
||||||
|
|
||||||
|
The maintained [request example](../examples/http-run.json) is a complete
|
||||||
|
copyable shape. The smallest valid shape is:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"prompt_id": "generic.markdown_summary",
|
||||||
|
"inputs": {
|
||||||
|
"transcript": {"type": "inline", "body": "Source text"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
| Field | Required | Meaning |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `prompt_id` | yes | Non-blank prompt ID. |
|
||||||
|
| `prompt_version` | no | Prompt version filter. |
|
||||||
|
| `profile_id` | no | Execution-profile ID; otherwise the prompt must set `default_profile`. |
|
||||||
|
| `inputs` | yes | Non-empty object mapping input names to references. |
|
||||||
|
| `vars` | no | Object mapping template-variable names to strings. |
|
||||||
|
| `model` | no | Runtime model-override object. |
|
||||||
|
| `include_raw_output` | no | Include `raw_model_output` when true. |
|
||||||
|
|
||||||
|
An input reference has a required `type` of `file` or `inline`. A `file`
|
||||||
|
reference requires `uri`; an `inline` reference requires `body`.
|
||||||
|
|
||||||
|
HTTP file references require a configured artifact root. Relative paths resolve
|
||||||
|
within that root. Absolute paths must be lexically within it; traversal outside
|
||||||
|
it is rejected with `400 artifact_not_allowed`. This lexical check does not
|
||||||
|
resolve symlinks: the operating system follows symlinks inside the root,
|
||||||
|
including ones that target outside it. Keep the root narrow and inaccessible to
|
||||||
|
untrusted writers.
|
||||||
|
|
||||||
|
The optional `model` object accepts `endpoint`, `model`, `temperature`,
|
||||||
|
`max_tokens`, `top_p`, `timeout_seconds`, `service_tier`,
|
||||||
|
`reasoning_effort`, `api_key_env`, and `extra_params`. Numeric ranges and
|
||||||
|
framework credential semantics are defined by the
|
||||||
|
[Promptkit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/formats.md).
|
||||||
|
Explicit zero values for the numeric fields are overrides; zero
|
||||||
|
`timeout_seconds` disables the per-generation deadline only, retaining the
|
||||||
|
request context and configured transport cap. The timeout layers are defined in
|
||||||
|
the [Promptkit outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/integrations/openai-compatible-chat.md#timeout-and-cancellation).
|
||||||
|
|
||||||
|
Raw API-key values are not accepted. `api_key` and any other unknown model
|
||||||
|
field cause `400 invalid_json`.
|
||||||
|
|
||||||
|
### Strict JSON
|
||||||
|
|
||||||
|
Request decoding rejects malformed JSON, unknown fields at every request level,
|
||||||
|
and trailing JSON tokens with `400 invalid_json`. A blank `prompt_id` or
|
||||||
|
empty `inputs` object returns `400 invalid_request`.
|
||||||
|
|
||||||
|
### Success Response
|
||||||
|
|
||||||
|
A completed run returns `200 OK`, including when generated content fails its
|
||||||
|
validation contract. The response contains:
|
||||||
|
|
||||||
|
- `artifact`: `name`, `content_type`, `body`, `size`, `hash`, and
|
||||||
|
optional `uri`;
|
||||||
|
- `validation`: `status`, `mode`, `repair_attempts`, `is_valid`, plus
|
||||||
|
optional `errors` and `schema_path`;
|
||||||
|
- `metadata`: run, prompt, rendered-prompt, profile, model, input-hash, usage,
|
||||||
|
timing, validation, and repair-attempt metadata; and
|
||||||
|
- optional `raw_model_output` when requested.
|
||||||
|
|
||||||
|
`metadata.model_params` has `endpoint`, `model`, `temperature`,
|
||||||
|
`max_tokens`, `top_p`, and `timeout_seconds`, plus optional
|
||||||
|
`service_tier`, `reasoning_effort`, `api_key_env`, and `extra_params`.
|
||||||
|
`metadata.usage` always includes `prompt_tokens`, `completion_tokens`,
|
||||||
|
`total_tokens`, `cached_tokens`, and `cache_write_tokens`; unavailable
|
||||||
|
cache usage is reported as zero.
|
||||||
|
|
||||||
|
A validation failure has `validation.status: "failed"`, `is_valid: false`,
|
||||||
|
and any available diagnostic errors, while still returning the artifact and
|
||||||
|
metadata.
|
||||||
|
|
||||||
|
## Error Responses
|
||||||
|
|
||||||
|
Errors have this shape:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"error":{"code":"invalid_request","message":"prompt_id is required"}}
|
||||||
|
```
|
||||||
|
|
||||||
|
Messages are concise and do not expose wrapped internal causes.
|
||||||
|
|
||||||
|
| Status | Code | Meaning |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `400` | `invalid_json` | Malformed JSON, unknown field, or trailing JSON. |
|
||||||
|
| `400` | `invalid_request` | Missing or invalid request data or runtime override. |
|
||||||
|
| `400` | `profile_required` | No profile ID and no prompt default profile. |
|
||||||
|
| `400` | `prompt_load_failed` | Prompt definition failed to load. |
|
||||||
|
| `400` | `profile_load_failed` | Profile failed to load. |
|
||||||
|
| `400` | `artifact_not_allowed` | HTTP file input is disabled or outside the artifact root. |
|
||||||
|
| `400` | `artifact_read_failed` | Input artifact is invalid or cannot be read. |
|
||||||
|
| `400` | `prompt_render_failed` | Prompt template rendering failed. |
|
||||||
|
| `400` | `api_key_env_missing` | The selected credential environment variable is unset or empty. |
|
||||||
|
| `404` | `not_found` | Route does not exist. |
|
||||||
|
| `404` | `prompt_not_found` | Prompt ID or version does not exist. |
|
||||||
|
| `404` | `profile_not_found` | Profile ID does not exist. |
|
||||||
|
| `405` | `method_not_allowed` | The route does not accept the method. |
|
||||||
|
| `413` | `request_too_large` | Encoded request exceeds its limit. |
|
||||||
|
| `413` | `artifact_too_large` | File input exceeds its limit. |
|
||||||
|
| `413` | `response_too_large` | Encoded response exceeds its limit. |
|
||||||
|
| `500` | `validation_runtime_failed` | Schema or validator runtime failure. |
|
||||||
|
| `500` | `internal_error` | Unclassified server failure. |
|
||||||
|
| `502` | `llm_failed` | Outbound model request failed. |
|
||||||
|
|
||||||
|
## Retry And Idempotency
|
||||||
|
|
||||||
|
Scriptorium provides no idempotency keys, pagination, caching headers, or rate
|
||||||
|
limits. Clients may retry transport failures or `5xx` responses only when
|
||||||
|
their workflow tolerates another model call: a retry can produce different
|
||||||
|
output and incur another provider request.
|
||||||
220
docs/cli.md
220
docs/cli.md
@@ -1,5 +1,10 @@
|
|||||||
# CLI Reference
|
# CLI Reference
|
||||||
|
|
||||||
|
This is the canonical contract for invoking Scriptorium. Configuration discovery,
|
||||||
|
precedence, application source locations, and server settings are defined in the
|
||||||
|
[configuration reference](config.md). The [HTTP API reference](api.md) owns
|
||||||
|
service request and response behavior.
|
||||||
|
|
||||||
## Shortest Useful Command
|
## Shortest Useful Command
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -10,139 +15,134 @@ go run ./cmd/scriptorium render \
|
|||||||
--input glossary=./examples/fixtures/glossary.yml
|
--input glossary=./examples/fixtures/glossary.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
`render` prepares and formats the prompt without calling an LLM.
|
`render` prepares a request without calling an LLM.
|
||||||
|
|
||||||
## Command Overview
|
## Commands
|
||||||
|
|
||||||
- `scriptorium run`: prepare prompt, call the configured LLM, write generated output, print a run summary.
|
- `scriptorium run`: prepare a prompt, call the configured LLM, and write the
|
||||||
- `scriptorium render`: prepare prompt only; write prepared-run output as `text` or `json`.
|
generated artifact.
|
||||||
|
- `scriptorium render`: prepare a prompt and write prepared-run output.
|
||||||
- `scriptorium serve`: start the HTTP server.
|
- `scriptorium serve`: start the HTTP server.
|
||||||
|
|
||||||
Integration references:
|
All commands accept `--config <path>` and reject positional arguments. An
|
||||||
|
effective `prompt_dir` is required for every command. Supply it through the
|
||||||
|
configuration contract or the command's `--prompt-dir` flag.
|
||||||
|
|
||||||
- [HTTP contract](integrations/http-api.md)
|
## `scriptorium run`
|
||||||
- [Narratio subprocess contract](integrations/narratio.md)
|
|
||||||
|
|
||||||
## Common Argument Rules
|
```text
|
||||||
|
scriptorium run [flags]
|
||||||
|
```
|
||||||
|
|
||||||
- `--config` is supported by `run`, `render`, and `serve`.
|
Required flags:
|
||||||
- `run` and `render` require:
|
|
||||||
- `--prompt`
|
|
||||||
- at least one `--input`
|
|
||||||
- an effective `prompt_dir` and `profile_dir` (from flags or config)
|
|
||||||
- `serve` requires an effective `prompt_dir` and `profile_dir` (from flags or config).
|
|
||||||
- Positional arguments are rejected.
|
|
||||||
|
|
||||||
## Flag Reference
|
| Flag | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `--prompt <id>` | Prompt ID to execute. |
|
||||||
|
| `--input name=path` | Input file mapping; repeat or use comma-separated mappings. |
|
||||||
|
|
||||||
### `scriptorium run`
|
Optional flags:
|
||||||
|
|
||||||
- `--config <path>`: app config file path.
|
| Flag | Meaning |
|
||||||
- `--prompt-dir <dir>`: prompt definition directory.
|
| --- | --- |
|
||||||
- `--profile-dir <dir>`: profile definition directory.
|
| `--config <path>` | Application configuration file. |
|
||||||
- `--schema-dir <dir>`: schema base directory for `json_schema` validation.
|
| `--prompt-dir <dir>` | Prompt-definition directory override. |
|
||||||
- `--prompt <id>`: prompt ID to execute. Required.
|
| `--profile-dir <dir>` | Custom profile-directory override. |
|
||||||
- `--prompt-id <id>`: deprecated alias for `--prompt`.
|
| `--schema-dir <dir>` | Schema base-directory override. |
|
||||||
- `--profile <id>`: explicit profile override.
|
| `--profile <id>` | Execution-profile override. |
|
||||||
- `--profile-id <id>`: deprecated alias for `--profile`.
|
| `--var name=value` | Template-variable mapping; repeat or use comma-separated mappings. |
|
||||||
- `--input name=path`: input mapping (repeatable, comma-separated accepted).
|
| `--out <path>` | Write generated content to this file instead of stdout. |
|
||||||
- `--var name=value`: template variable mapping (repeatable, comma-separated accepted).
|
| `--llm-base-url <url>` | Runtime endpoint override. |
|
||||||
- `--out <path>`: write artifact body to file instead of stdout.
|
| `--model <name>` | Runtime model override. |
|
||||||
- `--llm-base-url <url>`: runtime endpoint override.
|
| `--api-key-env <name>` | Runtime API-key environment-variable name override. |
|
||||||
- `--model <name>`: runtime model override.
|
| `--temperature <float>` | Runtime temperature override. |
|
||||||
- `--api-key-env <name>`: runtime API key environment-variable name override.
|
| `--max-tokens <int>` | Runtime maximum-token override. |
|
||||||
- `--temperature <float>`: runtime temperature override.
|
| `--top-p <float>` | Runtime top-p override. |
|
||||||
- `--max-tokens <int>`: runtime max tokens override.
|
| `--timeout <duration>` | Runtime timeout override using Go duration syntax. |
|
||||||
- `--top-p <float>`: runtime top-p override.
|
|
||||||
- `--timeout <duration>`: runtime timeout override (Go duration syntax, for example `30s`, `2m`).
|
|
||||||
|
|
||||||
### `scriptorium render`
|
Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for
|
||||||
|
`--profile`.
|
||||||
|
|
||||||
- Supports the same flags as `run`, except:
|
Omitted numeric runtime flags preserve the selected effective value; explicit
|
||||||
- no `--schema-dir` flag.
|
zero values override it. `--timeout 0s` disables the per-generation deadline
|
||||||
- Adds:
|
only; the caller context and configured transport cap remain active. CLI
|
||||||
- `--format text|json`: prepared-run output format.
|
durations are converted to whole seconds by truncation toward zero, so any
|
||||||
|
duration whose absolute value is below one second becomes an explicit
|
||||||
|
zero-second override. The timeout layers are defined in the
|
||||||
|
[Promptkit outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/integrations/openai-compatible-chat.md#timeout-and-cancellation).
|
||||||
|
|
||||||
Notes:
|
There is no raw API-key flag. Use `--api-key-env`.
|
||||||
- `render` still resolves profile and runtime settings.
|
|
||||||
- `render` still validates that `api_key_env` exists if the selected profile or overrides require it.
|
|
||||||
|
|
||||||
### `scriptorium serve`
|
## `scriptorium render`
|
||||||
|
|
||||||
- `--config <path>`: app config file path.
|
```text
|
||||||
- `--addr <listen-address>`: HTTP listen address.
|
scriptorium render [flags]
|
||||||
- `--prompt-dir <dir>`: prompt definition directory.
|
```
|
||||||
- `--profile-dir <dir>`: profile definition directory.
|
|
||||||
- `--schema-dir <dir>`: schema base directory for `json_schema` validation.
|
|
||||||
|
|
||||||
Notes:
|
`--prompt <id>` and at least one `--input name=path` are required. The
|
||||||
- `serve` does not accept runtime model override flags such as `--model` or `--llm-base-url`.
|
following optional flags are supported: `--config`, `--prompt-dir`,
|
||||||
|
`--profile-dir`, `--profile`, `--var`, `--out`, `--llm-base-url`,
|
||||||
|
`--model`, `--api-key-env`, `--temperature`, `--max-tokens`, `--top-p`,
|
||||||
|
`--timeout`, and `--format text|json`. Their meanings match the corresponding
|
||||||
|
`run` flags; `--format` selects prepared-run output and otherwise uses
|
||||||
|
`defaults.render_format`.
|
||||||
|
|
||||||
|
The same deprecated aliases and numeric/timeout behavior as `run` apply.
|
||||||
|
`render` does not accept `--schema-dir`; configure `schema_dir` through the
|
||||||
|
configuration file. It resolves profiles and schemas as part of preparation but
|
||||||
|
does not call an LLM.
|
||||||
|
|
||||||
|
## `scriptorium serve`
|
||||||
|
|
||||||
|
```text
|
||||||
|
scriptorium serve [flags]
|
||||||
|
```
|
||||||
|
|
||||||
|
Optional flags:
|
||||||
|
|
||||||
|
| Flag | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `--config <path>` | Application configuration file. |
|
||||||
|
| `--addr <listen-address>` | HTTP listen-address override. |
|
||||||
|
| `--prompt-dir <dir>` | Prompt-definition directory override. |
|
||||||
|
| `--profile-dir <dir>` | Custom profile-directory override. |
|
||||||
|
| `--schema-dir <dir>` | Schema base-directory override. |
|
||||||
|
| `--artifact-root <dir>` | Root for HTTP `file` input references. |
|
||||||
|
| `--max-request-bytes <n>` | Maximum encoded HTTP request-body bytes; `0` disables the limit. |
|
||||||
|
| `--max-artifact-bytes <n>` | Maximum HTTP file-input artifact bytes; `0` disables the limit. |
|
||||||
|
| `--max-response-bytes <n>` | Maximum encoded HTTP response bytes; `0` disables the limit. |
|
||||||
|
|
||||||
|
`serve` accepts no runtime model override flags. HTTP request fields, response
|
||||||
|
schemas, and error codes are defined in the [HTTP API reference](api.md).
|
||||||
|
|
||||||
## Input And Variable Syntax
|
## Input And Variable Syntax
|
||||||
|
|
||||||
- `--input name=path` maps prompt input names to local file paths.
|
`--input name=path` maps an input name to a local file; `--var name=value`
|
||||||
- `--var name=value` maps template variable names to values.
|
maps a template variable to a string. Both flags can be repeated or contain
|
||||||
- Both flags can be repeated.
|
comma-separated mappings. Values may contain `=` after the first separator.
|
||||||
- Both flags also support comma-separated batches, for example:
|
Empty names and values are rejected.
|
||||||
- `--input transcript=./t.md,glossary=./g.yml`
|
|
||||||
- `--var session_id=42,session_date=2026-05-04`
|
|
||||||
|
|
||||||
## Output Behavior
|
CLI inputs are file references. HTTP inline inputs are defined by the
|
||||||
|
[HTTP API reference](api.md).
|
||||||
|
|
||||||
`run`:
|
## Output And Exit Behavior
|
||||||
- Writes generated artifact content to stdout by default.
|
|
||||||
- Writes generated artifact content to `--out` when provided.
|
|
||||||
- Prints run summary metadata to stderr on success.
|
|
||||||
- Prints errors to stderr on failure.
|
|
||||||
|
|
||||||
`render`:
|
- `run` writes generated content to stdout, or to `--out` when supplied, and
|
||||||
- Writes prepared-run output to stdout by default.
|
writes a concise summary to stderr.
|
||||||
- Writes prepared-run output to `--out` when provided.
|
- `render` writes prepared-run output to stdout, or to `--out` when supplied,
|
||||||
- Does not print a success summary line.
|
without a success summary.
|
||||||
|
- `serve` writes startup and server errors to stderr.
|
||||||
|
|
||||||
`serve`:
|
Exit statuses:
|
||||||
- Logs startup and server errors to stderr.
|
|
||||||
|
|
||||||
## Exit Codes
|
| Status | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `0` | Success. |
|
||||||
|
| `1` | Parse, configuration, loading, rendering, generation, output-write, or other runtime error. |
|
||||||
|
| `2` | `run` generated and wrote output, but validation failed. |
|
||||||
|
|
||||||
- `0`: success.
|
## Workflows And Examples
|
||||||
- `1`: runtime/parse/config/load/render/generation/output-write error.
|
|
||||||
- `2`: `run` completed, output was generated, but validation status is `failed`.
|
|
||||||
|
|
||||||
When `run` exits `2`, output may already be written to stdout or `--out`.
|
The [maintained render script](../examples/render-markdown-summary.sh) is a
|
||||||
|
copyable render workflow. The [HTTP request example](../examples/http-run.json)
|
||||||
## Common Workflows
|
is for a running `serve` process.
|
||||||
|
|
||||||
Render prompt inputs and template variables as JSON:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium render \
|
|
||||||
--config ./examples/config.yml \
|
|
||||||
--prompt generic.markdown_summary \
|
|
||||||
--input transcript=./examples/fixtures/transcript.md \
|
|
||||||
--input glossary=./examples/fixtures/glossary.yml \
|
|
||||||
--var session_date=2026-05-04 \
|
|
||||||
--format json
|
|
||||||
```
|
|
||||||
|
|
||||||
Run a prompt with profile override and file output:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium run \
|
|
||||||
--config ./examples/config.yml \
|
|
||||||
--prompt generic.markdown_summary \
|
|
||||||
--profile local-fast \
|
|
||||||
--input transcript=./examples/fixtures/transcript.md \
|
|
||||||
--input glossary=./examples/fixtures/glossary.yml \
|
|
||||||
--out ./summary.md
|
|
||||||
```
|
|
||||||
|
|
||||||
Start the HTTP server with explicit config:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium serve --config ./examples/config.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Copyable example script:
|
|
||||||
|
|
||||||
- `examples/render-markdown-summary.sh`
|
|
||||||
|
|||||||
277
docs/config.md
277
docs/config.md
@@ -1,231 +1,86 @@
|
|||||||
# Configuration Reference
|
# Configuration Reference
|
||||||
|
|
||||||
## Config Discovery And Precedence
|
This is the canonical reference for Scriptorium application settings. Prompt,
|
||||||
|
profile, schema, execution-setting, built-in profile, and framework credential
|
||||||
|
semantics are defined by the
|
||||||
|
[Promptkit v0.1.0 format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/formats.md).
|
||||||
|
For command syntax, see the [CLI reference](cli.md); for HTTP request shapes and
|
||||||
|
outcomes, see the [HTTP API reference](api.md).
|
||||||
|
|
||||||
Application settings are loaded in this order:
|
## Discovery And Precedence
|
||||||
|
|
||||||
1. Built-in defaults
|
Application settings are resolved in this order:
|
||||||
2. `config.yml` values
|
|
||||||
3. CLI overrides
|
|
||||||
|
|
||||||
When `--config` is not provided, Scriptorium searches for config files in this order:
|
1. built-in Scriptorium defaults;
|
||||||
|
2. a configuration file; then
|
||||||
|
3. CLI overrides.
|
||||||
|
|
||||||
1. `/usr/local/etc/scriptorium/config.yml`
|
When `--config` is omitted, Scriptorium searches
|
||||||
2. `/etc/scriptorium/config.yml`
|
`/usr/local/etc/scriptorium/config.yml` and then `/etc/scriptorium/config.yml`.
|
||||||
|
If neither exists, it uses built-in defaults. An explicit `--config` path must
|
||||||
|
exist and decode successfully.
|
||||||
|
|
||||||
If neither file exists, Scriptorium continues with built-in defaults.
|
The maintained [minimal configuration](../examples/config.yml) and
|
||||||
|
[complete configuration](../examples/config.full.yml) are copyable examples.
|
||||||
|
|
||||||
When `--config <path>` is provided, that file is required.
|
## Application Configuration File
|
||||||
|
|
||||||
## Minimal App Config
|
Configuration is strict YAML: unknown fields are rejected. Empty string values
|
||||||
|
do not override a prior value. Raw API-key fields are not accepted.
|
||||||
|
|
||||||
```yaml
|
| Field | Default | Meaning |
|
||||||
prompt_dir: ./examples/prompts
|
| --- | --- | --- |
|
||||||
profile_dir: ./examples/profiles
|
| `prompt_dir` | unset | Promptkit prompt-definition source directory. `run`, `render`, and `serve` require an effective value. |
|
||||||
```
|
| `profile_dir` | unset | Optional custom Promptkit profile source directory overlaid on Promptkit built-ins. |
|
||||||
|
| `schema_dir` | `.` | Promptkit schema source directory for relative schema paths. |
|
||||||
|
| `server.addr` | `:8080` | Address used by `serve`. |
|
||||||
|
| `server.artifact_root` | unset | Root that enables HTTP `file` input references. |
|
||||||
|
| `server.max_request_bytes` | `16777216` | Maximum encoded HTTP request-body bytes; `0` disables the limit. |
|
||||||
|
| `server.max_artifact_bytes` | `16777216` | Maximum HTTP file-input artifact bytes; `0` disables the limit. |
|
||||||
|
| `server.max_response_bytes` | `16777216` | Maximum encoded HTTP response bytes; `0` disables the limit. |
|
||||||
|
| `defaults.render_format` | `text` | Default prepared-run output format: `text` or `json`. |
|
||||||
|
|
||||||
This is enough to use `run` and `render` when prompt/profile files are valid.
|
The size fields must be zero or greater. The [HTTP API](api.md) defines how
|
||||||
|
each limit is enforced and reported. `server.artifact_root` configures an HTTP
|
||||||
|
deployment boundary; see [operations](operations.md) for deployment handling.
|
||||||
|
|
||||||
## Production-Oriented App Config
|
## Framework Source Mapping
|
||||||
|
|
||||||
```yaml
|
Scriptorium passes `prompt_dir`, `profile_dir`, and `schema_dir` to Promptkit
|
||||||
prompt_dir: /opt/scriptorium/prompts
|
when constructing its engine. Scriptorium does not redefine or independently
|
||||||
profile_dir: /opt/scriptorium/profiles
|
parse those framework file formats.
|
||||||
schema_dir: /opt/scriptorium/schemas
|
|
||||||
|
|
||||||
server:
|
- Prompt selection, versions, message templates, inputs, output contracts, and
|
||||||
addr: 127.0.0.1:8080
|
session IDs are Promptkit contracts.
|
||||||
|
- Profile fields, numeric ranges, execution defaults, overlay precedence,
|
||||||
|
built-in profiles, and credential rules are Promptkit contracts.
|
||||||
|
- Schema path behavior and generated-content validation are Promptkit
|
||||||
|
contracts.
|
||||||
|
|
||||||
defaults:
|
See the
|
||||||
render_format: text
|
[tagged Promptkit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/formats.md)
|
||||||
```
|
for all of those definitions. The files under
|
||||||
|
[`examples/prompts`](../examples/prompts/),
|
||||||
|
[`examples/profiles`](../examples/profiles/), and
|
||||||
|
[`examples/schemas`](../examples/schemas/) are maintained Scriptorium
|
||||||
|
application inputs using that tagged format.
|
||||||
|
|
||||||
## App Config File (`config.yml`)
|
## Credentials And Outbound Behavior
|
||||||
|
|
||||||
Top-level fields:
|
Scriptorium maps `--api-key-env` and HTTP `model.api_key_env` into Promptkit
|
||||||
|
request overrides. Keep secret values in environment variables and store only
|
||||||
|
their names in configuration or framework source files. Do not place raw keys
|
||||||
|
in configuration, prompts, profiles, CLI arguments, examples, or HTTP
|
||||||
|
payloads.
|
||||||
|
|
||||||
- `prompt_dir` (optional): default prompt definition directory.
|
Promptkit's
|
||||||
- `profile_dir` (optional): default profile definition directory.
|
[OpenAI-compatible integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/integrations/openai-compatible-chat.md)
|
||||||
- `schema_dir` (optional): base directory for schema files used by `json_schema` validation.
|
defines outbound authentication, provider request mapping, transport limits,
|
||||||
- `server.addr` (optional): default listen address for `serve`.
|
and timeout layering.
|
||||||
- `defaults.render_format` (optional): default `render` output format (`text` or `json`).
|
|
||||||
|
|
||||||
Built-in defaults:
|
## Related References
|
||||||
|
|
||||||
- `schema_dir`: `.`
|
- [CLI reference](cli.md)
|
||||||
- `server.addr`: `:8080`
|
- [HTTP API reference](api.md)
|
||||||
- `defaults.render_format`: `text`
|
- [Operations guide](operations.md)
|
||||||
|
- [Promptkit framework formats](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/formats.md)
|
||||||
Validation behavior:
|
|
||||||
|
|
||||||
- Config decoding is strict; unknown YAML fields are rejected.
|
|
||||||
- Raw API key fields are not supported in `config.yml`.
|
|
||||||
|
|
||||||
## Prompt Definition Files
|
|
||||||
|
|
||||||
Prompt definitions are YAML files anywhere under `prompt_dir`, including nested subdirectories.
|
|
||||||
|
|
||||||
Subdirectories are organizational only. Callers still select prompts by the YAML `id`, not by file path. For example, `prompts/dnd/recap.yaml` may still declare `id: dnd.recap`, and callers use `--prompt dnd.recap`.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
id: generic.structured_events
|
|
||||||
version: "1.0.0"
|
|
||||||
default_profile: local-quality
|
|
||||||
description: Produce structured event JSON from a transcript.
|
|
||||||
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
content_type: text/markdown
|
|
||||||
description: Source transcript content
|
|
||||||
- name: glossary
|
|
||||||
required: false
|
|
||||||
content_type: text/yaml
|
|
||||||
description: Optional glossary context
|
|
||||||
|
|
||||||
messages:
|
|
||||||
- role: system
|
|
||||||
content_file: ./generic.structured_events.system.md
|
|
||||||
- role: user
|
|
||||||
content_file: ./generic.structured_events.user.md
|
|
||||||
|
|
||||||
output:
|
|
||||||
format: json
|
|
||||||
validation_mode: json_schema
|
|
||||||
schema_path: structured_events.schema.json
|
|
||||||
repair_attempts: 0
|
|
||||||
```
|
|
||||||
|
|
||||||
Field reference:
|
|
||||||
|
|
||||||
- `id` (required): prompt identifier.
|
|
||||||
- `version` (required): prompt version.
|
|
||||||
- `default_profile` (optional): profile ID used when request does not provide `profile_id`.
|
|
||||||
- `description` (optional): prompt description.
|
|
||||||
- `inputs` (optional list): expected named inputs.
|
|
||||||
- `messages` (required list): prompt message templates.
|
|
||||||
- `output` (required object): output contract.
|
|
||||||
|
|
||||||
`inputs[]` fields:
|
|
||||||
|
|
||||||
- `name` (required)
|
|
||||||
- `required` (optional, boolean)
|
|
||||||
- `content_type` (optional metadata)
|
|
||||||
- `description` (optional)
|
|
||||||
|
|
||||||
`messages[]` fields:
|
|
||||||
|
|
||||||
- `role` (required)
|
|
||||||
- `content` or `content_file` (exactly one is required)
|
|
||||||
|
|
||||||
Message rules:
|
|
||||||
|
|
||||||
- Repeated roles are allowed.
|
|
||||||
- `content_file` is resolved relative to the prompt YAML file location.
|
|
||||||
- Nested prompt files keep the same relative `content_file` behavior; `./recap.user.md` next to `dnd/recap.yaml` resolves from `dnd/`.
|
|
||||||
- Prompt decoding is strict; unknown YAML fields are rejected.
|
|
||||||
- Duplicate prompt IDs are invalid. If multiple files declare the requested prompt ID, Scriptorium fails instead of choosing one.
|
|
||||||
|
|
||||||
`output` fields:
|
|
||||||
|
|
||||||
- `format` (required): `text`, `markdown`, or `json`.
|
|
||||||
- `validation_mode` (required): `none`, `basic`, `json`, or `json_schema`.
|
|
||||||
- `schema_path` (required when `validation_mode: json_schema`).
|
|
||||||
- `repair_attempts` (required): integer `>= 0`.
|
|
||||||
|
|
||||||
Repair behavior boundary:
|
|
||||||
|
|
||||||
- `repair_attempts` is part of the prompt contract.
|
|
||||||
- CLI and HTTP currently construct the runner without a repairer, so normal `run`/`serve` execution does not perform output repair attempts.
|
|
||||||
|
|
||||||
## Profile Definition Files
|
|
||||||
|
|
||||||
Execution profiles are YAML files anywhere under `profile_dir`, including nested subdirectories.
|
|
||||||
|
|
||||||
Subdirectories are organizational only. Callers still select profiles by the YAML `id`, not by file path. For example, `profiles/local/local-quality.yaml` may still declare `id: local-quality`, and callers use `--profile local-quality`.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
id: local-fast
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: gpt-4o-mini
|
|
||||||
temperature: 0.2
|
|
||||||
max_tokens: 500
|
|
||||||
top_p: 1.0
|
|
||||||
timeout_seconds: 90
|
|
||||||
api_key_env: SCRIPTORIUM_API_KEY
|
|
||||||
service_tier: priority
|
|
||||||
```
|
|
||||||
|
|
||||||
Field reference:
|
|
||||||
|
|
||||||
- `id` (required)
|
|
||||||
- `endpoint` (required)
|
|
||||||
- `model` (required)
|
|
||||||
- `temperature` (optional): range `0..2`
|
|
||||||
- `max_tokens` (optional): `>= 0`
|
|
||||||
- `top_p` (optional): range `0..1`
|
|
||||||
- `timeout_seconds` (optional): `>= 0`
|
|
||||||
- `service_tier` (optional): provider-specific request tier such as OpenRouter `flex` or `priority`
|
|
||||||
- `reasoning_effort` (optional)
|
|
||||||
- `api_key_env` (optional)
|
|
||||||
- `extra_params` (optional map of strings)
|
|
||||||
|
|
||||||
Profile rules:
|
|
||||||
|
|
||||||
- Profile decoding is strict; unknown YAML fields are rejected.
|
|
||||||
- Raw `api_key` is rejected; use `api_key_env`.
|
|
||||||
- If `api_key_env` is set, that environment variable must be set when preparing/running.
|
|
||||||
- Duplicate profile IDs are invalid. If multiple files declare the requested profile ID, Scriptorium fails instead of choosing one.
|
|
||||||
|
|
||||||
Current outbound request behavior:
|
|
||||||
|
|
||||||
- The OpenAI-compatible client currently serializes: `model`, `messages`, `temperature`, `max_tokens`, `top_p`, `service_tier`, and optional `response_format` for `json_schema` prompts.
|
|
||||||
- `reasoning_effort` and `extra_params` are parsed and carried in effective settings, but are not currently serialized into outbound chat-completions requests.
|
|
||||||
|
|
||||||
## Schema Behavior
|
|
||||||
|
|
||||||
Schemas are JSON files, typically in `schema_dir`.
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
|
|
||||||
- `output.validation_mode: json_schema` requires `output.schema_path`.
|
|
||||||
- Relative `schema_path` values resolve from `schema_dir`, including explicit nested paths such as `dnd/structured_events.schema.json`.
|
|
||||||
- Absolute `schema_path` values are used directly.
|
|
||||||
- Scriptorium does not recursively search schemas by basename; nested schemas must be referenced by their relative path.
|
|
||||||
- Missing or invalid schema documents cause runtime validation errors.
|
|
||||||
- Invalid generated JSON causes validation status `failed` (not a runtime error).
|
|
||||||
|
|
||||||
Supported artifact reference types for request inputs are `file` and `inline`.
|
|
||||||
|
|
||||||
## Secrets Handling
|
|
||||||
|
|
||||||
- Keep secret values in environment variables.
|
|
||||||
- Store only environment-variable names in profile `api_key_env`.
|
|
||||||
- Do not put raw API keys in config, prompts, profiles, CLI flags, or HTTP request bodies.
|
|
||||||
|
|
||||||
## Maintained Examples
|
|
||||||
|
|
||||||
- App config: `examples/config.yml`
|
|
||||||
- Prompt examples: `examples/prompts/`
|
|
||||||
- Profile examples: `examples/profiles/`
|
|
||||||
- Schema examples: `examples/schemas/`
|
|
||||||
- Input fixtures: `examples/fixtures/`
|
|
||||||
- Render example script: `examples/render-markdown-summary.sh`
|
|
||||||
- HTTP request example: `examples/http-run.json`
|
|
||||||
|
|
||||||
Example organizational layout:
|
|
||||||
|
|
||||||
```text
|
|
||||||
examples/prompts/dnd/recap.yaml
|
|
||||||
examples/profiles/local/local-quality.yaml
|
|
||||||
examples/schemas/dnd/structured_events.schema.json
|
|
||||||
```
|
|
||||||
|
|
||||||
## Integration References
|
|
||||||
|
|
||||||
- [Inbound HTTP contract](integrations/http-api.md)
|
|
||||||
- [Outbound OpenAI-compatible contract](integrations/openai-compatible-chat.md)
|
|
||||||
|
|||||||
38
docs/consumers/api.md
Normal file
38
docs/consumers/api.md
Normal file
@@ -0,0 +1,38 @@
|
|||||||
|
# Consumer Integration Overview
|
||||||
|
|
||||||
|
Scriptorium exposes executable interfaces. Choose between a local subprocess
|
||||||
|
and the HTTP service according to the boundary your application needs.
|
||||||
|
|
||||||
|
| Interface | Use when |
|
||||||
|
| --- | --- |
|
||||||
|
| CLI subprocess | The consumer needs a synchronous local process boundary or prepared output. |
|
||||||
|
| HTTP API | The consumer needs a service boundary or remote access. |
|
||||||
|
|
||||||
|
- CLI subprocess: [subprocess integration](../integrations/subprocess.md)
|
||||||
|
- HTTP service: [HTTP API reference](../api.md)
|
||||||
|
- Application configuration: [configuration reference](../config.md)
|
||||||
|
|
||||||
|
Go applications that need an in-process prompt framework should import
|
||||||
|
Promptkit directly. The tagged
|
||||||
|
[Promptkit Go consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
owns that interface; Scriptorium does not provide a Go library package.
|
||||||
|
Consumers arriving from the former Scriptorium Go API should follow the
|
||||||
|
[migration guide](migrating-to-promptkit.md).
|
||||||
|
|
||||||
|
## Consumer Responsibilities
|
||||||
|
|
||||||
|
Consumers are responsible for:
|
||||||
|
|
||||||
|
- selecting and deploying prompt, profile, and schema assets;
|
||||||
|
- supplying required inputs and template variables;
|
||||||
|
- supplying credentials through the chosen interface;
|
||||||
|
- protecting rendered prompts and generated artifacts as potentially
|
||||||
|
sensitive;
|
||||||
|
- deciding whether validation-failed output is usable; and
|
||||||
|
- retrying only when another model call is acceptable.
|
||||||
|
|
||||||
|
Scriptorium does not persist run state. A retry can produce different output
|
||||||
|
and can incur another provider request. CLI exits belong to the
|
||||||
|
[CLI reference](../cli.md), HTTP status behavior belongs to the
|
||||||
|
[HTTP API reference](../api.md), and framework semantics belong to
|
||||||
|
[Promptkit v0.1.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/formats.md).
|
||||||
109
docs/consumers/migrating-to-promptkit.md
Normal file
109
docs/consumers/migrating-to-promptkit.md
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
# Migrate From Scriptorium To Promptkit
|
||||||
|
|
||||||
|
## Supported Migration Boundary
|
||||||
|
|
||||||
|
Scriptorium `v0.11.1` at
|
||||||
|
`gitea.maximumdirect.net/eric/scriptorium` is the final release that provides
|
||||||
|
the former in-process Go framework. Promptkit `v0.1.0` at
|
||||||
|
`gitea.maximumdirect.net/eric/promptkit` is the destination for that framework
|
||||||
|
API. Scriptorium `v0.12.0` and later provide the CLI and HTTP application only.
|
||||||
|
|
||||||
|
There is no Scriptorium compatibility facade, alias package, forwarding
|
||||||
|
package, or deprecated wrapper. A consumer that cannot migrate may remain
|
||||||
|
pinned to Scriptorium `v0.11.1`, but that framework-bearing line does not
|
||||||
|
provide the slim application release.
|
||||||
|
|
||||||
|
## Update A Go Consumer
|
||||||
|
|
||||||
|
Start from a clean consumer checkout and review the pending diff before
|
||||||
|
committing it. Add the published Promptkit module:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.1.0
|
||||||
|
```
|
||||||
|
|
||||||
|
For an ordinary consumer that imports the former root package under its
|
||||||
|
default name, replace the exact import and package qualifier, then format the
|
||||||
|
changed Go files:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git grep -l \
|
||||||
|
'"gitea.maximumdirect.net/eric/scriptorium"' \
|
||||||
|
-- '*.go' |
|
||||||
|
while IFS= read -r go_file
|
||||||
|
do
|
||||||
|
perl -pi -e \
|
||||||
|
's{"gitea.maximumdirect.net/eric/scriptorium"}{"gitea.maximumdirect.net/eric/promptkit"}g; s{\bscriptorium\.}{promptkit.}g' \
|
||||||
|
"$go_file"
|
||||||
|
gofmt -w "$go_file"
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
Inspect the resulting diff. Consumers that used an import alias should retain
|
||||||
|
or deliberately rename that alias instead of applying the qualifier
|
||||||
|
replacement mechanically.
|
||||||
|
|
||||||
|
Remove the now-unused Scriptorium requirement through module tidiness and run
|
||||||
|
the consumer's complete tests:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go mod tidy
|
||||||
|
go test ./...
|
||||||
|
```
|
||||||
|
|
||||||
|
Confirm that `go.mod` selects Promptkit `v0.1.0` and that no Go file imports
|
||||||
|
the former Scriptorium package:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test "$(
|
||||||
|
go list -m -f '{{.Path}}@{{.Version}}' \
|
||||||
|
gitea.maximumdirect.net/eric/promptkit
|
||||||
|
)" = 'gitea.maximumdirect.net/eric/promptkit@v0.1.0'
|
||||||
|
if git grep -n \
|
||||||
|
'gitea.maximumdirect.net/eric/scriptorium' \
|
||||||
|
-- '*.go'
|
||||||
|
then
|
||||||
|
printf '%s\n' 'a former Scriptorium Go import remains' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
```
|
||||||
|
|
||||||
|
## Compatibility And Additions
|
||||||
|
|
||||||
|
Promptkit preserves the established engine, request, result, profile,
|
||||||
|
source-option, model-client, artifact, validation-value, and public-error
|
||||||
|
shapes where practical. Exact declarations and current behavior belong to the
|
||||||
|
tagged [Promptkit consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
and Go source.
|
||||||
|
|
||||||
|
Promptkit also includes migration-relevant public contracts that were not in
|
||||||
|
Scriptorium `v0.11.1`:
|
||||||
|
|
||||||
|
- [`WithArtifactReader`](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/engine.go#L96-L105)
|
||||||
|
and the
|
||||||
|
[`ArtifactReader` declaration](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/types.go#L129-L135)
|
||||||
|
provide the artifact-reading extension described by the tagged
|
||||||
|
[extension-interface guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md#extension-interfaces).
|
||||||
|
- [`ErrProfileRequired` and `ErrAPIKeyEnvMissing`](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/engine.go#L28-L40)
|
||||||
|
provide the specific identities described by the tagged
|
||||||
|
[error guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md#errors).
|
||||||
|
|
||||||
|
Use those tagged owners for exact signatures, wrapping guarantees, and
|
||||||
|
extension behavior.
|
||||||
|
|
||||||
|
## Verify Consumer Behavior
|
||||||
|
|
||||||
|
Source compatibility is only the first check. Exercise the behavior the
|
||||||
|
consumer actually relies upon, especially:
|
||||||
|
|
||||||
|
- prompt, profile, and schema source selection;
|
||||||
|
- direct and environment-based credentials;
|
||||||
|
- caller, generation, and transport timeout layering;
|
||||||
|
- output validation and validation-failure handling;
|
||||||
|
- injected model-client and artifact-reader extensions; and
|
||||||
|
- every `errors.Is` branch used for recovery or classification.
|
||||||
|
|
||||||
|
Also verify any serialized values, redaction expectations, filesystem policy,
|
||||||
|
and provider integration behavior that crosses the consumer's own boundary.
|
||||||
|
Promptkit owns the in-process framework contract; Scriptorium owns only its
|
||||||
|
executable CLI and HTTP application interfaces.
|
||||||
56
docs/development.md
Normal file
56
docs/development.md
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
# Development
|
||||||
|
|
||||||
|
This is the contributor entry point for Scriptorium. Scriptorium is an
|
||||||
|
application that consumes the public
|
||||||
|
[Promptkit v0.1.0 package](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md);
|
||||||
|
framework implementation work belongs in Promptkit.
|
||||||
|
|
||||||
|
## Initial Orientation
|
||||||
|
|
||||||
|
Before starting work:
|
||||||
|
|
||||||
|
1. inspect the working tree and preserve unrelated changes;
|
||||||
|
2. read the [architecture policy](policy/architecture.md);
|
||||||
|
3. follow the task-specific contracts and internal documents below; and
|
||||||
|
4. inspect the relevant implementation and tests before changing them.
|
||||||
|
|
||||||
|
Also read the [documentation policy](policy/documentation.md) before changing
|
||||||
|
documentation and the [testing policy](policy/testing.md) before changing
|
||||||
|
tests.
|
||||||
|
|
||||||
|
## Task-Specific Reading Guide
|
||||||
|
|
||||||
|
| Task | Read before changing |
|
||||||
|
| --- | --- |
|
||||||
|
| Repository orientation or component responsibility | [Internal component overview](internal/overview.md) and [architecture policy](policy/architecture.md) |
|
||||||
|
| CLI commands, flags, output, or exit behavior | [CLI contract](cli.md) and [adapter internals](internal/adapters.md) |
|
||||||
|
| HTTP routes, DTOs, limits, status mapping, or artifact policy | [HTTP API contract](api.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) |
|
||||||
|
| Application configuration or precedence | [Configuration contract](config.md), [adapter internals](internal/adapters.md), and [source internals](internal/sources.md) |
|
||||||
|
| Prepared-run presentation | [CLI contract](cli.md), [adapter internals](internal/adapters.md), and `internal/format` |
|
||||||
|
| Prompt, profile, schema, generation, or validation semantics | [Promptkit framework formats](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/formats.md) and the [Promptkit consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md) |
|
||||||
|
| OpenAI-compatible outbound behavior or timeout layering | [Promptkit integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/integrations/openai-compatible-chat.md) |
|
||||||
|
| Subprocess behavior | [Subprocess integration](integrations/subprocess.md) and [CLI contract](cli.md) |
|
||||||
|
| Runtime operation or recovery | [Operations](operations.md) |
|
||||||
|
| Release packaging or publication | The [release procedure](release.md), [hosted release workflow](../.woodpecker/release.yml), and [architecture policy](policy/architecture.md) |
|
||||||
|
| Examples or copyable assets | The owning Scriptorium contract, the relevant [Promptkit format contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/formats.md), and the related files under `examples/` |
|
||||||
|
| Architecture decisions or future work | The [documentation policy](policy/documentation.md), relevant accepted ADRs, and relevant roadmap documents |
|
||||||
|
|
||||||
|
Cross-project changes land and release in Promptkit before Scriptorium adopts
|
||||||
|
the tagged version. Do not commit a Go workspace, local replacement, vendored
|
||||||
|
Promptkit source, or an import of a Promptkit `internal` package.
|
||||||
|
|
||||||
|
## Baseline Validation
|
||||||
|
|
||||||
|
For code changes, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go test ./...
|
||||||
|
go test -race ./...
|
||||||
|
go vet ./...
|
||||||
|
go build ./cmd/scriptorium
|
||||||
|
```
|
||||||
|
|
||||||
|
Check formatting with `gofmt`, run `git diff --check`, and validate affected
|
||||||
|
examples and documentation links. Documentation-only work does not require
|
||||||
|
unrelated new tests, but commands and examples changed by documentation must be
|
||||||
|
run.
|
||||||
@@ -1,192 +0,0 @@
|
|||||||
# HTTP API Integration
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
|
|
||||||
This document defines the implemented inbound HTTP contract for Scriptorium.
|
|
||||||
|
|
||||||
Current scope is only:
|
|
||||||
|
|
||||||
- `POST /v1/runs`
|
|
||||||
|
|
||||||
For CLI behavior, see the [CLI reference](../cli.md).
|
|
||||||
|
|
||||||
## Endpoint
|
|
||||||
|
|
||||||
- Method: `POST`
|
|
||||||
- Path: `/v1/runs`
|
|
||||||
- Content type: JSON request/response
|
|
||||||
|
|
||||||
Route behavior:
|
|
||||||
|
|
||||||
- unknown path: `404 not_found`
|
|
||||||
- unsupported method on `/v1/runs`: `405 method_not_allowed`
|
|
||||||
|
|
||||||
Copyable request example file:
|
|
||||||
|
|
||||||
- `examples/http-run.json`
|
|
||||||
|
|
||||||
## Request Body
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"prompt_id": "generic.structured_events",
|
|
||||||
"profile_id": "local-quality",
|
|
||||||
"prompt_version": "1.0.0",
|
|
||||||
"inputs": {
|
|
||||||
"transcript": {"type": "file", "uri": "./examples/fixtures/transcript.md"},
|
|
||||||
"glossary": {"type": "inline", "body": "party:\n - Rin"}
|
|
||||||
},
|
|
||||||
"vars": {
|
|
||||||
"session_date": "2026-05-04"
|
|
||||||
},
|
|
||||||
"model": {
|
|
||||||
"endpoint": "http://localhost:8000/v1",
|
|
||||||
"model": "gpt-4o-mini",
|
|
||||||
"temperature": 0.0,
|
|
||||||
"max_tokens": 800,
|
|
||||||
"top_p": 1.0,
|
|
||||||
"timeout_seconds": 120,
|
|
||||||
"service_tier": "priority",
|
|
||||||
"reasoning_effort": "medium",
|
|
||||||
"api_key_env": "SCRIPTORIUM_API_KEY",
|
|
||||||
"extra_params": {
|
|
||||||
"route": "primary"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"include_raw_output": false
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Required fields:
|
|
||||||
|
|
||||||
- `prompt_id`
|
|
||||||
- `inputs` (must contain at least one named input)
|
|
||||||
|
|
||||||
Input reference types currently supported by runtime artifact loading:
|
|
||||||
|
|
||||||
- `file`
|
|
||||||
- `inline`
|
|
||||||
|
|
||||||
## Strict JSON Rules
|
|
||||||
|
|
||||||
Request decoding uses strict JSON field checks:
|
|
||||||
|
|
||||||
- unknown request fields are rejected with `400 invalid_json`
|
|
||||||
- unknown `model` fields are rejected with `400 invalid_json`
|
|
||||||
- raw API-key payload fields such as `api_key` are rejected as unknown fields
|
|
||||||
|
|
||||||
## Success Response
|
|
||||||
|
|
||||||
Status: `200 OK`
|
|
||||||
|
|
||||||
Response shape:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"artifact": {
|
|
||||||
"name": "output",
|
|
||||||
"content_type": "application/json",
|
|
||||||
"body": "{\"summary\":\"...\"}",
|
|
||||||
"uri": "",
|
|
||||||
"size": 123,
|
|
||||||
"hash": "..."
|
|
||||||
},
|
|
||||||
"validation": {
|
|
||||||
"status": "passed",
|
|
||||||
"mode": "json_schema",
|
|
||||||
"errors": [],
|
|
||||||
"schema_path": "structured_events.schema.json",
|
|
||||||
"repair_attempts": 0,
|
|
||||||
"is_valid": true
|
|
||||||
},
|
|
||||||
"metadata": {
|
|
||||||
"run_id": "...",
|
|
||||||
"prompt_id": "generic.structured_events",
|
|
||||||
"prompt_version": "1.0.0",
|
|
||||||
"prompt_hash": "...",
|
|
||||||
"rendered_prompt_hash": "...",
|
|
||||||
"selected_profile_id": "local-quality",
|
|
||||||
"model_name": "gpt-4o-mini",
|
|
||||||
"endpoint": "http://localhost:8000/v1",
|
|
||||||
"model_params": {
|
|
||||||
"endpoint": "http://localhost:8000/v1",
|
|
||||||
"model": "gpt-4o-mini",
|
|
||||||
"temperature": 0,
|
|
||||||
"max_tokens": 800,
|
|
||||||
"top_p": 1,
|
|
||||||
"timeout_seconds": 120,
|
|
||||||
"service_tier": "priority",
|
|
||||||
"reasoning_effort": "medium",
|
|
||||||
"api_key_env": "SCRIPTORIUM_API_KEY",
|
|
||||||
"extra_params": {
|
|
||||||
"route": "primary"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"input_hashes": {
|
|
||||||
"transcript": "..."
|
|
||||||
},
|
|
||||||
"usage": {
|
|
||||||
"prompt_tokens": 11,
|
|
||||||
"completion_tokens": 22,
|
|
||||||
"total_tokens": 33
|
|
||||||
},
|
|
||||||
"start_time": "2026-05-04T12:00:00Z",
|
|
||||||
"end_time": "2026-05-04T12:00:01Z",
|
|
||||||
"duration_ms": 1000,
|
|
||||||
"validation_mode": "json_schema",
|
|
||||||
"validation_status": "passed",
|
|
||||||
"repair_attempts_used": 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
`raw_model_output` is omitted by default.
|
|
||||||
|
|
||||||
To include it, send:
|
|
||||||
|
|
||||||
- `"include_raw_output": true`
|
|
||||||
|
|
||||||
## Validation Failure Behavior
|
|
||||||
|
|
||||||
Validation content failures do not map to HTTP error status.
|
|
||||||
|
|
||||||
Behavior:
|
|
||||||
|
|
||||||
- status remains `200 OK`
|
|
||||||
- `validation.status` is `failed`
|
|
||||||
- validation errors are returned in `validation.errors`
|
|
||||||
|
|
||||||
## Error Responses
|
|
||||||
|
|
||||||
Error body shape:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"error": {
|
|
||||||
"code": "invalid_request",
|
|
||||||
"message": "prompt_id is required"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Current error mapping (non-exhaustive):
|
|
||||||
|
|
||||||
- `400 invalid_json`: malformed JSON or unknown JSON fields
|
|
||||||
- `400 invalid_request`: missing/invalid request fields
|
|
||||||
- `400 profile_required`: no explicit `profile_id` and prompt has no `default_profile`
|
|
||||||
- `400 prompt_load_failed`: prompt definition invalid/unloadable
|
|
||||||
- `400 profile_load_failed`: profile invalid/unloadable
|
|
||||||
- `400 artifact_read_failed`: input artifact loading failed
|
|
||||||
- `400 prompt_render_failed`: template render failed
|
|
||||||
- `400 api_key_env_missing`: named API-key environment variable is missing
|
|
||||||
- `404 prompt_not_found`
|
|
||||||
- `404 profile_not_found`
|
|
||||||
- `502 llm_failed`: outbound model request failed
|
|
||||||
- `500 validation_runtime_failed`: validator runtime/schema-load failure
|
|
||||||
- `500 internal_error`
|
|
||||||
|
|
||||||
## Security And Deployment Note
|
|
||||||
|
|
||||||
The HTTP adapter has no built-in authentication or authorization.
|
|
||||||
|
|
||||||
Deploy behind trusted controls (for example authenticated gateway/reverse proxy and network boundaries).
|
|
||||||
@@ -1,114 +0,0 @@
|
|||||||
# Narratio Subprocess Integration
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This document defines the supported subprocess contract for Narratio invoking Scriptorium through the public CLI.
|
|
||||||
|
|
||||||
This is a CLI contract, not an internal Go package integration.
|
|
||||||
|
|
||||||
## Supported Commands
|
|
||||||
|
|
||||||
Narratio should invoke:
|
|
||||||
|
|
||||||
- `scriptorium run`
|
|
||||||
- `scriptorium render`
|
|
||||||
|
|
||||||
Use `run` for generation.
|
|
||||||
|
|
||||||
Use `render` for preflight/debug output without LLM execution.
|
|
||||||
|
|
||||||
## Recommended Invocation Shapes
|
|
||||||
|
|
||||||
Run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scriptorium run \
|
|
||||||
--prompt <prompt_id> \
|
|
||||||
--input transcript=<path> \
|
|
||||||
--out <artifact_path>
|
|
||||||
```
|
|
||||||
|
|
||||||
Render:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
scriptorium render \
|
|
||||||
--prompt <prompt_id> \
|
|
||||||
--input transcript=<path> \
|
|
||||||
--format json
|
|
||||||
```
|
|
||||||
|
|
||||||
Narratio may add:
|
|
||||||
|
|
||||||
- `--config <path>`
|
|
||||||
- `--profile <profile_id>`
|
|
||||||
- repeatable `--input name=path`
|
|
||||||
- repeatable `--var name=value`
|
|
||||||
- runtime overrides when explicitly needed (`--model`, `--llm-base-url`, `--timeout`, etc.)
|
|
||||||
|
|
||||||
## Config And Directory Behavior
|
|
||||||
|
|
||||||
Narratio can rely on resolved app config or pass explicit paths.
|
|
||||||
|
|
||||||
- default config search order:
|
|
||||||
1. `/usr/local/etc/scriptorium/config.yml`
|
|
||||||
2. `/etc/scriptorium/config.yml`
|
|
||||||
- explicit `--config` requires file existence and valid syntax
|
|
||||||
- CLI flags override config values
|
|
||||||
|
|
||||||
## Profile Selection
|
|
||||||
|
|
||||||
Profile selection follows runner behavior:
|
|
||||||
|
|
||||||
1. explicit `--profile`
|
|
||||||
2. prompt `default_profile`
|
|
||||||
3. error if neither is available
|
|
||||||
|
|
||||||
Narratio should treat prompt/profile IDs as deployment configuration, not hardcoded logic.
|
|
||||||
|
|
||||||
## Input And Variable Contract
|
|
||||||
|
|
||||||
- Inputs use repeated `--input name=path`.
|
|
||||||
- Input names must match prompt definition input names.
|
|
||||||
- Variables use repeated `--var name=value` for small metadata values.
|
|
||||||
- Prefer file inputs for large content.
|
|
||||||
|
|
||||||
## Environment Contract
|
|
||||||
|
|
||||||
- Pass through required API-key environment variables referenced by `api_key_env`.
|
|
||||||
- Never pass raw API keys via CLI arguments.
|
|
||||||
- Keep subprocess environment scoped to required variables.
|
|
||||||
|
|
||||||
## Output And Error Handling
|
|
||||||
|
|
||||||
`run`:
|
|
||||||
|
|
||||||
- stdout: artifact body unless `--out` is used
|
|
||||||
- `--out`: writes artifact to file
|
|
||||||
- stderr: success summary and errors
|
|
||||||
|
|
||||||
`render`:
|
|
||||||
|
|
||||||
- stdout: prepared-run output unless `--out` is used
|
|
||||||
- stderr: errors
|
|
||||||
|
|
||||||
Narratio should capture stdout and stderr separately.
|
|
||||||
|
|
||||||
## Exit Status Contract
|
|
||||||
|
|
||||||
- `0`: success
|
|
||||||
- `1`: parse/config/load/render/generation/IO/runtime error
|
|
||||||
- `2`: run completed but validation failed
|
|
||||||
|
|
||||||
A `run` exit code `2` can still produce output (stdout or `--out`).
|
|
||||||
|
|
||||||
## Security Notes
|
|
||||||
|
|
||||||
- Treat generated artifacts and stderr logs as potentially sensitive.
|
|
||||||
- Avoid logging full rendered prompts by default in production contexts.
|
|
||||||
- Use controlled output paths and access controls for persisted artifacts.
|
|
||||||
|
|
||||||
## Canonical References
|
|
||||||
|
|
||||||
- CLI behavior: [CLI reference](../cli.md)
|
|
||||||
- Config behavior: [Configuration reference](../config.md)
|
|
||||||
- Operations and failure handling: [Operations guide](../operations.md), [Troubleshooting](../troubleshooting.md)
|
|
||||||
@@ -1,113 +0,0 @@
|
|||||||
# OpenAI-Compatible Chat Integration
|
|
||||||
|
|
||||||
## Scope
|
|
||||||
|
|
||||||
This document defines the outbound LLM contract implemented by `internal/llm/openai_compatible_client.go`.
|
|
||||||
|
|
||||||
It documents only fields and behaviors currently serialized by code.
|
|
||||||
|
|
||||||
## Endpoint Construction
|
|
||||||
|
|
||||||
Request endpoint is built as:
|
|
||||||
|
|
||||||
1. choose base URL:
|
|
||||||
- `GenerateRequest.Target.Endpoint` if set
|
|
||||||
- otherwise client config `BaseURL`
|
|
||||||
2. trim trailing slash
|
|
||||||
3. append `/chat/completions`
|
|
||||||
|
|
||||||
Example:
|
|
||||||
|
|
||||||
- base URL: `http://localhost:8000/v1`
|
|
||||||
- final URL: `http://localhost:8000/v1/chat/completions`
|
|
||||||
|
|
||||||
## Request Fields Sent
|
|
||||||
|
|
||||||
Serialized JSON fields:
|
|
||||||
|
|
||||||
- `model` (required after fallback resolution)
|
|
||||||
- `messages` (role/content pairs from rendered prompt)
|
|
||||||
- `temperature` (only when non-zero)
|
|
||||||
- `max_tokens` (only when non-zero)
|
|
||||||
- `top_p` (only when non-zero)
|
|
||||||
- `service_tier` (only when non-empty)
|
|
||||||
- `response_format` (only when structured output is provided)
|
|
||||||
|
|
||||||
`service_tier` is provider-specific. OpenRouter currently documents request values such as `flex` and `priority`; Scriptorium forwards any non-empty configured value and lets the backend validate support.
|
|
||||||
|
|
||||||
Structured output is currently `json_schema` only, serialized as:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"response_format": {
|
|
||||||
"type": "json_schema",
|
|
||||||
"json_schema": {
|
|
||||||
"name": "...",
|
|
||||||
"strict": true,
|
|
||||||
"schema": {"type": "object"}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Authentication Header
|
|
||||||
|
|
||||||
If `Target.APIKeyEnv` is set:
|
|
||||||
|
|
||||||
- resolve environment variable value at request time
|
|
||||||
- set `Authorization: Bearer <value>`
|
|
||||||
|
|
||||||
If the environment variable is unset/empty:
|
|
||||||
|
|
||||||
- request fails before HTTP call (`ErrInvalidRequest`)
|
|
||||||
|
|
||||||
If `Target.APIKeyEnv` is empty:
|
|
||||||
|
|
||||||
- no `Authorization` header is sent
|
|
||||||
|
|
||||||
## Timeout Behavior
|
|
||||||
|
|
||||||
Base timeout comes from client configuration.
|
|
||||||
|
|
||||||
Per-request override:
|
|
||||||
|
|
||||||
- if `Target.TimeoutSeconds > 0`, use that value for request timeout
|
|
||||||
- if `Target.TimeoutSeconds < 0`, request is rejected (`ErrInvalidRequest`)
|
|
||||||
|
|
||||||
## Response Expectations
|
|
||||||
|
|
||||||
Expected successful response shape (subset used):
|
|
||||||
|
|
||||||
- `choices[0].message.content`
|
|
||||||
- `usage.prompt_tokens`
|
|
||||||
- `usage.completion_tokens`
|
|
||||||
- `usage.total_tokens`
|
|
||||||
|
|
||||||
Malformed response conditions include:
|
|
||||||
|
|
||||||
- invalid JSON
|
|
||||||
- empty `choices`
|
|
||||||
- empty `choices[0].message.content`
|
|
||||||
|
|
||||||
Malformed responses return `ErrMalformedResponse`.
|
|
||||||
|
|
||||||
## Error Handling
|
|
||||||
|
|
||||||
- network/request-construction failures: `ErrRequestFailed`
|
|
||||||
- non-2xx HTTP status: `ErrUnexpectedStatus` (includes status code and trimmed response body snippet)
|
|
||||||
- malformed response shape/content: `ErrMalformedResponse`
|
|
||||||
|
|
||||||
## Unsupported Or Non-Serialized Fields
|
|
||||||
|
|
||||||
The following fields may exist in profile/effective settings but are not currently serialized into outbound chat-completions payloads:
|
|
||||||
|
|
||||||
- `reasoning_effort`
|
|
||||||
- `extra_params`
|
|
||||||
|
|
||||||
No built-in retries, tool-calls, or multi-request payload modes are implemented in this client.
|
|
||||||
|
|
||||||
## Relationship To Runner
|
|
||||||
|
|
||||||
When prompt validation mode is `json_schema`, runner prepares a structured-output schema spec and passes it to the client as `StructuredOutput`.
|
|
||||||
|
|
||||||
The client only serializes the provider request payload; it does not load schema files itself.
|
|
||||||
41
docs/integrations/subprocess.md
Normal file
41
docs/integrations/subprocess.md
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
# Subprocess Integration
|
||||||
|
|
||||||
|
This document covers process-boundary behavior for callers that invoke
|
||||||
|
Scriptorium as a child process. Command syntax, flags, output, and exit codes
|
||||||
|
are defined by the [CLI reference](../cli.md). Interface selection belongs in
|
||||||
|
the [consumer integration overview](../consumers/api.md).
|
||||||
|
|
||||||
|
## Process Contract
|
||||||
|
|
||||||
|
Use `scriptorium render` when the caller needs prepared output without a model
|
||||||
|
call, and `scriptorium run` for generation. Pass an explicit `--config` or
|
||||||
|
make the configuration search paths available to the child process; configuration
|
||||||
|
discovery, fields, profile selection, and credential mechanisms are defined in
|
||||||
|
the [configuration reference](../config.md).
|
||||||
|
|
||||||
|
Pass required API-key environment variables through the child environment. Do
|
||||||
|
not place raw API keys in arguments. Keep the environment limited to the values
|
||||||
|
needed for the selected profile.
|
||||||
|
|
||||||
|
## Streams And Output Ownership
|
||||||
|
|
||||||
|
Capture stdout and stderr separately. Stdout contains the requested artifact or
|
||||||
|
prepared output unless the caller selects an output file; stderr contains
|
||||||
|
summaries, diagnostics, and server messages. The exact destinations and status
|
||||||
|
meanings are part of the [CLI reference](../cli.md), not a stable stderr data
|
||||||
|
protocol.
|
||||||
|
|
||||||
|
When using `--out`, the caller owns the output path, its permissions, and
|
||||||
|
cleanup. Treat rendered prompts, generated artifacts, stdout, and stderr as
|
||||||
|
potentially sensitive.
|
||||||
|
|
||||||
|
## Cancellation And Recovery
|
||||||
|
|
||||||
|
A CLI invocation performs one synchronous request and creates no durable run
|
||||||
|
state. A supervising process that needs cancellation must terminate the child
|
||||||
|
process according to its own process-management policy. A later invocation is a
|
||||||
|
new request and can make another model call; there is no resume or checkpoint
|
||||||
|
protocol.
|
||||||
|
|
||||||
|
For deployment, filesystem permissions, and sensitive-artifact handling, see
|
||||||
|
the [operations guide](../operations.md).
|
||||||
@@ -1,144 +1,87 @@
|
|||||||
# Adapter And Repository Internals
|
# Adapter Internals
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
This document describes implemented adapter/repository boundaries and their current behavior.
|
Scriptorium adapters translate executable inputs into Promptkit public requests
|
||||||
|
and translate Promptkit results or errors back to CLI or HTTP behavior. They
|
||||||
|
own IO and presentation mechanics, not framework decisions.
|
||||||
|
|
||||||
## Adapter Map
|
External contracts are canonical in the [CLI reference](../cli.md) and
|
||||||
|
[HTTP API reference](../api.md). Promptkit's public engine contract is
|
||||||
|
described by its tagged
|
||||||
|
[Go consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md).
|
||||||
|
|
||||||
- `internal/adapter/cli`: CLI command parsing, app wiring, stdout/stderr handling, exit codes.
|
## Components And Collaborators
|
||||||
- `internal/adapter/http`: HTTP request/response mapping for `POST /v1/runs`.
|
|
||||||
- `internal/promptdef`: filesystem prompt-definition repository.
|
|
||||||
- `internal/profile`: filesystem execution-profile repository.
|
|
||||||
- `internal/artifact`: input artifact reader.
|
|
||||||
- `internal/prompt`: Go-template renderer.
|
|
||||||
- `internal/llm`: OpenAI-compatible LLM client implementation.
|
|
||||||
- `internal/validate`: output validator.
|
|
||||||
- `internal/format`: prepared-run formatters for `render` output.
|
|
||||||
|
|
||||||
## Inputs And Outputs
|
- `cmd/scriptorium` passes process arguments and streams to
|
||||||
|
`internal/adapter/cli`.
|
||||||
|
- `internal/adapter/cli` resolves settings through `internal/config`,
|
||||||
|
constructs `promptkit.Engine`, maps CLI values to `promptkit.RunRequest`,
|
||||||
|
and owns output files, summaries, and exit codes.
|
||||||
|
- `internal/adapter/http` strictly decodes request DTOs, maps them to Promptkit
|
||||||
|
public values, calls its adapter-owned `Runner` interface, and maps results
|
||||||
|
and errors to HTTP DTOs.
|
||||||
|
- `internal/format` renders `promptkit.PreparedRun` values as deterministic text
|
||||||
|
or JSON.
|
||||||
|
|
||||||
CLI adapter:
|
## Wiring Flows
|
||||||
|
|
||||||
- Input: process args, filesystem config/assets, environment.
|
### CLI
|
||||||
- Output: exit code, stdout artifact/prepared output, stderr summaries/errors.
|
|
||||||
|
|
||||||
HTTP adapter:
|
`run` calls `promptkit.Engine.Run`; `render` calls
|
||||||
|
`promptkit.Engine.Prepare`. Both share request mapping for prompt/profile
|
||||||
|
selection, file inputs, variables, and presence-aware execution overrides.
|
||||||
|
Omitted framework settings remain zero values so Promptkit resolves its own
|
||||||
|
defaults.
|
||||||
|
|
||||||
- Input: JSON request body (`runRequestDTO`).
|
`serve` constructs Scriptorium's restricted HTTP artifact reader, injects it
|
||||||
- Output: JSON success/error body with mapped status codes.
|
with `promptkit.WithArtifactReader`, passes the engine through the HTTP
|
||||||
|
adapter's consumer-owned `Runner` interface, and starts the server.
|
||||||
|
|
||||||
Filesystem repositories:
|
### HTTP
|
||||||
|
|
||||||
- Input: prompt/profile YAML files under configured directories.
|
The handler enforces transport limits and strict JSON decoding before mapping
|
||||||
- Output: normalized domain definitions/profiles or typed errors.
|
DTOs into `promptkit.RunRequest`, `promptkit.ArtifactRef`, and
|
||||||
|
`promptkit.ExecutionTargetOverride`. On success it reads Promptkit artifact,
|
||||||
|
validation, model, usage, and metadata values directly.
|
||||||
|
|
||||||
Artifact reader:
|
Failure mapping uses `errors.Is` against Promptkit's public sentinels and the
|
||||||
|
HTTP reader's Scriptorium-owned containment and size errors. Wrapped reader
|
||||||
|
errors preserve their identity through Promptkit's artifact-load boundary.
|
||||||
|
|
||||||
- Input: `domain.ArtifactRef`.
|
## Package-Local Guarantees
|
||||||
- Output: loaded `domain.Artifact`.
|
|
||||||
|
|
||||||
LLM adapter:
|
- Adapters contain no copied framework types or orchestration.
|
||||||
|
- Configuration is resolved before Promptkit engine construction.
|
||||||
|
- Explicit numeric overrides preserve presence, including zero.
|
||||||
|
- HTTP DTO and error mapping remains stable and transport-owned.
|
||||||
|
- Resolved secrets are not serialized or printed.
|
||||||
|
- No adapter creates durable run state.
|
||||||
|
|
||||||
- Input: `domain.GenerateRequest`.
|
## Verification
|
||||||
- Output: `domain.GenerateResponse`.
|
|
||||||
|
|
||||||
Validator:
|
Inspect:
|
||||||
|
|
||||||
- Input: artifact body + output contract.
|
|
||||||
- Output: validation result or runtime validation error.
|
|
||||||
|
|
||||||
## Boundaries
|
|
||||||
|
|
||||||
- Adapters convert external representations to domain requests and back.
|
|
||||||
- Use-case decisions remain in `internal/usecase`.
|
|
||||||
- External dependency details stay scoped to adapter packages.
|
|
||||||
|
|
||||||
## Config Fields Used
|
|
||||||
|
|
||||||
Primary app settings consumed by adapters:
|
|
||||||
|
|
||||||
- `prompt_dir`
|
|
||||||
- `profile_dir`
|
|
||||||
- `schema_dir`
|
|
||||||
- `server.addr`
|
|
||||||
- `defaults.render_format`
|
|
||||||
|
|
||||||
Execution profile/request settings used through runner:
|
|
||||||
|
|
||||||
- `endpoint`, `model`, `temperature`, `max_tokens`, `top_p`, `timeout_seconds`, `service_tier`, `api_key_env`, `reasoning_effort`, `extra_params`
|
|
||||||
|
|
||||||
## External Dependencies
|
|
||||||
|
|
||||||
- YAML decoding: `gopkg.in/yaml.v3` (strict known-fields mode in config/prompt/profile loaders).
|
|
||||||
- JSON Schema validation: `github.com/santhosh-tekuri/jsonschema/v6`.
|
|
||||||
- HTTP client/server: Go standard library.
|
|
||||||
|
|
||||||
## Failure Behavior
|
|
||||||
|
|
||||||
Strict decoding and input checks:
|
|
||||||
|
|
||||||
- config/prompt/profile loaders reject unknown YAML fields.
|
|
||||||
- prompt/profile repositories scan nested subdirectories recursively.
|
|
||||||
- prompt/profile lookup uses YAML `id` values; subdirectory paths are organizational only.
|
|
||||||
- duplicate prompt/profile IDs are invalid and fail instead of using first-match behavior.
|
|
||||||
- HTTP DTO decoder rejects unknown JSON fields.
|
|
||||||
- raw API key payload fields are rejected by strict decoding in profile/http paths.
|
|
||||||
|
|
||||||
Artifact refs:
|
|
||||||
|
|
||||||
- Supported reference types: `inline`, `file`.
|
|
||||||
- Unsupported types return `ErrUnsupportedRefType`.
|
|
||||||
|
|
||||||
LLM adapter:
|
|
||||||
|
|
||||||
- endpoint appends `/chat/completions`.
|
|
||||||
- non-2xx responses map to request failure errors.
|
|
||||||
- malformed responses (including missing/empty first choice content) are errors.
|
|
||||||
|
|
||||||
Validator:
|
|
||||||
|
|
||||||
- `basic`, `json`, `json_schema` content failures return `ValidationFailed` results.
|
|
||||||
- schema load/compile/path failures are runtime errors.
|
|
||||||
- schema lookup uses explicit `schema_path` values relative to `schema_dir`; it does not recursively search by basename.
|
|
||||||
|
|
||||||
HTTP error mapping:
|
|
||||||
|
|
||||||
- maps domain/use-case errors to stable HTTP code + error code/message.
|
|
||||||
- distinguishes missing profile selection and missing `api_key_env` variable using stable use-case sentinel errors.
|
|
||||||
- avoids returning internal wrapped-cause details in response payload.
|
|
||||||
|
|
||||||
## CLI Adapter Semantics
|
|
||||||
|
|
||||||
Implemented commands:
|
|
||||||
|
|
||||||
- `run`
|
|
||||||
- `render`
|
|
||||||
- `serve`
|
|
||||||
|
|
||||||
Behavior highlights:
|
|
||||||
|
|
||||||
- `run` exit `2` indicates validation failed after generation.
|
|
||||||
- `render` does not call the LLM.
|
|
||||||
- `serve` exposes HTTP handler only; no built-in auth.
|
|
||||||
- `render` supports `--format text|json`; `render` does not expose `--schema-dir`.
|
|
||||||
- deprecated aliases `--prompt-id` and `--profile-id` are still accepted.
|
|
||||||
|
|
||||||
## Tests To Inspect Before Changing
|
|
||||||
|
|
||||||
- `internal/adapter/cli/run_test.go`
|
- `internal/adapter/cli/run_test.go`
|
||||||
- `internal/adapter/http/handler_test.go`
|
- `internal/adapter/http/handler_test.go`
|
||||||
- `internal/promptdef/repository_test.go`
|
- `internal/adapter/http/artifact_reader_test.go`
|
||||||
- `internal/profile/repository_test.go`
|
|
||||||
- `internal/artifact/reader_test.go`
|
|
||||||
- `internal/prompt/renderer_test.go`
|
|
||||||
- `internal/llm/openai_compatible_client_test.go`
|
|
||||||
- `internal/validate/standard_validator_test.go`
|
|
||||||
- `internal/format/prepared_run_test.go`
|
- `internal/format/prepared_run_test.go`
|
||||||
|
- `internal/adapter/dependency_test.go`
|
||||||
|
|
||||||
## Architectural Invariants
|
The adapter tests protect parsing, configuration mapping, output, status
|
||||||
|
mapping, restricted artifacts, and representative real Promptkit-engine
|
||||||
|
workflows. The dependency test protects the repository boundary.
|
||||||
|
|
||||||
- Adapter packages do not own runner decision logic.
|
## Change Recipes
|
||||||
- External request/response strictness is part of contract stability.
|
|
||||||
- Prepared-render output never includes resolved API key values.
|
For a CLI or HTTP change:
|
||||||
- Outbound OpenAI-compatible request includes only currently serialized fields (`model`, `messages`, optional `temperature`, `max_tokens`, `top_p`, optional `service_tier`, optional `response_format`).
|
|
||||||
|
1. identify the Scriptorium-owned external contract;
|
||||||
|
2. map through Promptkit public values without copying framework semantics;
|
||||||
|
3. add or update the narrow application-owned test;
|
||||||
|
4. update the canonical Scriptorium contract; and
|
||||||
|
5. coordinate and tag Promptkit first if a required public capability is
|
||||||
|
genuinely absent.
|
||||||
|
|
||||||
|
Update [source internals](sources.md) when application source locations or HTTP
|
||||||
|
artifact containment changes.
|
||||||
|
|||||||
18
docs/internal/overview.md
Normal file
18
docs/internal/overview.md
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
# Internal Component Overview
|
||||||
|
|
||||||
|
This is the complete inventory of Scriptorium's implemented Go components.
|
||||||
|
The [architecture policy](../policy/architecture.md) owns normative boundaries;
|
||||||
|
public behavior belongs in the linked contracts.
|
||||||
|
|
||||||
|
| Component | Implemented responsibility | References |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `cmd/scriptorium` | Process entrypoint that delegates arguments and streams to the CLI adapter. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
||||||
|
| `internal/adapter/cli` | Parses commands, resolves application settings, constructs Promptkit engines, maps requests, and owns process output and exit behavior. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
||||||
|
| `internal/adapter/http` | Owns routes, DTOs, strict decoding, limits, Promptkit request/result mapping, public error mapping, and restricted HTTP artifact reading. | [HTTP API](../api.md), [adapter internals](adapters.md), [source internals](sources.md) |
|
||||||
|
| `internal/config` | Discovers and strictly decodes application configuration and applies built-in and CLI precedence. | [configuration contract](../config.md), [adapter internals](adapters.md) |
|
||||||
|
| `internal/defaults` | Holds Scriptorium-owned application and HTTP defaults. | [configuration contract](../config.md) |
|
||||||
|
| `internal/format` | Formats Promptkit prepared-run values for CLI text or JSON output. | [CLI contract](../cli.md), [adapter internals](adapters.md) |
|
||||||
|
|
||||||
|
Framework implementation packages are provided by
|
||||||
|
[Promptkit v0.1.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
and are not part of this repository.
|
||||||
@@ -1,155 +0,0 @@
|
|||||||
# Runner Internals
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
`internal/usecase.Runner` is the core use case orchestrator for prompt preparation and execution.
|
|
||||||
|
|
||||||
It owns request validation, prompt/profile resolution, runtime-parameter merge, artifact loading, prompt rendering, structured-output setup, LLM invocation, output validation, and result metadata.
|
|
||||||
|
|
||||||
## Inputs And Outputs
|
|
||||||
|
|
||||||
Primary input type:
|
|
||||||
|
|
||||||
- `domain.RunRequest`
|
|
||||||
|
|
||||||
Primary output types:
|
|
||||||
|
|
||||||
- `domain.PreparedRun` from `Prepare`
|
|
||||||
- `domain.RunResult` from `Run`
|
|
||||||
|
|
||||||
LLM boundary types:
|
|
||||||
|
|
||||||
- `domain.GenerateRequest`
|
|
||||||
- `domain.GenerateResponse`
|
|
||||||
|
|
||||||
## Boundaries
|
|
||||||
|
|
||||||
`Runner` coordinates the following interfaces:
|
|
||||||
|
|
||||||
- `promptdef.Repository`
|
|
||||||
- `profile.Repository`
|
|
||||||
- `artifact.Reader`
|
|
||||||
- `prompt.Renderer`
|
|
||||||
- `llm.Client`
|
|
||||||
- `validate.Validator`
|
|
||||||
- optional `usecase.OutputRepairer`
|
|
||||||
|
|
||||||
Transport concerns (CLI flags, HTTP DTO parsing, status-code mapping) stay outside runner.
|
|
||||||
|
|
||||||
## Config Fields Used
|
|
||||||
|
|
||||||
`Runner` does not read app config files directly.
|
|
||||||
|
|
||||||
It receives fully constructed repositories/readers/validators from adapters. Effective behavior depends on adapter wiring, including:
|
|
||||||
|
|
||||||
- prompt/profile directories
|
|
||||||
- schema base directory
|
|
||||||
- selected profile/runtime overrides in request
|
|
||||||
|
|
||||||
## External Adapters Used
|
|
||||||
|
|
||||||
`Runner` works with adapter implementations via interfaces. Current wiring from CLI/HTTP uses:
|
|
||||||
|
|
||||||
- filesystem prompt/profile repositories
|
|
||||||
- composite artifact reader
|
|
||||||
- Go-template prompt renderer
|
|
||||||
- OpenAI-compatible LLM client
|
|
||||||
- standard validator
|
|
||||||
|
|
||||||
## State And Resume Behavior
|
|
||||||
|
|
||||||
`Runner` is stateless across requests.
|
|
||||||
|
|
||||||
- No durable run-state storage.
|
|
||||||
- No built-in resume/skip checkpoints.
|
|
||||||
- Each `Run`/`Prepare` executes from request inputs and current repositories.
|
|
||||||
|
|
||||||
## Failure Behavior
|
|
||||||
|
|
||||||
Primary runner error classes:
|
|
||||||
|
|
||||||
- `ErrInvalidRequest`: invalid run request envelope.
|
|
||||||
- `ErrProfileRequired`: specific invalid-request reason when neither request `profile_id` nor prompt `default_profile` is available.
|
|
||||||
- `ErrAPIKeyEnvMissing`: specific invalid-request reason when `api_key_env` is set but the named environment variable is unset/empty.
|
|
||||||
- `ErrProfileLoad`: prompt/profile repository load failures.
|
|
||||||
- `ErrArtifactLoad`: artifact read failures.
|
|
||||||
- `ErrPromptRender`: template render failures.
|
|
||||||
- `ErrLLMGenerate`: outbound model request failures.
|
|
||||||
- `ErrValidation`: validation runtime failures (including structured-output schema load/compile failures).
|
|
||||||
|
|
||||||
Reason sentinel behavior:
|
|
||||||
|
|
||||||
- `ErrProfileRequired` and `ErrAPIKeyEnvMissing` are wrapped with `ErrInvalidRequest`.
|
|
||||||
- Adapters can use `errors.Is` for stable reason mapping without matching runner prose.
|
|
||||||
|
|
||||||
Validation content failures are not run errors:
|
|
||||||
|
|
||||||
- `Run` can succeed with `Validation.Status == failed`.
|
|
||||||
- CLI maps this to exit code `2`.
|
|
||||||
- HTTP returns `200` with failed validation details.
|
|
||||||
|
|
||||||
## Prepare Flow
|
|
||||||
|
|
||||||
`Prepare` performs:
|
|
||||||
|
|
||||||
1. validate request basics (prompt ID present).
|
|
||||||
2. load prompt definition by ID/version.
|
|
||||||
3. select profile ID:
|
|
||||||
- explicit request profile ID
|
|
||||||
- prompt `default_profile`
|
|
||||||
- otherwise return an invalid request with `ErrProfileRequired`
|
|
||||||
4. load execution profile.
|
|
||||||
5. merge effective runtime target:
|
|
||||||
- built-in execution defaults
|
|
||||||
- selected profile values
|
|
||||||
- request overrides
|
|
||||||
6. verify required `api_key_env` environment variable:
|
|
||||||
- missing/empty env value returns an invalid request with `ErrAPIKeyEnvMissing`
|
|
||||||
- only the environment-variable name is retained; secret value is never returned
|
|
||||||
7. resolve output contract and structured-output schema payload when `json_schema` mode is active.
|
|
||||||
8. read input artifacts.
|
|
||||||
9. render prompt messages.
|
|
||||||
10. compute prompt/input/render hashes and return `PreparedRun`.
|
|
||||||
|
|
||||||
`Prepare` does not call the LLM.
|
|
||||||
|
|
||||||
## Run Flow
|
|
||||||
|
|
||||||
`Run` performs:
|
|
||||||
|
|
||||||
1. generate run ID.
|
|
||||||
2. call `Prepare`.
|
|
||||||
3. call LLM with prepared messages/effective target/structured-output spec.
|
|
||||||
4. build output artifact content type from output format.
|
|
||||||
5. validate output.
|
|
||||||
6. optionally attempt bounded repair when repairer is injected and contract allows it.
|
|
||||||
7. return `RunResult` with artifact, raw output, validation, hashes, profile/model metadata, usage, and timestamps.
|
|
||||||
|
|
||||||
## Repair Hook Boundary
|
|
||||||
|
|
||||||
Repair attempts occur only when all are true:
|
|
||||||
|
|
||||||
- repairer is injected
|
|
||||||
- `repair_attempts > 0`
|
|
||||||
- validation status is `failed`
|
|
||||||
- validation mode is `json` or `json_schema`
|
|
||||||
|
|
||||||
Current production wiring boundary:
|
|
||||||
|
|
||||||
- CLI and HTTP adapters call `usecase.NewRunner(...)` (no repairer argument).
|
|
||||||
- Therefore normal CLI/HTTP execution does not perform repair attempts today.
|
|
||||||
|
|
||||||
## Tests To Inspect Before Changing
|
|
||||||
|
|
||||||
- `internal/usecase/runner_test.go`
|
|
||||||
- `internal/usecase/integration_test.go`
|
|
||||||
- `internal/adapter/cli/run_test.go`
|
|
||||||
- `internal/adapter/http/handler_test.go`
|
|
||||||
|
|
||||||
## Architectural Invariants
|
|
||||||
|
|
||||||
- `Run` reuses `Prepare`; prepare logic is not duplicated.
|
|
||||||
- Effective API-key environment-variable name may appear; resolved secret value must not.
|
|
||||||
- Structured-output schema document must load before LLM call for `json_schema` mode.
|
|
||||||
- Repair loops are bounded by `repair_attempts` and repairer presence.
|
|
||||||
- Runner stays transport-agnostic.
|
|
||||||
68
docs/internal/sources.md
Normal file
68
docs/internal/sources.md
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
# Source Internals
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document covers Scriptorium-owned source locations and the restricted HTTP
|
||||||
|
artifact reader. Prompt, profile, schema, and ordinary artifact semantics are
|
||||||
|
owned by the tagged
|
||||||
|
[Promptkit format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/formats.md).
|
||||||
|
|
||||||
|
## Application Source Locations
|
||||||
|
|
||||||
|
`internal/config` resolves `prompt_dir`, `profile_dir`, and `schema_dir` from
|
||||||
|
Scriptorium defaults, configuration files, and CLI overrides.
|
||||||
|
`internal/adapter/cli` passes those paths into `promptkit.Config` when
|
||||||
|
constructing the engine.
|
||||||
|
|
||||||
|
Scriptorium does not search, parse, validate, or overlay framework source files
|
||||||
|
itself. Promptkit owns prompt selection, profile built-ins and overlays, schema
|
||||||
|
resolution, ordinary file artifacts, and the related error identities.
|
||||||
|
|
||||||
|
The [configuration reference](../config.md) owns Scriptorium's source-location
|
||||||
|
fields and precedence. Maintained files under `examples/` are application
|
||||||
|
inputs that use Promptkit's tagged formats.
|
||||||
|
|
||||||
|
## Restricted HTTP Artifact Reader
|
||||||
|
|
||||||
|
`internal/adapter/http` implements `promptkit.ArtifactReader` for HTTP
|
||||||
|
requests. The `serve` path injects it with
|
||||||
|
`promptkit.WithArtifactReader`, replacing Promptkit's ordinary reader for
|
||||||
|
inbound HTTP inputs.
|
||||||
|
|
||||||
|
The reader:
|
||||||
|
|
||||||
|
- accepts inline references without an artifact root;
|
||||||
|
- denies file references when no root is configured;
|
||||||
|
- resolves relative paths below the configured root;
|
||||||
|
- accepts absolute paths only when they are lexically within that root;
|
||||||
|
- rejects lexical traversal outside the root;
|
||||||
|
- applies the configured file byte limit, with zero meaning unlimited;
|
||||||
|
- preserves content type, body, size, hash, name, and URI metadata; and
|
||||||
|
- honors context cancellation.
|
||||||
|
|
||||||
|
Containment is lexical and does not resolve symlinks. The operating system
|
||||||
|
follows symlinks after the check. The [HTTP API](../api.md) owns observable
|
||||||
|
request outcomes, and [operations](../operations.md) owns safe deployment
|
||||||
|
permissions and root selection.
|
||||||
|
|
||||||
|
Reader errors remain identifiable after Promptkit wraps them as artifact-load
|
||||||
|
failures, allowing the HTTP adapter to preserve Scriptorium status and error
|
||||||
|
codes.
|
||||||
|
|
||||||
|
## Verification And Change Recipe
|
||||||
|
|
||||||
|
Inspect:
|
||||||
|
|
||||||
|
- `internal/config/config_test.go`
|
||||||
|
- `internal/adapter/cli/run_test.go`
|
||||||
|
- `internal/adapter/http/artifact_reader_test.go`
|
||||||
|
- `internal/adapter/http/handler_test.go`
|
||||||
|
|
||||||
|
When changing an application source location or HTTP artifact policy:
|
||||||
|
|
||||||
|
1. preserve strict configuration precedence and the Promptkit public boundary;
|
||||||
|
2. keep containment and size policy in Scriptorium;
|
||||||
|
3. update focused configuration, reader, and handler tests;
|
||||||
|
4. update the [configuration](../config.md), [HTTP](../api.md), and
|
||||||
|
[operations](../operations.md) contracts as applicable; and
|
||||||
|
5. do not duplicate Promptkit loaders, formats, or ordinary artifact behavior.
|
||||||
@@ -1,125 +1,160 @@
|
|||||||
# Operations Guide
|
# Operations Guide
|
||||||
|
|
||||||
## Scope
|
## Scope And References
|
||||||
|
|
||||||
This document covers day-to-day operation of the CLI and HTTP service for currently implemented behavior.
|
This runbook covers deployment, normal operation, capacity planning, and safe
|
||||||
|
recovery for Scriptorium. It does not redefine invocation syntax, configuration
|
||||||
|
fields, or HTTP wire behavior.
|
||||||
|
|
||||||
For command syntax, see [CLI reference](cli.md). For file formats and defaults, see [Configuration reference](config.md).
|
- [CLI reference](cli.md): commands, output destinations, and exit codes.
|
||||||
|
- [Configuration reference](config.md): application settings, source
|
||||||
|
locations, defaults, and credential mapping.
|
||||||
|
- [Promptkit framework formats](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/formats.md):
|
||||||
|
prompt, profile, schema, execution-setting, and framework credential
|
||||||
|
contracts.
|
||||||
|
- [HTTP API reference](api.md): route, request/response schema, status codes,
|
||||||
|
limits, and HTTP artifact access.
|
||||||
|
- [Consumer integration overview](consumers/api.md): caller responsibilities.
|
||||||
|
|
||||||
## Operational Model
|
## Operational Model And State
|
||||||
|
|
||||||
Scriptorium executes one request at a time per CLI invocation or HTTP request.
|
Scriptorium handles one prompt request for each CLI invocation or HTTP request.
|
||||||
|
It has no durable run store, archive, checkpoint, cache, or resume mechanism.
|
||||||
|
A failed or interrupted request is recovered by correcting its inputs,
|
||||||
|
configuration, or environment and submitting a new request.
|
||||||
|
|
||||||
Important boundaries:
|
Generated artifacts, rendered prompts, model output, and run metadata are
|
||||||
|
caller-owned data. Retention, encryption, backup, and deletion are deployment
|
||||||
|
responsibilities.
|
||||||
|
|
||||||
- No durable run state is stored.
|
## Deploy The Filesystem And Process
|
||||||
- No built-in resume, checkpoint, archive, or backup workflow exists.
|
|
||||||
- Recovery is rerun-based: fix inputs/config, then rerun.
|
|
||||||
|
|
||||||
## Filesystem Layout And Config
|
Provide the process with readable configured Promptkit prompt, profile, and
|
||||||
|
schema sources that follow the tagged framework formats. For an HTTP deployment
|
||||||
|
that accepts file artifacts, use a dedicated, narrow artifact directory rather
|
||||||
|
than a general-purpose or sensitive filesystem tree.
|
||||||
|
|
||||||
Scriptorium depends on:
|
Run Scriptorium under an identity that can:
|
||||||
|
|
||||||
- prompt definition files (`prompt_dir`)
|
- read only the prompt, profile, schema, and allowed input-artifact paths it
|
||||||
- execution profile files (`profile_dir`)
|
needs;
|
||||||
- optional JSON schemas (`schema_dir`)
|
- read the required credential environment variables without writing them to
|
||||||
|
files or logs; and
|
||||||
|
- write only caller-selected output locations when CLI output files are used.
|
||||||
|
|
||||||
Config discovery order when `--config` is omitted:
|
Do not make the HTTP artifact directory writable by untrusted users. The HTTP
|
||||||
|
artifact containment behavior is lexical and the operating system follows
|
||||||
|
symlinks; account for that when choosing ownership and mount boundaries. See
|
||||||
|
the [HTTP API reference](api.md) for the externally observable behavior.
|
||||||
|
|
||||||
1. `/usr/local/etc/scriptorium/config.yml`
|
## Supply Credentials And Protect Runtime Data
|
||||||
2. `/etc/scriptorium/config.yml`
|
|
||||||
|
|
||||||
If neither exists, built-in defaults are used. If `--config <path>` is provided, that file must exist and parse successfully.
|
Set secret values in the process environment and configure only their
|
||||||
|
environment-variable names. Do not put raw keys in configuration, prompt or
|
||||||
|
profile files, process arguments, HTTP payloads, captured command lines, or
|
||||||
|
debug dumps.
|
||||||
|
|
||||||
Built-in defaults relevant to operations:
|
Treat stdout, stderr, prepared-run output, generated artifacts, and HTTP
|
||||||
|
responses as potentially sensitive. Send service logs to a controlled collector
|
||||||
|
and apply the same retention and access rules as for model input and output.
|
||||||
|
|
||||||
- `schema_dir: .`
|
## Run A Normal Workflow
|
||||||
- `server.addr: :8080`
|
|
||||||
- `defaults.render_format: text`
|
|
||||||
|
|
||||||
## Normal CLI Workflow
|
Before changing production inputs, profiles, or schemas:
|
||||||
|
|
||||||
Use `render` first when you need to verify prompt resolution and runtime settings without calling a model.
|
1. confirm the deployed configuration selects the intended sources and model
|
||||||
|
credentials;
|
||||||
|
2. use [`render`](cli.md) with the same request inputs and variables to confirm
|
||||||
|
preparation without a model call;
|
||||||
|
3. use [`run`](cli.md) for generation; and
|
||||||
|
4. retain or discard validation-failed output according to the caller's
|
||||||
|
policy.
|
||||||
|
|
||||||
Use `run` for generation.
|
The [maintained render script](../examples/render-markdown-summary.sh) is a
|
||||||
|
copyable preflight example. The CLI reference owns its complete invocation and
|
||||||
|
exit semantics.
|
||||||
|
|
||||||
Typical sequence:
|
## Expose The HTTP Service
|
||||||
|
|
||||||
1. Confirm prompt/profile directories resolve through config or flags.
|
The HTTP service has no built-in authentication or authorization. Place it on a
|
||||||
2. Confirm required input files exist and map to prompt input names.
|
trusted network or behind an authenticated reverse proxy, API gateway, or
|
||||||
3. Confirm required API-key environment variables are set.
|
equivalent access control. Restrict who can reach it and who can read the
|
||||||
4. Confirm the selected profile's model endpoint is reachable from the process environment.
|
artifact root.
|
||||||
5. Run `render` for preflight when changing prompt/profile/input wiring.
|
|
||||||
6. Run `run` for actual generation.
|
|
||||||
|
|
||||||
## Secrets Handling
|
Use a service manager or supervisor appropriate to the deployment to manage
|
||||||
|
process lifetime, restart policy, log capture, and environment injection. The
|
||||||
|
[HTTP API reference](api.md) owns client request shapes, status behavior, and
|
||||||
|
artifact-access outcomes.
|
||||||
|
|
||||||
Raw API keys are not accepted in config files, profile files as `api_key`, CLI flags, or HTTP request bodies.
|
## Plan Capacity And Limits
|
||||||
|
|
||||||
Operational pattern:
|
Capacity is primarily determined by concurrent model calls, input and output
|
||||||
|
sizes, schema complexity, provider latency, and network behavior. Size limits
|
||||||
|
protect request bodies, HTTP file artifacts, and encoded responses; configure
|
||||||
|
them through the [configuration reference](config.md) and rely on the
|
||||||
|
[HTTP API reference](api.md) for their response effects.
|
||||||
|
|
||||||
- Set environment variables that hold secret values.
|
Before increasing a limit:
|
||||||
- Set profile `api_key_env` (or runtime override `api_key_env`) to the environment variable name.
|
|
||||||
- Keep process environments scoped to only required variables.
|
|
||||||
|
|
||||||
## HTTP Service Operation
|
1. measure representative input, generated-output, and optional raw-output
|
||||||
|
sizes;
|
||||||
|
2. confirm memory, network, and upstream-provider capacity;
|
||||||
|
3. retain an upstream request-size and authentication boundary; and
|
||||||
|
4. test the intended workload in a non-production environment.
|
||||||
|
|
||||||
Start service with:
|
For large local inputs, prefer a controlled file-artifact directory over
|
||||||
|
placing arbitrary paths on the service host. Avoid disabling a limit unless an
|
||||||
|
equivalent trusted control exists elsewhere.
|
||||||
|
|
||||||
```bash
|
## Diagnose And Recover
|
||||||
go run ./cmd/scriptorium serve --config ./examples/config.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Current inbound API behavior:
|
### Preparation Or Configuration Failure
|
||||||
|
|
||||||
- Route: `POST /v1/runs`
|
Capture the CLI diagnostic or HTTP error response, then verify the selected
|
||||||
- JSON request parsing rejects unknown fields.
|
configuration, prompt ID, profile selection, source readability, and input
|
||||||
- Validation content failures still return `200 OK` with `validation.status: "failed"`.
|
mapping. Use `render` with the same request when it is unclear whether failure
|
||||||
|
occurs before model execution. Consult the [CLI reference](cli.md), the
|
||||||
|
[configuration reference](config.md), and the [HTTP API reference](api.md) for
|
||||||
|
the exact interface contract.
|
||||||
|
|
||||||
Security caveat:
|
### Credential Or Provider Failure
|
||||||
|
|
||||||
- `serve` has no built-in authentication or authorization.
|
Confirm that the process environment contains the configured credential name
|
||||||
- Deploy only behind trusted controls (private network boundary, authenticated reverse proxy, API gateway, or equivalent).
|
without printing the secret. Check endpoint reachability and provider health
|
||||||
|
from the process network. If preparation succeeds but generation fails, inspect
|
||||||
|
the selected model settings in prepared output and the service's controlled
|
||||||
|
logs. Correct the deployment or provider issue, then submit a new request.
|
||||||
|
|
||||||
## Output, Logs, And Exit Codes
|
### Artifact Or Permission Failure
|
||||||
|
|
||||||
`run` command:
|
Verify that the process can read the intended local input. For HTTP file
|
||||||
|
artifacts, verify the deployment's artifact root, ownership, path layout, and
|
||||||
|
file size. Do not widen filesystem permissions or the allowed root merely to
|
||||||
|
make an arbitrary path work; move or copy the required artifact into the
|
||||||
|
controlled location instead.
|
||||||
|
|
||||||
- Generated artifact body goes to stdout by default.
|
### Validation Failure
|
||||||
- `--out` writes generated artifact to a file.
|
|
||||||
- Summary metadata line is written to stderr on success.
|
|
||||||
- Exit code `2` means generation completed but validation failed.
|
|
||||||
|
|
||||||
`render` command:
|
A generated-content validation failure is distinct from a runtime failure.
|
||||||
|
CLI `run` reports the validation result and error count in its success summary;
|
||||||
|
it does not print the individual validation messages. For HTTP, inspect the
|
||||||
|
validation object in the response according to the [HTTP API reference](api.md).
|
||||||
|
|
||||||
- Prepared-run output goes to stdout by default.
|
Use rendered input and generated output to determine whether prompt instructions,
|
||||||
- `--out` writes prepared-run output to a file.
|
the selected model, or the schema needs correction. If schema loading or
|
||||||
- Exit code is `0` on success and `1` on failure.
|
compilation itself fails, correct the source deployment or schema document
|
||||||
|
before rerunning.
|
||||||
|
|
||||||
`serve` command:
|
### HTTP Limit Or Request Failure
|
||||||
|
|
||||||
- Startup and server errors are written to stderr.
|
Compare the request, artifact, or expected response size with the deployed
|
||||||
|
configuration, and validate the request against the [HTTP API reference](api.md).
|
||||||
|
Reduce the payload, use an appropriate controlled artifact source, omit
|
||||||
|
unneeded raw output, or adjust the deployment limit after capacity review.
|
||||||
|
|
||||||
## Validation Behavior In Operations
|
## Cleanup And Reruns
|
||||||
|
|
||||||
Validation modes (`none`, `basic`, `json`, `json_schema`) are defined by prompt output contract.
|
Because no run state is retained, cleanup concerns caller-owned output files,
|
||||||
|
logs, and artifacts only. Remove or rotate them using the deployment's normal
|
||||||
Operational interpretation:
|
retention policy. After a correction, rerun the request from the beginning;
|
||||||
|
there is no safe resume point.
|
||||||
- Validation runtime errors are hard failures (`run` exit `1`; HTTP error response).
|
|
||||||
- Validation content failures are soft failures (`run` exit `2`; HTTP `200` with failed status).
|
|
||||||
|
|
||||||
A failed validation run can still produce output. Decide whether to keep or discard that output in your surrounding workflow.
|
|
||||||
|
|
||||||
## Safe Recovery Steps
|
|
||||||
|
|
||||||
For failed runs or requests:
|
|
||||||
|
|
||||||
1. Capture stderr output or HTTP error code/message.
|
|
||||||
2. Confirm config path and directory settings.
|
|
||||||
3. Verify prompt/profile IDs and input mappings.
|
|
||||||
4. Verify API-key environment-variable presence when required.
|
|
||||||
5. Reproduce with `render --format json` when prompt/profile/input resolution is uncertain.
|
|
||||||
6. Rerun after correction.
|
|
||||||
|
|
||||||
Because Scriptorium does not persist run state, rerun is the canonical recovery path.
|
|
||||||
|
|||||||
@@ -1,112 +1,102 @@
|
|||||||
# Architecture
|
# Architecture
|
||||||
|
|
||||||
This document is the development architecture policy for Scriptorium.
|
This document defines Scriptorium's current application architecture and
|
||||||
|
durable development boundaries.
|
||||||
|
|
||||||
It is for developers and LLM coding agents. User-facing behavior belongs in `README.md` and the docs under `docs/` that target operators/users.
|
## System Shape
|
||||||
|
|
||||||
## Project Shape
|
Scriptorium is an executable application with three entry paths: CLI `run`, CLI
|
||||||
|
`render`, and the HTTP service started by `serve`. It does not expose a reusable
|
||||||
|
root Go package.
|
||||||
|
|
||||||
Scriptorium is a narrow prompt-execution application with three entry paths:
|
The application consumes
|
||||||
|
[Promptkit v0.1.0](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
through its supported root package. Promptkit owns prompt execution,
|
||||||
|
preparation, source formats, built-in profiles, model-client behavior, and
|
||||||
|
validation. Scriptorium owns application configuration, executable adapters,
|
||||||
|
prepared-run presentation, process behavior, and HTTP deployment policy.
|
||||||
|
|
||||||
- CLI `run`
|
The concrete package inventory is maintained in the
|
||||||
- CLI `render`
|
[internal overview](../internal/overview.md).
|
||||||
- HTTP `POST /v1/runs` through `serve`
|
|
||||||
|
|
||||||
Domain behavior is centralized in `internal/usecase` and `internal/domain`.
|
## Dependency Direction
|
||||||
|
|
||||||
## Core Principles
|
```text
|
||||||
|
cmd/scriptorium
|
||||||
|
|
|
||||||
|
v
|
||||||
|
CLI and HTTP adapters, configuration, defaults, and formatting
|
||||||
|
|
|
||||||
|
v
|
||||||
|
gitea.maximumdirect.net/eric/promptkit
|
||||||
|
```
|
||||||
|
|
||||||
- Keep orchestration narrow: Scriptorium executes one prompt request; it is not a multi-step workflow engine.
|
- Retained application packages may import Promptkit's root package.
|
||||||
- Keep adapter logic thin: adapters map external shapes to domain requests/results and should not hold domain decisions.
|
- They must not import Promptkit `internal` packages.
|
||||||
- Keep boundaries explicit: repositories/loaders/renderers/validators/LLM client stay behind package interfaces.
|
- They must not import the removed Scriptorium root facade or recreate former
|
||||||
- Keep config strict: YAML/JSON decoding for external inputs should reject unknown fields.
|
framework package families.
|
||||||
- Keep secrets out of payloads: raw API key values must not be accepted or emitted.
|
- Adapter-owned interfaces use Promptkit public values when a consumer-side
|
||||||
|
substitution boundary is needed.
|
||||||
|
- Scriptorium passes omitted framework settings as zero values so Promptkit
|
||||||
|
applies its own defaults.
|
||||||
|
|
||||||
## Package Boundaries
|
The repository architecture guard enforces these import and removal
|
||||||
|
invariants.
|
||||||
|
|
||||||
Current package map:
|
## Retained Boundaries
|
||||||
|
|
||||||
- `cmd/scriptorium`: process entrypoint.
|
- `internal/adapter/cli` owns commands, flags, configuration precedence,
|
||||||
- `internal/adapter/cli`: command parsing, app wiring for CLI commands, output behavior.
|
process streams, output files, summaries, and exit codes.
|
||||||
- `internal/adapter/http`: HTTP DTO mapping and error/status mapping.
|
- `internal/adapter/http` owns routes, strict JSON DTOs, size limits, response
|
||||||
- `internal/config`: application settings loading and CLI override precedence.
|
mapping, status mapping, and the restricted artifact reader.
|
||||||
- `internal/defaults`: compile-time default constants.
|
- `internal/config` owns discovery and strict decoding of Scriptorium
|
||||||
- `internal/domain`: core request/result and contract types.
|
application configuration.
|
||||||
- `internal/usecase`: `Runner` prepare/run orchestration and repair-hook boundary.
|
- `internal/defaults` owns Scriptorium application and HTTP defaults only.
|
||||||
- `internal/promptdef`: filesystem prompt-definition repository.
|
- `internal/format` owns deterministic prepared-run text and JSON presentation.
|
||||||
- `internal/profile`: filesystem execution-profile repository.
|
- Promptkit owns framework orchestration and contracts. Its
|
||||||
- `internal/artifact`: artifact reference readers.
|
[format reference](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/formats.md)
|
||||||
- `internal/prompt`: template renderer.
|
and
|
||||||
- `internal/llm`: provider-neutral LLM client interface and OpenAI-compatible implementation.
|
[outbound integration contract](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/integrations/openai-compatible-chat.md)
|
||||||
- `internal/validate`: validator interfaces and standard implementation.
|
are canonical.
|
||||||
- `internal/format`: prepared-run output formatting.
|
|
||||||
|
|
||||||
Detailed component behavior is documented in:
|
## HTTP Artifact Security Boundary
|
||||||
|
|
||||||
- `docs/internal/runner.md`
|
Ordinary CLI file loading is provided by Promptkit. Scriptorium's HTTP adapter
|
||||||
- `docs/internal/adapters.md`
|
injects a restricted `promptkit.ArtifactReader` for inbound HTTP requests.
|
||||||
|
That reader denies file references without an artifact root, enforces the
|
||||||
|
configured byte limit, and applies Scriptorium's lexical root-containment rule.
|
||||||
|
The operating system still follows symlinks after the lexical check.
|
||||||
|
|
||||||
## Configuration And Precedence
|
The [HTTP API](../api.md) owns observable request outcomes, and
|
||||||
|
[operations](../operations.md) owns deployment permissions and root selection.
|
||||||
|
|
||||||
Application settings are resolved as:
|
## State, Errors, And Secrets
|
||||||
|
|
||||||
1. built-in defaults
|
Scriptorium has no durable run-state store, checkpoint, cache, or resume
|
||||||
2. config file values
|
mechanism. Recovery is a new request after correcting inputs, configuration, or
|
||||||
3. CLI overrides
|
environment.
|
||||||
|
|
||||||
`config.yml` is for application wiring (directories, server address, render default format), not prompt/profile runtime execution settings.
|
Adapters map Promptkit public error identities into CLI exits or HTTP statuses
|
||||||
|
without classifying by message text. Raw API keys are not accepted in
|
||||||
Profile selection and runtime model resolution remain use-case concerns.
|
Scriptorium configuration, CLI arguments, or HTTP payloads, and resolved
|
||||||
|
secrets must not be emitted.
|
||||||
## State And Persistence Policy
|
|
||||||
|
|
||||||
Scriptorium has no durable run-state store.
|
|
||||||
|
|
||||||
- No built-in resume/checkpoint/archive behavior.
|
|
||||||
- Recovery model is rerun after correcting inputs/config/environment.
|
|
||||||
|
|
||||||
## External Integration Policy
|
|
||||||
|
|
||||||
Current external contracts:
|
|
||||||
|
|
||||||
- inbound HTTP contract: `POST /v1/runs`
|
|
||||||
- outbound model contract: OpenAI-compatible chat completions subset
|
|
||||||
- subprocess contract for integrators: CLI `run`/`render`
|
|
||||||
|
|
||||||
Integration docs belong under `docs/integrations/`.
|
|
||||||
|
|
||||||
## Error Handling And Logging
|
|
||||||
|
|
||||||
- Wrap errors with domain/operation context.
|
|
||||||
- Map domain errors to adapter-appropriate statuses/codes without leaking sensitive internals.
|
|
||||||
- Keep stderr summaries concise for CLI success/error paths.
|
|
||||||
- Never emit raw secret values.
|
|
||||||
|
|
||||||
## Testing Expectations
|
|
||||||
|
|
||||||
- Core runner behavior should be covered with isolated unit tests and fixture-based integration tests.
|
|
||||||
- Adapter behavior should be tested for parse/mapping/error semantics.
|
|
||||||
- Config parsing, prompt/profile loading, validator behavior, and LLM client error handling should remain covered by package tests.
|
|
||||||
- Repository-level docs/examples that claim runnable behavior should be validated by tests or smoke commands.
|
|
||||||
|
|
||||||
## Documentation Expectations
|
|
||||||
|
|
||||||
- Document implemented behavior only outside `docs/roadmap/`.
|
|
||||||
- Keep canonical reference locations stable (`docs/cli.md`, `docs/config.md`, `docs/operations.md`, `docs/troubleshooting.md`, `docs/internal/`).
|
|
||||||
- Update docs in the same change when architecture-relevant behavior changes.
|
|
||||||
|
|
||||||
## Architectural Invariants
|
## Architectural Invariants
|
||||||
|
|
||||||
- `Runner.Run` reuses `Runner.Prepare` flow.
|
- External YAML and JSON decoding remains strict.
|
||||||
- CLI and HTTP currently instantiate `Runner` without a repairer.
|
- CLI and HTTP behavior remains presentation and transport logic rather than
|
||||||
- Artifact reading supports `inline` and `file` references.
|
framework orchestration.
|
||||||
- Unknown input fields in config/prompt/profile/http JSON should be rejected by strict decoding.
|
- Explicit numeric request overrides preserve presence, including zero.
|
||||||
- Raw API key values must not be accepted through config/HTTP payloads.
|
- HTTP artifact containment and byte limits remain Scriptorium policy.
|
||||||
|
- No application package depends on Promptkit implementation packages.
|
||||||
|
|
||||||
## Non-Goals
|
## Non-Goals
|
||||||
|
|
||||||
- Do not move orchestration responsibilities from external callers into Scriptorium.
|
- Do not recreate an in-process Scriptorium framework API or compatibility
|
||||||
- Do not add adapter-specific business logic in `internal/adapter/*` packages.
|
facade.
|
||||||
- Do not bypass repository/renderer/validator/LLM boundaries by introducing cross-package coupling.
|
- Do not copy Promptkit types, defaults, built-in profiles, or implementation
|
||||||
|
into Scriptorium.
|
||||||
|
- Do not move CLI, inbound HTTP, process, or deployment policy into Promptkit.
|
||||||
|
- Do not add durable workflow, archive, or resume behavior.
|
||||||
|
|
||||||
Work that is not implemented belongs in `docs/roadmap/`.
|
Work that is not implemented belongs in `docs/roadmap/`.
|
||||||
|
|||||||
@@ -1,103 +0,0 @@
|
|||||||
# Development Guide
|
|
||||||
|
|
||||||
This document defines contributor workflow for Scriptorium.
|
|
||||||
|
|
||||||
## Repository Layout
|
|
||||||
|
|
||||||
- `cmd/scriptorium`: application entrypoint.
|
|
||||||
- `internal/domain`: core contracts.
|
|
||||||
- `internal/usecase`: runner orchestration.
|
|
||||||
- `internal/adapter/cli`: CLI adapter.
|
|
||||||
- `internal/adapter/http`: HTTP adapter.
|
|
||||||
- `internal/config`: application settings loading and precedence.
|
|
||||||
- `internal/defaults`: default constants.
|
|
||||||
- `internal/promptdef`: prompt-definition repository.
|
|
||||||
- `internal/profile`: execution-profile repository.
|
|
||||||
- `internal/artifact`: artifact readers.
|
|
||||||
- `internal/prompt`: prompt rendering.
|
|
||||||
- `internal/llm`: LLM client interface and OpenAI-compatible implementation.
|
|
||||||
- `internal/validate`: validation interfaces and implementation.
|
|
||||||
- `internal/format`: prepared-run formatting.
|
|
||||||
- `docs/`: canonical documentation.
|
|
||||||
- `examples/`: copyable maintained examples and fixtures.
|
|
||||||
|
|
||||||
## Common Commands
|
|
||||||
|
|
||||||
Build:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go build ./cmd/scriptorium
|
|
||||||
```
|
|
||||||
|
|
||||||
Test:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
Targeted test runs commonly used during changes:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go test ./internal/adapter/cli ./internal/adapter/http ./internal/usecase
|
|
||||||
```
|
|
||||||
|
|
||||||
## Coding Conventions
|
|
||||||
|
|
||||||
- Prefer small interfaces at package boundaries.
|
|
||||||
- Keep adapter packages focused on translation and IO concerns.
|
|
||||||
- Keep domain/use-case logic outside adapters.
|
|
||||||
- Wrap errors with operation context.
|
|
||||||
- Use strict decoding for user-provided YAML/JSON where applicable.
|
|
||||||
- Avoid introducing dependencies unless they materially reduce risk/complexity.
|
|
||||||
|
|
||||||
## Dependency Policy
|
|
||||||
|
|
||||||
- Prefer standard library unless an external library is clearly justified.
|
|
||||||
- Current non-stdlib dependencies are intentionally small:
|
|
||||||
- `gopkg.in/yaml.v3` for YAML decoding.
|
|
||||||
- `github.com/santhosh-tekuri/jsonschema/v6` for JSON Schema validation.
|
|
||||||
- Do not leak dependency-specific types across unrelated package boundaries.
|
|
||||||
|
|
||||||
## How To Add App Config Fields
|
|
||||||
|
|
||||||
1. Add fields in `internal/config/config.go` (`Config`, `AppSettings`, and/or `CLIOverrides` as needed).
|
|
||||||
2. Apply defaults in `BuiltInDefaults()` when required.
|
|
||||||
3. Parse and validate in `applyConfig` / `ApplyCLIOverrides`.
|
|
||||||
4. Wire the field through the consuming adapter(s).
|
|
||||||
5. Add/update config tests in `internal/config/config_test.go`.
|
|
||||||
6. Update canonical docs (`docs/config.md`, and other affected docs).
|
|
||||||
|
|
||||||
## How To Add CLI Flags
|
|
||||||
|
|
||||||
1. Add flags in `internal/adapter/cli/run.go` for the relevant command.
|
|
||||||
2. Ensure precedence behavior remains consistent with app config rules.
|
|
||||||
3. Keep `run`, `render`, and `serve` flag surfaces intentionally scoped.
|
|
||||||
4. Add/update parser and command tests in `internal/adapter/cli/run_test.go`.
|
|
||||||
5. Update `docs/cli.md` and any related docs/examples.
|
|
||||||
|
|
||||||
## How To Add Adapters Or Adapter Capabilities
|
|
||||||
|
|
||||||
1. Define or reuse the appropriate interface boundary in domain/use-case packages.
|
|
||||||
2. Implement adapter code under `internal/adapter/<name>` (or relevant boundary package).
|
|
||||||
3. Keep business decisions in `internal/usecase`.
|
|
||||||
4. Add focused adapter tests for mapping, parse, and error behavior.
|
|
||||||
5. Document the new/changed boundary in `docs/internal/adapters.md`.
|
|
||||||
6. If external contract changes, update `docs/integrations/` in the same change.
|
|
||||||
|
|
||||||
## How To Update Prompt/Profile/Schema Assets
|
|
||||||
|
|
||||||
1. Keep prompt/profile/schema files valid under strict loaders.
|
|
||||||
2. Keep examples secret-free.
|
|
||||||
3. Re-run tests that cover prompt/profile/validation behavior.
|
|
||||||
4. Update `docs/config.md` and any docs that reference changed contracts.
|
|
||||||
|
|
||||||
## Documentation Update Expectations
|
|
||||||
|
|
||||||
When behavior changes:
|
|
||||||
|
|
||||||
1. Update canonical doc locations, not duplicate files.
|
|
||||||
2. Keep non-roadmap docs limited to implemented behavior.
|
|
||||||
3. Update links after file moves/renames.
|
|
||||||
4. Re-run relevant tests and smoke commands.
|
|
||||||
|
|
||||||
Docs work is complete only when code/tests/examples/docs agree.
|
|
||||||
@@ -1,356 +1,165 @@
|
|||||||
# Go Project Documentation Policy
|
# Documentation Policy
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
Project documentation must help four audiences:
|
This policy assigns each documentation topic to one canonical owner. Its goal is
|
||||||
|
to keep this repository's documentation accurate, concise, discoverable, and
|
||||||
1. users who need to run the application;
|
resistant to drift for users, operators, developers, integrators, and LLM
|
||||||
2. administrators/operators who need to configure and operate it;
|
coding agents.
|
||||||
3. developers who need to understand and change it safely;
|
|
||||||
4. LLM coding agents that need clear scope, boundaries, and invariants.
|
|
||||||
|
|
||||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
|
||||||
|
|
||||||
## Core Rules
|
## Core Rules
|
||||||
|
|
||||||
### 1. Keep docs concise
|
### One Canonical Owner
|
||||||
|
|
||||||
|
Each authoritative fact belongs in one document. A non-owning document may give
|
||||||
|
a short, stable summary for orientation, but it must link to the canonical owner
|
||||||
|
instead of repeating volatile details.
|
||||||
|
|
||||||
|
Volatile details include commands, flags, configuration fields and defaults,
|
||||||
|
module keys, schemas, file names, paths, status codes, retry behavior, and
|
||||||
|
runtime guarantees. If readers could reasonably treat a statement as a
|
||||||
|
contract, maintain it only in the owning document.
|
||||||
|
|
||||||
Each document should cover a defined scope and only the essentials for that scope.
|
Minimal tested usage examples are allowed outside the owning contract when this
|
||||||
|
policy assigns them an orientation or instructional purpose. They must link to
|
||||||
Avoid:
|
the canonical contract and must not redefine complete syntax, defaults, or
|
||||||
- long background explanations;
|
semantics.
|
||||||
- repeated reference material;
|
|
||||||
- implementation detail in user-facing docs;
|
### Current And Future Behavior
|
||||||
- aspirational language outside roadmap docs;
|
|
||||||
- verbose examples where one minimal example is clearer.
|
Outside `docs/roadmap/`, documentation describes implemented behavior only.
|
||||||
|
Partial features may be described only to their implemented boundary.
|
||||||
### 2. Document only implemented behavior outside roadmap files
|
|
||||||
|
ADRs are the narrow exception: an ADR may record an accepted architectural
|
||||||
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
|
decision before implementation, but acceptance must not be presented as proof
|
||||||
|
that the behavior exists. The roadmap owns implementation status and sequencing
|
||||||
- `docs/roadmap/`
|
until the decision is implemented. Current architecture, user, operator,
|
||||||
|
integration, and internal documentation are updated when the behavior lands.
|
||||||
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
|
|
||||||
|
### Audience And Detail
|
||||||
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
|
|
||||||
|
Write for the document's stated audience and include only the detail needed for
|
||||||
### 3. Use canonical homes
|
its owned topic. User and operator docs should not expose implementation detail.
|
||||||
|
Developer docs should link to user-facing and external contracts rather than
|
||||||
Each type of information should have one canonical location.
|
restate them.
|
||||||
|
|
||||||
Canonical homes:
|
### Examples
|
||||||
|
|
||||||
- project purpose and quickstart: `README.md`
|
Complete copyable files belong in `examples/`. Documentation may use the
|
||||||
- development principles: `docs/policy/architecture.md`
|
smallest illustrative snippet needed to explain its owned topic, but should link
|
||||||
- configuration reference: `docs/config.md`
|
to maintained examples instead of embedding a second complete copy.
|
||||||
- CLI reference: `docs/cli.md`
|
|
||||||
- operations and recovery: `docs/operations.md`
|
Examples must be valid, secret-free, and tested where practical. Commands and
|
||||||
- troubleshooting: `docs/troubleshooting.md`
|
configuration used in documentation should match the application.
|
||||||
- implemented internals: `docs/internal/`
|
|
||||||
- future work: `docs/roadmap/`
|
### Security And Privacy
|
||||||
- contributor workflow: `docs/policy/development.md`
|
|
||||||
- copyable examples: `examples/`
|
Documentation and examples must not contain real credentials, private keys,
|
||||||
|
private environment dumps, sensitive source material, or private infrastructure
|
||||||
Other files should summarize briefly and link to the canonical source.
|
details unless intentionally public. Document secret-handling mechanisms, not
|
||||||
|
secret values.
|
||||||
### 4. Keep examples real
|
|
||||||
|
## Canonical Ownership
|
||||||
Examples should be valid, maintained, and free of secrets.
|
|
||||||
|
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||||
Where practical:
|
| --- | --- | --- | --- |
|
||||||
- example configs should load successfully;
|
| Product orientation and minimal end-to-end quickstart | `README.md` | What this project is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
|
||||||
- example commands should match real CLI syntax;
|
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, and detailed change recipes, which belong in the relevant internal component document. |
|
||||||
- important examples should be covered by tests.
|
| Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||||
|
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||||
## Documentation Profiles
|
| Testing policy | `docs/policy/testing.md` | Test philosophy, risk-based sufficiency, test boundaries, doubles, coverage guidance, regression-test policy, and criteria for adding, rewriting, or deleting tests. | Subsystem behavior, application contracts, subsystem-specific test inventories, and implementation plans. |
|
||||||
|
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||||
All projects require:
|
| Configuration contract | `docs/config.md` | Application discovery and precedence, source locations, server fields, render default, HTTP limits, and credential mapping. | Promptkit framework formats and defaults, complete example files, CLI syntax, runtime lifecycle, and implementation detail. |
|
||||||
|
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, output, cache, and debug handling, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||||
- `README.md`
|
| Release procedure | `docs/release.md` | Candidate validation, version and tag operations, hosted-workflow observation, and published-artifact verification. | Runtime operations, version-specific announcements, and complete application-interface contracts. |
|
||||||
- `docs/policy/architecture.md`
|
| Version-specific release notes | `docs/releases/` | Immutable release summaries, compatibility notices, and migration announcements for one published version. | Complete CLI, HTTP, configuration, operations, or dependency contracts. |
|
||||||
|
| Public HTTP contract | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||||
Additional docs depend on the project.
|
| Consumer guidance | `docs/consumers/` | Choosing between Scriptorium's executable interfaces and understanding consumer responsibilities. | HTTP wire semantics, CLI syntax, Promptkit's Go package, and internal implementation detail. |
|
||||||
|
| External and durable integration contracts | `docs/integrations/` | Scriptorium-owned process and executable integration contracts. | Promptkit framework formats and outbound provider protocols, physical runtime placement, internal transformations, CLI syntax, and configuration defaults. |
|
||||||
### Small library
|
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||||
|
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||||
Recommended:
|
| Architectural decision history | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||||
- `docs/policy/development.md`, if contributor conventions are non-obvious
|
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||||
|
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
|
||||||
### Simple CLI
|
|
||||||
|
Documents that do not exist are required only when the corresponding interface
|
||||||
Required:
|
or responsibility exists. Do not create placeholder API, consumer, integration,
|
||||||
- `docs/cli.md`
|
or operations documents for behavior the application does not have.
|
||||||
|
|
||||||
Recommended:
|
## Boundary Rules
|
||||||
- `docs/policy/development.md`
|
|
||||||
|
### Orientation
|
||||||
### Config-driven CLI
|
|
||||||
|
The README owns product orientation. The developer guide routes contributors.
|
||||||
Required:
|
Architecture owns normative structure. Internal overview owns the current
|
||||||
- `docs/cli.md`
|
concrete component map. These documents may link to one another but should not
|
||||||
- `docs/config.md`
|
maintain parallel package or behavior descriptions.
|
||||||
|
|
||||||
Recommended:
|
### Commands, Configuration, And Operations
|
||||||
- `examples/`
|
|
||||||
- `docs/policy/development.md`
|
CLI documentation answers how to invoke the application. Configuration
|
||||||
|
documentation answers what settings mean. Operations answers what happens to
|
||||||
### Stateful or operator-facing application
|
runtime state and how to operate or recover the application. When a workflow
|
||||||
|
crosses these topics, choose the document that owns the task and link to the
|
||||||
Required:
|
other contracts.
|
||||||
- `docs/cli.md`, if CLI-based
|
|
||||||
- `docs/config.md`, if config-driven
|
### Contracts And Implementation
|
||||||
- `docs/operations.md`
|
|
||||||
|
Integration and API documents define externally observable shapes and
|
||||||
Recommended:
|
semantics. Internal documents explain how this project implements or consumes
|
||||||
- `docs/troubleshooting.md`
|
those contracts. Internal docs may name a field, file, or protocol to identify
|
||||||
- `examples/`
|
a dependency, but must link to its canonical contract for the definition.
|
||||||
- `docs/policy/development.md`
|
|
||||||
|
### Security Topics
|
||||||
### Modular, staged, service-oriented, or orchestration application
|
|
||||||
|
This policy owns what documentation and examples may contain. Architecture owns
|
||||||
Required:
|
application security invariants. Configuration owns credential-supply
|
||||||
- `docs/cli.md`, if CLI-based
|
mechanisms. Operations owns permissions and handling of sensitive runtime
|
||||||
- `docs/config.md`, if config-driven
|
artifacts. Internal docs own implementation mechanisms only.
|
||||||
- `docs/operations.md`
|
|
||||||
- `docs/internal/`
|
## Architecture Decision Records
|
||||||
- `docs/policy/development.md`
|
|
||||||
|
Use sequentially numbered ADR filenames such as
|
||||||
Recommended:
|
`0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
|
||||||
- `docs/troubleshooting.md`
|
|
||||||
- validated examples under `examples/`
|
1. title;
|
||||||
|
2. status;
|
||||||
## Required Documents
|
3. date;
|
||||||
|
4. context;
|
||||||
### README.md
|
5. decision;
|
||||||
|
6. alternatives considered;
|
||||||
**Audience:** users, administrators, operators
|
7. consequences.
|
||||||
|
|
||||||
The README is the outward-facing project orientation page.
|
Use one of these statuses:
|
||||||
|
|
||||||
It should include, in order:
|
- **Proposed:** the decision is under consideration and may change;
|
||||||
|
- **Accepted:** the decision is approved, whether or not implementation is
|
||||||
1. concise description;
|
complete;
|
||||||
2. elevator pitch;
|
- **Rejected:** the proposed decision was considered and not adopted;
|
||||||
3. shortest useful command or usage example;
|
- **Superseded:** a later ADR replaces the accepted decision.
|
||||||
4. links to targeted docs.
|
|
||||||
|
A proposed ADR transitions to accepted or rejected. An accepted ADR transitions
|
||||||
The README should be short. It is not a manual.
|
to superseded only when a later accepted ADR replaces it. An ADR may be created
|
||||||
|
as accepted when the decision has already been made.
|
||||||
The “shortest useful command” means the simplest command that performs the project’s core use case. (It does not mean `app --help`.)
|
|
||||||
|
Treat the decision content of an accepted ADR as immutable. Its status and
|
||||||
### docs/policy/architecture.md
|
supersession metadata may be updated, but a changed decision requires a new ADR.
|
||||||
|
A superseded ADR must link to its replacement, and the replacement must link
|
||||||
**Audience:** developers, LLM coding agents
|
back to the superseded ADR. Rejected architectural alternatives belong in the
|
||||||
|
ADR; rejected product ideas belong in the roadmap.
|
||||||
`docs/policy/architecture.md` is required for every project.
|
|
||||||
|
## Maintenance
|
||||||
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
|
|
||||||
|
When behavior changes, update its canonical owner in the same change. If
|
||||||
It should include:
|
ownership moves, remove the old definition and replace it with a link where
|
||||||
|
navigation remains useful.
|
||||||
- project shape;
|
|
||||||
- core design principles;
|
Before completing documentation work:
|
||||||
- package and boundary philosophy;
|
|
||||||
- state/persistence philosophy, if applicable;
|
- verify affected behavior and examples;
|
||||||
- external integration philosophy, if applicable;
|
- check commands, flags, fields, defaults, schemas, and paths against their
|
||||||
- error-handling and logging principles;
|
implementation;
|
||||||
- testing expectations;
|
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
|
||||||
- documentation expectations;
|
- remove stale references and validate links;
|
||||||
- architectural invariants;
|
- confirm that non-owning documents summarize and link rather than redefine;
|
||||||
- explicit non-goals, if useful.
|
- confirm that no secrets or sensitive private data were added.
|
||||||
|
|
||||||
For small projects, this file may be brief. It may simply state that the project is intentionally narrow, monolithic, and dependency-light.
|
|
||||||
|
|
||||||
### docs/policy/development.md
|
|
||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
|
||||||
|
|
||||||
Required for projects maintained by humans and LLM coding agents.
|
|
||||||
|
|
||||||
It should include:
|
|
||||||
|
|
||||||
- repository layout;
|
|
||||||
- build/test commands;
|
|
||||||
- coding conventions;
|
|
||||||
- dependency policy;
|
|
||||||
- how to add config fields;
|
|
||||||
- how to add CLI flags;
|
|
||||||
- how to add stages/modules/adapters, if applicable;
|
|
||||||
- how to update examples;
|
|
||||||
- documentation update expectations.
|
|
||||||
|
|
||||||
### docs/config.md
|
|
||||||
|
|
||||||
**Audience:** administrators, operators, advanced users
|
|
||||||
|
|
||||||
Required for applications with configuration files.
|
|
||||||
|
|
||||||
It should include, in order:
|
|
||||||
|
|
||||||
1. config file locations and discovery precedence;
|
|
||||||
2. minimal working config;
|
|
||||||
3. production-oriented config;
|
|
||||||
4. full configuration reference;
|
|
||||||
5. secrets handling, if applicable;
|
|
||||||
6. links to maintained examples.
|
|
||||||
|
|
||||||
The full configuration reference should be canonical.
|
|
||||||
|
|
||||||
### docs/cli.md
|
|
||||||
|
|
||||||
**Audience:** users, administrators, operators
|
|
||||||
|
|
||||||
Required for CLI applications.
|
|
||||||
|
|
||||||
It should include, in order:
|
|
||||||
|
|
||||||
1. shortest useful command;
|
|
||||||
2. command overview;
|
|
||||||
3. complete flag reference;
|
|
||||||
4. common workflows;
|
|
||||||
5. diagnostic or recovery commands, if applicable.
|
|
||||||
|
|
||||||
Explain when commands are useful, not just their syntax.
|
|
||||||
|
|
||||||
### docs/operations.md
|
|
||||||
|
|
||||||
**Audience:** administrators, operators
|
|
||||||
|
|
||||||
Required for applications that maintain state, support resume behavior, run multiple stages, write durable artifacts, use remote storage, or require recovery procedures.
|
|
||||||
|
|
||||||
It should cover:
|
|
||||||
|
|
||||||
- normal workflow;
|
|
||||||
- filesystem layout;
|
|
||||||
- remote storage layout, if applicable;
|
|
||||||
- logs and manifests;
|
|
||||||
- resume/retry behavior;
|
|
||||||
- cleanup behavior;
|
|
||||||
- archive/backup behavior;
|
|
||||||
- safe recovery procedures;
|
|
||||||
- operational caveats.
|
|
||||||
|
|
||||||
### docs/troubleshooting.md
|
|
||||||
|
|
||||||
**Audience:** administrators, operators
|
|
||||||
|
|
||||||
Recommended once recurring failure modes exist.
|
|
||||||
|
|
||||||
Each entry should include:
|
|
||||||
|
|
||||||
- symptom;
|
|
||||||
- likely cause;
|
|
||||||
- diagnostic command or inspection step;
|
|
||||||
- safe fix;
|
|
||||||
- relevant links.
|
|
||||||
|
|
||||||
### docs/internal/
|
|
||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
|
||||||
|
|
||||||
Required for modular, staged, service-oriented, or orchestration projects.
|
|
||||||
|
|
||||||
This directory describes implemented internal components. It is not the roadmap.
|
|
||||||
|
|
||||||
Use one file per major component where useful.
|
|
||||||
|
|
||||||
Each component doc should include:
|
|
||||||
|
|
||||||
1. purpose;
|
|
||||||
2. inputs and outputs;
|
|
||||||
3. boundaries;
|
|
||||||
4. config fields used;
|
|
||||||
5. external adapters used;
|
|
||||||
6. state or manifest behavior, if applicable;
|
|
||||||
7. skip/resume behavior, if applicable;
|
|
||||||
8. failure behavior;
|
|
||||||
9. tests to inspect before changing;
|
|
||||||
10. architectural invariants.
|
|
||||||
|
|
||||||
### docs/roadmap/
|
|
||||||
|
|
||||||
**Audience:** maintainers, developers, LLM coding agents
|
|
||||||
|
|
||||||
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
|
|
||||||
|
|
||||||
Roadmap docs should clearly distinguish:
|
|
||||||
|
|
||||||
- proposed work;
|
|
||||||
- accepted plans;
|
|
||||||
- deferred ideas;
|
|
||||||
- rejected ideas;
|
|
||||||
- implementation prompts or task breakdowns, if useful.
|
|
||||||
|
|
||||||
Roadmap docs should not be confused with current behavior.
|
|
||||||
|
|
||||||
### docs/integrations/
|
|
||||||
|
|
||||||
**Audience:** developers, LLM coding agents
|
|
||||||
|
|
||||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
|
||||||
|
|
||||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses.
|
|
||||||
|
|
||||||
Use one file per integration where useful.
|
|
||||||
|
|
||||||
## Examples Directory
|
|
||||||
|
|
||||||
Projects with non-trivial configuration or workflows should include `examples/`.
|
|
||||||
|
|
||||||
Useful examples include:
|
|
||||||
|
|
||||||
- minimal working config;
|
|
||||||
- production-oriented config;
|
|
||||||
- full annotated config;
|
|
||||||
- local development config;
|
|
||||||
- remote/object-storage config;
|
|
||||||
- minimal session/input file.
|
|
||||||
|
|
||||||
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
|
|
||||||
|
|
||||||
## Security and Privacy
|
|
||||||
|
|
||||||
Docs and examples must not include:
|
|
||||||
|
|
||||||
- real API keys;
|
|
||||||
- tokens;
|
|
||||||
- passwords;
|
|
||||||
- private keys;
|
|
||||||
- private environment dumps;
|
|
||||||
- sensitive user data;
|
|
||||||
- raw private transcripts;
|
|
||||||
- private infrastructure details unless intentionally public.
|
|
||||||
|
|
||||||
Document secret-handling mechanisms, not actual secret values.
|
|
||||||
|
|
||||||
## Maintenance Rules
|
|
||||||
|
|
||||||
When docs change, verify the affected behavior.
|
|
||||||
|
|
||||||
Where practical:
|
|
||||||
|
|
||||||
- load example config files in tests;
|
|
||||||
- test CLI examples or command parser behavior;
|
|
||||||
- validate documented flags against real flags;
|
|
||||||
- remove stale references;
|
|
||||||
- update links after renames;
|
|
||||||
- keep roadmap content out of non-roadmap docs.
|
|
||||||
|
|
||||||
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
|
|
||||||
|
|
||||||
Documentation is complete only when it matches the current code.
|
|
||||||
|
|
||||||
## Documentation Change Checklist
|
|
||||||
|
|
||||||
Before merging documentation changes, verify:
|
|
||||||
|
|
||||||
- README is concise and orientation-focused.
|
|
||||||
- `docs/policy/architecture.md` describes development principles.
|
|
||||||
- Future work appears only under `docs/roadmap/`.
|
|
||||||
- User-facing docs avoid unnecessary internals.
|
|
||||||
- Developer-facing docs preserve boundaries and invariants.
|
|
||||||
- Config examples match the schema.
|
|
||||||
- CLI examples match real commands and flags.
|
|
||||||
- Defaults appear in the canonical config reference.
|
|
||||||
- No secrets or private data are included.
|
|
||||||
- Links are accurate.
|
|
||||||
|
|||||||
301
docs/policy/testing.md
Normal file
301
docs/policy/testing.md
Normal file
@@ -0,0 +1,301 @@
|
|||||||
|
# Testing Policy
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Our tests exist to make **incorrect changes expensive and correct changes cheap**.
|
||||||
|
|
||||||
|
We do not optimize for test count, line coverage, exhaustive isolation, or the fewest possible tests. We optimize for sufficient confidence in important behavior while imposing as little unnecessary friction as possible on future development.
|
||||||
|
|
||||||
|
## Every test has a cost
|
||||||
|
|
||||||
|
Testing is not an unqualified good. Every test imposes both an immediate cost and a continuing lifetime cost.
|
||||||
|
|
||||||
|
A test must be:
|
||||||
|
|
||||||
|
- written and reviewed;
|
||||||
|
- understood by future maintainers and coding agents;
|
||||||
|
- executed in local and CI workflows;
|
||||||
|
- diagnosed when it fails;
|
||||||
|
- updated when legitimate behavior changes;
|
||||||
|
- maintained as fixtures, APIs, and dependencies evolve; and
|
||||||
|
- removed or rewritten when it becomes redundant, brittle, misleading, or obsolete.
|
||||||
|
|
||||||
|
Tests also create cognitive and architectural friction. They can constrain refactoring, duplicate policy, slow feedback loops, add noise to failures, and cause harmless implementation changes to require unrelated edits across the suite.
|
||||||
|
|
||||||
|
A test is warranted only when the confidence it provides justifies these costs.
|
||||||
|
|
||||||
|
Apply this cost-benefit analysis at two levels:
|
||||||
|
|
||||||
|
1. **Per test:** What realistic defect does this test detect, how consequential would that defect be, and is that protection worth the test's lifetime cost?
|
||||||
|
2. **Across the suite:** Does this collection provide materially more confidence than a smaller, simpler suite would?
|
||||||
|
|
||||||
|
The preferred test suite is a **lean suite that provides sufficient confidence in the risks that matter, without redundant or low-value tests**. We seek sufficient confidence with the least unnecessary testing friction, not the fewest possible tests.
|
||||||
|
|
||||||
|
Some friction is intentional. Tests should make dangerous changes—such as breaking compatibility, corrupting data, violating security boundaries, or reintroducing subtle bugs—require deliberate review. They should not make ordinary internal changes needlessly expensive.
|
||||||
|
|
||||||
|
The cost of a test is not a reason to omit testing by default. Do not cite maintenance cost abstractly. When omitting a plausible test, be able to state why the protected failure is low-risk, already covered, obvious, reversible, or cheaper to detect elsewhere. For consequential, subtle, or difficult-to-observe behavior, the presumption should favor testing.
|
||||||
|
|
||||||
|
## Default testing style
|
||||||
|
|
||||||
|
Use a **classical/Detroit-style** approach:
|
||||||
|
|
||||||
|
- Test observable behavior, resulting state, contracts, and invariants.
|
||||||
|
- Use real internal collaborators when they are fast and deterministic.
|
||||||
|
- Use fakes, stubs, or mocks primarily at expensive, nondeterministic, destructive, or external boundaries.
|
||||||
|
- Prefer package-level behavioral tests over tests coupled to private helpers or internal call sequences.
|
||||||
|
- Treat exact collaborator interactions as testable behavior only when the interaction itself is a requirement.
|
||||||
|
|
||||||
|
Examples of appropriate seams include clocks, randomness, subprocesses, remote APIs, object storage, email, and paid LLM calls.
|
||||||
|
|
||||||
|
## Test execution requirements
|
||||||
|
|
||||||
|
Tests in the default suite must be deterministic, offline, and independent of real credentials. They must not invoke paid APIs or depend on mutable external services. Tests that require live infrastructure must be explicitly opt-in and clearly separated from the default suite.
|
||||||
|
|
||||||
|
Control clocks, randomness, environment variables, and other process-global or machine-specific state when they affect behavior. Tests should be safe to run repeatedly and alongside other tests without depending on execution order or state left by an earlier test.
|
||||||
|
|
||||||
|
## What deserves tests
|
||||||
|
|
||||||
|
Prioritize tests for:
|
||||||
|
|
||||||
|
1. Public and package-level contracts.
|
||||||
|
2. Domain rules and important invariants.
|
||||||
|
3. Boundary conditions and malformed input.
|
||||||
|
4. Failure handling, cancellation, retries, recovery, and partial success.
|
||||||
|
5. Serialization, schemas, compatibility, and round trips.
|
||||||
|
6. Previously observed or plausible regressions.
|
||||||
|
7. Representative integration and end-to-end workflows.
|
||||||
|
|
||||||
|
A package-level contract is behavior relied upon by another package or major collaborator, not every observable detail of a package implementation.
|
||||||
|
|
||||||
|
For behavior involving **data integrity, destructive operations, compatibility, security, concurrency, idempotency, or recovery**, presume that durable tests are required unless the behavior is already credibly protected at another layer.
|
||||||
|
|
||||||
|
Do not add tests merely because a function, branch, or line exists. Do not add a test when the same meaningful risk is already adequately protected elsewhere.
|
||||||
|
|
||||||
|
## Choose the right test boundary
|
||||||
|
|
||||||
|
Test through the narrowest stable boundary that expresses the behavior clearly.
|
||||||
|
|
||||||
|
This is often the package API, but it may instead be:
|
||||||
|
|
||||||
|
- a smaller pure function when dense application logic is most clearly isolated there;
|
||||||
|
- a package-level operation when several internal collaborators jointly produce the behavior; or
|
||||||
|
- a larger integration boundary when correctness emerges from interaction with a real dependency.
|
||||||
|
|
||||||
|
Do not force all behavior through oversized end-to-end tests. Do not test every private helper merely because it exists. Choose the boundary that gives durable confidence with the least incidental coupling.
|
||||||
|
|
||||||
|
## Test behavior, not implementation
|
||||||
|
|
||||||
|
A test should protect a decision, contract, or invariant—not memorialize the current implementation.
|
||||||
|
|
||||||
|
Before adding or retaining a test, ask:
|
||||||
|
|
||||||
|
> What realistic defect would this test catch?
|
||||||
|
|
||||||
|
A test is suspect when its main purpose is to detect that someone:
|
||||||
|
|
||||||
|
- changed an internal constant;
|
||||||
|
- renamed or split a private helper;
|
||||||
|
- reordered equivalent internal operations;
|
||||||
|
- changed incidental formatting;
|
||||||
|
- replaced one correct algorithm with another; or
|
||||||
|
- refactored internal object structure without changing behavior.
|
||||||
|
|
||||||
|
Refactoring should normally require no test edits unless the refactored structure is itself part of the contract.
|
||||||
|
|
||||||
|
A test can be factually correct and still have negative value. Accurately describing current behavior is not enough; the protected behavior must be important enough to justify the future friction.
|
||||||
|
|
||||||
|
## Expected effects of different changes
|
||||||
|
|
||||||
|
Use the following expectations when evaluating test failures and test maintenance:
|
||||||
|
|
||||||
|
| Change | Expected effect on tests |
|
||||||
|
|---|---|
|
||||||
|
| Internal refactor that preserves behavior | Existing tests should normally remain unchanged and continue to pass. |
|
||||||
|
| Change to an internal default with no contractual significance | Behavioral tests should normally remain unchanged; tests should derive expectations from configuration or relationships rather than duplicate the old value. |
|
||||||
|
| Intentional change to public behavior, policy, schema, or compatibility guarantees | The relevant tests should be reviewed and changed deliberately. |
|
||||||
|
| Accidental violation of a contract or invariant | Tests should fail; fix the production code rather than rewriting the tests to accept the defect. |
|
||||||
|
|
||||||
|
A test failing is not the same as a test needing to be edited. Many tests may correctly fail because of one production defect. The maintenance smell is a correct internal change that requires unrelated expectation updates throughout the suite.
|
||||||
|
|
||||||
|
## Separate mechanism from policy
|
||||||
|
|
||||||
|
Configurable thresholds and defaults must not be duplicated throughout the test suite.
|
||||||
|
|
||||||
|
For example, do not encode an internal concurrency limit indirectly:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Production policy:
|
||||||
|
const maxConcurrency = 4
|
||||||
|
|
||||||
|
// Brittle test:
|
||||||
|
err := startProcesses(5)
|
||||||
|
require.Error(t, err)
|
||||||
|
```
|
||||||
|
|
||||||
|
Instead, test the mechanism relationally:
|
||||||
|
|
||||||
|
```go
|
||||||
|
const limit = 2
|
||||||
|
runner := NewRunner(limit)
|
||||||
|
|
||||||
|
require.NoError(t, runner.Start(limit))
|
||||||
|
require.ErrorIs(t, runner.Start(limit+1), ErrTooMuchConcurrency)
|
||||||
|
```
|
||||||
|
|
||||||
|
The test should prove:
|
||||||
|
|
||||||
|
- the configured limit is accepted; and
|
||||||
|
- one beyond the configured limit is rejected.
|
||||||
|
|
||||||
|
The production default should be tested exactly only when its literal value is itself a public, operational, safety, protocol, or compatibility requirement.
|
||||||
|
|
||||||
|
Apply the same rule to limits, timeouts, capacities, retry counts, and ranges: test relationships and behavior, not duplicated literals.
|
||||||
|
|
||||||
|
For concurrency limits, test both kinds of behavior when relevant:
|
||||||
|
|
||||||
|
1. **Configuration enforcement:** invalid or excessive requested values are handled correctly.
|
||||||
|
2. **Runtime enforcement:** observed peak concurrency never exceeds the configured limit.
|
||||||
|
|
||||||
|
Use a test-controlled limit and measure the behavior relative to that limit. Do not merely assert today's default value.
|
||||||
|
|
||||||
|
## Avoid semantic duplication across layers
|
||||||
|
|
||||||
|
Each behavior should have a clear test owner.
|
||||||
|
|
||||||
|
- Configuration tests own application YAML, discovery, precedence, and
|
||||||
|
application defaults.
|
||||||
|
- CLI tests own argument mapping, streams, summaries, exit behavior, and
|
||||||
|
representative command workflows.
|
||||||
|
- HTTP tests own DTOs, strict decoding, limits, status mapping, and restricted
|
||||||
|
artifact policy.
|
||||||
|
- Formatter tests own prepared-run text and JSON presentation.
|
||||||
|
- Architecture tests own dependency direction and removal invariants.
|
||||||
|
- Promptkit owns framework parsing, orchestration, validation, profiles, and
|
||||||
|
model-client behavior.
|
||||||
|
|
||||||
|
Higher-level tests should not repeat every lower-level case. A single intentional policy change should not require unrelated edits across many test files.
|
||||||
|
|
||||||
|
Tests that are individually reasonable may still be collectively redundant. Evaluate the marginal value of each additional test in light of the protection already provided by the rest of the suite.
|
||||||
|
|
||||||
|
## Use test doubles deliberately
|
||||||
|
|
||||||
|
Choose the least elaborate test double that provides the required control or observation.
|
||||||
|
|
||||||
|
As a default:
|
||||||
|
|
||||||
|
1. Prefer real collaborators when they are fast and deterministic.
|
||||||
|
2. Use small in-memory fakes when realistic stateful behavior is helpful.
|
||||||
|
3. Use stubs when a dependency only needs to provide controlled responses.
|
||||||
|
4. Use mocks when the interaction itself is contractual.
|
||||||
|
|
||||||
|
Mocks are appropriate when the contract includes facts such as:
|
||||||
|
|
||||||
|
- a notification is sent exactly once;
|
||||||
|
- a transaction is committed only after successful writes;
|
||||||
|
- cancellation reaches a subprocess;
|
||||||
|
- an expensive API is called no more than once; or
|
||||||
|
- a security audit event is emitted.
|
||||||
|
|
||||||
|
Do not use mocks merely to isolate every object or reproduce the implementation's call graph.
|
||||||
|
|
||||||
|
## Go-specific guidance
|
||||||
|
|
||||||
|
Use:
|
||||||
|
|
||||||
|
- table-driven tests for meaningful behavioral categories and boundaries;
|
||||||
|
- `t.TempDir()` for real filesystem behavior;
|
||||||
|
- `httptest.Server` for realistic HTTP interactions;
|
||||||
|
- fuzz tests for parsers, normalization, path handling, and broad input spaces;
|
||||||
|
- golden files only when the complete output is intentionally stable;
|
||||||
|
- integration tests where correctness depends on component interaction; and
|
||||||
|
- a small number of representative end-to-end tests.
|
||||||
|
|
||||||
|
Avoid exact error-string assertions unless the wording is itself contractual. Prefer `errors.Is`, `errors.As`, typed errors, or structured error fields.
|
||||||
|
|
||||||
|
At CLI boundaries, prefer exit classifications, structured output, and the smallest stable semantic fragment needed to identify the error. Do not snapshot complete diagnostic wording unless it is contractual.
|
||||||
|
|
||||||
|
Golden-file updates must require an explicit local flag. CI must not update golden files automatically, and reviewers must inspect the semantic diff before accepting an update.
|
||||||
|
|
||||||
|
Keep tests readable and direct. Test helpers and fixture frameworks must earn their own maintenance cost; do not build elaborate test infrastructure for small or isolated needs.
|
||||||
|
|
||||||
|
## Coverage
|
||||||
|
|
||||||
|
Coverage is a diagnostic, not a target.
|
||||||
|
|
||||||
|
Use it to find untested critical branches and unexpectedly weak packages. Do not write low-value tests solely to increase a percentage, and do not infer test quality from coverage alone.
|
||||||
|
|
||||||
|
Security-sensitive HTTP containment and external mappings may warrant denser
|
||||||
|
coverage than straightforward process wiring. Uneven coverage is acceptable
|
||||||
|
when it reflects risk.
|
||||||
|
|
||||||
|
Increasing coverage is valuable only when the newly covered behavior protects a meaningful risk at an acceptable cost.
|
||||||
|
|
||||||
|
## Regression tests
|
||||||
|
|
||||||
|
A bug fix should normally include a regression test that fails before the fix and passes afterward.
|
||||||
|
|
||||||
|
Retain the test when the defect could realistically recur and its consequences justify the ongoing cost. Prefer the narrowest durable test of the violated contract or invariant; do not preserve accidental implementation details from the original bug.
|
||||||
|
|
||||||
|
Not every historical bug requires a permanent test. If the underlying design has made recurrence impossible, the test has become redundant, or a stronger invariant test now subsumes it, remove or consolidate it.
|
||||||
|
|
||||||
|
## Deleting or rewriting tests
|
||||||
|
|
||||||
|
Tests are maintained code, not permanent historical artifacts.
|
||||||
|
|
||||||
|
Delete or rewrite a test when its maintenance cost exceeds the confidence it provides.
|
||||||
|
|
||||||
|
Strong candidates include tests that:
|
||||||
|
|
||||||
|
- require updates after harmless internal changes;
|
||||||
|
- directly assert private constants without protecting a real contract;
|
||||||
|
- duplicate the same policy across several layers;
|
||||||
|
- verify mock choreography rather than outcomes;
|
||||||
|
- snapshot large amounts of incidental output;
|
||||||
|
- test trivial private helpers already exercised through stable package behavior;
|
||||||
|
- protect risks already covered more effectively elsewhere;
|
||||||
|
- are flaky, misleading, obsolete, or disproportionately expensive to diagnose; or
|
||||||
|
- no longer correspond to a plausible failure mode.
|
||||||
|
|
||||||
|
Several brittle tests may encode one genuine requirement. Replace them with one durable behavior-level or invariant test rather than preserving all of them.
|
||||||
|
|
||||||
|
Deleting a low-value test can improve the quality of the suite by reducing noise, maintenance burden, and friction around legitimate change.
|
||||||
|
|
||||||
|
## Reviewing a proposed test
|
||||||
|
|
||||||
|
Use the following questions when the value, boundary, or durability of a proposed test is not self-evident. Significant test additions should be reviewable against them, but written answers are not required for every routine test.
|
||||||
|
|
||||||
|
1. What realistic defect would it catch?
|
||||||
|
2. How likely is that defect?
|
||||||
|
3. How consequential would it be?
|
||||||
|
4. Is the behavior already protected elsewhere?
|
||||||
|
5. At which layer should this behavior be owned?
|
||||||
|
6. Does the test assert a durable contract or an incidental implementation detail?
|
||||||
|
7. Could the implementation be refactored without changing the behavior and without editing this test?
|
||||||
|
8. What should cause this test to fail?
|
||||||
|
9. What legitimate changes should not cause this test to fail?
|
||||||
|
10. What ongoing maintenance, execution, and diagnostic cost will the test impose?
|
||||||
|
11. Is there a smaller or more direct test that protects the same risk?
|
||||||
|
|
||||||
|
Do not add the test when its expected lifetime cost exceeds its expected protective value.
|
||||||
|
|
||||||
|
When deciding not to test plausible behavior, record or be able to explain why the risk is low, already protected, obvious, reversible, or cheaper to detect elsewhere.
|
||||||
|
|
||||||
|
## Definition of sufficient
|
||||||
|
|
||||||
|
A test suite is sufficient when:
|
||||||
|
|
||||||
|
- important contracts and invariants are protected;
|
||||||
|
- meaningful boundaries and failure modes are exercised;
|
||||||
|
- realistic and consequential regressions are credibly protected against silent recurrence;
|
||||||
|
- behavior involving data integrity, destructive operations, compatibility, security, concurrency, idempotency, and recovery is credibly protected;
|
||||||
|
- important external boundaries have realistic integration coverage;
|
||||||
|
- representative complete workflows are tested;
|
||||||
|
- failures provide useful signal rather than redundant noise;
|
||||||
|
- legitimate internal changes usually do not require test edits; and
|
||||||
|
- additional tests would mostly repeat existing protection or preserve inconsequential implementation details.
|
||||||
|
|
||||||
|
Sufficiency is a risk judgment, not a coverage percentage or test count. Reassess it as the application, its users, and the consequences of failure evolve.
|
||||||
|
|
||||||
|
The governing rule is:
|
||||||
|
|
||||||
|
> Test heavily where failure is consequential, subtle, or difficult to detect after the fact. Test lightly where failure is obvious, reversible, and inexpensive—and retain no test whose lifetime cost exceeds the confidence it provides.
|
||||||
373
docs/release.md
Normal file
373
docs/release.md
Normal file
@@ -0,0 +1,373 @@
|
|||||||
|
# Release Procedure
|
||||||
|
|
||||||
|
## Release Model And Status
|
||||||
|
|
||||||
|
Scriptorium publishes annotated semantic tags and tag-triggered Linux binary
|
||||||
|
releases. The hosted
|
||||||
|
[release workflow](../.woodpecker/release.yml) builds `amd64` and `arm64`
|
||||||
|
executables, publishes their SHA-256 checksums, and uses the matching file
|
||||||
|
under `docs/releases/` as the hosted release body.
|
||||||
|
|
||||||
|
`v0.12.0` is the selected version for the pending first application-only
|
||||||
|
release. It remains an unreleased candidate until its annotated tag is
|
||||||
|
published, the hosted workflow succeeds, and every published artifact is
|
||||||
|
verified. Later releases select a new `vMAJOR.MINOR.PATCH` version according to
|
||||||
|
the intended compatibility change.
|
||||||
|
|
||||||
|
Run this procedure from the Scriptorium repository root. A release must not
|
||||||
|
depend on a Go workspace, module replacement, vendor tree, sibling checkout,
|
||||||
|
unpublished dependency, or unpushed source commit.
|
||||||
|
|
||||||
|
## Establish The Candidate
|
||||||
|
|
||||||
|
For the pending application-only release, start a POSIX shell and select:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export RELEASE_VERSION=v0.12.0
|
||||||
|
```
|
||||||
|
|
||||||
|
For a later release, export its not-yet-published semantic version instead.
|
||||||
|
Then run the following guard in that same shell:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
: "${RELEASE_VERSION:?export an unpublished vMAJOR.MINOR.PATCH version}"
|
||||||
|
if ! printf '%s\n' "$RELEASE_VERSION" |
|
||||||
|
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
|
||||||
|
then
|
||||||
|
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
RELEASE_COMMIT=$(git rev-parse --verify 'HEAD^{commit}')
|
||||||
|
export RELEASE_COMMIT
|
||||||
|
|
||||||
|
check_release_candidate() {
|
||||||
|
test "$(git branch --show-current)" = main
|
||||||
|
test -z "$(git status --porcelain)"
|
||||||
|
|
||||||
|
gowork_value=$(go env GOWORK)
|
||||||
|
case "$gowork_value" in
|
||||||
|
''|off) ;;
|
||||||
|
*)
|
||||||
|
printf '%s\n' "active Go workspace: $gowork_value" >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
test -z "$(git ls-files go.work go.work.sum)"
|
||||||
|
test ! -e vendor
|
||||||
|
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||||
|
then
|
||||||
|
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
git fetch origin main --tags
|
||||||
|
test "$RELEASE_COMMIT" = \
|
||||||
|
"$(git rev-parse --verify 'refs/remotes/origin/main^{commit}')"
|
||||||
|
|
||||||
|
if git show-ref --verify --quiet "refs/tags/$RELEASE_VERSION"
|
||||||
|
then
|
||||||
|
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if test -n "$(
|
||||||
|
git ls-remote --tags origin \
|
||||||
|
"refs/tags/$RELEASE_VERSION" \
|
||||||
|
"refs/tags/$RELEASE_VERSION^{}"
|
||||||
|
)"
|
||||||
|
then
|
||||||
|
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_release_candidate
|
||||||
|
```
|
||||||
|
|
||||||
|
Do not continue unless this guard succeeds. It deliberately requires the
|
||||||
|
candidate to be the exact clean commit already published at `origin/main`.
|
||||||
|
|
||||||
|
## Verify Modules And Repository Boundaries
|
||||||
|
|
||||||
|
Confirm the module path and declared Go version:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test "$(
|
||||||
|
GOWORK=off go list -m -f '{{.Path}} {{.GoVersion}}'
|
||||||
|
)" = 'gitea.maximumdirect.net/eric/scriptorium 1.25.5'
|
||||||
|
```
|
||||||
|
|
||||||
|
Require Promptkit `v0.1.0` as both the direct module-graph edge and the selected
|
||||||
|
module version:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
direct_promptkit=$(
|
||||||
|
GOWORK=off go mod graph |
|
||||||
|
awk '
|
||||||
|
$1 == "gitea.maximumdirect.net/eric/scriptorium" &&
|
||||||
|
$2 ~ /^gitea\.maximumdirect\.net\/eric\/promptkit@/ {
|
||||||
|
print $2
|
||||||
|
}
|
||||||
|
'
|
||||||
|
)
|
||||||
|
test "$direct_promptkit" = \
|
||||||
|
'gitea.maximumdirect.net/eric/promptkit@v0.1.0'
|
||||||
|
test "$(
|
||||||
|
GOWORK=off go list -m -f '{{.Path}}@{{.Version}}' \
|
||||||
|
gitea.maximumdirect.net/eric/promptkit
|
||||||
|
)" = 'gitea.maximumdirect.net/eric/promptkit@v0.1.0'
|
||||||
|
GOWORK=off go list -m all
|
||||||
|
```
|
||||||
|
|
||||||
|
Require tidy module metadata and recheck the repository exclusions:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
GOWORK=off go mod tidy -diff
|
||||||
|
test -z "$(git ls-files go.work go.work.sum)"
|
||||||
|
test ! -e vendor
|
||||||
|
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||||
|
then
|
||||||
|
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
test -z "$(git status --porcelain)"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Validate The Application
|
||||||
|
|
||||||
|
Run the complete application validation:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
GOWORK=off go test ./...
|
||||||
|
GOWORK=off go test -race ./...
|
||||||
|
GOWORK=off go vet ./...
|
||||||
|
validation_build_dir=$(mktemp -d)
|
||||||
|
GOWORK=off go build \
|
||||||
|
-o "$validation_build_dir/scriptorium" \
|
||||||
|
./cmd/scriptorium
|
||||||
|
```
|
||||||
|
|
||||||
|
The ordinary test run includes the architecture guard that rejects a root Go
|
||||||
|
package, former framework package families, and imports of Promptkit internal
|
||||||
|
packages. Inspect the repository for generated binaries, credentials,
|
||||||
|
temporary output, sibling paths, and other files that do not belong in the
|
||||||
|
tracked release source.
|
||||||
|
|
||||||
|
Check every tracked Go file. This command must produce no output:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
unformatted=$(
|
||||||
|
git ls-files '*.go' |
|
||||||
|
while IFS= read -r go_file
|
||||||
|
do
|
||||||
|
gofmt -l "$go_file"
|
||||||
|
done
|
||||||
|
)
|
||||||
|
test -z "$unformatted"
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the maintained render script and smoke-test both maintained configuration
|
||||||
|
examples without a model call:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
GOWORK=off ./examples/render-markdown-summary.sh
|
||||||
|
for config_file in examples/config.yml examples/config.full.yml
|
||||||
|
do
|
||||||
|
GOWORK=off go run ./cmd/scriptorium render \
|
||||||
|
--config "$config_file" \
|
||||||
|
--prompt generic.markdown_summary \
|
||||||
|
--input transcript=./examples/fixtures/transcript.md \
|
||||||
|
--input glossary=./examples/fixtures/glossary.yml \
|
||||||
|
--format json >/dev/null
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
Exercise usage output and offline rendering with the temporary native
|
||||||
|
executable:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
usage_output="$validation_build_dir/usage.txt"
|
||||||
|
if "$validation_build_dir/scriptorium" >"$usage_output" 2>&1
|
||||||
|
then
|
||||||
|
printf '%s\n' 'expected an invocation without a command to fail' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
grep -F 'usage: scriptorium' "$usage_output"
|
||||||
|
"$validation_build_dir/scriptorium" render \
|
||||||
|
--config ./examples/config.yml \
|
||||||
|
--prompt generic.markdown_summary \
|
||||||
|
--input transcript=./examples/fixtures/transcript.md \
|
||||||
|
--input glossary=./examples/fixtures/glossary.yml \
|
||||||
|
--format text >/dev/null
|
||||||
|
```
|
||||||
|
|
||||||
|
Follow every maintained local, Promptkit-tagged, and other external Markdown
|
||||||
|
link. Confirm that all repository-relative link targets exist. Finish the
|
||||||
|
application checks with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git diff --check
|
||||||
|
test -z "$(git status --porcelain)"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Require Release Notes And Reproduce Packaging
|
||||||
|
|
||||||
|
The immutable release note must exist before tagging:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
release_notes="docs/releases/$RELEASE_VERSION.md"
|
||||||
|
test -f "$release_notes"
|
||||||
|
test -s "$release_notes"
|
||||||
|
```
|
||||||
|
|
||||||
|
Validate every local and currently published link in the note. For links
|
||||||
|
pinned to the candidate Scriptorium tag, confirm that the corresponding
|
||||||
|
repository-relative path exists even though its tag URL is not live yet.
|
||||||
|
|
||||||
|
Reproduce the hosted build flags, targets, and filenames in a temporary
|
||||||
|
directory:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
release_dist=$(mktemp -d)
|
||||||
|
release_package='gitea.maximumdirect.net/eric/scriptorium/cmd/scriptorium'
|
||||||
|
|
||||||
|
build_release_binary() {
|
||||||
|
target_os="$1"
|
||||||
|
target_arch="$2"
|
||||||
|
output="$release_dist/scriptorium-$RELEASE_VERSION-$target_os-$target_arch"
|
||||||
|
|
||||||
|
CGO_ENABLED=0 GOOS="$target_os" GOARCH="$target_arch" GOWORK=off \
|
||||||
|
go build -trimpath -ldflags '-s -w' \
|
||||||
|
-o "$output" "$release_package"
|
||||||
|
}
|
||||||
|
|
||||||
|
build_release_binary linux amd64
|
||||||
|
build_release_binary linux arm64
|
||||||
|
|
||||||
|
test -s "$release_dist/scriptorium-$RELEASE_VERSION-linux-amd64"
|
||||||
|
test -s "$release_dist/scriptorium-$RELEASE_VERSION-linux-arm64"
|
||||||
|
file "$release_dist/scriptorium-$RELEASE_VERSION-linux-amd64"
|
||||||
|
file "$release_dist/scriptorium-$RELEASE_VERSION-linux-arm64"
|
||||||
|
```
|
||||||
|
|
||||||
|
Require `file` to identify Linux executables for `x86-64` and `ARM aarch64`,
|
||||||
|
respectively. Inspect the
|
||||||
|
[hosted workflow](../.woodpecker/release.yml) and confirm that it uses the
|
||||||
|
same build flags and names, copies the selected release note to
|
||||||
|
`dist/RELEASE_NOTES.md`, publishes only `dist/scriptorium-*`, and keeps
|
||||||
|
checksum generation enabled.
|
||||||
|
|
||||||
|
## Create And Publish The Tag
|
||||||
|
|
||||||
|
Run the candidate guard again immediately before creating the tag:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
check_release_candidate
|
||||||
|
test -f "$release_notes"
|
||||||
|
test -s "$release_notes"
|
||||||
|
```
|
||||||
|
|
||||||
|
Create an annotated tag explicitly bound to the validated commit, using the
|
||||||
|
version-specific release note as its message:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git tag --annotate "$RELEASE_VERSION" \
|
||||||
|
--file "$release_notes" \
|
||||||
|
"$RELEASE_COMMIT"
|
||||||
|
```
|
||||||
|
|
||||||
|
Inspect the tag and require it to resolve to the validated source:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
test "$(git cat-file -t "refs/tags/$RELEASE_VERSION")" = tag
|
||||||
|
git show --no-patch --decorate "refs/tags/$RELEASE_VERSION"
|
||||||
|
test "$(
|
||||||
|
git rev-parse --verify "refs/tags/$RELEASE_VERSION^{commit}"
|
||||||
|
)" = "$RELEASE_COMMIT"
|
||||||
|
```
|
||||||
|
|
||||||
|
If inspection finds an error, delete only the unpublished local tag, correct
|
||||||
|
the candidate, and repeat the complete validation. Never move or recreate a
|
||||||
|
published tag.
|
||||||
|
|
||||||
|
Push only the selected tag ref:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git push origin \
|
||||||
|
"refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Observe And Verify Publication
|
||||||
|
|
||||||
|
Open the hosted workflow run for the selected tag. Require
|
||||||
|
`build-release-assets` to succeed before `publish-release`, then require the
|
||||||
|
publication step and hosted release to succeed. A queued, running, failed, or
|
||||||
|
partially published workflow is not a verified release.
|
||||||
|
|
||||||
|
Compare the local and remote annotated-tag objects and their source commits:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
remote_tag=$(
|
||||||
|
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" |
|
||||||
|
awk 'NR == 1 { print $1 }'
|
||||||
|
)
|
||||||
|
remote_commit=$(
|
||||||
|
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION^{}" |
|
||||||
|
awk 'NR == 1 { print $1 }'
|
||||||
|
)
|
||||||
|
test -n "$remote_tag"
|
||||||
|
test "$remote_tag" = \
|
||||||
|
"$(git rev-parse --verify "refs/tags/$RELEASE_VERSION")"
|
||||||
|
test "$remote_commit" = "$RELEASE_COMMIT"
|
||||||
|
```
|
||||||
|
|
||||||
|
Download the hosted binaries and checksum file into a temporary directory:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
release_base="https://gitea.maximumdirect.net/eric/scriptorium/releases/download/$RELEASE_VERSION"
|
||||||
|
download_dir=$(mktemp -d)
|
||||||
|
(
|
||||||
|
cd "$download_dir"
|
||||||
|
for asset in \
|
||||||
|
"scriptorium-$RELEASE_VERSION-linux-amd64" \
|
||||||
|
"scriptorium-$RELEASE_VERSION-linux-arm64" \
|
||||||
|
SHA256SUMS
|
||||||
|
do
|
||||||
|
curl --fail --location --remote-name "$release_base/$asset"
|
||||||
|
done
|
||||||
|
|
||||||
|
test -s "scriptorium-$RELEASE_VERSION-linux-amd64"
|
||||||
|
test -s "scriptorium-$RELEASE_VERSION-linux-arm64"
|
||||||
|
test -s SHA256SUMS
|
||||||
|
sha256sum --check SHA256SUMS
|
||||||
|
test "$(wc -l < SHA256SUMS | tr -d ' ')" = 2
|
||||||
|
file "scriptorium-$RELEASE_VERSION-linux-amd64"
|
||||||
|
file "scriptorium-$RELEASE_VERSION-linux-arm64"
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Require the same Linux architectures observed in the local packaging check and
|
||||||
|
confirm that the hosted release contains no unexpected asset. On a compatible
|
||||||
|
Linux host, make the matching downloaded binary executable and repeat the
|
||||||
|
usage-output and offline-render smoke checks against it.
|
||||||
|
|
||||||
|
Only after the tag, workflow, release body, binaries, architectures, and
|
||||||
|
checksums all pass verification is the candidate a verified published release.
|
||||||
|
|
||||||
|
## Handle Failures
|
||||||
|
|
||||||
|
Before tag publication, correct the release commit or note and restart the
|
||||||
|
complete procedure. After tag publication, never delete, move, overwrite, or
|
||||||
|
recreate the tag. A transient hosted failure may be retried only against the
|
||||||
|
same immutable tag and commit and only when doing so cannot overwrite or
|
||||||
|
silently retain partial assets. A source, packaging, note, or artifact defect
|
||||||
|
requires a new corrective semantic version from a new validated commit.
|
||||||
|
|
||||||
|
Record the selected version, validated commit, tag object, workflow result,
|
||||||
|
artifact names, checksum result, and smoke-check outcome in the release
|
||||||
|
checkpoint. Keep temporary builds and downloaded assets outside the repository,
|
||||||
|
and require a clean `main` synchronized with `origin/main` when verification
|
||||||
|
is complete.
|
||||||
36
docs/releases/v0.12.0.md
Normal file
36
docs/releases/v0.12.0.md
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
# Scriptorium v0.12.0
|
||||||
|
|
||||||
|
## Breaking Project Boundary
|
||||||
|
|
||||||
|
Scriptorium is now an executable-only CLI and HTTP application. This is a
|
||||||
|
breaking change for Go consumers: the former root Go package is not included,
|
||||||
|
and no compatibility facade is provided.
|
||||||
|
|
||||||
|
Scriptorium `v0.11.1` was the final framework-bearing release. Former Go
|
||||||
|
consumers should follow the
|
||||||
|
[migration guide](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/consumers/migrating-to-promptkit.md)
|
||||||
|
and adopt
|
||||||
|
[Promptkit `v0.1.0`](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
for in-process prompt preparation and execution.
|
||||||
|
|
||||||
|
## Application Interfaces
|
||||||
|
|
||||||
|
The Scriptorium command-line and HTTP application interfaces remain. Their
|
||||||
|
canonical documentation defines the supported commands, configuration,
|
||||||
|
requests, responses, operational behavior, and deployment responsibilities:
|
||||||
|
|
||||||
|
- [CLI reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/cli.md)
|
||||||
|
- [HTTP API reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/api.md)
|
||||||
|
- [Configuration reference](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/config.md)
|
||||||
|
- [Operations guide](https://gitea.maximumdirect.net/eric/scriptorium/src/tag/v0.12.0/docs/operations.md)
|
||||||
|
|
||||||
|
## Framework Dependency And Consumers
|
||||||
|
|
||||||
|
The released Scriptorium binaries use Promptkit `v0.1.0` as their framework
|
||||||
|
dependency. Promptkit owns the reusable engine, source formats, profiles,
|
||||||
|
generation boundary, and validation contracts. See the
|
||||||
|
[Promptkit Go consumer guide](https://gitea.maximumdirect.net/eric/promptkit/src/tag/v0.1.0/docs/consumers/pkg-promptkit.md)
|
||||||
|
for that supported API.
|
||||||
|
|
||||||
|
All known downstream Go consumers were migrated to Promptkit before this
|
||||||
|
release.
|
||||||
493
docs/roadmap/implementation.md
Normal file
493
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,493 @@
|
|||||||
|
# Step 9 Implementation Plan
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Ready for implementation.
|
||||||
|
|
||||||
|
This plan implements the target state and policy decisions in
|
||||||
|
[Step 9](step9.md). Execute the stages in order. A later stage may begin only
|
||||||
|
after the preceding stage's completion checks pass and any required
|
||||||
|
cross-repository commit is published.
|
||||||
|
|
||||||
|
## Fixed Decisions And Baseline
|
||||||
|
|
||||||
|
- Promptkit `v0.1.0` at commit
|
||||||
|
`9e68a2bbf779545995270c47842048a3bc6c85dc` is the published framework
|
||||||
|
baseline. Do not create a new Promptkit tag for documentation-only work.
|
||||||
|
- Scriptorium `v0.11.1` is the final framework-bearing release.
|
||||||
|
- Scriptorium `v0.12.0` is the intended first application-only release.
|
||||||
|
- Scriptorium will not restore a Go facade, compatibility package, alias, or
|
||||||
|
forwarding API.
|
||||||
|
- Promptkit retains maintainer-run validation and tag-only source releases. Do
|
||||||
|
not add hosted Promptkit CI or binary packaging.
|
||||||
|
- Scriptorium retains hosted, tag-triggered Linux `amd64` and `arm64` binary
|
||||||
|
releases with SHA-256 checksums.
|
||||||
|
- Versioned Scriptorium release notes live at
|
||||||
|
`docs/releases/<tag>.md`. The release workflow copies the file matching
|
||||||
|
`CI_COMMIT_TAG` into its build workspace and supplies that copy to the
|
||||||
|
Woodpecker release plugin's documented
|
||||||
|
[`note` setting](https://woodpecker-ci.org/plugins/release).
|
||||||
|
- The obsolete linker assignment to the removed
|
||||||
|
`internal/buildinfo.Version` symbol is deleted. Do not introduce a version
|
||||||
|
command or replacement build-information package.
|
||||||
|
- Temporary roadmaps remain present through tag and hosted-release
|
||||||
|
verification. They are removed only after completion is recorded in Git
|
||||||
|
history.
|
||||||
|
|
||||||
|
Repository paths for this work are:
|
||||||
|
|
||||||
|
- Scriptorium: `/Users/eric/Code/scriptorium/scriptorium`
|
||||||
|
- Promptkit: `/Users/eric/Code/scriptorium/promptkit`
|
||||||
|
- Notarius verification checkout: `/Users/eric/Code/notarius`
|
||||||
|
|
||||||
|
Do not commit `go.work`, `go.work.sum`, a `replace` directive, vendored
|
||||||
|
Promptkit source, generated binaries, downloaded release assets, credentials,
|
||||||
|
or temporary validation output. Use directories created with `mktemp -d` for
|
||||||
|
clones, caches, builds, and downloads, and remove or leave those directories
|
||||||
|
outside the repositories.
|
||||||
|
|
||||||
|
## Stage 1: Correct Promptkit Release And Project Documentation
|
||||||
|
|
||||||
|
Work only in the Promptkit repository during this stage.
|
||||||
|
|
||||||
|
### Changes
|
||||||
|
|
||||||
|
1. Rewrite `docs/release.md` as a current, reusable procedure:
|
||||||
|
- state that `v0.1.0` is the initial published release rather than a planned
|
||||||
|
tag;
|
||||||
|
- retain semantic `vMAJOR.MINOR.PATCH` tags, pre-`v1` compatibility notes,
|
||||||
|
maintainer-run validation, and source-only publication;
|
||||||
|
- require the maintainer to provide a not-yet-published
|
||||||
|
`RELEASE_VERSION`, validate it as a semantic version, and derive the
|
||||||
|
release commit from `HEAD`;
|
||||||
|
- retain clean-checkout, no-workspace, no-replacement, module identity,
|
||||||
|
tests, race tests, vet, build, maintained example, formatting, link,
|
||||||
|
whitespace, and repository-hygiene checks;
|
||||||
|
- use the annotated tag message as Promptkit's source-only release notes,
|
||||||
|
recording the version, validated commit, compatibility summary, public
|
||||||
|
API changes, and any required consumer action;
|
||||||
|
- retain annotated-tag creation, exact-ref publication, remote tag-object
|
||||||
|
comparison, source-commit verification, and ordinary Go module
|
||||||
|
resolution checks; and
|
||||||
|
- state the Promptkit-before-consumer release ordering without restating
|
||||||
|
Scriptorium's release procedure.
|
||||||
|
2. Use guarded shell examples. A missing or already-existing release version,
|
||||||
|
a dirty checkout, an active workspace, a replacement, an unpushed release
|
||||||
|
commit, or an existing local or remote tag must stop the procedure before
|
||||||
|
tag creation. Publish only the selected tag ref, never all local tags.
|
||||||
|
3. Add a short “Related project” entry to Promptkit's `README.md` linking to
|
||||||
|
the Scriptorium repository as the CLI and HTTP application built on
|
||||||
|
Promptkit. Do not copy Scriptorium invocation or transport contracts into
|
||||||
|
Promptkit.
|
||||||
|
4. Audit Promptkit README, consumer, architecture, development, and internal
|
||||||
|
documents for stale extraction or pre-release language. Change only claims
|
||||||
|
that are no longer true; exact public API and format contracts remain with
|
||||||
|
their existing canonical owners.
|
||||||
|
|
||||||
|
### Validation
|
||||||
|
|
||||||
|
- Follow every changed local and cross-project Markdown link.
|
||||||
|
- Check all changed shell fragments for valid POSIX shell syntax without
|
||||||
|
executing tag or push commands.
|
||||||
|
- Run `git diff --check`.
|
||||||
|
- Confirm no code, module, example, architecture boundary, or public contract
|
||||||
|
changed.
|
||||||
|
|
||||||
|
### Completion State
|
||||||
|
|
||||||
|
Commit the Promptkit documentation changes with a plain-English message and
|
||||||
|
push the commit to `origin/main`. Require a clean Promptkit working tree with
|
||||||
|
`main` synchronized to `origin/main` before Stage 2. Do not create a Promptkit
|
||||||
|
tag.
|
||||||
|
|
||||||
|
## Stage 2: Establish Scriptorium Release Ownership And Mechanics
|
||||||
|
|
||||||
|
Work only in the Scriptorium repository during this stage.
|
||||||
|
|
||||||
|
### Documentation Ownership
|
||||||
|
|
||||||
|
1. Add two canonical-owner rows to `docs/policy/documentation.md`:
|
||||||
|
- `docs/release.md` owns the Scriptorium release procedure, including
|
||||||
|
candidate validation, version and tag operations, hosted-workflow
|
||||||
|
observation, and artifact verification;
|
||||||
|
- `docs/releases/` owns immutable version-specific release notes and
|
||||||
|
migration announcements, not complete interface contracts.
|
||||||
|
2. Update the release-packaging row in `docs/development.md` to route
|
||||||
|
contributors to `docs/release.md`, `.woodpecker/release.yml`, and the
|
||||||
|
architecture policy.
|
||||||
|
3. Create `docs/release.md` with an end-to-end procedure for:
|
||||||
|
- selecting `v0.12.0` for this release and a semantic version for later
|
||||||
|
releases;
|
||||||
|
- requiring a clean, pushed release commit on `main`, outside a Go
|
||||||
|
workspace and without a module replacement or vendor tree;
|
||||||
|
- confirming module identity, Go version, direct Promptkit version, module
|
||||||
|
graph, module tidiness, and tag availability;
|
||||||
|
- running the complete application and release-equivalent validation
|
||||||
|
defined in Stage 4;
|
||||||
|
- requiring `docs/releases/<tag>.md`;
|
||||||
|
- creating and inspecting an annotated tag bound to the validated commit;
|
||||||
|
- pushing only that exact tag ref;
|
||||||
|
- monitoring the hosted workflow;
|
||||||
|
- comparing local and remote tag objects and commits;
|
||||||
|
- downloading and verifying release assets and checksums; and
|
||||||
|
- handling failures without moving or recreating a published tag.
|
||||||
|
The procedure must distinguish an unreleased candidate from a verified
|
||||||
|
published release and must not claim that `v0.12.0` exists before its tag
|
||||||
|
and hosted artifacts are confirmed.
|
||||||
|
4. Keep operator workflows in `docs/operations.md`; add a release-procedure
|
||||||
|
link there only if a concrete operator task needs it.
|
||||||
|
|
||||||
|
### Release Workflow
|
||||||
|
|
||||||
|
Update `.woodpecker/release.yml` as follows:
|
||||||
|
|
||||||
|
1. Keep tag-only execution, Go `1.25`, `CGO_ENABLED=0`, `-trimpath`, `-s -w`,
|
||||||
|
the existing Linux architectures, filenames, release plugin, and checksum
|
||||||
|
policy.
|
||||||
|
2. Remove only the stale
|
||||||
|
`-X gitea.maximumdirect.net/eric/scriptorium/internal/buildinfo.Version=...`
|
||||||
|
linker assignment.
|
||||||
|
3. Before building, require a release-notes source at
|
||||||
|
`docs/releases/$CI_COMMIT_TAG.md`. Fail with a clear diagnostic when it is
|
||||||
|
absent.
|
||||||
|
4. Copy that source to a stable path under the workflow's `dist` directory,
|
||||||
|
such as `dist/RELEASE_NOTES.md`.
|
||||||
|
5. Set the release plugin's `note` value to the copied file. Keep release-note
|
||||||
|
content out of the binary asset glob so only the two binaries are
|
||||||
|
checksummed and uploaded alongside `SHA256SUMS`.
|
||||||
|
6. Preserve `overwrite: false` and non-prerelease publication. Do not add
|
||||||
|
automatic tag generation or release mutation.
|
||||||
|
|
||||||
|
Do not add a Go test that parses or snapshots the workflow. The material risks
|
||||||
|
are protected by release-equivalent builds, missing-note failure behavior,
|
||||||
|
hosted execution, and artifact inspection.
|
||||||
|
|
||||||
|
### Validation
|
||||||
|
|
||||||
|
- Inspect the YAML and embedded shell for the exact intended paths and
|
||||||
|
dependency between build and publish steps.
|
||||||
|
- In a temporary directory, reproduce the build step with
|
||||||
|
`CI_COMMIT_TAG=v0.12.0` and a temporary output directory. Confirm both
|
||||||
|
release-equivalent binaries build and the release-note selection fails for
|
||||||
|
a nonexistent tag.
|
||||||
|
- Use `file` or an equivalent binary inspector to confirm Linux `amd64` and
|
||||||
|
Linux `arm64` targets.
|
||||||
|
- Run `git diff --check` and validate all new local documentation links.
|
||||||
|
|
||||||
|
### Completion State
|
||||||
|
|
||||||
|
The release contract and workflow are internally consistent, reusable after
|
||||||
|
`v0.12.0`, and contain no reference to a removed package. Do not tag or publish
|
||||||
|
Scriptorium in this stage.
|
||||||
|
|
||||||
|
## Stage 3: Publish Migration Guidance And Release Notes
|
||||||
|
|
||||||
|
Work only in the Scriptorium repository during this stage.
|
||||||
|
|
||||||
|
### Go Consumer Migration Guide
|
||||||
|
|
||||||
|
Create `docs/consumers/migrating-to-promptkit.md` as the canonical guide for
|
||||||
|
leaving the former Scriptorium Go package. It must:
|
||||||
|
|
||||||
|
1. Identify Scriptorium `v0.11.1` and
|
||||||
|
`gitea.maximumdirect.net/eric/scriptorium` as the source, Promptkit `v0.1.0`
|
||||||
|
and `gitea.maximumdirect.net/eric/promptkit` as the destination, and
|
||||||
|
Scriptorium `v0.12.0` as executable-only.
|
||||||
|
2. Give copyable commands that:
|
||||||
|
- add Promptkit `v0.1.0`;
|
||||||
|
- replace Go imports and package qualifiers;
|
||||||
|
- remove the old Scriptorium dependency through `go mod tidy`; and
|
||||||
|
- run the consumer's tests.
|
||||||
|
3. Explain at a high level that the established engine, requests, results,
|
||||||
|
profiles, source options, model-client boundary, artifacts, validation
|
||||||
|
values, and error identities were preserved where practical.
|
||||||
|
4. Identify the Promptkit additions relevant to migration:
|
||||||
|
`WithArtifactReader`, `ArtifactReader`, `ErrProfileRequired`, and
|
||||||
|
`ErrAPIKeyEnvMissing`. Link exact declarations and behavior to Promptkit's
|
||||||
|
tagged consumer guide and Go source rather than reproducing them.
|
||||||
|
5. Tell consumers to verify the behavior they actually rely upon, including
|
||||||
|
configuration sources, credentials, timeout layering, validation,
|
||||||
|
injected extensions, and `errors.Is` handling.
|
||||||
|
6. State that no compatibility facade exists. Consumers unable to migrate may
|
||||||
|
remain pinned to Scriptorium `v0.11.1`, but that line does not provide the
|
||||||
|
slim application release.
|
||||||
|
|
||||||
|
### Scriptorium Navigation
|
||||||
|
|
||||||
|
1. Add the migration guide to the documentation list in `README.md`.
|
||||||
|
2. Add a concise link from `docs/consumers/api.md` for readers arriving from
|
||||||
|
the former Go API.
|
||||||
|
3. Audit all Scriptorium Markdown and Go package comments:
|
||||||
|
- current product statements must describe a CLI and HTTP application;
|
||||||
|
- framework contracts must link to tagged Promptkit `v0.1.0` owners;
|
||||||
|
- no permanent document may imply that Scriptorium still exports a Go
|
||||||
|
package;
|
||||||
|
- no document may copy Promptkit's exact fields, defaults, or API
|
||||||
|
declarations merely to explain the split.
|
||||||
|
4. Keep general cross-project navigation pointed at project entry pages and
|
||||||
|
contract links pinned to the consumed Promptkit tag.
|
||||||
|
|
||||||
|
### Versioned Release Notes
|
||||||
|
|
||||||
|
Create `docs/releases/v0.12.0.md`. Write it so the same Markdown works both as
|
||||||
|
a checked-in document and as the hosted release body. It must:
|
||||||
|
|
||||||
|
- identify the executable-only boundary as a breaking change;
|
||||||
|
- direct former Go consumers to the migration guide and Promptkit `v0.1.0`;
|
||||||
|
- identify `v0.11.1` as the last framework-bearing Scriptorium release;
|
||||||
|
- summarize the retained CLI and HTTP application interfaces without
|
||||||
|
duplicating their contracts;
|
||||||
|
- state that all known downstream Go consumers were migrated;
|
||||||
|
- name Promptkit `v0.1.0` as the released binary's framework dependency; and
|
||||||
|
- link to canonical CLI, HTTP, configuration, operations, migration, and
|
||||||
|
Promptkit consumer documents.
|
||||||
|
|
||||||
|
Use absolute tag URLs for links that must work from the hosted release page.
|
||||||
|
Scriptorium `v0.12.0` tag URLs will not exist before publication; validate
|
||||||
|
their corresponding repository-relative targets locally before tagging and
|
||||||
|
the actual URLs in Stage 5. Promptkit `v0.1.0` links must already resolve.
|
||||||
|
|
||||||
|
### Validation
|
||||||
|
|
||||||
|
- Run every copyable non-destructive migration command against a temporary Go
|
||||||
|
module or a temporary copy of the maintained historical consumer shape.
|
||||||
|
Confirm it selects Promptkit `v0.1.0`, contains no Scriptorium import, and
|
||||||
|
compiles.
|
||||||
|
- Validate every local and currently published cross-project link.
|
||||||
|
- Confirm every prospective Scriptorium `v0.12.0` URL maps to an existing path
|
||||||
|
in the candidate tree.
|
||||||
|
- Run `git diff --check`.
|
||||||
|
- Re-audit for stale project identity, framework ownership, pre-cutover, and
|
||||||
|
planned-release claims.
|
||||||
|
|
||||||
|
### Completion State
|
||||||
|
|
||||||
|
Commit the combined Scriptorium release-workflow, release-procedure, migration,
|
||||||
|
release-note, policy, and navigation changes with one or more plain-English
|
||||||
|
messages. Push them to `origin/main`. Require a clean Scriptorium working tree
|
||||||
|
with `main` synchronized to `origin/main` before Stage 4. Do not tag the release
|
||||||
|
yet.
|
||||||
|
|
||||||
|
## Stage 4: Perform The Independent Pre-Release Acceptance Run
|
||||||
|
|
||||||
|
This is a validation and release-readiness stage. Do not modify product code,
|
||||||
|
move tags, or publish a release while running it.
|
||||||
|
|
||||||
|
### Promptkit Main And Published Tag
|
||||||
|
|
||||||
|
1. Confirm Promptkit's local `main`, `origin/main`, local annotated `v0.1.0`
|
||||||
|
tag, remote tag object, and resolved tag commit.
|
||||||
|
2. On Promptkit `main`, run:
|
||||||
|
- `go test ./...`;
|
||||||
|
- `go test -race ./...`;
|
||||||
|
- `go vet ./...`;
|
||||||
|
- `go build ./...`;
|
||||||
|
- `go run ./examples/go-library/prepare`;
|
||||||
|
- `gofmt -l` over tracked Go files;
|
||||||
|
- local and cross-project Markdown-link checks;
|
||||||
|
- `git diff --check`;
|
||||||
|
- module identity, no-workspace, no-replacement, no-vendor, and tracked-tree
|
||||||
|
hygiene checks.
|
||||||
|
3. Clone the remote Promptkit `v0.1.0` tag into a temporary directory with
|
||||||
|
fresh task-specific `GOMODCACHE` and `GOCACHE` directories and `GOWORK=off`.
|
||||||
|
Run the complete documented Promptkit validation there.
|
||||||
|
4. From a separate temporary consumer module, resolve
|
||||||
|
`gitea.maximumdirect.net/eric/promptkit@v0.1.0` through the ordinary remote
|
||||||
|
module path and confirm its origin metadata and expected source commit
|
||||||
|
without reading the sibling Promptkit checkout.
|
||||||
|
|
||||||
|
### Scriptorium Release Candidate
|
||||||
|
|
||||||
|
Clone the pushed Scriptorium release commit from `origin/main` into a temporary
|
||||||
|
directory. Use fresh task-specific module and build caches and `GOWORK=off`.
|
||||||
|
Perform all checks from that clone:
|
||||||
|
|
||||||
|
1. Confirm:
|
||||||
|
- module path and Go version;
|
||||||
|
- direct Promptkit requirement exactly `v0.1.0`;
|
||||||
|
- module graph selection of Promptkit `v0.1.0`;
|
||||||
|
- no `replace`, `go.work`, `go.work.sum`, vendor tree, root Go package,
|
||||||
|
former framework packages, Promptkit internal import, or sibling path;
|
||||||
|
- `go mod tidy -diff` produces no changes; and
|
||||||
|
- local and remote `v0.12.0` tags do not exist.
|
||||||
|
2. Run:
|
||||||
|
- `go test ./...`;
|
||||||
|
- `go test -race ./...`;
|
||||||
|
- `go vet ./...`;
|
||||||
|
- `gofmt -l` over tracked Go files;
|
||||||
|
- `git diff --check`;
|
||||||
|
- architecture and tracked-tree hygiene checks;
|
||||||
|
- the maintained render script;
|
||||||
|
- render smoke checks with both configuration examples; and
|
||||||
|
- every maintained local, Promptkit-tagged, and other existing external
|
||||||
|
documentation link.
|
||||||
|
3. Build a temporary native executable and exercise basic help and offline
|
||||||
|
render behavior.
|
||||||
|
4. Reproduce the release workflow with `CI_COMMIT_TAG=v0.12.0` into temporary
|
||||||
|
output:
|
||||||
|
- select and copy `docs/releases/v0.12.0.md`;
|
||||||
|
- build stripped, trimmed Linux `amd64` and `arm64` binaries;
|
||||||
|
- verify exact filenames, nonempty files, target operating systems and
|
||||||
|
architectures, and absence of the obsolete linker assignment;
|
||||||
|
- compute SHA-256 checksums in the same basename-only form expected from the
|
||||||
|
plugin; and
|
||||||
|
- confirm no extra file matches the upload glob.
|
||||||
|
5. Confirm repository-relative targets for the not-yet-live Scriptorium
|
||||||
|
`v0.12.0` release-note links.
|
||||||
|
|
||||||
|
### Downstream Confirmation
|
||||||
|
|
||||||
|
In the Notarius checkout:
|
||||||
|
|
||||||
|
- require a clean `main` synchronized with `origin/main`;
|
||||||
|
- confirm `go.mod` directly requires Promptkit and no tracked file imports the
|
||||||
|
former Scriptorium Go package; and
|
||||||
|
- run `go test ./...`.
|
||||||
|
|
||||||
|
If another downstream consumer is discovered, stop and disposition it under
|
||||||
|
the Step 8 gate before publication.
|
||||||
|
|
||||||
|
### Release Checkpoint
|
||||||
|
|
||||||
|
Record the exact Scriptorium release commit, Promptkit tag object and commit,
|
||||||
|
validation commands, and results. Confirm both source repositories remain
|
||||||
|
clean after validation.
|
||||||
|
|
||||||
|
Any failure returns work to the owning earlier stage. Repeat the complete
|
||||||
|
Stage 4 acceptance run after the fix is committed and pushed. Proceed only
|
||||||
|
when the validated Scriptorium commit is the exact clean `origin/main` commit
|
||||||
|
intended for `v0.12.0`.
|
||||||
|
|
||||||
|
## Stage 5: Publish And Verify Scriptorium `v0.12.0`
|
||||||
|
|
||||||
|
This stage performs external, tag-triggered publication. Do not start it
|
||||||
|
without confirmed repository access, the hosted release secret and workflow,
|
||||||
|
and the complete Stage 4 evidence.
|
||||||
|
|
||||||
|
### Tag Publication
|
||||||
|
|
||||||
|
1. Fetch `origin/main` and all tags without changing files.
|
||||||
|
2. Reconfirm that `v0.12.0` is absent locally and remotely and that the working
|
||||||
|
tree is clean at the validated release commit.
|
||||||
|
3. Create an annotated `v0.12.0` tag. Its message must identify Scriptorium
|
||||||
|
`v0.12.0`, the exact release commit, and that the documented release
|
||||||
|
validation passed.
|
||||||
|
4. Inspect the tag object and resolved commit.
|
||||||
|
5. Push only `refs/tags/v0.12.0`.
|
||||||
|
6. Compare the local and remote annotated tag object IDs and resolved commit
|
||||||
|
IDs.
|
||||||
|
|
||||||
|
Never move, delete, or recreate a published release tag. A transient hosted
|
||||||
|
workflow failure may be retried against the same tag. A source or workflow
|
||||||
|
defect discovered after tag publication requires a fix on `main` and an
|
||||||
|
explicit roadmap/version decision before a later patch release; do not silently
|
||||||
|
retag `v0.12.0` or mark Step 9 complete.
|
||||||
|
|
||||||
|
### Hosted Release Verification
|
||||||
|
|
||||||
|
1. Monitor the Woodpecker tag pipeline until it reaches a terminal state.
|
||||||
|
2. Require both the build and publish steps to succeed.
|
||||||
|
3. Inspect the Gitea release and require:
|
||||||
|
- a non-prerelease `v0.12.0` release;
|
||||||
|
- release body content sourced from `docs/releases/v0.12.0.md`;
|
||||||
|
- exactly the intended Linux `amd64` binary, Linux `arm64` binary, and
|
||||||
|
`SHA256SUMS` release assets; and
|
||||||
|
- no source-copy, Promptkit binary, release-note file, or stale artifact
|
||||||
|
uploaded by the workflow.
|
||||||
|
4. Download all three assets into a temporary directory.
|
||||||
|
5. Verify `SHA256SUMS` against both binaries, confirm file types and target
|
||||||
|
architectures, and inspect that neither file is empty or malformed.
|
||||||
|
6. Run basic `--help` and offline render smoke behavior on a downloaded binary
|
||||||
|
when the host or an available isolated Linux runtime supports its
|
||||||
|
architecture. When execution is unavailable, record that limitation and
|
||||||
|
rely on the matching pre-release binary smoke plus downloaded-artifact
|
||||||
|
format and checksum verification.
|
||||||
|
7. Verify every Scriptorium `v0.12.0` and Promptkit `v0.1.0` URL in the hosted
|
||||||
|
release notes.
|
||||||
|
8. Resolve Scriptorium `v0.12.0` and Promptkit `v0.1.0` through fresh remote Go
|
||||||
|
module queries and confirm their expected tag commits and dependency edge.
|
||||||
|
|
||||||
|
### Completion State
|
||||||
|
|
||||||
|
Retain the publication evidence needed for the final completion record. Remove
|
||||||
|
temporary downloads and confirm no release outputs entered either repository.
|
||||||
|
Do not retire the roadmaps until every hosted verification passes.
|
||||||
|
|
||||||
|
## Stage 6: Record Completion And Retire Migration Roadmaps
|
||||||
|
|
||||||
|
Work in Scriptorium only after Stage 5 is fully complete.
|
||||||
|
|
||||||
|
### Completion Record
|
||||||
|
|
||||||
|
1. Update `docs/roadmap/migration.md` to mark Step 9 and the overall Promptkit
|
||||||
|
migration complete.
|
||||||
|
2. Update `docs/roadmap/step9.md` and this implementation plan with a concise
|
||||||
|
completion summary containing:
|
||||||
|
- Promptkit tag and commit;
|
||||||
|
- Scriptorium tag and commit;
|
||||||
|
- hosted release and asset results;
|
||||||
|
- downstream status;
|
||||||
|
- independent validation results; and
|
||||||
|
- any explicitly accepted platform limitation in downloaded-binary
|
||||||
|
execution.
|
||||||
|
3. Commit and push that completion record with a plain-English message. This
|
||||||
|
commit provides the historical completion snapshot before temporary files
|
||||||
|
are removed.
|
||||||
|
|
||||||
|
### Roadmap Retirement
|
||||||
|
|
||||||
|
In a subsequent commit:
|
||||||
|
|
||||||
|
1. Move any still-useful current contract or procedure into its already
|
||||||
|
designated permanent owner.
|
||||||
|
2. Remove all completed Promptkit-migration roadmaps:
|
||||||
|
- `docs/roadmap/migration.md`;
|
||||||
|
- `docs/roadmap/step7.md`;
|
||||||
|
- `docs/roadmap/step9.md`; and
|
||||||
|
- `docs/roadmap/implementation.md`.
|
||||||
|
3. Remove the empty `docs/roadmap/` directory if no unrelated active roadmap
|
||||||
|
remains.
|
||||||
|
4. Search the full repository for incoming links or prose references to the
|
||||||
|
removed files and repair them by linking to ADRs, the release procedure,
|
||||||
|
versioned release notes, migration guide, or another canonical owner.
|
||||||
|
5. Do not change accepted ADR decision content. Git history, tags, hosted
|
||||||
|
release notes, permanent migration guidance, and the completion-record
|
||||||
|
commit retain the durable history.
|
||||||
|
|
||||||
|
### Final Validation
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
- all local Markdown-link checks;
|
||||||
|
- all maintained cross-project links;
|
||||||
|
- `git diff --check`;
|
||||||
|
- a stale-reference search for removed roadmap paths and pre-cutover claims;
|
||||||
|
- tracked-tree hygiene checks in both repositories; and
|
||||||
|
- `git status --short --branch` in Scriptorium, Promptkit, and Notarius.
|
||||||
|
|
||||||
|
The cleanup is documentation-only and does not require repeating unrelated Go
|
||||||
|
tests unless the cleanup changes a command, example, or behavior-bearing file.
|
||||||
|
Commit and push the retirement with a plain-English message.
|
||||||
|
|
||||||
|
## Final Completion State
|
||||||
|
|
||||||
|
The implementation is complete when:
|
||||||
|
|
||||||
|
- Promptkit main contains accurate reusable release guidance and still
|
||||||
|
publishes `v0.1.0` as an independently validated source-only Go module;
|
||||||
|
- Scriptorium main contains a canonical release procedure, permanent migration
|
||||||
|
guide, versioned `v0.12.0` release notes, and a release workflow matching the
|
||||||
|
current application;
|
||||||
|
- Scriptorium `v0.12.0` is published from the independently validated commit
|
||||||
|
with verified Linux `amd64` and `arm64` binaries and SHA-256 checksums;
|
||||||
|
- the hosted release body matches the checked-in versioned notes;
|
||||||
|
- Notarius remains migrated and no other downstream Go consumer remains;
|
||||||
|
- neither repository or release depends on local multi-repository state;
|
||||||
|
- both repositories are clean and synchronized with their remotes; and
|
||||||
|
- the completed temporary migration roadmaps have been removed after their
|
||||||
|
completion record was committed.
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
None. The Step 9 feature roadmap, accepted ADRs, current release topology, and
|
||||||
|
repository policies determine the required implementation and release choices.
|
||||||
341
docs/roadmap/migration.md
Normal file
341
docs/roadmap/migration.md
Normal file
@@ -0,0 +1,341 @@
|
|||||||
|
# Promptkit Migration Roadmap
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Accepted plan. Steps 1 through 8 are complete. Step 9 remains proposed and is
|
||||||
|
not yet implemented.
|
||||||
|
|
||||||
|
## Objective
|
||||||
|
|
||||||
|
Split the current repository into two projects:
|
||||||
|
|
||||||
|
- **Promptkit**: the reusable Go framework, public Go facade, execution engine,
|
||||||
|
source and validation support, OpenAI-compatible client, extension
|
||||||
|
interfaces, and built-in execution-profile registry.
|
||||||
|
- **Scriptorium**: a slim runnable application that imports Promptkit and
|
||||||
|
provides the CLI and HTTP interfaces.
|
||||||
|
|
||||||
|
Scriptorium will become another downstream Promptkit consumer rather than the
|
||||||
|
owner of the framework.
|
||||||
|
|
||||||
|
## Compatibility And Migration Policy
|
||||||
|
|
||||||
|
This is an intentionally breaking change.
|
||||||
|
|
||||||
|
- New and migrated Go consumers must import Promptkit instead of Scriptorium.
|
||||||
|
- Scriptorium will not retain type aliases, forwarding packages, deprecated
|
||||||
|
facade APIs, or other source-compatibility shims.
|
||||||
|
- Existing consumers may continue using a previously tagged Scriptorium module
|
||||||
|
version until they are migrated.
|
||||||
|
- The migration does not need to preserve compatibility between intermediate
|
||||||
|
development states. Each completed phase must instead leave the affected
|
||||||
|
repository internally consistent and tested.
|
||||||
|
- Promptkit should initially preserve the useful shape and behavior of the
|
||||||
|
current public Go facade where doing so reduces extraction risk. Broader API
|
||||||
|
redesign should follow the split unless required to establish the new
|
||||||
|
boundary.
|
||||||
|
|
||||||
|
## Target Ownership
|
||||||
|
|
||||||
|
Promptkit should own application-neutral framework behavior:
|
||||||
|
|
||||||
|
- public engine, request, result, option, extension, and error APIs;
|
||||||
|
- prompt-definition loading and rendering;
|
||||||
|
- execution profiles, overlays, and the built-in profile registry;
|
||||||
|
- artifact-loading interfaces and general-purpose `file` and `inline` support;
|
||||||
|
- schema loading and output validation;
|
||||||
|
- LLM client boundaries and the OpenAI-compatible implementation;
|
||||||
|
- preparation and execution orchestration;
|
||||||
|
- framework and execution defaults.
|
||||||
|
|
||||||
|
Scriptorium should own executable and transport concerns:
|
||||||
|
|
||||||
|
- the `scriptorium` command and its `run`, `render`, and `serve` interfaces;
|
||||||
|
- CLI parsing, output formatting, exit codes, and process behavior;
|
||||||
|
- application-config discovery and CLI precedence;
|
||||||
|
- HTTP routing, request and response DTOs, limits, and error/status mapping;
|
||||||
|
- HTTP artifact-root and deployment security policy;
|
||||||
|
- server and adapter defaults;
|
||||||
|
- executable examples, operations guidance, and transport documentation.
|
||||||
|
|
||||||
|
The intended dependency direction is:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Scriptorium CLI and HTTP adapters
|
||||||
|
|
|
||||||
|
v
|
||||||
|
Promptkit
|
||||||
|
|
|
||||||
|
v
|
||||||
|
consumer-supplied sources and clients
|
||||||
|
```
|
||||||
|
|
||||||
|
Scriptorium must use Promptkit's public API. It must not depend on Promptkit
|
||||||
|
implementation packages or reproduce framework orchestration.
|
||||||
|
|
||||||
|
## Migration Steps
|
||||||
|
|
||||||
|
### Step 1: Refresh And Synchronize Documentation
|
||||||
|
|
||||||
|
Perform a repository-wide documentation refresh before migration development.
|
||||||
|
|
||||||
|
At minimum:
|
||||||
|
|
||||||
|
- reconcile all current-behavior documentation with the code, tests, examples,
|
||||||
|
defaults, and current public contracts;
|
||||||
|
- introduce the planned documentation-policy updates;
|
||||||
|
- establish an architecture decision record policy and canonical ADR location;
|
||||||
|
- resolve stale, duplicated, or misplaced material;
|
||||||
|
- validate documentation links and maintained examples;
|
||||||
|
- leave future migration behavior in `docs/roadmap/` until implemented.
|
||||||
|
|
||||||
|
**Gate:** Do not begin architectural migration work until the documentation
|
||||||
|
refresh and policy updates are merged and the repository has an agreed,
|
||||||
|
accurate baseline.
|
||||||
|
|
||||||
|
**Gate status:** Complete as of 2026-07-26. The completed documentation
|
||||||
|
refresh, follow-up verification, and layered-timeout correction remain recorded
|
||||||
|
in repository history. Step 1 remains complete after that validation.
|
||||||
|
|
||||||
|
### Step 2: Record The Architectural Decision And Detailed Boundary
|
||||||
|
|
||||||
|
Create an ADR, under the policy established in Step 1, that records:
|
||||||
|
|
||||||
|
- the decision to split Promptkit from Scriptorium;
|
||||||
|
- the target ownership and dependency direction;
|
||||||
|
- the selected Promptkit repository and Go module paths;
|
||||||
|
- the breaking-change and versioning policy;
|
||||||
|
- ownership of configuration fields and defaults;
|
||||||
|
- artifact-reader and HTTP containment responsibilities;
|
||||||
|
- local multi-repository development and release coordination;
|
||||||
|
- documentation ownership after the split.
|
||||||
|
|
||||||
|
Use the ADR to resolve any remaining public-boundary decisions before code is
|
||||||
|
moved.
|
||||||
|
|
||||||
|
**Gate:** The ADR is accepted, and every existing package, public contract,
|
||||||
|
configuration category, and maintained asset has a target owner.
|
||||||
|
|
||||||
|
**Gate status:** Complete as of 2026-07-26.
|
||||||
|
[ADR 0002: Split Promptkit From Scriptorium](../adr/0002-split-promptkit-from-scriptorium.md)
|
||||||
|
is accepted and records the required ownership and coordination decisions.
|
||||||
|
|
||||||
|
### Step 3: Characterize Existing Framework Behavior
|
||||||
|
|
||||||
|
Strengthen or add contract-focused tests where needed so extraction can be
|
||||||
|
verified without relying on package placement.
|
||||||
|
|
||||||
|
The completed Step 3 gate records the accepted implementation scope and
|
||||||
|
intended completion state.
|
||||||
|
|
||||||
|
Preserve coverage of:
|
||||||
|
|
||||||
|
- `Prepare` and `Run` behavior;
|
||||||
|
- prompt, profile, execution-default, and request-override precedence;
|
||||||
|
- presence-aware numeric overrides;
|
||||||
|
- built-in profile fallback and custom-profile overlays;
|
||||||
|
- strict YAML and JSON decoding;
|
||||||
|
- prompt, profile, schema, and artifact source behavior;
|
||||||
|
- structured-output requests and output validation;
|
||||||
|
- validation failures versus validation runtime errors;
|
||||||
|
- secret handling and redaction;
|
||||||
|
- public error classification;
|
||||||
|
- HTTP artifact restrictions and transport mappings.
|
||||||
|
|
||||||
|
**Gate:** Current framework and adapter contracts are represented by passing
|
||||||
|
tests sufficient to detect behavioral regressions during the split.
|
||||||
|
|
||||||
|
**Gate status:** Complete as of 2026-07-27. The framework contract corpus,
|
||||||
|
public `Engine` characterization, ownership audit, full test and vet suites,
|
||||||
|
temporary executable build, and maintained offline examples passed.
|
||||||
|
|
||||||
|
### Step 4: Make Scriptorium Adapters Consume The Public Facade
|
||||||
|
|
||||||
|
Within the current repository, refactor the CLI and HTTP adapters to use the
|
||||||
|
public framework facade rather than constructing or importing internal runner
|
||||||
|
components directly.
|
||||||
|
|
||||||
|
Add only the minimum public capabilities needed to support this boundary. These
|
||||||
|
may include:
|
||||||
|
|
||||||
|
- a small `Run`/`Prepare` consumer interface;
|
||||||
|
- injectable artifact-reading behavior for Scriptorium's restricted HTTP
|
||||||
|
policy;
|
||||||
|
- source options currently available only through internal constructors;
|
||||||
|
- prepared-run formatting based on public types;
|
||||||
|
- stable public error classification required by CLI and HTTP mappings.
|
||||||
|
|
||||||
|
Do not broadly export internal repositories, domain types, or use-case
|
||||||
|
implementations.
|
||||||
|
|
||||||
|
**Gate:** The CLI and HTTP adapters use only the public framework API for
|
||||||
|
framework behavior, and all tests and documented smoke commands pass.
|
||||||
|
|
||||||
|
**Gate status:** Complete as of 2026-07-28. CLI `run`, `render`, and `serve`,
|
||||||
|
the HTTP handler, and prepared-run formatting use the public facade; the
|
||||||
|
restricted HTTP reader is injected through the public extension point. The
|
||||||
|
post-implementation public-error, deterministic MIME, and recursive
|
||||||
|
dependency-guard corrections passed full tests, vet, build, race checks,
|
||||||
|
maintained examples, and configuration smoke checks.
|
||||||
|
|
||||||
|
### Step 5: Create The Promptkit Repository
|
||||||
|
|
||||||
|
Promptkit was established as an independent repository and Go module through
|
||||||
|
the completed out-of-band workflow recorded in repository history. Its
|
||||||
|
foundation includes:
|
||||||
|
|
||||||
|
- confirmed repository access, governance, origin, and default-branch tracking;
|
||||||
|
- module `gitea.maximumdirect.net/eric/promptkit` at Go `1.25.5`;
|
||||||
|
- a minimal root `promptkit` public package boundary with no placeholder
|
||||||
|
framework packages;
|
||||||
|
- library-specific development, architecture, documentation, testing, and
|
||||||
|
release policies;
|
||||||
|
- documented maintainer-run test, vet, build, formatting, documentation-link,
|
||||||
|
and repository-hygiene validation;
|
||||||
|
- source-commit and semantic Go module tag releases without runnable binaries
|
||||||
|
or binary packaging; and
|
||||||
|
- temporary workspace and uncommitted replacement workflows for coordinated
|
||||||
|
development without committed repository coupling.
|
||||||
|
|
||||||
|
[ADR 0003](../adr/0003-use-maintainer-run-validation-and-tag-only-releases-for-promptkit.md)
|
||||||
|
records the controlling Promptkit validation and release decision.
|
||||||
|
|
||||||
|
**Gate status:** Complete as of 2026-07-28. Promptkit passed its documented
|
||||||
|
validation independently, all maintained links and repository-hygiene checks
|
||||||
|
passed, and no workspace, replacement, CI configuration, binary, tag, command,
|
||||||
|
or placeholder package was added. The completed repository foundation
|
||||||
|
supported the Step 6 extraction.
|
||||||
|
|
||||||
|
### Step 6: Extract And Stabilize Promptkit
|
||||||
|
|
||||||
|
Move the application-neutral framework and built-in profile assets into
|
||||||
|
Promptkit. Preserve implementation packages as internal where practical.
|
||||||
|
|
||||||
|
The initial public API should remain focused on the established engine workflow
|
||||||
|
and the source and client extension points required by real consumers. Avoid
|
||||||
|
combining the extraction with unrelated API redesign.
|
||||||
|
|
||||||
|
Move or recreate the relevant:
|
||||||
|
|
||||||
|
- framework implementation;
|
||||||
|
- public package tests and framework contract tests;
|
||||||
|
- built-in profile assets and registry tests;
|
||||||
|
- Go consumer examples;
|
||||||
|
- framework, consumer, configuration-format, and integration documentation.
|
||||||
|
|
||||||
|
Verify that Promptkit can be built, tested, and consumed independently of the
|
||||||
|
Scriptorium repository.
|
||||||
|
|
||||||
|
**Gate:** Promptkit independently provides the agreed framework contract,
|
||||||
|
passes its documented validation, and has published its first versioned tag
|
||||||
|
before Scriptorium or another consumer adopts it, as required by
|
||||||
|
[ADR 0003](../adr/0003-use-maintainer-run-validation-and-tag-only-releases-for-promptkit.md).
|
||||||
|
|
||||||
|
**Gate status:** Complete as of 2026-07-28. Repository history records source
|
||||||
|
Scriptorium commit `c7263ab2a8e58f7fb97280082d327a820c7cece7`,
|
||||||
|
accepted Promptkit commit
|
||||||
|
`9e68a2bbf779545995270c47842048a3bc6c85dc`, independently passing acceptance,
|
||||||
|
published annotated tag `v0.1.0`, and successful remote-consumer validation.
|
||||||
|
Scriptorium remains unchanged at its pre-cutover boundary. Step 7 adoption of
|
||||||
|
the tagged module and removal of the duplicated framework is the next gate.
|
||||||
|
|
||||||
|
### Step 7: Slim Scriptorium And Adopt Promptkit
|
||||||
|
|
||||||
|
Update Scriptorium to import the tagged Promptkit module and remove the
|
||||||
|
framework implementation and public Go facade that Promptkit replaces.
|
||||||
|
|
||||||
|
Retain only Scriptorium-owned executable and transport behavior. In particular:
|
||||||
|
|
||||||
|
- wire CLI and HTTP requests through Promptkit's public API;
|
||||||
|
- keep application config and transport defaults in Scriptorium;
|
||||||
|
- keep restricted HTTP artifact policy in Scriptorium while injecting it
|
||||||
|
through Promptkit's supported boundary;
|
||||||
|
- remove obsolete framework packages, tests, and documentation;
|
||||||
|
- update Scriptorium examples and docs to describe the CLI and HTTP application;
|
||||||
|
- direct Go framework consumers to Promptkit without providing compatibility
|
||||||
|
aliases or forwarding APIs.
|
||||||
|
|
||||||
|
**Gate:** Scriptorium builds and passes all tests using a tagged Promptkit
|
||||||
|
dependency, contains no duplicate framework implementation, and its current
|
||||||
|
documentation describes only the slimmed application.
|
||||||
|
|
||||||
|
**Gate status:** Complete as of 2026-07-28. Scriptorium directly resolves
|
||||||
|
Promptkit `v0.1.0`, no longer contains the framework copy or public Go facade,
|
||||||
|
and retains only its application, adapter, configuration, presentation,
|
||||||
|
transport, packaging, and executable-example responsibilities. Release-grade
|
||||||
|
Scriptorium validation passed with a fresh remote dependency cache, and the
|
||||||
|
published Promptkit tag passed its documented validation independently. The
|
||||||
|
application is ready for the downstream-consumer migrations in Step 8.
|
||||||
|
|
||||||
|
### Step 8: Migrate Downstream Consumers To Promptkit
|
||||||
|
|
||||||
|
Inventory downstream Go consumers and migrate each from the Scriptorium package
|
||||||
|
to Promptkit. This work may occur in external repositories and must be tracked
|
||||||
|
explicitly.
|
||||||
|
|
||||||
|
For each consumer:
|
||||||
|
|
||||||
|
- update module imports and dependencies;
|
||||||
|
- adapt to any intentionally changed public API;
|
||||||
|
- run its tests and relevant integration or smoke checks;
|
||||||
|
- confirm configuration, source, validation, and error behavior;
|
||||||
|
- release or deploy the migrated consumer through its normal process.
|
||||||
|
|
||||||
|
Consumers that cannot migrate immediately may remain pinned to the last
|
||||||
|
framework-bearing Scriptorium tag. No compatibility work is required in the new
|
||||||
|
Scriptorium project for those consumers.
|
||||||
|
|
||||||
|
**Gate:** All in-scope downstream consumers are either migrated and verified or
|
||||||
|
explicitly recorded as remaining on the previous Scriptorium version with an
|
||||||
|
owner and follow-up plan. Do not declare the ecosystem migration complete until
|
||||||
|
the required out-of-band consumer changes are confirmed.
|
||||||
|
|
||||||
|
**Gate status:** Complete as of 2026-07-28. The maintainer confirmed that
|
||||||
|
Notarius was the only downstream consumer of Scriptorium's former Go package.
|
||||||
|
Its clean, synchronized main branch now directly requires Promptkit `v0.1.0`,
|
||||||
|
all relevant Go imports use Promptkit rather than Scriptorium, and its full Go
|
||||||
|
test suite passes. No downstream consumer remains to migrate or disposition.
|
||||||
|
|
||||||
|
### Step 9: Complete Release And Documentation Cutover
|
||||||
|
|
||||||
|
Complete the coordinated project transition:
|
||||||
|
|
||||||
|
- publish Promptkit before dependent Scriptorium releases;
|
||||||
|
- release the breaking Scriptorium version against the tagged Promptkit
|
||||||
|
dependency;
|
||||||
|
- publish migration guidance that maps the former Scriptorium Go API to
|
||||||
|
Promptkit;
|
||||||
|
- update cross-project links, examples, package documentation, and release
|
||||||
|
notes;
|
||||||
|
- verify that no release artifact depends on local workspaces or replacements;
|
||||||
|
- archive completed roadmap material according to the documentation policy in
|
||||||
|
effect at that time.
|
||||||
|
|
||||||
|
**Gate:** Promptkit and Scriptorium are independently releasable, their
|
||||||
|
documentation has distinct and accurate ownership, and the migration status of
|
||||||
|
all identified downstream consumers is recorded.
|
||||||
|
|
||||||
|
## Cross-Cutting Constraints
|
||||||
|
|
||||||
|
- Preserve the invariant that execution orchestration remains narrow and
|
||||||
|
application-neutral.
|
||||||
|
- Keep adapter-specific decisions out of Promptkit.
|
||||||
|
- Keep Scriptorium dependent only on Promptkit's supported public API.
|
||||||
|
- Preserve strict external decoding, error classification, validation
|
||||||
|
semantics, and secret redaction throughout the migration.
|
||||||
|
- Keep each repository buildable and testable at merged phase boundaries.
|
||||||
|
- Coordinate cross-repository changes through tagged dependencies and explicit
|
||||||
|
gates rather than assuming atomic commits across repositories.
|
||||||
|
- Document only implemented behavior outside roadmap files.
|
||||||
|
|
||||||
|
## Completion Criteria
|
||||||
|
|
||||||
|
The migration is complete when:
|
||||||
|
|
||||||
|
- Promptkit is the independent owner of the reusable framework and built-in
|
||||||
|
profiles;
|
||||||
|
- Scriptorium is a slim CLI and HTTP consumer of Promptkit;
|
||||||
|
- Scriptorium no longer exposes or maintains the former public Go framework;
|
||||||
|
- all required downstream migrations and external repository work have been
|
||||||
|
completed or explicitly dispositioned;
|
||||||
|
- both repositories build, test, document, version, and release independently.
|
||||||
282
docs/roadmap/step9.md
Normal file
282
docs/roadmap/step9.md
Normal file
@@ -0,0 +1,282 @@
|
|||||||
|
# Migration Step 9: Complete Release And Documentation Cutover
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Proposed. Steps 1 through 8 of the
|
||||||
|
[migration roadmap](migration.md) are complete. This is the final migration
|
||||||
|
gate.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Finish the Promptkit split as a released, documented, and independently
|
||||||
|
maintainable project boundary. Step 9 turns the already-implemented repository
|
||||||
|
state into the supported public release state, gives former Scriptorium Go
|
||||||
|
consumers a durable migration path, reconciles release guidance in both
|
||||||
|
repositories, and retires the temporary migration records once their work is
|
||||||
|
complete.
|
||||||
|
|
||||||
|
This roadmap defines the intended end state.
|
||||||
|
|
||||||
|
## Release Baseline And Version Decisions
|
||||||
|
|
||||||
|
The coordinated release boundary is:
|
||||||
|
|
||||||
|
- Promptkit `v0.1.0`, already published from commit
|
||||||
|
`9e68a2bbf779545995270c47842048a3bc6c85dc`, is the framework version consumed
|
||||||
|
by Scriptorium;
|
||||||
|
- Scriptorium `v0.11.1` is the final published framework-bearing Scriptorium
|
||||||
|
release; and
|
||||||
|
- Scriptorium `v0.12.0` is the first slim application-only release.
|
||||||
|
|
||||||
|
The `v0.12.0` version satisfies
|
||||||
|
[ADR 0002](../adr/0002-split-promptkit-from-scriptorium.md), which requires
|
||||||
|
the first slim pre-`v1` Scriptorium release to advance the minor version beyond
|
||||||
|
the final framework-bearing release.
|
||||||
|
|
||||||
|
Promptkit does not need another tag merely to complete this migration.
|
||||||
|
Promptkit documentation corrections that do not change the library contract
|
||||||
|
may land on its main branch without changing Scriptorium's dependency. If Step
|
||||||
|
9 discovers that a Promptkit code or consumer-visible contract change is
|
||||||
|
required, Promptkit must instead publish an appropriate later semantic version
|
||||||
|
first, and Scriptorium must adopt and validate that tag before `v0.12.0` is
|
||||||
|
published.
|
||||||
|
|
||||||
|
No release may depend on a Go workspace, local module replacement, vendored
|
||||||
|
sibling source, unpublished commit, or an unpushed tag.
|
||||||
|
|
||||||
|
## Scriptorium Release Readiness
|
||||||
|
|
||||||
|
Scriptorium must have a durable, canonical release procedure appropriate to
|
||||||
|
its hosted tag-triggered binary workflow. The procedure and contributor
|
||||||
|
reading guide must collectively define:
|
||||||
|
|
||||||
|
- semantic-version selection and the clean-checkout preconditions;
|
||||||
|
- validation outside a Go workspace and without a module replacement;
|
||||||
|
- module identity, dependency resolution, module tidiness, tests, race tests,
|
||||||
|
vet, formatting, link, example, and repository-hygiene checks;
|
||||||
|
- release-equivalent Linux `amd64` and `arm64` builds;
|
||||||
|
- annotated tag creation and publication;
|
||||||
|
- observation of the hosted release workflow;
|
||||||
|
- verification of published binaries and checksums; and
|
||||||
|
- post-publication smoke checks using downloaded release artifacts where the
|
||||||
|
execution platform permits.
|
||||||
|
|
||||||
|
The documentation policy must assign the release procedure one canonical
|
||||||
|
owner, and `docs/development.md` must route release work to it. Operations
|
||||||
|
documentation should link to release material only when an operator task
|
||||||
|
requires it; it must not become a second release procedure.
|
||||||
|
|
||||||
|
The tag workflow must accurately describe the current application. In
|
||||||
|
particular, remove the obsolete linker assignment to the deleted
|
||||||
|
`internal/buildinfo.Version` symbol. The release continues to use stripped,
|
||||||
|
trimmed binaries unless a separate supported application-version interface is
|
||||||
|
introduced. Adding a new `--version` command or other product behavior is not
|
||||||
|
part of this migration.
|
||||||
|
|
||||||
|
Release validation must exercise the same build commands and artifact names as
|
||||||
|
the hosted workflow. The resulting `v0.12.0` release must contain the supported
|
||||||
|
Linux `amd64` and `arm64` Scriptorium binaries and published SHA-256 checksums,
|
||||||
|
with no framework source or Promptkit binary artifact.
|
||||||
|
|
||||||
|
## Promptkit Release Readiness
|
||||||
|
|
||||||
|
Promptkit remains governed by
|
||||||
|
[ADR 0003](../adr/0003-use-maintainer-run-validation-and-tag-only-releases-for-promptkit.md):
|
||||||
|
maintainer-run validation, semantic Go module tags, and no hosted CI or binary
|
||||||
|
release artifacts.
|
||||||
|
|
||||||
|
Its release procedure must be corrected from pre-release language to current,
|
||||||
|
reusable guidance. It must no longer describe `v0.1.0` as an uncreated planned
|
||||||
|
release or instruct a maintainer to create an existing tag. It should:
|
||||||
|
|
||||||
|
- acknowledge `v0.1.0` as the initial published release;
|
||||||
|
- use version-agnostic instructions for later releases;
|
||||||
|
- retain the clean-checkout, full-validation, tag-ordering, and remote-tag
|
||||||
|
verification requirements; and
|
||||||
|
- require pre-`v1` release notes to identify public API changes and consumer
|
||||||
|
migration requirements.
|
||||||
|
|
||||||
|
Before Scriptorium `v0.12.0` is tagged, independently reconfirm that the local
|
||||||
|
and remote Promptkit `v0.1.0` tags resolve to the expected source commit, that
|
||||||
|
the tagged module is available through ordinary Go module resolution, and that
|
||||||
|
Promptkit passes its documented release validation without Scriptorium or
|
||||||
|
sibling-repository state.
|
||||||
|
|
||||||
|
## Go Consumer Migration Guidance
|
||||||
|
|
||||||
|
Scriptorium must publish a permanent migration guide under `docs/consumers/`
|
||||||
|
for consumers of the former Go package. Scriptorium owns this guide because it
|
||||||
|
describes departure from Scriptorium's removed API; Promptkit's declarations,
|
||||||
|
GoDoc, format reference, and consumer guide remain canonical for the
|
||||||
|
destination contract.
|
||||||
|
|
||||||
|
The guide must identify the supported migration baseline:
|
||||||
|
|
||||||
|
- source: Scriptorium `v0.11.1` and import path
|
||||||
|
`gitea.maximumdirect.net/eric/scriptorium`;
|
||||||
|
- destination: Promptkit `v0.1.0` and import path
|
||||||
|
`gitea.maximumdirect.net/eric/promptkit`; and
|
||||||
|
- Scriptorium `v0.12.0` and later: executable application only, with no root Go
|
||||||
|
package or compatibility facade.
|
||||||
|
|
||||||
|
It must provide a minimal, copyable migration workflow:
|
||||||
|
|
||||||
|
1. replace the Scriptorium module requirement and Go imports with Promptkit
|
||||||
|
`v0.1.0`;
|
||||||
|
2. update package qualifiers from `scriptorium` to `promptkit`;
|
||||||
|
3. run `go mod tidy`;
|
||||||
|
4. compile and test the consuming project; and
|
||||||
|
5. verify prompt, profile, schema, credential, timeout, validation, injected
|
||||||
|
client, and error-handling behavior relevant to that consumer.
|
||||||
|
|
||||||
|
The guide should explain that the established engine, request, result, profile,
|
||||||
|
source-option, model-client, artifact, validation, and error shapes were
|
||||||
|
intentionally preserved where practical, while Promptkit also owns the
|
||||||
|
post-extraction public error identities and artifact-reader extension point.
|
||||||
|
It must direct exact API questions to Promptkit's tagged GoDoc and consumer
|
||||||
|
guide rather than duplicating the declaration reference.
|
||||||
|
|
||||||
|
The guide must also state the deliberate compatibility policy: there are no
|
||||||
|
Scriptorium aliases, forwarding packages, or deprecated wrappers. A consumer
|
||||||
|
that cannot migrate may remain pinned to `v0.11.1`, but it will not receive the
|
||||||
|
application-only Scriptorium line through that package API.
|
||||||
|
|
||||||
|
## Documentation And Project Identity Cutover
|
||||||
|
|
||||||
|
Review both repositories as separate products and reconcile every maintained
|
||||||
|
link, example, package comment, and current-state statement with the released
|
||||||
|
boundary.
|
||||||
|
|
||||||
|
Scriptorium documentation must:
|
||||||
|
|
||||||
|
- present Scriptorium as a CLI and HTTP application, not a Go framework;
|
||||||
|
- link in-process Go consumers and framework contract questions to tagged
|
||||||
|
Promptkit `v0.1.0` documentation;
|
||||||
|
- link former Scriptorium Go consumers to the migration guide;
|
||||||
|
- keep CLI, HTTP, application configuration, operations, subprocess, and
|
||||||
|
executable examples under Scriptorium ownership; and
|
||||||
|
- avoid reproducing Promptkit fields, defaults, public declarations, or
|
||||||
|
integration contracts.
|
||||||
|
|
||||||
|
Promptkit documentation must:
|
||||||
|
|
||||||
|
- present Promptkit as the reusable Go framework and owner of its root API,
|
||||||
|
file formats, built-in profiles, validation, and outbound integration;
|
||||||
|
- retain Scriptorium only as a downstream application example or related
|
||||||
|
project, not as a framework owner or dependency;
|
||||||
|
- link to Scriptorium only for executable CLI and HTTP workflows when that
|
||||||
|
navigation is useful; and
|
||||||
|
- contain no stale extraction, planned-first-release, or pre-cutover claims.
|
||||||
|
|
||||||
|
Cross-project links must point to the canonical owner. Scriptorium links that
|
||||||
|
define the framework version it consumes remain pinned to Promptkit `v0.1.0`;
|
||||||
|
general project-navigation links may point to the other repository's current
|
||||||
|
project entry point. Maintained examples must stay repository-local and must
|
||||||
|
not require a sibling checkout.
|
||||||
|
|
||||||
|
## Release Notes And Public Communication
|
||||||
|
|
||||||
|
The Scriptorium `v0.12.0` release notes must clearly identify the release as a
|
||||||
|
breaking project-boundary change. They must:
|
||||||
|
|
||||||
|
- state that Scriptorium is now an executable-only CLI and HTTP application;
|
||||||
|
- state that the former Go framework moved to Promptkit;
|
||||||
|
- link the Scriptorium migration guide and Promptkit `v0.1.0` consumer
|
||||||
|
documentation;
|
||||||
|
- identify `v0.11.1` as the final framework-bearing Scriptorium release;
|
||||||
|
- summarize the retained Scriptorium interfaces and the removed root package;
|
||||||
|
- record that all known downstream Go consumers were migrated before release;
|
||||||
|
and
|
||||||
|
- identify the Promptkit version used by the released binary.
|
||||||
|
|
||||||
|
Release notes must not serve as a duplicate CLI, HTTP, configuration, or
|
||||||
|
Promptkit API reference. They should route readers to the corresponding
|
||||||
|
canonical documents.
|
||||||
|
|
||||||
|
## Independent Release And Artifact Verification
|
||||||
|
|
||||||
|
The final acceptance run must treat the repositories as independent remote
|
||||||
|
projects:
|
||||||
|
|
||||||
|
- validate Promptkit from its exact published tag without Scriptorium;
|
||||||
|
- validate Scriptorium from its intended release commit outside any workspace
|
||||||
|
and with a fresh module and build cache that cannot read the sibling
|
||||||
|
Promptkit checkout;
|
||||||
|
- confirm the Scriptorium module graph selects the intended published Promptkit
|
||||||
|
tag;
|
||||||
|
- verify both working trees contain no tracked workspace, replacement, vendored
|
||||||
|
cross-project source, generated binary, credential, or temporary release
|
||||||
|
residue;
|
||||||
|
- publish and verify the annotated Scriptorium `v0.12.0` tag;
|
||||||
|
- verify the hosted release completes and publishes the expected binaries and
|
||||||
|
checksums;
|
||||||
|
- download the published artifacts into a temporary location, verify their
|
||||||
|
checksums, file types, target architectures, and basic executable behavior;
|
||||||
|
and
|
||||||
|
- recheck maintained local and cross-project documentation links after
|
||||||
|
publication.
|
||||||
|
|
||||||
|
Ordinary ignored developer files, including an ignored local Scriptorium
|
||||||
|
binary, do not fail repository hygiene. Acceptance concerns tracked content,
|
||||||
|
release inputs, generated files introduced by the release work, and published
|
||||||
|
artifacts.
|
||||||
|
|
||||||
|
## Roadmap Retirement
|
||||||
|
|
||||||
|
Roadmaps are temporary coordination documents. After every Step 9 completion
|
||||||
|
criterion is satisfied and durable release and migration records exist:
|
||||||
|
|
||||||
|
- mark Step 9 and the overall migration complete before cleanup;
|
||||||
|
- preserve any still-useful current contract in its canonical permanent owner;
|
||||||
|
- rely on ADRs, Git history, tags, release notes, and the migration guide for
|
||||||
|
durable decision and release history;
|
||||||
|
- remove completed migration, step, and implementation roadmaps rather than
|
||||||
|
retaining them as a second current-state reference; and
|
||||||
|
- repair every incoming link affected by that removal.
|
||||||
|
|
||||||
|
The roadmap files must remain until the out-of-band tag and hosted release have
|
||||||
|
been verified. Creating a release candidate or merging documentation is not
|
||||||
|
enough to declare the migration complete.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
Step 9 does not:
|
||||||
|
|
||||||
|
- redesign Promptkit's public API or Scriptorium's CLI or HTTP contracts;
|
||||||
|
- restore a Scriptorium Go facade or add compatibility shims;
|
||||||
|
- add hosted CI or binary artifacts to Promptkit;
|
||||||
|
- add new Scriptorium target platforms beyond the existing Linux `amd64` and
|
||||||
|
`arm64` release policy;
|
||||||
|
- introduce an application version command solely to preserve a stale linker
|
||||||
|
flag;
|
||||||
|
- redo the completed downstream migration inventory; or
|
||||||
|
- require a new Promptkit release when no Promptkit contract change is needed.
|
||||||
|
|
||||||
|
## Completion Criteria
|
||||||
|
|
||||||
|
Step 9 is complete only when all of the following are true:
|
||||||
|
|
||||||
|
- Promptkit `v0.1.0` remains independently available, validated, and correctly
|
||||||
|
documented as the published framework dependency;
|
||||||
|
- Scriptorium has an accurate, canonical, and tested release procedure;
|
||||||
|
- the Scriptorium release workflow contains no reference to removed framework
|
||||||
|
or build-information packages and produces only the intended application
|
||||||
|
artifacts;
|
||||||
|
- the permanent Go-consumer migration guide is complete, copyable, and linked
|
||||||
|
from appropriate Scriptorium entry points;
|
||||||
|
- both repositories' permanent documentation, examples, package comments, and
|
||||||
|
cross-project links reflect distinct and canonical ownership;
|
||||||
|
- Scriptorium `v0.12.0` is published from a clean, independently validated
|
||||||
|
commit that directly requires a published Promptkit tag;
|
||||||
|
- the hosted Scriptorium release publishes verified Linux `amd64` and `arm64`
|
||||||
|
binaries and SHA-256 checksums;
|
||||||
|
- the `v0.12.0` release notes communicate the breaking package move and link to
|
||||||
|
the migration path;
|
||||||
|
- all known downstream Go consumers remain migrated or explicitly
|
||||||
|
dispositioned;
|
||||||
|
- neither release depends on local multi-repository state; and
|
||||||
|
- completed migration roadmaps are removed after their useful content and
|
||||||
|
completion evidence have durable owners.
|
||||||
|
|
||||||
|
When these criteria are satisfied, the Promptkit split is complete and both
|
||||||
|
projects can evolve, validate, version, document, and release independently.
|
||||||
@@ -1,334 +0,0 @@
|
|||||||
# Troubleshooting
|
|
||||||
|
|
||||||
This guide lists recurring implemented failure modes and safe fixes.
|
|
||||||
|
|
||||||
For command syntax, see [CLI reference](cli.md). For configuration and file formats, see [Configuration reference](config.md). For operational behavior, see [Operations guide](operations.md).
|
|
||||||
|
|
||||||
## Missing Or Invalid Config File
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI errors such as `application config error: config file not found` or `invalid config YAML`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- `--config` points to a missing file.
|
|
||||||
- Config YAML has syntax errors or unknown fields.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium render --config /path/to/config.yml --prompt generic.markdown_summary --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Correct file path.
|
|
||||||
- Remove unknown fields.
|
|
||||||
- Fix YAML syntax.
|
|
||||||
- Keep secrets out of config.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
|
|
||||||
## Missing Prompt/Profile Directory Settings
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI parse errors saying prompt directory or profile directory is required.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Neither CLI flags nor config provide effective `prompt_dir` / `profile_dir`.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Run the failing command with explicit `--prompt-dir` and `--profile-dir` once to verify.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Set `prompt_dir` and `profile_dir` in config, or always pass both flags.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
|
|
||||||
## Unknown Or Unsupported Flags
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI parse error for an unknown flag.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Typo or command mismatch (for example, `serve` with runtime model override flags).
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Compare command against the command-specific flag list.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Remove unsupported flags.
|
|
||||||
- Use `run`/`render` for runtime model overrides.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
|
|
||||||
## Prompt Definition Load Failures
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI run/render error from prompt loading.
|
|
||||||
- HTTP `404 prompt_not_found` or `400 prompt_load_failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Prompt ID not found.
|
|
||||||
- Invalid prompt YAML.
|
|
||||||
- Invalid prompt contract (for example bad validation mode, message content/content_file rule violation, missing schema path for `json_schema`).
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium render --config ./examples/config.yml --prompt <prompt-id> --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml --format json
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Correct prompt ID.
|
|
||||||
- Fix prompt YAML and contract fields.
|
|
||||||
- Ensure referenced `content_file` paths exist.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
|
|
||||||
## Profile Definition Load Failures
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI run/render error from profile loading.
|
|
||||||
- HTTP `404 profile_not_found` or `400 profile_load_failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Profile ID missing/not found.
|
|
||||||
- Invalid profile YAML.
|
|
||||||
- Invalid profile values.
|
|
||||||
- Raw `api_key` field present (rejected).
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
go run ./cmd/scriptorium render --config ./examples/config.yml --prompt generic.markdown_summary --profile <profile-id> --input transcript=./examples/fixtures/transcript.md --input glossary=./examples/fixtures/glossary.yml
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Correct profile ID.
|
|
||||||
- Fix profile YAML and value ranges.
|
|
||||||
- Replace `api_key` with `api_key_env`.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
|
|
||||||
## Input Artifact Read Failures
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI run/render error reading input artifacts.
|
|
||||||
- HTTP `400 artifact_read_failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- File path in input mapping does not exist or is unreadable.
|
|
||||||
- Unsupported artifact reference type in HTTP request.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Verify every mapped file path exists and is readable by the process.
|
|
||||||
- For HTTP, verify each input uses supported `type` values.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Correct file paths and permissions.
|
|
||||||
- Use supported input types (`file`, `inline`).
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
|
|
||||||
## Prompt Template Render Failures
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI run/render error from prompt rendering.
|
|
||||||
- HTTP `400 prompt_render_failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Template references missing input names.
|
|
||||||
- Template syntax or data reference issues.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Run `render --format json` with the same prompt, inputs, vars, and profile selection.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Align template `{{input "name"}}` references with actual input mappings.
|
|
||||||
- Fix template syntax and variable names.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
|
|
||||||
## Missing API-Key Environment Variable
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI run/render invalid request error about missing API-key environment variable.
|
|
||||||
- HTTP `400 api_key_env_missing`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Selected profile or override sets `api_key_env`, but that environment variable is unset/empty.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
printenv SCRIPTORIUM_API_KEY
|
|
||||||
```
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Set the required environment variable before invoking CLI/service.
|
|
||||||
- Or use a profile that does not require API key auth for the target endpoint.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [Operations guide](operations.md)
|
|
||||||
|
|
||||||
## LLM Request Failures
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI `run` fails with LLM generation errors.
|
|
||||||
- HTTP returns `502 llm_failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Endpoint unreachable.
|
|
||||||
- Non-2xx response from provider.
|
|
||||||
- Timeout.
|
|
||||||
- Malformed provider response.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Confirm endpoint URL and model in selected profile/overrides.
|
|
||||||
- Retry with `render` first to confirm pre-LLM preparation works.
|
|
||||||
- Check provider/network logs for non-2xx responses and timeouts.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Correct endpoint/model settings.
|
|
||||||
- Adjust timeout if needed.
|
|
||||||
- Resolve provider-side or network issues.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [Operations guide](operations.md)
|
|
||||||
|
|
||||||
## Validation Status Failed (`run` Exit 2 Or HTTP 200 With Failed Status)
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI exits with code `2`.
|
|
||||||
- HTTP returns `200`, but `validation.status` is `failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Generated output failed `basic`, `json`, or `json_schema` content validation.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Inspect validation mode and validation errors in CLI summary/HTTP response.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Refine prompt constraints.
|
|
||||||
- Tighten schema or adjust model/profile settings.
|
|
||||||
- Rerun after correction.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [Operations guide](operations.md)
|
|
||||||
|
|
||||||
## Validation Runtime Failure
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- CLI `run` fails with validation runtime error.
|
|
||||||
- HTTP returns `500 validation_runtime_failed`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- `json_schema` schema file missing/inaccessible.
|
|
||||||
- Invalid schema JSON document.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Verify `schema_dir` and `output.schema_path` resolution.
|
|
||||||
- Check schema file readability and valid JSON syntax.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Correct schema path.
|
|
||||||
- Fix schema JSON content.
|
|
||||||
- Rerun.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Configuration reference](config.md)
|
|
||||||
- [Operations guide](operations.md)
|
|
||||||
|
|
||||||
## HTTP Request Parsing/Contract Errors
|
|
||||||
|
|
||||||
Symptom:
|
|
||||||
|
|
||||||
- HTTP `400 invalid_json` or `400 invalid_request`.
|
|
||||||
|
|
||||||
Likely cause:
|
|
||||||
|
|
||||||
- Malformed JSON body.
|
|
||||||
- Unknown JSON fields.
|
|
||||||
- Missing required `prompt_id` or `inputs`.
|
|
||||||
|
|
||||||
Diagnostic step:
|
|
||||||
|
|
||||||
- Revalidate request JSON.
|
|
||||||
- Confirm required request fields are present.
|
|
||||||
|
|
||||||
Safe fix:
|
|
||||||
|
|
||||||
- Send valid JSON with only supported fields.
|
|
||||||
- Ensure `prompt_id` and at least one input mapping are included.
|
|
||||||
|
|
||||||
Relevant links:
|
|
||||||
|
|
||||||
- [Operations guide](operations.md)
|
|
||||||
- [CLI reference](cli.md)
|
|
||||||
13
examples/config.full.yml
Normal file
13
examples/config.full.yml
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
prompt_dir: ./examples/prompts
|
||||||
|
profile_dir: ./examples/profiles
|
||||||
|
schema_dir: ./examples/schemas
|
||||||
|
|
||||||
|
server:
|
||||||
|
addr: 127.0.0.1:8080
|
||||||
|
artifact_root: .
|
||||||
|
max_request_bytes: 16777216
|
||||||
|
max_artifact_bytes: 16777216
|
||||||
|
max_response_bytes: 16777216
|
||||||
|
|
||||||
|
defaults:
|
||||||
|
render_format: text
|
||||||
@@ -4,6 +4,7 @@ schema_dir: ./examples/schemas
|
|||||||
|
|
||||||
server:
|
server:
|
||||||
addr: :8080
|
addr: :8080
|
||||||
|
artifact_root: .
|
||||||
|
|
||||||
defaults:
|
defaults:
|
||||||
render_format: text
|
render_format: text
|
||||||
|
|||||||
7
go.mod
7
go.mod
@@ -3,8 +3,11 @@ module gitea.maximumdirect.net/eric/scriptorium
|
|||||||
go 1.25.5
|
go 1.25.5
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
gitea.maximumdirect.net/eric/promptkit v0.1.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
)
|
)
|
||||||
|
|
||||||
require golang.org/x/text v0.14.0 // indirect
|
require (
|
||||||
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||||
|
golang.org/x/text v0.14.0 // indirect
|
||||||
|
)
|
||||||
|
|||||||
2
go.sum
2
go.sum
@@ -1,3 +1,5 @@
|
|||||||
|
gitea.maximumdirect.net/eric/promptkit v0.1.0 h1:vuKeBxkiY8E54LRFbLQFjlJJCiOfMvB1++DYBCrD/ug=
|
||||||
|
gitea.maximumdirect.net/eric/promptkit v0.1.0/go.mod h1:R95NM6fbMDGDC0/UomgnSBP6ui2ns+8SZb8bESNvrDQ=
|
||||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||||
|
|||||||
@@ -12,18 +12,11 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
httpadapter "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http"
|
httpadapter "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http"
|
||||||
artifactadapter "gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
|
||||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -33,8 +26,7 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
errPromptDirRequired = "prompt directory is required; provide --prompt-dir or config.yml prompt_dir"
|
errPromptDirRequired = "prompt directory is required; provide --prompt-dir or config.yml prompt_dir"
|
||||||
errProfileDirRequired = "profile directory is required; provide --profile-dir or config.yml profile_dir"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type runConfig struct {
|
type runConfig struct {
|
||||||
@@ -75,10 +67,14 @@ type renderConfig struct {
|
|||||||
type serveConfig struct {
|
type serveConfig struct {
|
||||||
configPath string
|
configPath string
|
||||||
|
|
||||||
addr string
|
addr string
|
||||||
promptDir string
|
promptDir string
|
||||||
profileDir string
|
profileDir string
|
||||||
schemaDir string
|
schemaDir string
|
||||||
|
artifactRoot string
|
||||||
|
maxRequestBytes int64
|
||||||
|
maxArtifactBytes int64
|
||||||
|
maxResponseBytes int64
|
||||||
}
|
}
|
||||||
|
|
||||||
type commonCommandSettings struct {
|
type commonCommandSettings struct {
|
||||||
@@ -86,6 +82,10 @@ type commonCommandSettings struct {
|
|||||||
profileDir string
|
profileDir string
|
||||||
schemaDir string
|
schemaDir string
|
||||||
serverAddr string
|
serverAddr string
|
||||||
|
artifactRoot string
|
||||||
|
maxRequestBytes int64
|
||||||
|
maxArtifactBytes int64
|
||||||
|
maxResponseBytes int64
|
||||||
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
defaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,15 +133,13 @@ func runCommand(args []string, stdout, stderr io.Writer) int {
|
|||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
llmClient, err := newOpenAIClient()
|
engine, err := newEngine(cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "llm client error: %v\n", err)
|
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, llmClient)
|
res, runErr := engine.Run(context.Background(), req)
|
||||||
|
|
||||||
res, runErr := runner.Run(context.Background(), req)
|
|
||||||
if runErr != nil {
|
if runErr != nil {
|
||||||
fmt.Fprintf(stderr, "run error: %v\n", runErr)
|
fmt.Fprintf(stderr, "run error: %v\n", runErr)
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
@@ -169,9 +167,13 @@ func renderCommand(args []string, stdout, stderr io.Writer) int {
|
|||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, nil)
|
engine, err := newEngine(&cfg.runConfig)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
|
||||||
prepared, prepErr := runner.Prepare(context.Background(), req)
|
prepared, prepErr := engine.Prepare(context.Background(), req)
|
||||||
if prepErr != nil {
|
if prepErr != nil {
|
||||||
fmt.Fprintf(stderr, "render error: %v\n", prepErr)
|
fmt.Fprintf(stderr, "render error: %v\n", prepErr)
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
@@ -197,15 +199,26 @@ func serveCommand(args []string, stderr io.Writer) int {
|
|||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
llmClient, err := newOpenAIClient()
|
artifactReader, err := httpadapter.NewRestrictedArtifactReader(cfg.artifactRoot, cfg.maxArtifactBytes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(stderr, "llm client error: %v\n", err)
|
fmt.Fprintf(stderr, "artifact root error: %v\n", err)
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
|
|
||||||
runner := newRunner(cfg.promptDir, cfg.profileDir, cfg.schemaDir, llmClient)
|
engine, err := newEngine(&runConfig{
|
||||||
|
promptDir: cfg.promptDir,
|
||||||
|
profileDir: cfg.profileDir,
|
||||||
|
schemaDir: cfg.schemaDir,
|
||||||
|
}, promptkit.WithArtifactReader(artifactReader))
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(stderr, "engine error: %v\n", err)
|
||||||
|
return ExitRuntimeError
|
||||||
|
}
|
||||||
|
|
||||||
h := httpadapter.NewHandler(runner)
|
h := httpadapter.NewHandlerWithOptions(engine, httpadapter.HandlerOptions{
|
||||||
|
MaxRequestBytes: cfg.maxRequestBytes,
|
||||||
|
MaxResponseBytes: cfg.maxResponseBytes,
|
||||||
|
})
|
||||||
srv := &http.Server{
|
srv := &http.Server{
|
||||||
Addr: cfg.addr,
|
Addr: cfg.addr,
|
||||||
Handler: h,
|
Handler: h,
|
||||||
@@ -282,6 +295,10 @@ func parseServeArgs(args []string) (*serveConfig, error) {
|
|||||||
fs.StringVar(&cfg.promptDir, "prompt-dir", "", "directory containing prompt definition YAML files")
|
fs.StringVar(&cfg.promptDir, "prompt-dir", "", "directory containing prompt definition YAML files")
|
||||||
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing execution profile YAML files")
|
fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing execution profile YAML files")
|
||||||
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
fs.StringVar(&cfg.schemaDir, "schema-dir", "", "base directory for validation schemas")
|
||||||
|
fs.StringVar(&cfg.artifactRoot, "artifact-root", "", "base directory for HTTP file input artifacts")
|
||||||
|
fs.Int64Var(&cfg.maxRequestBytes, "max-request-bytes", 0, "maximum HTTP request body bytes; 0 disables the limit")
|
||||||
|
fs.Int64Var(&cfg.maxArtifactBytes, "max-artifact-bytes", 0, "maximum HTTP file artifact bytes; 0 disables the limit")
|
||||||
|
fs.Int64Var(&cfg.maxResponseBytes, "max-response-bytes", 0, "maximum HTTP response body bytes; 0 disables the limit")
|
||||||
|
|
||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -291,10 +308,14 @@ func parseServeArgs(args []string) (*serveConfig, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
settings, err := resolveCommonSettings(fs, cfg.configPath, appconfig.CLIOverrides{
|
settings, err := resolveCommonSettings(fs, cfg.configPath, appconfig.CLIOverrides{
|
||||||
PromptDir: cfg.promptDirIfSet(fs),
|
PromptDir: cfg.promptDirIfSet(fs),
|
||||||
ProfileDir: cfg.profileDirIfSet(fs),
|
ProfileDir: cfg.profileDirIfSet(fs),
|
||||||
SchemaDir: cfg.schemaDirIfSet(fs),
|
SchemaDir: cfg.schemaDirIfSet(fs),
|
||||||
ServerAddr: cfg.addrIfSet(fs),
|
ServerAddr: cfg.addrIfSet(fs),
|
||||||
|
ArtifactRoot: cfg.artifactRootIfSet(fs),
|
||||||
|
MaxRequestBytes: cfg.maxRequestBytesIfSet(fs),
|
||||||
|
MaxArtifactBytes: cfg.maxArtifactBytesIfSet(fs),
|
||||||
|
MaxResponseBytes: cfg.maxResponseBytesIfSet(fs),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -304,14 +325,23 @@ func parseServeArgs(args []string) (*serveConfig, error) {
|
|||||||
cfg.profileDir = settings.profileDir
|
cfg.profileDir = settings.profileDir
|
||||||
cfg.schemaDir = settings.schemaDir
|
cfg.schemaDir = settings.schemaDir
|
||||||
cfg.addr = settings.serverAddr
|
cfg.addr = settings.serverAddr
|
||||||
|
cfg.artifactRoot = settings.artifactRoot
|
||||||
|
cfg.maxRequestBytes = settings.maxRequestBytes
|
||||||
|
cfg.maxArtifactBytes = settings.maxArtifactBytes
|
||||||
|
cfg.maxResponseBytes = settings.maxResponseBytes
|
||||||
|
|
||||||
if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
|
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
||||||
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
if strings.TrimSpace(cfg.profileDir) != "" {
|
||||||
|
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
||||||
|
}
|
||||||
cfg.schemaDir = filepath.Clean(cfg.schemaDir)
|
cfg.schemaDir = filepath.Clean(cfg.schemaDir)
|
||||||
|
if strings.TrimSpace(cfg.artifactRoot) != "" {
|
||||||
|
cfg.artifactRoot = filepath.Clean(cfg.artifactRoot)
|
||||||
|
}
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -329,7 +359,7 @@ func registerExecutionRequestFlags(fs *flag.FlagSet, cfg *runConfig) {
|
|||||||
fs.Float64Var(&cfg.temperature, "temperature", 0, "optional temperature override")
|
fs.Float64Var(&cfg.temperature, "temperature", 0, "optional temperature override")
|
||||||
fs.IntVar(&cfg.maxTokens, "max-tokens", 0, "optional max tokens override")
|
fs.IntVar(&cfg.maxTokens, "max-tokens", 0, "optional max tokens override")
|
||||||
fs.Float64Var(&cfg.topP, "top-p", 0, "optional top_p override")
|
fs.Float64Var(&cfg.topP, "top-p", 0, "optional top_p override")
|
||||||
fs.DurationVar(&cfg.timeout, "timeout", defaults.LLMRequestTimeoutDefault, "LLM request timeout")
|
fs.DurationVar(&cfg.timeout, "timeout", 0, "LLM request timeout")
|
||||||
fs.StringVar(&cfg.promptID, "prompt-id", "", "deprecated alias for --prompt")
|
fs.StringVar(&cfg.promptID, "prompt-id", "", "deprecated alias for --prompt")
|
||||||
fs.StringVar(&cfg.profileID, "profile-id", "", "deprecated alias for --profile")
|
fs.StringVar(&cfg.profileID, "profile-id", "", "deprecated alias for --profile")
|
||||||
}
|
}
|
||||||
@@ -353,7 +383,7 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
|||||||
cfg.schemaDir = settings.schemaDir
|
cfg.schemaDir = settings.schemaDir
|
||||||
cfg.defaultRenderFormat = settings.defaultRenderFormat
|
cfg.defaultRenderFormat = settings.defaultRenderFormat
|
||||||
|
|
||||||
if err := validateRequiredLibraryDirs(cfg.promptDir, cfg.profileDir); err != nil {
|
if err := validateRequiredLibraryDirs(cfg.promptDir); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(cfg.promptID) == "" {
|
if strings.TrimSpace(cfg.promptID) == "" {
|
||||||
@@ -363,7 +393,9 @@ func finalizeExecutionRequestConfig(fs *flag.FlagSet, cfg *runConfig) error {
|
|||||||
return errors.New("at least one --input is required")
|
return errors.New("at least one --input is required")
|
||||||
}
|
}
|
||||||
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
cfg.promptDir = filepath.Clean(cfg.promptDir)
|
||||||
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
if strings.TrimSpace(cfg.profileDir) != "" {
|
||||||
|
cfg.profileDir = filepath.Clean(cfg.profileDir)
|
||||||
|
}
|
||||||
if cfg.outputPath != "" {
|
if cfg.outputPath != "" {
|
||||||
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
cfg.outputPath = filepath.Clean(cfg.outputPath)
|
||||||
}
|
}
|
||||||
@@ -426,6 +458,34 @@ func (c *serveConfig) addrIfSet(fs *flag.FlagSet) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *serveConfig) artifactRootIfSet(fs *flag.FlagSet) string {
|
||||||
|
if flagWasSet(fs, "artifact-root") {
|
||||||
|
return c.artifactRoot
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *serveConfig) maxRequestBytesIfSet(fs *flag.FlagSet) *int64 {
|
||||||
|
if flagWasSet(fs, "max-request-bytes") {
|
||||||
|
return &c.maxRequestBytes
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *serveConfig) maxArtifactBytesIfSet(fs *flag.FlagSet) *int64 {
|
||||||
|
if flagWasSet(fs, "max-artifact-bytes") {
|
||||||
|
return &c.maxArtifactBytes
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *serveConfig) maxResponseBytesIfSet(fs *flag.FlagSet) *int64 {
|
||||||
|
if flagWasSet(fs, "max-response-bytes") {
|
||||||
|
return &c.maxResponseBytes
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func registerConfigPathFlag(fs *flag.FlagSet, target *string) {
|
func registerConfigPathFlag(fs *flag.FlagSet, target *string) {
|
||||||
fs.StringVar(
|
fs.StringVar(
|
||||||
target,
|
target,
|
||||||
@@ -463,72 +523,71 @@ func resolveCommonSettings(fs *flag.FlagSet, configPath string, overrides appcon
|
|||||||
profileDir: settings.ProfileDir,
|
profileDir: settings.ProfileDir,
|
||||||
schemaDir: settings.SchemaDir,
|
schemaDir: settings.SchemaDir,
|
||||||
serverAddr: settings.ServerAddr,
|
serverAddr: settings.ServerAddr,
|
||||||
|
artifactRoot: settings.ArtifactRoot,
|
||||||
|
maxRequestBytes: settings.MaxRequestBytes,
|
||||||
|
maxArtifactBytes: settings.MaxArtifactBytes,
|
||||||
|
maxResponseBytes: settings.MaxResponseBytes,
|
||||||
defaultRenderFormat: settings.DefaultRenderFormat,
|
defaultRenderFormat: settings.DefaultRenderFormat,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateRequiredLibraryDirs(promptDir, profileDir string) error {
|
func validateRequiredLibraryDirs(promptDir string) error {
|
||||||
if strings.TrimSpace(promptDir) == "" {
|
if strings.TrimSpace(promptDir) == "" {
|
||||||
return errors.New(errPromptDirRequired)
|
return errors.New(errPromptDirRequired)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(profileDir) == "" {
|
|
||||||
return errors.New(errProfileDirRequired)
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func newRunner(promptDir, profileDir, schemaDir string, llmClient llm.Client) *usecase.Runner {
|
func newEngine(cfg *runConfig, options ...promptkit.Option) (*promptkit.Engine, error) {
|
||||||
return usecase.NewRunner(
|
return promptkit.NewEngine(promptkit.Config{
|
||||||
promptdef.NewFilesystemRepository(promptDir),
|
PromptDir: cfg.promptDir,
|
||||||
profile.NewFilesystemRepository(profileDir),
|
ProfileDir: cfg.profileDir,
|
||||||
artifactadapter.NewCompositeReader(),
|
SchemaDir: cfg.schemaDir,
|
||||||
prompt.NewGoRenderer(),
|
}, options...)
|
||||||
llmClient,
|
|
||||||
validate.NewStandardValidator(schemaDir),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newOpenAIClient() (*llm.OpenAICompatibleClient, error) {
|
func buildRunRequestFromConfig(cfg *runConfig) (promptkit.RunRequest, error) {
|
||||||
return llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{
|
|
||||||
Timeout: defaults.LLMRequestTimeoutDefault,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildRunRequestFromConfig(cfg *runConfig) (domain.RunRequest, error) {
|
|
||||||
inputMappings, err := parseMappings(cfg.inputRaw, false)
|
inputMappings, err := parseMappings(cfg.inputRaw, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.RunRequest{}, fmt.Errorf("input parse error: %w", err)
|
return promptkit.RunRequest{}, fmt.Errorf("input parse error: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
varMappings := map[string]string{}
|
varMappings := map[string]string{}
|
||||||
if len(cfg.varRaw) > 0 {
|
if len(cfg.varRaw) > 0 {
|
||||||
varMappings, err = parseMappings(cfg.varRaw, false)
|
varMappings, err = parseMappings(cfg.varRaw, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.RunRequest{}, fmt.Errorf("var parse error: %w", err)
|
return promptkit.RunRequest{}, fmt.Errorf("var parse error: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
inputs := make(map[string]domain.ArtifactRef, len(inputMappings))
|
inputs := make(map[string]promptkit.ArtifactRef, len(inputMappings))
|
||||||
for name, path := range inputMappings {
|
for name, path := range inputMappings {
|
||||||
inputs[name] = domain.ArtifactRef{Type: domain.ArtifactRefFile, URI: path}
|
inputs[name] = promptkit.File(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
var modelOverride *domain.ExecutionTarget
|
var modelOverride *promptkit.ExecutionTargetOverride
|
||||||
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.topPSet || cfg.apiKeyEnvSet || cfg.timeoutSet {
|
||||||
modelOverride = &domain.ExecutionTarget{
|
modelOverride = &promptkit.ExecutionTargetOverride{
|
||||||
Endpoint: cfg.llmBaseURL,
|
Endpoint: cfg.llmBaseURL,
|
||||||
Model: cfg.model,
|
Model: cfg.model,
|
||||||
Temperature: cfg.temperature,
|
APIKeyEnv: cfg.apiKeyEnv,
|
||||||
MaxTokens: cfg.maxTokens,
|
}
|
||||||
TopP: cfg.topP,
|
if cfg.temperatureSet {
|
||||||
APIKeyEnv: cfg.apiKeyEnv,
|
modelOverride.Temperature = &cfg.temperature
|
||||||
|
}
|
||||||
|
if cfg.maxTokensSet {
|
||||||
|
modelOverride.MaxTokens = &cfg.maxTokens
|
||||||
|
}
|
||||||
|
if cfg.topPSet {
|
||||||
|
modelOverride.TopP = &cfg.topP
|
||||||
}
|
}
|
||||||
if cfg.timeoutSet {
|
if cfg.timeoutSet {
|
||||||
modelOverride.TimeoutSeconds = int(cfg.timeout.Seconds())
|
timeoutSeconds := int(cfg.timeout.Seconds())
|
||||||
|
modelOverride.TimeoutSeconds = &timeoutSeconds
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return domain.RunRequest{
|
return promptkit.RunRequest{
|
||||||
PromptID: cfg.promptID,
|
PromptID: cfg.promptID,
|
||||||
ProfileID: cfg.profileID,
|
ProfileID: cfg.profileID,
|
||||||
Inputs: inputs,
|
Inputs: inputs,
|
||||||
@@ -592,21 +651,21 @@ func writeOutput(stdout io.Writer, outputPath string, body []byte) error {
|
|||||||
return os.WriteFile(outputPath, body, 0644)
|
return os.WriteFile(outputPath, body, 0644)
|
||||||
}
|
}
|
||||||
|
|
||||||
func determineExitCode(runErr error, result *domain.RunResult) int {
|
func determineExitCode(runErr error, result *promptkit.RunResult) int {
|
||||||
if runErr != nil {
|
if runErr != nil {
|
||||||
return ExitRuntimeError
|
return ExitRuntimeError
|
||||||
}
|
}
|
||||||
if result != nil && result.Validation.Status == domain.ValidationFailed {
|
if result != nil && result.Validation.Status == promptkit.ValidationFailed {
|
||||||
return ExitValidationFailed
|
return ExitValidationFailed
|
||||||
}
|
}
|
||||||
return ExitOK
|
return ExitOK
|
||||||
}
|
}
|
||||||
|
|
||||||
func printSummary(stderr io.Writer, res *domain.RunResult) {
|
func printSummary(stderr io.Writer, res *promptkit.RunResult) {
|
||||||
if res == nil {
|
if res == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d\n",
|
fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d",
|
||||||
res.PromptID,
|
res.PromptID,
|
||||||
res.PromptVersion,
|
res.PromptVersion,
|
||||||
res.SelectedProfileID,
|
res.SelectedProfileID,
|
||||||
@@ -620,11 +679,15 @@ func printSummary(stderr io.Writer, res *domain.RunResult) {
|
|||||||
res.Usage.CompletionTokens,
|
res.Usage.CompletionTokens,
|
||||||
res.Usage.TotalTokens,
|
res.Usage.TotalTokens,
|
||||||
)
|
)
|
||||||
|
if res.Usage.CachedTokens != 0 || res.Usage.CacheWriteTokens != 0 {
|
||||||
|
fmt.Fprintf(stderr, " cached_tokens=%d cache_write_tokens=%d", res.Usage.CachedTokens, res.Usage.CacheWriteTokens)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(stderr)
|
||||||
}
|
}
|
||||||
|
|
||||||
func printUsage(w io.Writer) {
|
func printUsage(w io.Writer) {
|
||||||
fmt.Fprintln(w, "usage: scriptorium <run|render|serve> ...")
|
fmt.Fprintln(w, "usage: scriptorium <run|render|serve> ...")
|
||||||
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID --input name=path [--input ...] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--out path] [--timeout 10m]")
|
fmt.Fprintln(w, " run: scriptorium run [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID --input name=path [--input ...] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--out path] [--timeout 10m]")
|
||||||
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID --input name=path [--input ...] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
|
fmt.Fprintln(w, " render: scriptorium render [--config PATH] [--prompt-dir DIR] [--profile-dir DIR] --prompt ID --input name=path [--input ...] [--profile ID] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--temperature N] [--max-tokens N] [--top-p N] [--var k=v] [--format text|json] [--out path] [--timeout 10m]")
|
||||||
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR]\n", defaults.HTTPAddrDefault)
|
fmt.Fprintf(w, " serve: scriptorium serve [--config PATH] [--addr %s] [--prompt-dir DIR] [--profile-dir DIR] [--schema-dir DIR] [--artifact-root DIR] [--max-request-bytes N] [--max-artifact-bytes N] [--max-response-bytes N]\n", defaults.HTTPAddrDefault)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,9 +17,9 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
appconfig "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -74,12 +74,12 @@ func TestParseRunArgsRequiredFlags(t *testing.T) {
|
|||||||
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"})
|
cfg, err := parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--prompt", "p", "--input", "a=b"})
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("expected missing --profile-dir error")
|
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "profile directory is required") {
|
if cfg.profileDir != "" {
|
||||||
t.Fatalf("expected clear profile-dir guidance, got %v", err)
|
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"})
|
_, err = parseRunArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles", "--input", "a=b"})
|
||||||
@@ -161,17 +161,12 @@ func TestParseServeArgsRequiredFlags(t *testing.T) {
|
|||||||
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
t.Fatalf("expected clear prompt-dir guidance, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"})
|
cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts"})
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected missing --profile-dir error")
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "profile directory is required") {
|
|
||||||
t.Fatalf("expected clear profile-dir guidance, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg, err := parseServeArgs([]string{"--config", configPath, "--prompt-dir", "./prompts", "--profile-dir", "./profiles"})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected valid serve args, got %v", err)
|
t.Fatalf("expected missing --profile-dir to be accepted, got %v", err)
|
||||||
|
}
|
||||||
|
if cfg.profileDir != "" {
|
||||||
|
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
||||||
}
|
}
|
||||||
if cfg.addr != defaults.HTTPAddrDefault {
|
if cfg.addr != defaults.HTTPAddrDefault {
|
||||||
t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr)
|
t.Fatalf("expected default addr %s, got %q", defaults.HTTPAddrDefault, cfg.addr)
|
||||||
@@ -197,6 +192,26 @@ func TestParseServeArgsRejectsRuntimeOverrideFlags(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestUsageIncludesServeFileAndSizeLimitFlags(t *testing.T) {
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
code := Run(nil, io.Discard, &stderr)
|
||||||
|
if code != ExitRuntimeError {
|
||||||
|
t.Fatalf("expected usage path to return runtime error, got %d", code)
|
||||||
|
}
|
||||||
|
|
||||||
|
usage := stderr.String()
|
||||||
|
for _, want := range []string{
|
||||||
|
"--artifact-root",
|
||||||
|
"--max-request-bytes",
|
||||||
|
"--max-artifact-bytes",
|
||||||
|
"--max-response-bytes",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(usage, want) {
|
||||||
|
t.Fatalf("expected usage to include %q, got %q", want, usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseRunArgsTimeout(t *testing.T) {
|
func TestParseRunArgsTimeout(t *testing.T) {
|
||||||
cfg, err := parseRunArgs([]string{
|
cfg, err := parseRunArgs([]string{
|
||||||
"--prompt-dir", "./prompts",
|
"--prompt-dir", "./prompts",
|
||||||
@@ -207,8 +222,8 @@ func TestParseRunArgsTimeout(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected valid run args, got %v", err)
|
t.Fatalf("expected valid run args, got %v", err)
|
||||||
}
|
}
|
||||||
if cfg.timeout != defaults.LLMRequestTimeoutDefault {
|
if cfg.timeout != 0 {
|
||||||
t.Fatalf("expected default timeout %s, got %s", defaults.LLMRequestTimeoutDefault, cfg.timeout)
|
t.Fatalf("expected omitted timeout to remain unset, got %s", cfg.timeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg, err = parseRunArgs([]string{
|
cfg, err = parseRunArgs([]string{
|
||||||
@@ -440,11 +455,19 @@ profile_dir: ./from-config/profiles
|
|||||||
schema_dir: ./from-config/schemas
|
schema_dir: ./from-config/schemas
|
||||||
server:
|
server:
|
||||||
addr: 127.0.0.1:9000
|
addr: 127.0.0.1:9000
|
||||||
|
artifact_root: ./from-config/artifacts
|
||||||
|
max_request_bytes: 1024
|
||||||
|
max_artifact_bytes: 2048
|
||||||
|
max_response_bytes: 4096
|
||||||
`)
|
`)
|
||||||
|
|
||||||
cfg, err := parseServeArgs([]string{
|
cfg, err := parseServeArgs([]string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"--addr", ":7777",
|
"--addr", ":7777",
|
||||||
|
"--artifact-root", "./from-cli/artifacts",
|
||||||
|
"--max-request-bytes", "0",
|
||||||
|
"--max-artifact-bytes", "8192",
|
||||||
|
"--max-response-bytes", "16384",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected valid args, got %v", err)
|
t.Fatalf("expected valid args, got %v", err)
|
||||||
@@ -462,6 +485,18 @@ server:
|
|||||||
if cfg.addr != ":7777" {
|
if cfg.addr != ":7777" {
|
||||||
t.Fatalf("expected CLI addr override, got %q", cfg.addr)
|
t.Fatalf("expected CLI addr override, got %q", cfg.addr)
|
||||||
}
|
}
|
||||||
|
if cfg.artifactRoot != filepath.Clean("./from-cli/artifacts") {
|
||||||
|
t.Fatalf("expected CLI artifact root override, got %q", cfg.artifactRoot)
|
||||||
|
}
|
||||||
|
if cfg.maxRequestBytes != 0 {
|
||||||
|
t.Fatalf("expected CLI max request bytes override, got %d", cfg.maxRequestBytes)
|
||||||
|
}
|
||||||
|
if cfg.maxArtifactBytes != 8192 {
|
||||||
|
t.Fatalf("expected CLI max artifact bytes override, got %d", cfg.maxArtifactBytes)
|
||||||
|
}
|
||||||
|
if cfg.maxResponseBytes != 16384 {
|
||||||
|
t.Fatalf("expected CLI max response bytes override, got %d", cfg.maxResponseBytes)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseServeArgsWithConfigProvidesRequiredDirectoriesAndAddr(t *testing.T) {
|
func TestParseServeArgsWithConfigProvidesRequiredDirectoriesAndAddr(t *testing.T) {
|
||||||
@@ -471,6 +506,10 @@ profile_dir: ./from-config/profiles
|
|||||||
schema_dir: ./from-config/schemas
|
schema_dir: ./from-config/schemas
|
||||||
server:
|
server:
|
||||||
addr: 127.0.0.1:9000
|
addr: 127.0.0.1:9000
|
||||||
|
artifact_root: ./from-config/artifacts
|
||||||
|
max_request_bytes: 1024
|
||||||
|
max_artifact_bytes: 2048
|
||||||
|
max_response_bytes: 4096
|
||||||
`)
|
`)
|
||||||
|
|
||||||
cfg, err := parseServeArgs([]string{
|
cfg, err := parseServeArgs([]string{
|
||||||
@@ -492,6 +531,75 @@ server:
|
|||||||
if cfg.addr != "127.0.0.1:9000" {
|
if cfg.addr != "127.0.0.1:9000" {
|
||||||
t.Fatalf("expected addr from config, got %q", cfg.addr)
|
t.Fatalf("expected addr from config, got %q", cfg.addr)
|
||||||
}
|
}
|
||||||
|
if cfg.artifactRoot != filepath.Clean("./from-config/artifacts") {
|
||||||
|
t.Fatalf("expected artifact root from config, got %q", cfg.artifactRoot)
|
||||||
|
}
|
||||||
|
if cfg.maxRequestBytes != 1024 {
|
||||||
|
t.Fatalf("expected max request bytes from config, got %d", cfg.maxRequestBytes)
|
||||||
|
}
|
||||||
|
if cfg.maxArtifactBytes != 2048 {
|
||||||
|
t.Fatalf("expected max artifact bytes from config, got %d", cfg.maxArtifactBytes)
|
||||||
|
}
|
||||||
|
if cfg.maxResponseBytes != 4096 {
|
||||||
|
t.Fatalf("expected max response bytes from config, got %d", cfg.maxResponseBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseServeArgsRejectsNegativeSizeLimits(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
flag string
|
||||||
|
}{
|
||||||
|
{name: "request", flag: "--max-request-bytes"},
|
||||||
|
{name: "artifact", flag: "--max-artifact-bytes"},
|
||||||
|
{name: "response", flag: "--max-response-bytes"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := parseServeArgs([]string{
|
||||||
|
"--prompt-dir", "./prompts",
|
||||||
|
tc.flag, "-1",
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected negative size limit error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunAndRenderRejectServeSizeLimitFlags(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
parse func([]string) error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "run",
|
||||||
|
parse: func(args []string) error {
|
||||||
|
_, err := parseRunArgs(args)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "render",
|
||||||
|
parse: func(args []string) error {
|
||||||
|
_, err := parseRenderArgs(args)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
err := tc.parse([]string{
|
||||||
|
"--prompt-dir", "./prompts",
|
||||||
|
"--prompt", "p",
|
||||||
|
"--input", "a=b",
|
||||||
|
"--max-request-bytes", "1024",
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected unsupported flag error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) {
|
func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *testing.T) {
|
||||||
@@ -547,6 +655,40 @@ func TestRunAndRenderBuildEquivalentRuntimeOverrideRequestsForSharedFlags(t *tes
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuildRunRequestPreservesNumericOverridePresence(t *testing.T) {
|
||||||
|
omitted, err := buildRunRequestFromConfig(&runConfig{
|
||||||
|
promptID: "prompt-1",
|
||||||
|
inputRaw: []string{"transcript=./transcript.md"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected omitted override request to build, got %v", err)
|
||||||
|
}
|
||||||
|
if omitted.Execution != nil {
|
||||||
|
t.Fatalf("expected omitted numeric flags to leave execution override nil, got %#v", omitted.Execution)
|
||||||
|
}
|
||||||
|
|
||||||
|
explicitZeros, err := buildRunRequestFromConfig(&runConfig{
|
||||||
|
promptID: "prompt-1",
|
||||||
|
inputRaw: []string{"transcript=./transcript.md"},
|
||||||
|
temperatureSet: true,
|
||||||
|
maxTokensSet: true,
|
||||||
|
topPSet: true,
|
||||||
|
timeoutSet: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected explicit zero override request to build, got %v", err)
|
||||||
|
}
|
||||||
|
if explicitZeros.Execution == nil {
|
||||||
|
t.Fatal("expected explicit numeric flags to create execution override")
|
||||||
|
}
|
||||||
|
if explicitZeros.Execution.Temperature == nil || explicitZeros.Execution.MaxTokens == nil || explicitZeros.Execution.TopP == nil || explicitZeros.Execution.TimeoutSeconds == nil {
|
||||||
|
t.Fatalf("expected explicit zero numeric overrides to remain non-nil, got %#v", explicitZeros.Execution)
|
||||||
|
}
|
||||||
|
if *explicitZeros.Execution.Temperature != 0 || *explicitZeros.Execution.MaxTokens != 0 || *explicitZeros.Execution.TopP != 0 || *explicitZeros.Execution.TimeoutSeconds != 0 {
|
||||||
|
t.Fatalf("expected explicit numeric overrides to retain zero values, got %#v", explicitZeros.Execution)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
|
func TestParseRunArgsFailsClearlyWhenNoEffectivePromptDir(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
profile_dir: ./profiles
|
profile_dir: ./profiles
|
||||||
@@ -565,21 +707,21 @@ profile_dir: ./profiles
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRunArgsFailsClearlyWhenNoEffectiveProfileDir(t *testing.T) {
|
func TestParseRunArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
prompt_dir: ./prompts
|
prompt_dir: ./prompts
|
||||||
`)
|
`)
|
||||||
|
|
||||||
_, err := parseRunArgs([]string{
|
cfg, err := parseRunArgs([]string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("expected missing profile_dir error")
|
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "profile directory is required") || !strings.Contains(err.Error(), "config.yml profile_dir") {
|
if cfg.profileDir != "" {
|
||||||
t.Fatalf("expected clear profile_dir guidance, got %v", err)
|
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -601,21 +743,21 @@ profile_dir: ./profiles
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseRenderArgsFailsClearlyWhenNoEffectiveProfileDir(t *testing.T) {
|
func TestParseRenderArgsAcceptsMissingEffectiveProfileDir(t *testing.T) {
|
||||||
configPath := writeAppConfigFile(t, `
|
configPath := writeAppConfigFile(t, `
|
||||||
prompt_dir: ./prompts
|
prompt_dir: ./prompts
|
||||||
`)
|
`)
|
||||||
|
|
||||||
_, err := parseRenderArgs([]string{
|
cfg, err := parseRenderArgs([]string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"--prompt", "p",
|
"--prompt", "p",
|
||||||
"--input", "a=b",
|
"--input", "a=b",
|
||||||
})
|
})
|
||||||
if err == nil {
|
if err != nil {
|
||||||
t.Fatal("expected missing profile_dir error")
|
t.Fatalf("expected missing profile_dir to be accepted, got %v", err)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "profile directory is required") || !strings.Contains(err.Error(), "config.yml profile_dir") {
|
if cfg.profileDir != "" {
|
||||||
t.Fatalf("expected clear profile_dir guidance, got %v", err)
|
t.Fatalf("expected empty profile dir for built-ins, got %q", cfg.profileDir)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -623,13 +765,13 @@ func TestDetermineExitCode(t *testing.T) {
|
|||||||
if got := determineExitCode(errors.New("boom"), nil); got != ExitRuntimeError {
|
if got := determineExitCode(errors.New("boom"), nil); got != ExitRuntimeError {
|
||||||
t.Fatalf("expected runtime exit code, got %d", got)
|
t.Fatalf("expected runtime exit code, got %d", got)
|
||||||
}
|
}
|
||||||
if got := determineExitCode(nil, &domain.RunResult{Validation: domain.ValidationResult{Status: domain.ValidationFailed}}); got != ExitValidationFailed {
|
if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationFailed}}); got != ExitValidationFailed {
|
||||||
t.Fatalf("expected validation exit code, got %d", got)
|
t.Fatalf("expected validation exit code, got %d", got)
|
||||||
}
|
}
|
||||||
if got := determineExitCode(nil, &domain.RunResult{Validation: domain.ValidationResult{Status: domain.ValidationPassed}}); got != ExitOK {
|
if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed}}); got != ExitOK {
|
||||||
t.Fatalf("expected success exit code for passed validation, got %d", got)
|
t.Fatalf("expected success exit code for passed validation, got %d", got)
|
||||||
}
|
}
|
||||||
if got := determineExitCode(nil, &domain.RunResult{Validation: domain.ValidationResult{Status: domain.ValidationSkipped}}); got != ExitOK {
|
if got := determineExitCode(nil, &promptkit.RunResult{Validation: promptkit.ValidationResult{Status: promptkit.ValidationSkipped}}); got != ExitOK {
|
||||||
t.Fatalf("expected success exit code for skipped validation, got %d", got)
|
t.Fatalf("expected success exit code for skipped validation, got %d", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -747,6 +889,35 @@ func TestRenderCommandDefaultFormatTextIncludesPreparedDetailsAndNoSecrets(t *te
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandExplicitZeroTemperatureReachesEffectiveSettings(t *testing.T) {
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
|
|
||||||
|
writePromptFile(t, lib.promptDir, "prompt.render", "local-default")
|
||||||
|
profile := `id: local-default
|
||||||
|
endpoint: http://127.0.0.1:1/v1
|
||||||
|
model: profile-model
|
||||||
|
temperature: 0.7
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(filepath.Join(lib.profileDir, "local-default.yaml"), []byte(profile), 0o644); err != nil {
|
||||||
|
t.Fatalf("failed to write profile fixture: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--prompt-dir", lib.promptDir,
|
||||||
|
"--profile-dir", lib.profileDir,
|
||||||
|
"--prompt", "prompt.render",
|
||||||
|
"--input", "transcript=" + inputPath,
|
||||||
|
"--temperature", "0",
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, "\n temperature: 0\n") {
|
||||||
|
t.Fatalf("expected explicit zero temperature in effective settings, got:\n%s", stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
func TestRenderCommandSucceedsWithPromptAndProfileDirsFromConfig(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello transcript")
|
||||||
@@ -900,6 +1071,29 @@ func TestRenderCommandPromptDefaultProfileWorksThroughCLIPath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderCommandUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
||||||
|
t.Setenv("OPENROUTER_API_KEY", "test-key")
|
||||||
|
lib := newCLITestLibrary(t)
|
||||||
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
|
|
||||||
|
writePromptFile(t, lib.promptDir, "prompt.builtin", "mistral-small-3")
|
||||||
|
|
||||||
|
code, stdout, stderr := runCLICommand(t, renderCommand, []string{
|
||||||
|
"--prompt-dir", lib.promptDir,
|
||||||
|
"--prompt", "prompt.builtin",
|
||||||
|
"--input", "transcript=" + inputPath,
|
||||||
|
})
|
||||||
|
if code != ExitOK {
|
||||||
|
t.Fatalf("expected ExitOK, got %d stderr=%q", code, stderr)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, "selected_profile_id: mistral-small-3") {
|
||||||
|
t.Fatalf("expected built-in selected profile, got %q", stdout)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout, "model: mistralai/mistral-small-3.2-24b-instruct") {
|
||||||
|
t.Fatalf("expected built-in model, got %q", stdout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
func TestRenderCommandExplicitProfileOverridesPromptDefault(t *testing.T) {
|
||||||
lib := newCLITestLibrary(t)
|
lib := newCLITestLibrary(t)
|
||||||
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
inputPath := lib.writeInputFile(t, "transcript.md", "hello")
|
||||||
@@ -1048,12 +1242,12 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
|
|||||||
if err := writeOutput(&stdout, "", []byte("artifact-body")); err != nil {
|
if err := writeOutput(&stdout, "", []byte("artifact-body")); err != nil {
|
||||||
t.Fatalf("unexpected writeOutput error: %v", err)
|
t.Fatalf("unexpected writeOutput error: %v", err)
|
||||||
}
|
}
|
||||||
printSummary(&stderr, &domain.RunResult{
|
printSummary(&stderr, &promptkit.RunResult{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
PromptVersion: "1",
|
PromptVersion: "1",
|
||||||
SelectedProfileID: "exec",
|
SelectedProfileID: "exec",
|
||||||
ModelName: "m",
|
ModelName: "m",
|
||||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
|
||||||
RenderedPromptHash: "h",
|
RenderedPromptHash: "h",
|
||||||
InputHashes: map[string]string{"in": "x"},
|
InputHashes: map[string]string{"in": "x"},
|
||||||
})
|
})
|
||||||
@@ -1064,6 +1258,38 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) {
|
|||||||
if !strings.Contains(stderr.String(), "prompt=p@1") {
|
if !strings.Contains(stderr.String(), "prompt=p@1") {
|
||||||
t.Fatalf("expected summary on stderr, got %q", stderr.String())
|
t.Fatalf("expected summary on stderr, got %q", stderr.String())
|
||||||
}
|
}
|
||||||
|
if strings.Contains(stderr.String(), "cached_tokens=") || strings.Contains(stderr.String(), "cache_write_tokens=") {
|
||||||
|
t.Fatalf("expected zero cache usage to be omitted from summary, got %q", stderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrintSummaryIncludesCacheUsageWhenPresent(t *testing.T) {
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
printSummary(&stderr, &promptkit.RunResult{
|
||||||
|
PromptID: "p",
|
||||||
|
PromptVersion: "1",
|
||||||
|
SelectedProfileID: "exec",
|
||||||
|
ModelName: "m",
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic},
|
||||||
|
RenderedPromptHash: "h",
|
||||||
|
InputHashes: map[string]string{"in": "x"},
|
||||||
|
Usage: promptkit.TokenUsage{
|
||||||
|
PromptTokens: 10,
|
||||||
|
CompletionTokens: 5,
|
||||||
|
TotalTokens: 15,
|
||||||
|
CachedTokens: 0,
|
||||||
|
CacheWriteTokens: 3,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
summary := stderr.String()
|
||||||
|
if !strings.Contains(summary, "usage=10/5/15") {
|
||||||
|
t.Fatalf("expected base usage summary, got %q", summary)
|
||||||
|
}
|
||||||
|
if !strings.Contains(summary, "cached_tokens=0 cache_write_tokens=3") {
|
||||||
|
t.Fatalf("expected cache usage in summary, got %q", summary)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type cliTestLibrary struct {
|
type cliTestLibrary struct {
|
||||||
|
|||||||
296
internal/adapter/dependency_test.go
Normal file
296
internal/adapter/dependency_test.go
Normal file
@@ -0,0 +1,296 @@
|
|||||||
|
package adapter_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"go/parser"
|
||||||
|
"go/token"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
scriptoriumModulePath = "gitea.maximumdirect.net/eric/scriptorium"
|
||||||
|
promptkitInternalPath = "gitea.maximumdirect.net/eric/promptkit/internal"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
removedFrameworkPackageRoots = []string{
|
||||||
|
scriptoriumModulePath + "/internal/artifact",
|
||||||
|
scriptoriumModulePath + "/internal/domain",
|
||||||
|
scriptoriumModulePath + "/internal/filecatalog",
|
||||||
|
scriptoriumModulePath + "/internal/llm",
|
||||||
|
scriptoriumModulePath + "/internal/profile",
|
||||||
|
scriptoriumModulePath + "/internal/prompt",
|
||||||
|
scriptoriumModulePath + "/internal/promptdef",
|
||||||
|
scriptoriumModulePath + "/internal/usecase",
|
||||||
|
scriptoriumModulePath + "/internal/validate",
|
||||||
|
}
|
||||||
|
removedFrameworkDirectories = []string{
|
||||||
|
"internal/artifact",
|
||||||
|
"internal/domain",
|
||||||
|
"internal/filecatalog",
|
||||||
|
"internal/llm",
|
||||||
|
"internal/profile",
|
||||||
|
"internal/prompt",
|
||||||
|
"internal/promptdef",
|
||||||
|
"internal/usecase",
|
||||||
|
"internal/validate",
|
||||||
|
}
|
||||||
|
nonSourceDirectories = map[string]struct{}{
|
||||||
|
".cache": {},
|
||||||
|
".codebase-memory": {},
|
||||||
|
".git": {},
|
||||||
|
"build": {},
|
||||||
|
"coverage": {},
|
||||||
|
"dist": {},
|
||||||
|
"node_modules": {},
|
||||||
|
"out": {},
|
||||||
|
"testdata": {},
|
||||||
|
"vendor": {},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
type forbiddenImport struct {
|
||||||
|
filePath string
|
||||||
|
importPath string
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplicationBoundary(t *testing.T) {
|
||||||
|
moduleRoot := moduleRootFromTestFile(t)
|
||||||
|
|
||||||
|
violations, err := findForbiddenProductionImports(moduleRoot)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("scan production imports: %v", err)
|
||||||
|
}
|
||||||
|
for _, violation := range violations {
|
||||||
|
t.Errorf("%s imports forbidden package %s", violation.filePath, violation.importPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
assertFrameworkImplementationAbsent(t, moduleRoot)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForbiddenImportScannerDetectsFormerRoot(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
sourcePath := writeGoSource(t, root, "nested/consumer/root.go", scriptoriumModulePath)
|
||||||
|
|
||||||
|
violations, err := findForbiddenProductionImports(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("scan source fixture: %v", err)
|
||||||
|
}
|
||||||
|
assertSingleViolation(t, violations, sourcePath, scriptoriumModulePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForbiddenImportScannerDetectsFormerFrameworkFamily(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
importPath := scriptoriumModulePath + "/internal/profile/builtin"
|
||||||
|
sourcePath := writeGoSource(t, root, "nested/consumer/profile.go", importPath)
|
||||||
|
|
||||||
|
violations, err := findForbiddenProductionImports(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("scan source fixture: %v", err)
|
||||||
|
}
|
||||||
|
assertSingleViolation(t, violations, sourcePath, importPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForbiddenImportScannerDetectsPromptkitInternalPackages(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
importPath string
|
||||||
|
}{
|
||||||
|
{name: "exact internal root", importPath: promptkitInternalPath},
|
||||||
|
{name: "internal descendant", importPath: promptkitInternalPath + "/domain"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
sourcePath := writeGoSource(t, root, "nested/consumer/promptkit.go", tc.importPath)
|
||||||
|
|
||||||
|
violations, err := findForbiddenProductionImports(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("scan source fixture: %v", err)
|
||||||
|
}
|
||||||
|
assertSingleViolation(t, violations, sourcePath, tc.importPath)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForbiddenImportScannerAllowsRetainedApplicationPackages(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
sourcePath := filepath.Join(root, "nested/consumer/application.go")
|
||||||
|
if err := os.MkdirAll(filepath.Dir(sourcePath), 0o755); err != nil {
|
||||||
|
t.Fatalf("create source fixture directory: %v", err)
|
||||||
|
}
|
||||||
|
source := `package consumer
|
||||||
|
|
||||||
|
import (
|
||||||
|
_ "gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
_ "gitea.maximumdirect.net/eric/scriptorium/internal/adapter/http"
|
||||||
|
_ "gitea.maximumdirect.net/eric/scriptorium/internal/config"
|
||||||
|
_ "gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
|
_ "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||||
|
)
|
||||||
|
`
|
||||||
|
if err := os.WriteFile(sourcePath, []byte(source), 0o644); err != nil {
|
||||||
|
t.Fatalf("write source fixture: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
violations, err := findForbiddenProductionImports(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("scan source fixture: %v", err)
|
||||||
|
}
|
||||||
|
if len(violations) != 0 {
|
||||||
|
t.Fatalf("expected retained application imports to be allowed, got %#v", violations)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func findForbiddenProductionImports(root string) ([]forbiddenImport, error) {
|
||||||
|
var violations []forbiddenImport
|
||||||
|
err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if entry.IsDir() {
|
||||||
|
if path != root && shouldSkipSourceDirectory(entry.Name()) {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(entry.Name(), ".go") || strings.HasSuffix(entry.Name(), "_test.go") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("parse imports in %s: %w", path, err)
|
||||||
|
}
|
||||||
|
for _, imported := range file.Imports {
|
||||||
|
importPath, err := strconv.Unquote(imported.Path.Value)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("parse import path in %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if isForbiddenProductionImport(importPath) {
|
||||||
|
violations = append(violations, forbiddenImport{
|
||||||
|
filePath: path,
|
||||||
|
importPath: importPath,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("walk repository root %s: %w", root, err)
|
||||||
|
}
|
||||||
|
return violations, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func shouldSkipSourceDirectory(name string) bool {
|
||||||
|
_, skip := nonSourceDirectories[name]
|
||||||
|
return skip
|
||||||
|
}
|
||||||
|
|
||||||
|
func isForbiddenProductionImport(importPath string) bool {
|
||||||
|
if importPath == scriptoriumModulePath {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if importPath == promptkitInternalPath || strings.HasPrefix(importPath, promptkitInternalPath+"/") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, root := range removedFrameworkPackageRoots {
|
||||||
|
if importPath == root || strings.HasPrefix(importPath, root+"/") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func moduleRootFromTestFile(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, testFile, _, ok := runtime.Caller(0)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("locate dependency guard source")
|
||||||
|
}
|
||||||
|
root, err := findModuleRoot(filepath.Dir(testFile))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return root
|
||||||
|
}
|
||||||
|
|
||||||
|
func findModuleRoot(start string) (string, error) {
|
||||||
|
dir, err := filepath.Abs(start)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("resolve module search path: %w", err)
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
goMod := filepath.Join(dir, "go.mod")
|
||||||
|
if info, err := os.Stat(goMod); err == nil && !info.IsDir() {
|
||||||
|
return dir, nil
|
||||||
|
} else if err != nil && !os.IsNotExist(err) {
|
||||||
|
return "", fmt.Errorf("inspect %s: %w", goMod, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
parent := filepath.Dir(dir)
|
||||||
|
if parent == dir {
|
||||||
|
return "", fmt.Errorf("locate go.mod from %s", start)
|
||||||
|
}
|
||||||
|
dir = parent
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertFrameworkImplementationAbsent(t *testing.T, moduleRoot string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(moduleRoot)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read module root: %v", err)
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".go") && !strings.HasSuffix(entry.Name(), "_test.go") {
|
||||||
|
t.Errorf("module root contains production Go file %s", entry.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, relativePath := range removedFrameworkDirectories {
|
||||||
|
path := filepath.Join(moduleRoot, filepath.FromSlash(relativePath))
|
||||||
|
if _, err := os.Stat(path); err == nil {
|
||||||
|
t.Errorf("removed framework directory still exists: %s", relativePath)
|
||||||
|
} else if !os.IsNotExist(err) {
|
||||||
|
t.Errorf("inspect removed framework directory %s: %v", relativePath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeGoSource(t *testing.T, root, relativePath, importPath string) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
sourcePath := filepath.Join(root, filepath.FromSlash(relativePath))
|
||||||
|
if err := os.MkdirAll(filepath.Dir(sourcePath), 0o755); err != nil {
|
||||||
|
t.Fatalf("create source fixture directory: %v", err)
|
||||||
|
}
|
||||||
|
source := fmt.Sprintf("package consumer\n\nimport _ %q\n", importPath)
|
||||||
|
if err := os.WriteFile(sourcePath, []byte(source), 0o644); err != nil {
|
||||||
|
t.Fatalf("write source fixture: %v", err)
|
||||||
|
}
|
||||||
|
return sourcePath
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertSingleViolation(t *testing.T, violations []forbiddenImport, sourcePath, importPath string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if len(violations) != 1 {
|
||||||
|
t.Fatalf("expected one forbidden import, got %#v", violations)
|
||||||
|
}
|
||||||
|
if violations[0].filePath != sourcePath {
|
||||||
|
t.Fatalf("unexpected importing file: %q", violations[0].filePath)
|
||||||
|
}
|
||||||
|
if violations[0].importPath != importPath {
|
||||||
|
t.Fatalf("unexpected forbidden import: %q", violations[0].importPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
165
internal/adapter/http/artifact_reader.go
Normal file
165
internal/adapter/http/artifact_reader.go
Normal file
@@ -0,0 +1,165 @@
|
|||||||
|
package httpadapter
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"mime"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrFileNotAllowed = errors.New("file artifact references are not allowed")
|
||||||
|
ErrFileOutsideRoot = errors.New("file artifact path is outside artifact root")
|
||||||
|
ErrFileTooLarge = errors.New("file artifact exceeds size limit")
|
||||||
|
)
|
||||||
|
|
||||||
|
const fallbackArtifactContentType = "text/plain"
|
||||||
|
|
||||||
|
// NewRestrictedArtifactReader creates the HTTP artifact reader for a rooted
|
||||||
|
// filesystem and optional byte limit. An empty root permits inline artifacts
|
||||||
|
// but denies file references; a zero limit permits artifacts of any size.
|
||||||
|
func NewRestrictedArtifactReader(root string, maxBytes int64) (promptkit.ArtifactReader, error) {
|
||||||
|
if maxBytes < 0 {
|
||||||
|
return nil, fmt.Errorf("artifact size limit must be greater than or equal to 0")
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanRoot := strings.TrimSpace(root)
|
||||||
|
if cleanRoot == "" {
|
||||||
|
return &restrictedArtifactReader{maxBytes: maxBytes}, nil
|
||||||
|
}
|
||||||
|
absRoot, err := filepath.Abs(filepath.Clean(cleanRoot))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("resolve artifact root: %w", err)
|
||||||
|
}
|
||||||
|
return &restrictedArtifactReader{root: absRoot, maxBytes: maxBytes}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type restrictedArtifactReader struct {
|
||||||
|
root string
|
||||||
|
maxBytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ promptkit.ArtifactReader = (*restrictedArtifactReader)(nil)
|
||||||
|
|
||||||
|
func (r *restrictedArtifactReader) Read(ctx context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
switch ref.Type {
|
||||||
|
case promptkit.ArtifactRefInline:
|
||||||
|
return readInlineArtifact(ref)
|
||||||
|
case promptkit.ArtifactRefFile:
|
||||||
|
return r.readFileArtifact(ref)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported artifact reference type %q", ref.Type)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readInlineArtifact(ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
|
if ref.Body == "" {
|
||||||
|
return nil, errors.New("inline artifact body is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
body := []byte(ref.Body)
|
||||||
|
return &promptkit.Artifact{
|
||||||
|
ContentType: fallbackArtifactContentType,
|
||||||
|
Body: body,
|
||||||
|
Size: int64(len(body)),
|
||||||
|
Hash: artifactHash(body),
|
||||||
|
URI: ref.URI,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *restrictedArtifactReader) readFileArtifact(ref promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||||
|
if ref.URI == "" {
|
||||||
|
return nil, errors.New("file artifact path is required")
|
||||||
|
}
|
||||||
|
if r.root == "" {
|
||||||
|
return nil, ErrFileNotAllowed
|
||||||
|
}
|
||||||
|
|
||||||
|
path, err := r.resolveLexicalPath(ref.URI)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return readArtifactFile(path, r.maxBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveLexicalPath checks cleaned path containment without resolving symlinks.
|
||||||
|
func (r *restrictedArtifactReader) resolveLexicalPath(rawPath string) (string, error) {
|
||||||
|
cleanPath := filepath.Clean(strings.TrimSpace(rawPath))
|
||||||
|
candidate := cleanPath
|
||||||
|
if !filepath.IsAbs(cleanPath) {
|
||||||
|
candidate = filepath.Join(r.root, cleanPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
absCandidate, err := filepath.Abs(candidate)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("resolve artifact path: %w", err)
|
||||||
|
}
|
||||||
|
absCandidate = filepath.Clean(absCandidate)
|
||||||
|
|
||||||
|
rel, err := filepath.Rel(r.root, absCandidate)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("compare artifact path to root: %w", err)
|
||||||
|
}
|
||||||
|
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
|
||||||
|
return "", ErrFileOutsideRoot
|
||||||
|
}
|
||||||
|
return absCandidate, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readArtifactFile(path string, maxBytes int64) (*promptkit.Artifact, error) {
|
||||||
|
file, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
info, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to stat file %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if maxBytes > 0 && info.Size() > maxBytes {
|
||||||
|
return nil, ErrFileTooLarge
|
||||||
|
}
|
||||||
|
|
||||||
|
var reader io.Reader = file
|
||||||
|
if maxBytes > 0 {
|
||||||
|
reader = io.LimitReader(file, maxBytes+1)
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(reader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read file %s: %w", path, err)
|
||||||
|
}
|
||||||
|
if maxBytes > 0 && int64(len(body)) > maxBytes {
|
||||||
|
return nil, ErrFileTooLarge
|
||||||
|
}
|
||||||
|
|
||||||
|
contentType := mime.TypeByExtension(filepath.Ext(path))
|
||||||
|
if contentType == "" {
|
||||||
|
contentType = fallbackArtifactContentType
|
||||||
|
}
|
||||||
|
return &promptkit.Artifact{
|
||||||
|
Name: filepath.Base(path),
|
||||||
|
ContentType: contentType,
|
||||||
|
Body: body,
|
||||||
|
URI: path,
|
||||||
|
Size: int64(len(body)),
|
||||||
|
Hash: artifactHash(body),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func artifactHash(body []byte) string {
|
||||||
|
return fmt.Sprintf("%x", sha256.Sum256(body))
|
||||||
|
}
|
||||||
185
internal/adapter/http/artifact_reader_test.go
Normal file
185
internal/adapter/http/artifact_reader_test.go
Normal file
@@ -0,0 +1,185 @@
|
|||||||
|
package httpadapter
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"mime"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRestrictedArtifactReaderReadsContainedFiles(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
outside := t.TempDir()
|
||||||
|
inputPath := filepath.Join(root, "input.html")
|
||||||
|
if err := os.WriteFile(inputPath, []byte("allowed"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "input.unknown"), []byte("unknown type"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("denied"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
expectedContentType := mime.TypeByExtension(filepath.Ext(inputPath))
|
||||||
|
if expectedContentType == "" {
|
||||||
|
t.Fatal("expected built-in HTML content type")
|
||||||
|
}
|
||||||
|
|
||||||
|
reader, err := NewRestrictedArtifactReader(root, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct restricted reader: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ref := range []promptkit.ArtifactRef{
|
||||||
|
{Type: promptkit.ArtifactRefFile, URI: "nested/../input.html"},
|
||||||
|
{Type: promptkit.ArtifactRefFile, URI: inputPath},
|
||||||
|
} {
|
||||||
|
artifact, err := reader.Read(context.Background(), ref)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read contained path %q: %v", ref.URI, err)
|
||||||
|
}
|
||||||
|
if artifact.Name != "input.html" || artifact.URI != inputPath || artifact.Size != int64(len("allowed")) || string(artifact.Body) != "allowed" {
|
||||||
|
t.Fatalf("unexpected artifact metadata: %#v", artifact)
|
||||||
|
}
|
||||||
|
if artifact.ContentType != expectedContentType {
|
||||||
|
t.Fatalf("unexpected artifact content type: got %q, want %q", artifact.ContentType, expectedContentType)
|
||||||
|
}
|
||||||
|
if artifact.Hash != artifactHash([]byte("allowed")) {
|
||||||
|
t.Fatalf("unexpected artifact hash: %q", artifact.Hash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
artifact, err := reader.Read(context.Background(), promptkit.File("input.unknown"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read unknown-extension path: %v", err)
|
||||||
|
}
|
||||||
|
if artifact.ContentType != fallbackArtifactContentType {
|
||||||
|
t.Fatalf("unexpected fallback content type: %q", artifact.ContentType)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ref := range []promptkit.ArtifactRef{
|
||||||
|
{Type: promptkit.ArtifactRefFile, URI: filepath.Join("..", filepath.Base(outside), "secret.txt")},
|
||||||
|
{Type: promptkit.ArtifactRefFile, URI: filepath.Join(outside, "secret.txt")},
|
||||||
|
} {
|
||||||
|
_, err := reader.Read(context.Background(), ref)
|
||||||
|
if !errors.Is(err, ErrFileOutsideRoot) {
|
||||||
|
t.Fatalf("expected ErrFileOutsideRoot for %q, got %v", ref.URI, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestrictedArtifactReaderFollowsSymlinkAfterLexicalCheck(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
outside := t.TempDir()
|
||||||
|
target := filepath.Join(outside, "linked.txt")
|
||||||
|
if err := os.WriteFile(target, []byte("linked outside root"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(target, filepath.Join(root, "linked.txt")); err != nil {
|
||||||
|
t.Skipf("symlink creation unavailable: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
reader, err := NewRestrictedArtifactReader(root, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct restricted reader: %v", err)
|
||||||
|
}
|
||||||
|
artifact, err := reader.Read(context.Background(), promptkit.File("linked.txt"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read symlink inside root: %v", err)
|
||||||
|
}
|
||||||
|
if string(artifact.Body) != "linked outside root" {
|
||||||
|
t.Fatalf("unexpected symlink artifact body: %q", artifact.Body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestrictedArtifactReaderWithoutRootDeniesFiles(t *testing.T) {
|
||||||
|
reader, err := NewRestrictedArtifactReader("", 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct rootless reader: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
artifact, err := reader.Read(context.Background(), promptkit.Inline("inline"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read inline artifact: %v", err)
|
||||||
|
}
|
||||||
|
if artifact.ContentType != fallbackArtifactContentType || string(artifact.Body) != "inline" || artifact.Hash != artifactHash([]byte("inline")) {
|
||||||
|
t.Fatalf("unexpected inline artifact: %#v", artifact)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = reader.Read(context.Background(), promptkit.File("input.txt"))
|
||||||
|
if !errors.Is(err, ErrFileNotAllowed) {
|
||||||
|
t.Fatalf("expected ErrFileNotAllowed, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestrictedArtifactReaderEnforcesLimits(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "exact.txt"), []byte("12345"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "large.txt"), []byte("123456"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
reader, err := NewRestrictedArtifactReader(root, 5)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct limited reader: %v", err)
|
||||||
|
}
|
||||||
|
artifact, err := reader.Read(context.Background(), promptkit.File("exact.txt"))
|
||||||
|
if err != nil || string(artifact.Body) != "12345" {
|
||||||
|
t.Fatalf("expected exact-limit artifact, got %#v and %v", artifact, err)
|
||||||
|
}
|
||||||
|
_, err = reader.Read(context.Background(), promptkit.File("large.txt"))
|
||||||
|
if !errors.Is(err, ErrFileTooLarge) {
|
||||||
|
t.Fatalf("expected ErrFileTooLarge, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
unlimited, err := NewRestrictedArtifactReader(root, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct unlimited reader: %v", err)
|
||||||
|
}
|
||||||
|
artifact, err = unlimited.Read(context.Background(), promptkit.File("large.txt"))
|
||||||
|
if err != nil || string(artifact.Body) != "123456" {
|
||||||
|
t.Fatalf("expected unlimited artifact, got %#v and %v", artifact, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := NewRestrictedArtifactReader(root, -1); err == nil {
|
||||||
|
t.Fatal("expected negative limit to fail")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestrictedArtifactReaderRejectsCanceledAndMalformedReferences(t *testing.T) {
|
||||||
|
reader, err := NewRestrictedArtifactReader(t.TempDir(), 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct reader: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
for _, ref := range []promptkit.ArtifactRef{
|
||||||
|
promptkit.Inline("input"),
|
||||||
|
promptkit.File("input.txt"),
|
||||||
|
} {
|
||||||
|
_, err := reader.Read(canceledCtx, ref)
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("expected cancellation for %#v, got %v", ref, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, ref := range []promptkit.ArtifactRef{
|
||||||
|
{Type: promptkit.ArtifactRefType("unsupported")},
|
||||||
|
{Type: promptkit.ArtifactRefInline},
|
||||||
|
{Type: promptkit.ArtifactRefFile},
|
||||||
|
} {
|
||||||
|
if _, err := reader.Read(context.Background(), ref); err == nil {
|
||||||
|
t.Fatalf("expected malformed reference %#v to fail", ref)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -21,16 +21,16 @@ type inputRefDTO struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type modelOverrideRequestDTO struct {
|
type modelOverrideRequestDTO struct {
|
||||||
Endpoint string `json:"endpoint,omitempty"`
|
Endpoint string `json:"endpoint,omitempty"`
|
||||||
Model string `json:"model,omitempty"`
|
Model string `json:"model,omitempty"`
|
||||||
Temperature float64 `json:"temperature,omitempty"`
|
Temperature *float64 `json:"temperature,omitempty"`
|
||||||
MaxTokens int `json:"max_tokens,omitempty"`
|
MaxTokens *int `json:"max_tokens,omitempty"`
|
||||||
TopP float64 `json:"top_p,omitempty"`
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
|
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||||
ExtraParams map[string]string `json:"extra_params,omitempty"`
|
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type runResponseDTO struct {
|
type runResponseDTO struct {
|
||||||
@@ -70,22 +70,24 @@ type metadataDTO struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type modelParamsDTO struct {
|
type modelParamsDTO struct {
|
||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
Model string `json:"model"`
|
Model string `json:"model"`
|
||||||
Temperature float64 `json:"temperature"`
|
Temperature float64 `json:"temperature"`
|
||||||
MaxTokens int `json:"max_tokens"`
|
MaxTokens int `json:"max_tokens"`
|
||||||
TopP float64 `json:"top_p"`
|
TopP float64 `json:"top_p"`
|
||||||
TimeoutSeconds int `json:"timeout_seconds"`
|
TimeoutSeconds int `json:"timeout_seconds"`
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||||
ExtraParams map[string]string `json:"extra_params,omitempty"`
|
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type tokenUsageDTO struct {
|
type tokenUsageDTO struct {
|
||||||
PromptTokens int `json:"prompt_tokens"`
|
PromptTokens int `json:"prompt_tokens"`
|
||||||
CompletionTokens int `json:"completion_tokens"`
|
CompletionTokens int `json:"completion_tokens"`
|
||||||
TotalTokens int `json:"total_tokens"`
|
TotalTokens int `json:"total_tokens"`
|
||||||
|
CachedTokens int `json:"cached_tokens"`
|
||||||
|
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type validationDTO struct {
|
type validationDTO struct {
|
||||||
|
|||||||
@@ -4,25 +4,37 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type Runner interface {
|
type Runner interface {
|
||||||
Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error)
|
Run(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type Handler struct {
|
type Handler struct {
|
||||||
runner Runner
|
runner Runner
|
||||||
|
options HandlerOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
type HandlerOptions struct {
|
||||||
|
MaxRequestBytes int64
|
||||||
|
MaxResponseBytes int64
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewHandler(runner Runner) *Handler {
|
func NewHandler(runner Runner) *Handler {
|
||||||
return &Handler{runner: runner}
|
return NewHandlerWithOptions(runner, HandlerOptions{
|
||||||
|
MaxRequestBytes: defaults.HTTPMaxRequestBytesDefault,
|
||||||
|
MaxResponseBytes: defaults.HTTPMaxResponseBytesDefault,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewHandlerWithOptions(runner Runner, options HandlerOptions) *Handler {
|
||||||
|
return &Handler{runner: runner, options: options}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -36,9 +48,26 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var req runRequestDTO
|
var req runRequestDTO
|
||||||
dec := json.NewDecoder(r.Body)
|
body := r.Body
|
||||||
|
if h.options.MaxRequestBytes > 0 {
|
||||||
|
body = http.MaxBytesReader(w, r.Body, h.options.MaxRequestBytes)
|
||||||
|
}
|
||||||
|
dec := json.NewDecoder(body)
|
||||||
dec.DisallowUnknownFields()
|
dec.DisallowUnknownFields()
|
||||||
if err := dec.Decode(&req); err != nil {
|
if err := dec.Decode(&req); err != nil {
|
||||||
|
if isRequestTooLarge(err) {
|
||||||
|
writeError(w, http.StatusRequestEntityTooLarge, "request_too_large", "request body is too large")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid_json", "invalid JSON request body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var trailing any
|
||||||
|
if err := dec.Decode(&trailing); err != io.EOF {
|
||||||
|
if isRequestTooLarge(err) {
|
||||||
|
writeError(w, http.StatusRequestEntityTooLarge, "request_too_large", "request body is too large")
|
||||||
|
return
|
||||||
|
}
|
||||||
writeError(w, http.StatusBadRequest, "invalid_json", "invalid JSON request body")
|
writeError(w, http.StatusBadRequest, "invalid_json", "invalid JSON request body")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -52,21 +81,21 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
mappedInputs := make(map[string]domain.ArtifactRef, len(req.Inputs))
|
mappedInputs := make(map[string]promptkit.ArtifactRef, len(req.Inputs))
|
||||||
for name, in := range req.Inputs {
|
for name, in := range req.Inputs {
|
||||||
mappedInputs[name] = domain.ArtifactRef{
|
mappedInputs[name] = promptkit.ArtifactRef{
|
||||||
Type: domain.ArtifactRefType(in.Type),
|
Type: promptkit.ArtifactRefType(in.Type),
|
||||||
URI: in.URI,
|
URI: in.URI,
|
||||||
Body: in.Body,
|
Body: in.Body,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var model *domain.ExecutionTarget
|
var model *promptkit.ExecutionTargetOverride
|
||||||
if req.Model != nil {
|
if req.Model != nil {
|
||||||
model = executionTargetFromModelOverrideDTO(req.Model)
|
model = executionTargetOverrideFromModelOverrideDTO(req.Model)
|
||||||
}
|
}
|
||||||
|
|
||||||
res, err := h.runner.Run(r.Context(), domain.RunRequest{
|
res, err := h.runner.Run(r.Context(), promptkit.RunRequest{
|
||||||
PromptID: req.PromptID,
|
PromptID: req.PromptID,
|
||||||
PromptVersion: req.PromptVersion,
|
PromptVersion: req.PromptVersion,
|
||||||
ProfileID: req.ProfileID,
|
ProfileID: req.ProfileID,
|
||||||
@@ -105,6 +134,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
PromptTokens: res.Usage.PromptTokens,
|
PromptTokens: res.Usage.PromptTokens,
|
||||||
CompletionTokens: res.Usage.CompletionTokens,
|
CompletionTokens: res.Usage.CompletionTokens,
|
||||||
TotalTokens: res.Usage.TotalTokens,
|
TotalTokens: res.Usage.TotalTokens,
|
||||||
|
CachedTokens: res.Usage.CachedTokens,
|
||||||
|
CacheWriteTokens: res.Usage.CacheWriteTokens,
|
||||||
},
|
},
|
||||||
StartTime: res.StartTime,
|
StartTime: res.StartTime,
|
||||||
EndTime: res.EndTime,
|
EndTime: res.EndTime,
|
||||||
@@ -118,14 +149,14 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
raw := res.RawOutput
|
raw := res.RawOutput
|
||||||
resp.RawModelOutput = &raw
|
resp.RawModelOutput = &raw
|
||||||
}
|
}
|
||||||
writeJSON(w, http.StatusOK, resp)
|
writeLimitedJSON(w, http.StatusOK, resp, h.options.MaxResponseBytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
func executionTargetFromModelOverrideDTO(dto *modelOverrideRequestDTO) *domain.ExecutionTarget {
|
func executionTargetOverrideFromModelOverrideDTO(dto *modelOverrideRequestDTO) *promptkit.ExecutionTargetOverride {
|
||||||
if dto == nil {
|
if dto == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return &domain.ExecutionTarget{
|
return &promptkit.ExecutionTargetOverride{
|
||||||
Endpoint: dto.Endpoint,
|
Endpoint: dto.Endpoint,
|
||||||
Model: dto.Model,
|
Model: dto.Model,
|
||||||
Temperature: dto.Temperature,
|
Temperature: dto.Temperature,
|
||||||
@@ -139,7 +170,7 @@ func executionTargetFromModelOverrideDTO(dto *modelOverrideRequestDTO) *domain.E
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func modelParamsDTOFromExecutionTarget(target domain.ExecutionTarget) modelParamsDTO {
|
func modelParamsDTOFromExecutionTarget(target promptkit.ExecutionTarget) modelParamsDTO {
|
||||||
return modelParamsDTO{
|
return modelParamsDTO{
|
||||||
Endpoint: target.Endpoint,
|
Endpoint: target.Endpoint,
|
||||||
Model: target.Model,
|
Model: target.Model,
|
||||||
@@ -154,7 +185,7 @@ func modelParamsDTOFromExecutionTarget(target domain.ExecutionTarget) modelParam
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func mapValidation(v domain.ValidationResult) validationDTO {
|
func mapValidation(v promptkit.ValidationResult) validationDTO {
|
||||||
return validationDTO{
|
return validationDTO{
|
||||||
Status: string(v.Status),
|
Status: string(v.Status),
|
||||||
Mode: string(v.Mode),
|
Mode: string(v.Mode),
|
||||||
@@ -167,29 +198,31 @@ func mapValidation(v domain.ValidationResult) validationDTO {
|
|||||||
|
|
||||||
func mapRunError(err error) (int, string, string) {
|
func mapRunError(err error) (int, string, string) {
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, promptdef.ErrPromptDefinitionNotFound):
|
case errors.Is(err, promptkit.ErrPromptNotFound):
|
||||||
return http.StatusNotFound, "prompt_not_found", "prompt definition not found"
|
return http.StatusNotFound, "prompt_not_found", "prompt definition not found"
|
||||||
case errors.Is(err, profile.ErrProfileNotFound):
|
case errors.Is(err, promptkit.ErrProfileNotFound):
|
||||||
return http.StatusNotFound, "profile_not_found", "execution profile not found"
|
return http.StatusNotFound, "profile_not_found", "execution profile not found"
|
||||||
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
|
case errors.Is(err, promptkit.ErrProfileRequired):
|
||||||
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
|
||||||
case errors.Is(err, profile.ErrInvalidYAML), errors.Is(err, profile.ErrInvalidProfile):
|
|
||||||
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
|
||||||
case errors.Is(err, usecase.ErrProfileRequired):
|
|
||||||
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
|
return http.StatusBadRequest, "profile_required", "profile_id is required when prompt default_profile is not set"
|
||||||
case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
|
case errors.Is(err, promptkit.ErrAPIKeyEnvMissing):
|
||||||
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
|
return http.StatusBadRequest, "api_key_env_missing", "api_key_env is set but the environment variable is missing"
|
||||||
case errors.Is(err, usecase.ErrInvalidRequest):
|
case errors.Is(err, promptkit.ErrPromptLoad):
|
||||||
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
|
||||||
case errors.Is(err, usecase.ErrProfileLoad):
|
|
||||||
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition"
|
||||||
case errors.Is(err, usecase.ErrArtifactLoad):
|
case errors.Is(err, promptkit.ErrProfileLoad):
|
||||||
|
return http.StatusBadRequest, "profile_load_failed", "failed to load execution profile"
|
||||||
|
case errors.Is(err, promptkit.ErrInvalidRequest):
|
||||||
|
return http.StatusBadRequest, "invalid_request", "invalid run request"
|
||||||
|
case errors.Is(err, ErrFileNotAllowed), errors.Is(err, ErrFileOutsideRoot):
|
||||||
|
return http.StatusBadRequest, "artifact_not_allowed", "file input artifact is not allowed"
|
||||||
|
case errors.Is(err, ErrFileTooLarge):
|
||||||
|
return http.StatusRequestEntityTooLarge, "artifact_too_large", "file input artifact is too large"
|
||||||
|
case errors.Is(err, promptkit.ErrArtifactLoad):
|
||||||
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
|
return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact"
|
||||||
case errors.Is(err, usecase.ErrPromptRender):
|
case errors.Is(err, promptkit.ErrPromptRender):
|
||||||
return http.StatusBadRequest, "prompt_render_failed", "failed to render prompt"
|
return http.StatusBadRequest, "prompt_render_failed", "failed to render prompt"
|
||||||
case errors.Is(err, usecase.ErrLLMGenerate):
|
case errors.Is(err, promptkit.ErrLLMGenerate):
|
||||||
return http.StatusBadGateway, "llm_failed", "model generation request failed"
|
return http.StatusBadGateway, "llm_failed", "model generation request failed"
|
||||||
case errors.Is(err, usecase.ErrValidation):
|
case errors.Is(err, promptkit.ErrValidation):
|
||||||
return http.StatusInternalServerError, "validation_runtime_failed", "validation runtime failed"
|
return http.StatusInternalServerError, "validation_runtime_failed", "validation runtime failed"
|
||||||
default:
|
default:
|
||||||
return http.StatusInternalServerError, "internal_error", "internal server error"
|
return http.StatusInternalServerError, "internal_error", "internal server error"
|
||||||
@@ -197,9 +230,23 @@ func mapRunError(err error) (int, string, string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func writeJSON(w http.ResponseWriter, status int, v any) {
|
func writeJSON(w http.ResponseWriter, status int, v any) {
|
||||||
|
writeLimitedJSON(w, status, v, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeLimitedJSON(w http.ResponseWriter, status int, v any, maxBytes int64) {
|
||||||
|
data, err := json.Marshal(v)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, "internal_error", "internal server error")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
data = append(data, '\n')
|
||||||
|
if maxBytes > 0 && int64(len(data)) > maxBytes {
|
||||||
|
writeError(w, http.StatusRequestEntityTooLarge, "response_too_large", "response body is too large")
|
||||||
|
return
|
||||||
|
}
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
w.WriteHeader(status)
|
w.WriteHeader(status)
|
||||||
_ = json.NewEncoder(w).Encode(v)
|
_, _ = w.Write(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
func writeError(w http.ResponseWriter, status int, code, message string) {
|
func writeError(w http.ResponseWriter, status int, code, message string) {
|
||||||
@@ -210,3 +257,8 @@ func writeError(w http.ResponseWriter, status int, code, message string) {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isRequestTooLarge(err error) bool {
|
||||||
|
var maxBytesErr *http.MaxBytesError
|
||||||
|
return errors.As(err, &maxBytesErr)
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,24 +7,23 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/usecase"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type fakeRunner struct {
|
type fakeRunner struct {
|
||||||
result *domain.RunResult
|
result *promptkit.RunResult
|
||||||
err error
|
err error
|
||||||
last domain.RunRequest
|
last promptkit.RunRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeRunner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
|
func (f *fakeRunner) Run(ctx context.Context, req promptkit.RunRequest) (*promptkit.RunResult, error) {
|
||||||
f.last = req
|
f.last = req
|
||||||
if f.err != nil {
|
if f.err != nil {
|
||||||
return nil, f.err
|
return nil, f.err
|
||||||
@@ -32,22 +31,60 @@ func (f *fakeRunner) Run(ctx context.Context, req domain.RunRequest) (*domain.Ru
|
|||||||
return f.result, nil
|
return f.result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaintainedHTTPRunExampleMatchesRequestContract(t *testing.T) {
|
||||||
|
body, err := os.ReadFile(filepath.Join("..", "..", "..", "examples", "http-run.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read maintained HTTP request example: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
runner := &fakeRunner{result: &promptkit.RunResult{}}
|
||||||
|
h := NewHandler(runner)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(body))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected maintained HTTP request example to be accepted, got %d: %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
var invalidExample map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(body, &invalidExample); err != nil {
|
||||||
|
t.Fatalf("decode maintained HTTP request example: %v", err)
|
||||||
|
}
|
||||||
|
invalidExample["unexpected"] = json.RawMessage(`true`)
|
||||||
|
invalidBody, err := json.Marshal(invalidExample)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("encode structurally invalid request example: %v", err)
|
||||||
|
}
|
||||||
|
invalidReq := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewReader(invalidBody))
|
||||||
|
invalidW := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(invalidW, invalidReq)
|
||||||
|
assertHTTPErrorCode(t, invalidW, http.StatusBadRequest, "invalid_json")
|
||||||
|
}
|
||||||
|
|
||||||
|
type handlerLLMClient struct{}
|
||||||
|
|
||||||
|
func (handlerLLMClient) Generate(ctx context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||||
|
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
||||||
start := time.Now().UTC()
|
start := time.Now().UTC()
|
||||||
end := start.Add(2 * time.Second)
|
end := start.Add(2 * time.Second)
|
||||||
const envName = "SCRIPTORIUM_API_KEY"
|
const envName = "SCRIPTORIUM_API_KEY"
|
||||||
const secret = "never-include-me"
|
const secret = "never-include-me"
|
||||||
|
|
||||||
r := &fakeRunner{result: &domain.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
RunID: "11111111-1111-4111-8111-111111111111",
|
RunID: "11111111-1111-4111-8111-111111111111",
|
||||||
Artifact: domain.Artifact{
|
Artifact: promptkit.Artifact{
|
||||||
Name: "output",
|
Name: "output",
|
||||||
ContentType: "text/plain",
|
ContentType: "text/plain",
|
||||||
Body: []byte("hello"),
|
Body: []byte("hello"),
|
||||||
Size: 5,
|
Size: 5,
|
||||||
Hash: "abc",
|
Hash: "abc",
|
||||||
},
|
},
|
||||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
PromptID: "prompt-1",
|
PromptID: "prompt-1",
|
||||||
PromptVersion: "1.0.0",
|
PromptVersion: "1.0.0",
|
||||||
PromptHash: "phash",
|
PromptHash: "phash",
|
||||||
@@ -55,7 +92,7 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
SelectedProfileID: "exec-default",
|
SelectedProfileID: "exec-default",
|
||||||
ModelName: "m1",
|
ModelName: "m1",
|
||||||
Endpoint: "http://llm/v1",
|
Endpoint: "http://llm/v1",
|
||||||
EffectiveModelParams: domain.ExecutionTarget{
|
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||||
Endpoint: "http://llm/v1",
|
Endpoint: "http://llm/v1",
|
||||||
Model: "m1",
|
Model: "m1",
|
||||||
Temperature: 0.2,
|
Temperature: 0.2,
|
||||||
@@ -66,11 +103,17 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
APIKeyEnv: envName,
|
APIKeyEnv: envName,
|
||||||
},
|
},
|
||||||
InputHashes: map[string]string{"transcript": "h1"},
|
InputHashes: map[string]string{"transcript": "h1"},
|
||||||
Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3},
|
Usage: promptkit.TokenUsage{
|
||||||
StartTime: start,
|
PromptTokens: 1,
|
||||||
EndTime: end,
|
CompletionTokens: 2,
|
||||||
Duration: 2 * time.Second,
|
TotalTokens: 3,
|
||||||
RawOutput: "hello",
|
CachedTokens: 4,
|
||||||
|
CacheWriteTokens: 5,
|
||||||
|
},
|
||||||
|
StartTime: start,
|
||||||
|
EndTime: end,
|
||||||
|
Duration: 2 * time.Second,
|
||||||
|
RawOutput: "hello",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
h := NewHandler(r)
|
h := NewHandler(r)
|
||||||
@@ -111,6 +154,13 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
|
if metadata["model_name"] != "m1" || metadata["endpoint"] != "http://llm/v1" {
|
||||||
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
|
t.Fatalf("unexpected model metadata: name=%#v endpoint=%#v", metadata["model_name"], metadata["endpoint"])
|
||||||
}
|
}
|
||||||
|
usage := metadata["usage"].(map[string]any)
|
||||||
|
if usage["prompt_tokens"] != float64(1) || usage["completion_tokens"] != float64(2) || usage["total_tokens"] != float64(3) {
|
||||||
|
t.Fatalf("unexpected base usage metadata: %#v", usage)
|
||||||
|
}
|
||||||
|
if usage["cached_tokens"] != float64(4) || usage["cache_write_tokens"] != float64(5) {
|
||||||
|
t.Fatalf("unexpected cache usage metadata: %#v", usage)
|
||||||
|
}
|
||||||
modelParams := metadata["model_params"].(map[string]any)
|
modelParams := metadata["model_params"].(map[string]any)
|
||||||
if modelParams["api_key_env"] != envName {
|
if modelParams["api_key_env"] != envName {
|
||||||
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
t.Fatalf("expected model_params.api_key_env=%q, got %#v", envName, modelParams["api_key_env"])
|
||||||
@@ -134,7 +184,7 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" {
|
if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" {
|
||||||
t.Fatalf("expected model override, got %#v", r.last.Execution)
|
t.Fatalf("expected model override, got %#v", r.last.Execution)
|
||||||
}
|
}
|
||||||
if r.last.Execution.TimeoutSeconds != 120 {
|
if r.last.Execution.TimeoutSeconds == nil || *r.last.Execution.TimeoutSeconds != 120 {
|
||||||
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution)
|
t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution)
|
||||||
}
|
}
|
||||||
if r.last.Execution.ServiceTier != "flex" {
|
if r.last.Execution.ServiceTier != "flex" {
|
||||||
@@ -142,14 +192,113 @@ func TestHandlerPostRunsSuccessWithExplicitProfileID(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlerInlineRefsWorkWithoutArtifactRoot(t *testing.T) {
|
||||||
|
h := newArtifactRootHandler(t, "")
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"inline","body":"inline body"}}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerFileRefsWithoutArtifactRootAreRejected(t *testing.T) {
|
||||||
|
h := newArtifactRootHandler(t, "")
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"file","uri":"input.txt"}}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "artifact_not_allowed")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerFileRefsUnderArtifactRootWork(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "input.txt"), []byte("allowed"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h := newArtifactRootHandler(t, root)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"file","uri":"input.txt"}}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerFileRefsAboveArtifactLimitAreRejected(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "large.txt"), []byte("123456"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h := newArtifactRootHandlerWithLimit(t, root, 5)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"file","uri":"large.txt"}}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusRequestEntityTooLarge, "artifact_too_large")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerFileRefsOutsideArtifactRootAreRejected(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
outside := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(outside, "secret.txt"), []byte("denied"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h := newArtifactRootHandler(t, root)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
uri string
|
||||||
|
}{
|
||||||
|
{name: "relative traversal", uri: filepath.Join("..", filepath.Base(outside), "secret.txt")},
|
||||||
|
{name: "absolute outside root", uri: filepath.Join(outside, "secret.txt")},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
body := fmt.Sprintf(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"file","uri":%q}}
|
||||||
|
}`, tc.uri)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(body))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "artifact_not_allowed")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
||||||
r := &fakeRunner{result: &domain.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: domain.Artifact{Body: []byte("ok")},
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
PromptID: "prompt-1",
|
PromptID: "prompt-1",
|
||||||
PromptVersion: "1.0.0",
|
PromptVersion: "1.0.0",
|
||||||
SelectedProfileID: "prompt-default",
|
SelectedProfileID: "prompt-default",
|
||||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
}}
|
}}
|
||||||
h := NewHandler(r)
|
h := NewHandler(r)
|
||||||
|
|
||||||
@@ -171,13 +320,17 @@ func TestHandlerPostRunsSuccessUsingPromptDefaultProfile(t *testing.T) {
|
|||||||
if metadata["selected_profile_id"] != "prompt-default" {
|
if metadata["selected_profile_id"] != "prompt-default" {
|
||||||
t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"])
|
t.Fatalf("expected selected_profile_id from result, got %#v", metadata["selected_profile_id"])
|
||||||
}
|
}
|
||||||
|
usage := metadata["usage"].(map[string]any)
|
||||||
|
if usage["cached_tokens"] != float64(0) || usage["cache_write_tokens"] != float64(0) {
|
||||||
|
t.Fatalf("expected zero cache usage fields to be included, got %#v", usage)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
||||||
r := &fakeRunner{result: &domain.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: domain.Artifact{Body: []byte("ok")},
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
EffectiveModelParams: domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
}}
|
}}
|
||||||
h := NewHandler(r)
|
h := NewHandler(r)
|
||||||
|
|
||||||
@@ -211,31 +364,147 @@ func TestHandlerModelOverrideMapsAllSupportedExecutionFields(t *testing.T) {
|
|||||||
got := r.last.Execution
|
got := r.last.Execution
|
||||||
if got.Endpoint != "http://override/v1" ||
|
if got.Endpoint != "http://override/v1" ||
|
||||||
got.Model != "override-model" ||
|
got.Model != "override-model" ||
|
||||||
got.Temperature != 0.6 ||
|
|
||||||
got.MaxTokens != 250 ||
|
|
||||||
got.TopP != 0.85 ||
|
|
||||||
got.TimeoutSeconds != 33 ||
|
|
||||||
got.ServiceTier != "flex" ||
|
got.ServiceTier != "flex" ||
|
||||||
got.ReasoningEffort != "medium" ||
|
got.ReasoningEffort != "medium" ||
|
||||||
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
got.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
||||||
t.Fatalf("unexpected mapped execution target: %+v", got)
|
t.Fatalf("unexpected mapped execution target: %+v", got)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(got.ExtraParams, map[string]string{"provider_option": "on"}) {
|
if got.Temperature == nil || *got.Temperature != 0.6 {
|
||||||
|
t.Fatalf("unexpected mapped temperature: %#v", got.Temperature)
|
||||||
|
}
|
||||||
|
if got.MaxTokens == nil || *got.MaxTokens != 250 {
|
||||||
|
t.Fatalf("unexpected mapped max_tokens: %#v", got.MaxTokens)
|
||||||
|
}
|
||||||
|
if got.TopP == nil || *got.TopP != 0.85 {
|
||||||
|
t.Fatalf("unexpected mapped top_p: %#v", got.TopP)
|
||||||
|
}
|
||||||
|
if got.TimeoutSeconds == nil || *got.TimeoutSeconds != 33 {
|
||||||
|
t.Fatalf("unexpected mapped timeout_seconds: %#v", got.TimeoutSeconds)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got.ExtraParams, map[string]any{"provider_option": "on"}) {
|
||||||
t.Fatalf("unexpected mapped extra_params: %#v", got.ExtraParams)
|
t.Fatalf("unexpected mapped extra_params: %#v", got.ExtraParams)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlerModelOverrideAcceptsJSONCompatibleExtraParams(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
reqBody := `{
|
||||||
|
"prompt_id": "prompt-1",
|
||||||
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||||
|
"model": {
|
||||||
|
"extra_params": {
|
||||||
|
"string_value": "enabled",
|
||||||
|
"number_value": 42,
|
||||||
|
"boolean_value": true,
|
||||||
|
"object_value": {"nested": "value", "count": 2},
|
||||||
|
"array_value": ["first", 3, false]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}`
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(reqBody))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.Execution == nil {
|
||||||
|
t.Fatal("expected execution override in run request")
|
||||||
|
}
|
||||||
|
want := map[string]any{
|
||||||
|
"string_value": "enabled",
|
||||||
|
"number_value": float64(42),
|
||||||
|
"boolean_value": true,
|
||||||
|
"object_value": map[string]any{"nested": "value", "count": float64(2)},
|
||||||
|
"array_value": []any{"first", float64(3), false},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(r.last.Execution.ExtraParams, want) {
|
||||||
|
t.Fatalf("unexpected mapped extra_params:\ngot=%#v\nwant=%#v", r.last.Execution.ExtraParams, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerModelOverrideExplicitZeroTemperatureMapsAsPresent(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id": "prompt-1",
|
||||||
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||||
|
"model": {"temperature": 0}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.Execution == nil || r.last.Execution.Temperature == nil {
|
||||||
|
t.Fatalf("expected temperature override to be present, got %#v", r.last.Execution)
|
||||||
|
}
|
||||||
|
if *r.last.Execution.Temperature != 0 {
|
||||||
|
t.Fatalf("expected zero temperature override, got %v", *r.last.Execution.Temperature)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerModelOverrideOmittedTemperatureMapsAsAbsent(t *testing.T) {
|
||||||
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.7},
|
||||||
|
}}
|
||||||
|
h := NewHandler(r)
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id": "prompt-1",
|
||||||
|
"inputs": {"transcript": {"type": "file", "uri": "./t.md"}},
|
||||||
|
"model": {"model": "override-model"}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected 200, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if r.last.Execution == nil {
|
||||||
|
t.Fatal("expected model override")
|
||||||
|
}
|
||||||
|
if r.last.Execution.Temperature != nil {
|
||||||
|
t.Fatalf("expected omitted temperature to remain absent, got %#v", r.last.Execution.Temperature)
|
||||||
|
}
|
||||||
|
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
metadata := resp["metadata"].(map[string]any)
|
||||||
|
params := metadata["model_params"].(map[string]any)
|
||||||
|
if params["temperature"] != 0.7 {
|
||||||
|
t.Fatalf("expected effective profile/default temperature in response, got %#v", params["temperature"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
|
func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing.T) {
|
||||||
r := &fakeRunner{result: &domain.RunResult{
|
r := &fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: domain.Artifact{
|
Artifact: promptkit.Artifact{
|
||||||
Name: "output",
|
Name: "output",
|
||||||
ContentType: "text/plain",
|
ContentType: "text/plain",
|
||||||
Body: []byte("ok"),
|
Body: []byte("ok"),
|
||||||
Size: 2,
|
Size: 2,
|
||||||
Hash: "abc",
|
Hash: "abc",
|
||||||
},
|
},
|
||||||
Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true},
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
EffectiveModelParams: domain.ExecutionTarget{
|
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||||
Endpoint: "http://llm/v1",
|
Endpoint: "http://llm/v1",
|
||||||
Model: "gpt-test",
|
Model: "gpt-test",
|
||||||
Temperature: 0.4,
|
Temperature: 0.4,
|
||||||
@@ -245,8 +514,10 @@ func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
ReasoningEffort: "high",
|
ReasoningEffort: "high",
|
||||||
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
||||||
ExtraParams: map[string]string{
|
ExtraParams: map[string]any{
|
||||||
"provider_option": "on",
|
"provider_option": "on",
|
||||||
|
"number_value": 42,
|
||||||
|
"object_value": map[string]any{"nested": "value"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
@@ -301,6 +572,13 @@ func TestHandlerResponseMetadataModelParamsIncludesAllSupportedFields(t *testing
|
|||||||
if extraParams["provider_option"] != "on" {
|
if extraParams["provider_option"] != "on" {
|
||||||
t.Fatalf("unexpected extra_params.provider_option: %#v", extraParams["provider_option"])
|
t.Fatalf("unexpected extra_params.provider_option: %#v", extraParams["provider_option"])
|
||||||
}
|
}
|
||||||
|
if extraParams["number_value"] != float64(42) {
|
||||||
|
t.Fatalf("unexpected extra_params.number_value: %#v", extraParams["number_value"])
|
||||||
|
}
|
||||||
|
objectValue, ok := extraParams["object_value"].(map[string]any)
|
||||||
|
if !ok || objectValue["nested"] != "value" {
|
||||||
|
t.Fatalf("unexpected extra_params.object_value: %#v", extraParams["object_value"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerInvalidJSON(t *testing.T) {
|
func TestHandlerInvalidJSON(t *testing.T) {
|
||||||
@@ -315,6 +593,69 @@ func TestHandlerInvalidJSON(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandlerRejectsTrailingJSON(t *testing.T) {
|
||||||
|
h := NewHandler(&fakeRunner{})
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}} {}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_json")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerRequestTooLarge(t *testing.T) {
|
||||||
|
h := NewHandlerWithOptions(&fakeRunner{}, HandlerOptions{MaxRequestBytes: 12})
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusRequestEntityTooLarge, "request_too_large")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerMalformedJSONBelowLimitStillBadRequest(t *testing.T) {
|
||||||
|
h := NewHandlerWithOptions(&fakeRunner{}, HandlerOptions{MaxRequestBytes: 1024})
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString("{"))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusBadRequest, "invalid_json")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerResponseTooLarge(t *testing.T) {
|
||||||
|
h := NewHandlerWithOptions(&fakeRunner{result: &promptkit.RunResult{
|
||||||
|
Artifact: promptkit.Artifact{Body: []byte(strings.Repeat("x", 128))},
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
}}, HandlerOptions{MaxRequestBytes: 1024, MaxResponseBytes: 64})
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusRequestEntityTooLarge, "response_too_large")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerRawOutputDoesNotBypassResponseLimit(t *testing.T) {
|
||||||
|
h := NewHandlerWithOptions(&fakeRunner{result: &promptkit.RunResult{
|
||||||
|
Artifact: promptkit.Artifact{Body: []byte("ok")},
|
||||||
|
RawOutput: strings.Repeat("raw", 80),
|
||||||
|
Validation: promptkit.ValidationResult{Status: promptkit.ValidationPassed, Mode: promptkit.ValidationBasic, IsValid: true},
|
||||||
|
EffectiveModelParams: promptkit.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m1"},
|
||||||
|
}}, HandlerOptions{MaxRequestBytes: 1024, MaxResponseBytes: 128})
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"file","uri":"a"}},
|
||||||
|
"include_raw_output":true
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assertHTTPErrorCode(t, w, http.StatusRequestEntityTooLarge, "response_too_large")
|
||||||
|
}
|
||||||
|
|
||||||
func TestHandlerMissingPromptID(t *testing.T) {
|
func TestHandlerMissingPromptID(t *testing.T) {
|
||||||
h := NewHandler(&fakeRunner{})
|
h := NewHandler(&fakeRunner{})
|
||||||
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"inputs":{"x":{"type":"file","uri":"a"}}}`))
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"inputs":{"x":{"type":"file","uri":"a"}}}`))
|
||||||
@@ -335,7 +676,32 @@ func TestHandlerMissingPromptID(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerUsecaseErrorMapping(t *testing.T) {
|
func TestHandlerReservedExtraParamsThroughEngineMapsToInvalidRequest(t *testing.T) {
|
||||||
|
h := NewHandler(newHandlerEngineWithDefaultClient(t))
|
||||||
|
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{
|
||||||
|
"prompt_id":"p",
|
||||||
|
"inputs":{"x":{"type":"inline","body":"input"}},
|
||||||
|
"model":{"extra_params":{"model":"collision"}}
|
||||||
|
}`))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("expected 400, got %d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
errBody := resp["error"].(map[string]any)
|
||||||
|
if errBody["code"] != "invalid_request" {
|
||||||
|
t.Fatalf("expected invalid_request code, got %#v", errBody["code"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandlerPublicErrorMapping(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
err error
|
err error
|
||||||
@@ -344,16 +710,20 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) {
|
|||||||
message string
|
message string
|
||||||
avoidCause string
|
avoidCause string
|
||||||
}{
|
}{
|
||||||
{name: "prompt not found", err: wrap(usecase.ErrProfileLoad, promptdef.ErrPromptDefinitionNotFound), status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
|
{name: "prompt not found", err: promptkit.ErrPromptNotFound, status: http.StatusNotFound, code: "prompt_not_found", message: "prompt definition not found"},
|
||||||
{name: "prompt load invalid", err: wrap(usecase.ErrProfileLoad, promptdef.ErrInvalidPromptDefinition), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition"},
|
{name: "prompt load", err: wrap(promptkit.ErrPromptLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "prompt_load_failed", message: "failed to load prompt definition", avoidCause: "read failed"},
|
||||||
{name: "missing profile/default", err: wrap(usecase.ErrInvalidRequest, usecase.ErrProfileRequired), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
|
{name: "missing profile/default", err: wrap(promptkit.ErrProfileRequired, promptkit.ErrInvalidRequest), status: http.StatusBadRequest, code: "profile_required", message: "profile_id is required when prompt default_profile is not set"},
|
||||||
{name: "profile not found", err: wrap(usecase.ErrProfileLoad, profile.ErrProfileNotFound), status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
|
{name: "profile not found", err: promptkit.ErrProfileNotFound, status: http.StatusNotFound, code: "profile_not_found", message: "execution profile not found"},
|
||||||
{name: "profile invalid", err: wrap(usecase.ErrProfileLoad, profile.ErrInvalidProfile), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile"},
|
{name: "profile load", err: wrap(promptkit.ErrProfileLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "profile_load_failed", message: "failed to load execution profile", avoidCause: "read failed"},
|
||||||
{name: "api key env missing", err: wrap(usecase.ErrInvalidRequest, usecase.ErrAPIKeyEnvMissing), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
|
{name: "api key env missing", err: wrap(promptkit.ErrAPIKeyEnvMissing, promptkit.ErrInvalidRequest), status: http.StatusBadRequest, code: "api_key_env_missing", message: "api_key_env is set but the environment variable is missing"},
|
||||||
{name: "artifact", err: wrap(usecase.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
{name: "invalid request", err: promptkit.ErrInvalidRequest, status: http.StatusBadRequest, code: "invalid_request", message: "invalid run request"},
|
||||||
{name: "prompt render", err: wrap(usecase.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
{name: "file denied", err: ErrFileNotAllowed, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
|
||||||
{name: "llm", err: wrap(usecase.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"},
|
{name: "file outside root", err: ErrFileOutsideRoot, status: http.StatusBadRequest, code: "artifact_not_allowed", message: "file input artifact is not allowed"},
|
||||||
{name: "validation runtime", err: wrap(usecase.ErrValidation, fmt.Errorf("validator broke")), status: http.StatusInternalServerError, code: "validation_runtime_failed", message: "validation runtime failed", avoidCause: "validator broke"},
|
{name: "file too large", err: ErrFileTooLarge, status: http.StatusRequestEntityTooLarge, code: "artifact_too_large", message: "file input artifact is too large"},
|
||||||
|
{name: "artifact", err: wrap(promptkit.ErrArtifactLoad, fmt.Errorf("read failed")), status: http.StatusBadRequest, code: "artifact_read_failed", message: "failed to read input artifact", avoidCause: "read failed"},
|
||||||
|
{name: "prompt render", err: wrap(promptkit.ErrPromptRender, fmt.Errorf("render failed")), status: http.StatusBadRequest, code: "prompt_render_failed", message: "failed to render prompt", avoidCause: "render failed"},
|
||||||
|
{name: "llm", err: wrap(promptkit.ErrLLMGenerate, fmt.Errorf("llm failed")), status: http.StatusBadGateway, code: "llm_failed", message: "model generation request failed", avoidCause: "llm failed"},
|
||||||
|
{name: "validation runtime", err: wrap(promptkit.ErrValidation, fmt.Errorf("validator broke")), status: http.StatusInternalServerError, code: "validation_runtime_failed", message: "validation runtime failed", avoidCause: "validator broke"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
@@ -406,12 +776,12 @@ func TestHandlerRawAPIKeyRejectedByStrictJSON(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestHandlerValidationFailureStillSuccessAndRawOutputOptIn(t *testing.T) {
|
func TestHandlerValidationFailureStillSuccessAndRawOutputOptIn(t *testing.T) {
|
||||||
h := NewHandler(&fakeRunner{result: &domain.RunResult{
|
h := NewHandler(&fakeRunner{result: &promptkit.RunResult{
|
||||||
Artifact: domain.Artifact{Body: []byte("bad json")},
|
Artifact: promptkit.Artifact{Body: []byte("bad json")},
|
||||||
RawOutput: "bad json",
|
RawOutput: "bad json",
|
||||||
Validation: domain.ValidationResult{
|
Validation: promptkit.ValidationResult{
|
||||||
Status: domain.ValidationFailed,
|
Status: promptkit.ValidationFailed,
|
||||||
Mode: domain.ValidationJSON,
|
Mode: promptkit.ValidationJSON,
|
||||||
Errors: []string{"invalid JSON"},
|
Errors: []string{"invalid JSON"},
|
||||||
},
|
},
|
||||||
}})
|
}})
|
||||||
@@ -455,3 +825,82 @@ func TestHandlerValidationFailureStillSuccessAndRawOutputOptIn(t *testing.T) {
|
|||||||
func wrap(stage error, cause error) error {
|
func wrap(stage error, cause error) error {
|
||||||
return fmt.Errorf("%w: %w", stage, cause)
|
return fmt.Errorf("%w: %w", stage, cause)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newArtifactRootHandler(t *testing.T, root string) *Handler {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
return newArtifactRootHandlerWithLimit(t, root, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newArtifactRootHandlerWithLimit(t *testing.T, root string, maxArtifactBytes int64) *Handler {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
reader, err := NewRestrictedArtifactReader(root, maxArtifactBytes)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected restricted artifact reader: %v", err)
|
||||||
|
}
|
||||||
|
return NewHandler(newHandlerEngine(t, promptkit.WithArtifactReader(reader)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func newHandlerEngine(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
return newHandlerEngineWithOptions(t, append(options, promptkit.WithLLMClient(handlerLLMClient{}))...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newHandlerEngineWithDefaultClient(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
return newHandlerEngineWithOptions(t, options...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newHandlerEngineWithOptions(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
promptDir := t.TempDir()
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(promptDir, "prompt.yaml"), []byte(`id: p
|
||||||
|
version: "1"
|
||||||
|
default_profile: exec
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: "hi"
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: 0
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write prompt fixture: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(profileDir, "profile.yaml"), []byte(`id: exec
|
||||||
|
endpoint: http://example.invalid/v1
|
||||||
|
model: model
|
||||||
|
`), 0o644); err != nil {
|
||||||
|
t.Fatalf("write profile fixture: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: promptDir,
|
||||||
|
ProfileDir: profileDir,
|
||||||
|
}, options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct public engine: %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertHTTPErrorCode(t *testing.T, w *httptest.ResponseRecorder, status int, code string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
if w.Code != status {
|
||||||
|
t.Fatalf("expected %d, got %d body=%s", status, w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
var resp map[string]any
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
|
t.Fatalf("invalid JSON response: %v", err)
|
||||||
|
}
|
||||||
|
errBody := resp["error"].(map[string]any)
|
||||||
|
if errBody["code"] != code {
|
||||||
|
t.Fatalf("expected code %q, got %#v", code, errBody["code"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,110 +0,0 @@
|
|||||||
package artifact
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
"mime"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrUnsupportedRefType = errors.New("unsupported artifact reference type")
|
|
||||||
ErrMissingInlineBody = errors.New("missing body for inline artifact")
|
|
||||||
ErrMissingFilePath = errors.New("missing file path for file artifact")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Reader resolves artifact references into actual artifacts.
|
|
||||||
type Reader interface {
|
|
||||||
Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// CompositeReader routes artifact resolution based on the reference type.
|
|
||||||
type CompositeReader struct {
|
|
||||||
inlineReader *inlineReader
|
|
||||||
fileReader *fileReader
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewCompositeReader() Reader {
|
|
||||||
return &CompositeReader{
|
|
||||||
inlineReader: &inlineReader{},
|
|
||||||
fileReader: &fileReader{},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *CompositeReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
switch ref.Type {
|
|
||||||
case domain.ArtifactRefInline:
|
|
||||||
return c.inlineReader.Read(ctx, ref)
|
|
||||||
case domain.ArtifactRefFile:
|
|
||||||
return c.fileReader.Read(ctx, ref)
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedRefType, ref.Type)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type inlineReader struct{}
|
|
||||||
|
|
||||||
func (r *inlineReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
if ref.Body == "" {
|
|
||||||
return nil, ErrMissingInlineBody
|
|
||||||
}
|
|
||||||
|
|
||||||
body := []byte(ref.Body)
|
|
||||||
return &domain.Artifact{
|
|
||||||
ContentType: defaults.ContentTypeTextPlain,
|
|
||||||
Body: body,
|
|
||||||
Size: int64(len(body)),
|
|
||||||
Hash: fmt.Sprintf("%x", sha256.Sum256(body)),
|
|
||||||
URI: ref.URI,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type fileReader struct{}
|
|
||||||
|
|
||||||
func (r *fileReader) Read(ctx context.Context, ref domain.ArtifactRef) (*domain.Artifact, error) {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
if ref.URI == "" {
|
|
||||||
return nil, ErrMissingFilePath
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := os.ReadFile(ref.URI)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read file %s: %w", ref.URI, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
contentType := mime.TypeByExtension(filepath.Ext(ref.URI))
|
|
||||||
if contentType == "" {
|
|
||||||
contentType = defaults.ContentTypeTextPlain
|
|
||||||
}
|
|
||||||
|
|
||||||
return &domain.Artifact{
|
|
||||||
Name: filepath.Base(ref.URI),
|
|
||||||
ContentType: contentType,
|
|
||||||
Body: data,
|
|
||||||
URI: ref.URI,
|
|
||||||
Size: int64(len(data)),
|
|
||||||
Hash: fmt.Sprintf("%x", sha256.Sum256(data)),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
@@ -1,105 +0,0 @@
|
|||||||
package artifact
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCompositeReader_Read(t *testing.T) {
|
|
||||||
reader := NewCompositeReader()
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
t.Run("inline artifact", func(t *testing.T) {
|
|
||||||
ref := domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefInline,
|
|
||||||
Body: "hello world",
|
|
||||||
}
|
|
||||||
art, err := reader.Read(ctx, ref)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if string(art.Body) != "hello world" {
|
|
||||||
t.Errorf("expected 'hello world', got %s", string(art.Body))
|
|
||||||
}
|
|
||||||
if art.ContentType != "text/plain" {
|
|
||||||
t.Errorf("expected text/plain content type, got %q", art.ContentType)
|
|
||||||
}
|
|
||||||
if art.Hash != "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" {
|
|
||||||
t.Errorf("unexpected hash: %s", art.Hash)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("inline artifact missing body", func(t *testing.T) {
|
|
||||||
ref := domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefInline,
|
|
||||||
Body: "",
|
|
||||||
}
|
|
||||||
_, err := reader.Read(ctx, ref)
|
|
||||||
if !errors.Is(err, ErrMissingInlineBody) {
|
|
||||||
t.Errorf("expected ErrMissingInlineBody, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("unsupported ref type", func(t *testing.T) {
|
|
||||||
ref := domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefType("unsupported"),
|
|
||||||
URI: "unsupported://bucket/key",
|
|
||||||
}
|
|
||||||
_, err := reader.Read(ctx, ref)
|
|
||||||
if !errors.Is(err, ErrUnsupportedRefType) {
|
|
||||||
t.Error("expected error for unsupported type")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFileReader_Read(t *testing.T) {
|
|
||||||
content := []byte("test file content")
|
|
||||||
tmpFile, err := os.CreateTemp("", "artifact_test_*.txt")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer os.Remove(tmpFile.Name())
|
|
||||||
|
|
||||||
if _, err := tmpFile.Write(content); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
tmpFile.Close()
|
|
||||||
|
|
||||||
reader := NewCompositeReader()
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
t.Run("file artifact loading", func(t *testing.T) {
|
|
||||||
ref := domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefFile,
|
|
||||||
URI: tmpFile.Name(),
|
|
||||||
}
|
|
||||||
art, err := reader.Read(ctx, ref)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if string(art.Body) != string(content) {
|
|
||||||
t.Errorf("expected %s, got %s", string(content), string(art.Body))
|
|
||||||
}
|
|
||||||
if art.Name == "" {
|
|
||||||
t.Error("expected name to be inferred from filename")
|
|
||||||
}
|
|
||||||
if art.Hash != "60f5237ed4049f0382661ef009d2bc42e48c3ceb3edb6600f7024e7ab3b838f3" {
|
|
||||||
t.Errorf("unexpected hash: %s", art.Hash)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing file path", func(t *testing.T) {
|
|
||||||
ref := domain.ArtifactRef{
|
|
||||||
Type: domain.ArtifactRefFile,
|
|
||||||
URI: "",
|
|
||||||
}
|
|
||||||
_, err := reader.Read(ctx, ref)
|
|
||||||
if !errors.Is(err, ErrMissingFilePath) {
|
|
||||||
t.Errorf("expected ErrMissingFilePath, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -40,7 +40,11 @@ type Config struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ServerConfig struct {
|
type ServerConfig struct {
|
||||||
Addr string `yaml:"addr"`
|
Addr string `yaml:"addr"`
|
||||||
|
ArtifactRoot string `yaml:"artifact_root"`
|
||||||
|
MaxRequestBytes *int64 `yaml:"max_request_bytes"`
|
||||||
|
MaxArtifactBytes *int64 `yaml:"max_artifact_bytes"`
|
||||||
|
MaxResponseBytes *int64 `yaml:"max_response_bytes"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type DefaultsConfig struct {
|
type DefaultsConfig struct {
|
||||||
@@ -53,16 +57,24 @@ type AppSettings struct {
|
|||||||
ProfileDir string
|
ProfileDir string
|
||||||
SchemaDir string
|
SchemaDir string
|
||||||
ServerAddr string
|
ServerAddr string
|
||||||
|
ArtifactRoot string
|
||||||
|
MaxRequestBytes int64
|
||||||
|
MaxArtifactBytes int64
|
||||||
|
MaxResponseBytes int64
|
||||||
DefaultRenderFormat renderformat.PreparedRunOutputFormat
|
DefaultRenderFormat renderformat.PreparedRunOutputFormat
|
||||||
}
|
}
|
||||||
|
|
||||||
// CLIOverrides can be applied after config load to enforce precedence.
|
// CLIOverrides can be applied after config load to enforce precedence.
|
||||||
type CLIOverrides struct {
|
type CLIOverrides struct {
|
||||||
PromptDir string
|
PromptDir string
|
||||||
ProfileDir string
|
ProfileDir string
|
||||||
SchemaDir string
|
SchemaDir string
|
||||||
ServerAddr string
|
ServerAddr string
|
||||||
RenderFormat string
|
ArtifactRoot string
|
||||||
|
MaxRequestBytes *int64
|
||||||
|
MaxArtifactBytes *int64
|
||||||
|
MaxResponseBytes *int64
|
||||||
|
RenderFormat string
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuiltInDefaults returns compile-time application defaults.
|
// BuiltInDefaults returns compile-time application defaults.
|
||||||
@@ -70,6 +82,9 @@ func BuiltInDefaults() AppSettings {
|
|||||||
return AppSettings{
|
return AppSettings{
|
||||||
SchemaDir: defaults.SchemaDirDefault,
|
SchemaDir: defaults.SchemaDirDefault,
|
||||||
ServerAddr: defaults.HTTPAddrDefault,
|
ServerAddr: defaults.HTTPAddrDefault,
|
||||||
|
MaxRequestBytes: defaults.HTTPMaxRequestBytesDefault,
|
||||||
|
MaxArtifactBytes: defaults.HTTPMaxArtifactBytesDefault,
|
||||||
|
MaxResponseBytes: defaults.HTTPMaxResponseBytesDefault,
|
||||||
DefaultRenderFormat: renderformat.DefaultPreparedRunOutputFormat,
|
DefaultRenderFormat: renderformat.DefaultPreparedRunOutputFormat,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -142,6 +157,27 @@ func ApplyCLIOverrides(base AppSettings, overrides CLIOverrides) (AppSettings, e
|
|||||||
if v := strings.TrimSpace(overrides.ServerAddr); v != "" {
|
if v := strings.TrimSpace(overrides.ServerAddr); v != "" {
|
||||||
out.ServerAddr = v
|
out.ServerAddr = v
|
||||||
}
|
}
|
||||||
|
if v := strings.TrimSpace(overrides.ArtifactRoot); v != "" {
|
||||||
|
out.ArtifactRoot = filepath.Clean(v)
|
||||||
|
}
|
||||||
|
if overrides.MaxRequestBytes != nil {
|
||||||
|
if *overrides.MaxRequestBytes < 0 {
|
||||||
|
return AppSettings{}, fmt.Errorf("%w: server.max_request_bytes must be greater than or equal to 0", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
out.MaxRequestBytes = *overrides.MaxRequestBytes
|
||||||
|
}
|
||||||
|
if overrides.MaxArtifactBytes != nil {
|
||||||
|
if *overrides.MaxArtifactBytes < 0 {
|
||||||
|
return AppSettings{}, fmt.Errorf("%w: server.max_artifact_bytes must be greater than or equal to 0", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
out.MaxArtifactBytes = *overrides.MaxArtifactBytes
|
||||||
|
}
|
||||||
|
if overrides.MaxResponseBytes != nil {
|
||||||
|
if *overrides.MaxResponseBytes < 0 {
|
||||||
|
return AppSettings{}, fmt.Errorf("%w: server.max_response_bytes must be greater than or equal to 0", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
out.MaxResponseBytes = *overrides.MaxResponseBytes
|
||||||
|
}
|
||||||
if rawFormat := strings.TrimSpace(overrides.RenderFormat); rawFormat != "" {
|
if rawFormat := strings.TrimSpace(overrides.RenderFormat); rawFormat != "" {
|
||||||
parsed, err := renderformat.ParsePreparedRunOutputFormat(rawFormat)
|
parsed, err := renderformat.ParsePreparedRunOutputFormat(rawFormat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -181,6 +217,27 @@ func applyConfig(base AppSettings, cfg Config) (AppSettings, error) {
|
|||||||
if v := strings.TrimSpace(cfg.Server.Addr); v != "" {
|
if v := strings.TrimSpace(cfg.Server.Addr); v != "" {
|
||||||
out.ServerAddr = v
|
out.ServerAddr = v
|
||||||
}
|
}
|
||||||
|
if v := strings.TrimSpace(cfg.Server.ArtifactRoot); v != "" {
|
||||||
|
out.ArtifactRoot = filepath.Clean(v)
|
||||||
|
}
|
||||||
|
if cfg.Server.MaxRequestBytes != nil {
|
||||||
|
if *cfg.Server.MaxRequestBytes < 0 {
|
||||||
|
return AppSettings{}, fmt.Errorf("%w: server.max_request_bytes must be greater than or equal to 0", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
out.MaxRequestBytes = *cfg.Server.MaxRequestBytes
|
||||||
|
}
|
||||||
|
if cfg.Server.MaxArtifactBytes != nil {
|
||||||
|
if *cfg.Server.MaxArtifactBytes < 0 {
|
||||||
|
return AppSettings{}, fmt.Errorf("%w: server.max_artifact_bytes must be greater than or equal to 0", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
out.MaxArtifactBytes = *cfg.Server.MaxArtifactBytes
|
||||||
|
}
|
||||||
|
if cfg.Server.MaxResponseBytes != nil {
|
||||||
|
if *cfg.Server.MaxResponseBytes < 0 {
|
||||||
|
return AppSettings{}, fmt.Errorf("%w: server.max_response_bytes must be greater than or equal to 0", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
out.MaxResponseBytes = *cfg.Server.MaxResponseBytes
|
||||||
|
}
|
||||||
if rawFormat := strings.TrimSpace(cfg.Defaults.RenderFormat); rawFormat != "" {
|
if rawFormat := strings.TrimSpace(cfg.Defaults.RenderFormat); rawFormat != "" {
|
||||||
parsed, err := renderformat.ParsePreparedRunOutputFormat(rawFormat)
|
parsed, err := renderformat.ParsePreparedRunOutputFormat(rawFormat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
||||||
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
renderformat "gitea.maximumdirect.net/eric/scriptorium/internal/format"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -24,6 +25,20 @@ func TestLoadConfigMissingImplicitPathUsesBuiltInDefaults(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuiltInDefaultsIncludeHTTPSizeLimits(t *testing.T) {
|
||||||
|
got := BuiltInDefaults()
|
||||||
|
|
||||||
|
if got.MaxRequestBytes != defaults.HTTPMaxRequestBytesDefault {
|
||||||
|
t.Fatalf("unexpected max request bytes: %d", got.MaxRequestBytes)
|
||||||
|
}
|
||||||
|
if got.MaxArtifactBytes != defaults.HTTPMaxArtifactBytesDefault {
|
||||||
|
t.Fatalf("unexpected max artifact bytes: %d", got.MaxArtifactBytes)
|
||||||
|
}
|
||||||
|
if got.MaxResponseBytes != defaults.HTTPMaxResponseBytesDefault {
|
||||||
|
t.Fatalf("unexpected max response bytes: %d", got.MaxResponseBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadConfigMissingExplicitPathReturnsError(t *testing.T) {
|
func TestLoadConfigMissingExplicitPathReturnsError(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
tmp := t.TempDir()
|
||||||
missing := filepath.Join(tmp, "missing.yml")
|
missing := filepath.Join(tmp, "missing.yml")
|
||||||
@@ -92,6 +107,10 @@ profile_dir: ./profiles
|
|||||||
schema_dir: ./schemas
|
schema_dir: ./schemas
|
||||||
server:
|
server:
|
||||||
addr: 127.0.0.1:9090
|
addr: 127.0.0.1:9090
|
||||||
|
artifact_root: ./artifacts
|
||||||
|
max_request_bytes: 1024
|
||||||
|
max_artifact_bytes: 2048
|
||||||
|
max_response_bytes: 4096
|
||||||
defaults:
|
defaults:
|
||||||
render_format: json
|
render_format: json
|
||||||
`)
|
`)
|
||||||
@@ -113,11 +132,61 @@ defaults:
|
|||||||
if got.ServerAddr != "127.0.0.1:9090" {
|
if got.ServerAddr != "127.0.0.1:9090" {
|
||||||
t.Fatalf("unexpected server.addr: %q", got.ServerAddr)
|
t.Fatalf("unexpected server.addr: %q", got.ServerAddr)
|
||||||
}
|
}
|
||||||
|
if got.ArtifactRoot != filepath.Clean("./artifacts") {
|
||||||
|
t.Fatalf("unexpected server.artifact_root: %q", got.ArtifactRoot)
|
||||||
|
}
|
||||||
|
if got.MaxRequestBytes != 1024 {
|
||||||
|
t.Fatalf("unexpected server.max_request_bytes: %d", got.MaxRequestBytes)
|
||||||
|
}
|
||||||
|
if got.MaxArtifactBytes != 2048 {
|
||||||
|
t.Fatalf("unexpected server.max_artifact_bytes: %d", got.MaxArtifactBytes)
|
||||||
|
}
|
||||||
|
if got.MaxResponseBytes != 4096 {
|
||||||
|
t.Fatalf("unexpected server.max_response_bytes: %d", got.MaxResponseBytes)
|
||||||
|
}
|
||||||
if got.DefaultRenderFormat != renderformat.PreparedRunFormatJSON {
|
if got.DefaultRenderFormat != renderformat.PreparedRunFormatJSON {
|
||||||
t.Fatalf("unexpected defaults.render_format: %q", got.DefaultRenderFormat)
|
t.Fatalf("unexpected defaults.render_format: %q", got.DefaultRenderFormat)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadConfigAcceptsZeroHTTPSizeLimits(t *testing.T) {
|
||||||
|
path := writeConfigFile(t, "config.yml", `
|
||||||
|
server:
|
||||||
|
max_request_bytes: 0
|
||||||
|
max_artifact_bytes: 0
|
||||||
|
max_response_bytes: 0
|
||||||
|
`)
|
||||||
|
|
||||||
|
got, err := LoadConfig(path, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if got.MaxRequestBytes != 0 || got.MaxArtifactBytes != 0 || got.MaxResponseBytes != 0 {
|
||||||
|
t.Fatalf("expected zero limits to be preserved, got request=%d artifact=%d response=%d", got.MaxRequestBytes, got.MaxArtifactBytes, got.MaxResponseBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadConfigRejectsNegativeHTTPSizeLimits(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
body string
|
||||||
|
}{
|
||||||
|
{name: "request", body: "server:\n max_request_bytes: -1\n"},
|
||||||
|
{name: "artifact", body: "server:\n max_artifact_bytes: -1\n"},
|
||||||
|
{name: "response", body: "server:\n max_response_bytes: -1\n"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
path := writeConfigFile(t, "config.yml", tc.body)
|
||||||
|
_, err := LoadConfig(path, true)
|
||||||
|
if !errors.Is(err, ErrInvalidConfig) {
|
||||||
|
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadConfigEmptyFileResolvesToBuiltInDefaults(t *testing.T) {
|
func TestLoadConfigEmptyFileResolvesToBuiltInDefaults(t *testing.T) {
|
||||||
path := writeConfigFile(t, "config.yml", "")
|
path := writeConfigFile(t, "config.yml", "")
|
||||||
|
|
||||||
@@ -185,15 +254,26 @@ func TestApplyCLIOverridesAppliesPrecedence(t *testing.T) {
|
|||||||
ProfileDir: "/from/config/profiles",
|
ProfileDir: "/from/config/profiles",
|
||||||
SchemaDir: "/from/config/schemas",
|
SchemaDir: "/from/config/schemas",
|
||||||
ServerAddr: ":1234",
|
ServerAddr: ":1234",
|
||||||
|
ArtifactRoot: "/from/config/artifacts",
|
||||||
|
MaxRequestBytes: 111,
|
||||||
|
MaxArtifactBytes: 222,
|
||||||
|
MaxResponseBytes: 333,
|
||||||
DefaultRenderFormat: renderformat.PreparedRunFormatJSON,
|
DefaultRenderFormat: renderformat.PreparedRunFormatJSON,
|
||||||
}
|
}
|
||||||
|
maxRequestBytes := int64(0)
|
||||||
|
maxArtifactBytes := int64(444)
|
||||||
|
maxResponseBytes := int64(555)
|
||||||
|
|
||||||
got, err := ApplyCLIOverrides(base, CLIOverrides{
|
got, err := ApplyCLIOverrides(base, CLIOverrides{
|
||||||
PromptDir: "./prompts-cli",
|
PromptDir: "./prompts-cli",
|
||||||
ProfileDir: "./profiles-cli",
|
ProfileDir: "./profiles-cli",
|
||||||
SchemaDir: "./schemas-cli",
|
SchemaDir: "./schemas-cli",
|
||||||
ServerAddr: ":8081",
|
ServerAddr: ":8081",
|
||||||
RenderFormat: "text",
|
ArtifactRoot: "./artifacts-cli",
|
||||||
|
MaxRequestBytes: &maxRequestBytes,
|
||||||
|
MaxArtifactBytes: &maxArtifactBytes,
|
||||||
|
MaxResponseBytes: &maxResponseBytes,
|
||||||
|
RenderFormat: "text",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
@@ -211,11 +291,45 @@ func TestApplyCLIOverridesAppliesPrecedence(t *testing.T) {
|
|||||||
if got.ServerAddr != ":8081" {
|
if got.ServerAddr != ":8081" {
|
||||||
t.Fatalf("unexpected server addr: %q", got.ServerAddr)
|
t.Fatalf("unexpected server addr: %q", got.ServerAddr)
|
||||||
}
|
}
|
||||||
|
if got.ArtifactRoot != filepath.Clean("./artifacts-cli") {
|
||||||
|
t.Fatalf("unexpected artifact root: %q", got.ArtifactRoot)
|
||||||
|
}
|
||||||
|
if got.MaxRequestBytes != 0 {
|
||||||
|
t.Fatalf("unexpected max request bytes: %d", got.MaxRequestBytes)
|
||||||
|
}
|
||||||
|
if got.MaxArtifactBytes != 444 {
|
||||||
|
t.Fatalf("unexpected max artifact bytes: %d", got.MaxArtifactBytes)
|
||||||
|
}
|
||||||
|
if got.MaxResponseBytes != 555 {
|
||||||
|
t.Fatalf("unexpected max response bytes: %d", got.MaxResponseBytes)
|
||||||
|
}
|
||||||
if got.DefaultRenderFormat != renderformat.PreparedRunFormatText {
|
if got.DefaultRenderFormat != renderformat.PreparedRunFormatText {
|
||||||
t.Fatalf("unexpected render format: %q", got.DefaultRenderFormat)
|
t.Fatalf("unexpected render format: %q", got.DefaultRenderFormat)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestApplyCLIOverridesRejectsNegativeHTTPSizeLimits(t *testing.T) {
|
||||||
|
negative := int64(-1)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
overrides CLIOverrides
|
||||||
|
}{
|
||||||
|
{name: "request", overrides: CLIOverrides{MaxRequestBytes: &negative}},
|
||||||
|
{name: "artifact", overrides: CLIOverrides{MaxArtifactBytes: &negative}},
|
||||||
|
{name: "response", overrides: CLIOverrides{MaxResponseBytes: &negative}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := ApplyCLIOverrides(BuiltInDefaults(), tc.overrides)
|
||||||
|
if !errors.Is(err, ErrInvalidConfig) {
|
||||||
|
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplyCLIOverridesInvalidRenderFormatReturnsError(t *testing.T) {
|
func TestApplyCLIOverridesInvalidRenderFormatReturnsError(t *testing.T) {
|
||||||
_, err := ApplyCLIOverrides(BuiltInDefaults(), CLIOverrides{RenderFormat: "yaml"})
|
_, err := ApplyCLIOverrides(BuiltInDefaults(), CLIOverrides{RenderFormat: "yaml"})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
|||||||
@@ -1,36 +1,13 @@
|
|||||||
package defaults
|
package defaults
|
||||||
|
|
||||||
import (
|
import "time"
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
HTTPAddrDefault = ":8080"
|
HTTPAddrDefault = ":8080"
|
||||||
SchemaDirDefault = "."
|
SchemaDirDefault = "."
|
||||||
OutputArtifactName = "output"
|
HTTPMaxRequestBytesDefault = 16 * 1024 * 1024
|
||||||
ContentTypeTextPlain = "text/plain"
|
HTTPMaxArtifactBytesDefault = 16 * 1024 * 1024
|
||||||
ContentTypeTextMarkdown = "text/markdown"
|
HTTPMaxResponseBytesDefault = 16 * 1024 * 1024
|
||||||
ContentTypeApplicationJSON = "application/json"
|
|
||||||
OpenAIChatCompletionsPath = "/chat/completions"
|
|
||||||
|
|
||||||
ExecutionDefaultTemperature = 0.0
|
|
||||||
ExecutionDefaultMaxTokens = 0
|
|
||||||
ExecutionDefaultTopP = 1.0
|
|
||||||
ExecutionDefaultTimeoutSeconds = 600
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var HTTPReadHeaderTimeoutDefault = 10 * time.Second
|
||||||
LLMRequestTimeoutDefault = 10 * time.Minute
|
|
||||||
HTTPReadHeaderTimeoutDefault = 10 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
func ExecutionTargetDefault() domain.ExecutionTarget {
|
|
||||||
return domain.ExecutionTarget{
|
|
||||||
Temperature: ExecutionDefaultTemperature,
|
|
||||||
MaxTokens: ExecutionDefaultMaxTokens,
|
|
||||||
TopP: ExecutionDefaultTopP,
|
|
||||||
TimeoutSeconds: ExecutionDefaultTimeoutSeconds,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,255 +0,0 @@
|
|||||||
package domain
|
|
||||||
|
|
||||||
import (
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ArtifactRefType defines how an artifact is referenced.
|
|
||||||
type ArtifactRefType string
|
|
||||||
|
|
||||||
const (
|
|
||||||
ArtifactRefInline ArtifactRefType = "inline"
|
|
||||||
ArtifactRefFile ArtifactRefType = "file"
|
|
||||||
)
|
|
||||||
|
|
||||||
// OutputFormat defines the desired format of the generated artifact.
|
|
||||||
type OutputFormat string
|
|
||||||
|
|
||||||
const (
|
|
||||||
FormatText OutputFormat = "text"
|
|
||||||
FormatMarkdown OutputFormat = "markdown"
|
|
||||||
FormatJSON OutputFormat = "json"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ValidationMode defines how the output should be validated.
|
|
||||||
type ValidationMode string
|
|
||||||
|
|
||||||
const (
|
|
||||||
ValidationNone ValidationMode = "none"
|
|
||||||
ValidationBasic ValidationMode = "basic"
|
|
||||||
ValidationJSON ValidationMode = "json"
|
|
||||||
ValidationJSONSchema ValidationMode = "json_schema"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ValidationStatus defines the result of a validation check.
|
|
||||||
type ValidationStatus string
|
|
||||||
|
|
||||||
const (
|
|
||||||
ValidationPassed ValidationStatus = "passed"
|
|
||||||
ValidationFailed ValidationStatus = "failed"
|
|
||||||
ValidationSkipped ValidationStatus = "skipped"
|
|
||||||
)
|
|
||||||
|
|
||||||
// RunRequest represents a request to generate a single artifact.
|
|
||||||
type RunRequest struct {
|
|
||||||
PromptID string
|
|
||||||
PromptVersion string
|
|
||||||
ProfileID string
|
|
||||||
Inputs map[string]ArtifactRef
|
|
||||||
Vars map[string]string
|
|
||||||
Execution *ExecutionTarget
|
|
||||||
Validation *OutputContract
|
|
||||||
Metadata map[string]string
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunResult represents the complete result of a prompt execution run.
|
|
||||||
type RunResult struct {
|
|
||||||
RunID string
|
|
||||||
Artifact Artifact
|
|
||||||
RawOutput string
|
|
||||||
Validation ValidationResult
|
|
||||||
PromptID string
|
|
||||||
PromptVersion string
|
|
||||||
PromptHash string
|
|
||||||
RenderedPromptHash string
|
|
||||||
SelectedProfileID string
|
|
||||||
ModelName string
|
|
||||||
Endpoint string
|
|
||||||
EffectiveModelParams ExecutionTarget
|
|
||||||
InputHashes map[string]string
|
|
||||||
Usage TokenUsage
|
|
||||||
StartTime time.Time
|
|
||||||
EndTime time.Time
|
|
||||||
Duration time.Duration
|
|
||||||
Error error
|
|
||||||
}
|
|
||||||
|
|
||||||
// PreparedRun contains pre-LLM execution state from the prepare/render phase.
|
|
||||||
// It must never include resolved API key values, model output, or validation data.
|
|
||||||
type PreparedRun struct {
|
|
||||||
PromptID string `json:"prompt_id"`
|
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
|
||||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
|
||||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
|
||||||
Messages []RenderedMessage `json:"messages"`
|
|
||||||
StartTime time.Time `json:"start_time,omitempty"`
|
|
||||||
EndTime time.Time `json:"end_time,omitempty"`
|
|
||||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ArtifactRef represents a reference to an input artifact.
|
|
||||||
type ArtifactRef struct {
|
|
||||||
Type ArtifactRefType
|
|
||||||
URI string
|
|
||||||
Body string // Used for inline
|
|
||||||
}
|
|
||||||
|
|
||||||
// Artifact represents the actual loaded content of a reference.
|
|
||||||
type Artifact struct {
|
|
||||||
Name string
|
|
||||||
ContentType string
|
|
||||||
Body []byte
|
|
||||||
URI string
|
|
||||||
Size int64
|
|
||||||
Hash string
|
|
||||||
}
|
|
||||||
|
|
||||||
// PromptDefinition represents a configured prompt execution definition.
|
|
||||||
type PromptDefinition struct {
|
|
||||||
ID string `yaml:"id"`
|
|
||||||
Version string `yaml:"version"`
|
|
||||||
DefaultProfile string `yaml:"default_profile"`
|
|
||||||
Description string `yaml:"description"`
|
|
||||||
Inputs []PromptInput `yaml:"inputs"`
|
|
||||||
Templates []PromptMessageTemplate `yaml:"templates"`
|
|
||||||
OutputFormat OutputFormat `yaml:"output_format"`
|
|
||||||
Validation OutputContract `yaml:"validation"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// PromptInput describes one named input expected by a prompt definition.
|
|
||||||
type PromptInput struct {
|
|
||||||
Name string `yaml:"name"`
|
|
||||||
Required bool `yaml:"required"`
|
|
||||||
ContentType string `yaml:"content_type"`
|
|
||||||
Description string `yaml:"description"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// PromptMessageTemplate defines a template for a chat message.
|
|
||||||
type PromptMessageTemplate struct {
|
|
||||||
Role string `yaml:"role"`
|
|
||||||
Content string `yaml:"content"`
|
|
||||||
ContentFile string `yaml:"content_file"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExecutionProfile describes how and where to execute a model.
|
|
||||||
type ExecutionProfile struct {
|
|
||||||
ID string `yaml:"id"`
|
|
||||||
Endpoint string `yaml:"endpoint"`
|
|
||||||
Model string `yaml:"model"`
|
|
||||||
Temperature float64 `yaml:"temperature"`
|
|
||||||
MaxTokens int `yaml:"max_tokens"`
|
|
||||||
TopP float64 `yaml:"top_p"`
|
|
||||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
|
||||||
ServiceTier string `yaml:"service_tier"`
|
|
||||||
ReasoningEffort string `yaml:"reasoning_effort"`
|
|
||||||
APIKeyEnv string `yaml:"api_key_env"`
|
|
||||||
ExtraParams map[string]string `yaml:"extra_params"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExecutionTarget represents effective model runtime settings for a run.
|
|
||||||
type ExecutionTarget struct {
|
|
||||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
|
||||||
Model string `yaml:"model" json:"model"`
|
|
||||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
|
||||||
MaxTokens int `yaml:"max_tokens" json:"max_tokens"`
|
|
||||||
TopP float64 `yaml:"top_p" json:"top_p"`
|
|
||||||
TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"`
|
|
||||||
ServiceTier string `yaml:"service_tier" json:"service_tier"`
|
|
||||||
ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"`
|
|
||||||
APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"`
|
|
||||||
ExtraParams map[string]string `yaml:"extra_params" json:"extra_params"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// OutputContract defines the requirements for the output artifact.
|
|
||||||
type OutputContract struct {
|
|
||||||
Format OutputFormat `yaml:"format"`
|
|
||||||
ValidationMode ValidationMode `yaml:"validation_mode"`
|
|
||||||
SchemaPath string `yaml:"schema_path"`
|
|
||||||
RepairAttempts int `yaml:"repair_attempts"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// RenderedPrompt represents the prompt after template application.
|
|
||||||
type RenderedPrompt struct {
|
|
||||||
Messages []RenderedMessage `json:"messages"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// RenderedMessage is a single message in a rendered prompt.
|
|
||||||
type RenderedMessage struct {
|
|
||||||
Role string `json:"role"`
|
|
||||||
Content string `json:"content"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// GenerateRequest is the internal request passed to the LLM client.
|
|
||||||
type GenerateRequest struct {
|
|
||||||
Prompt RenderedPrompt
|
|
||||||
Target ExecutionTarget
|
|
||||||
StructuredOutput *StructuredOutputSpec
|
|
||||||
}
|
|
||||||
|
|
||||||
// StructuredOutputType indicates which provider-level output mode is requested.
|
|
||||||
type StructuredOutputType string
|
|
||||||
|
|
||||||
const (
|
|
||||||
StructuredOutputJSONSchema StructuredOutputType = "json_schema"
|
|
||||||
)
|
|
||||||
|
|
||||||
// StructuredOutputSpec describes provider-level structured output requirements.
|
|
||||||
type StructuredOutputSpec struct {
|
|
||||||
Type StructuredOutputType `json:"type"`
|
|
||||||
JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// StructuredOutputJSONSpec contains json_schema output constraints.
|
|
||||||
type StructuredOutputJSONSpec struct {
|
|
||||||
Name string `json:"name"`
|
|
||||||
Strict bool `json:"strict"`
|
|
||||||
Schema any `json:"schema"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// GenerateResponse is the response received from the LLM client.
|
|
||||||
type GenerateResponse struct {
|
|
||||||
Content string
|
|
||||||
Usage TokenUsage
|
|
||||||
}
|
|
||||||
|
|
||||||
// TokenUsage tracks token consumption.
|
|
||||||
type TokenUsage struct {
|
|
||||||
PromptTokens int
|
|
||||||
CompletionTokens int
|
|
||||||
TotalTokens int
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValidationResult represents the outcome of an output validation.
|
|
||||||
type ValidationResult struct {
|
|
||||||
Status ValidationStatus
|
|
||||||
Mode ValidationMode
|
|
||||||
Errors []string
|
|
||||||
SchemaPath string
|
|
||||||
RepairAttempts int
|
|
||||||
IsValid bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunMetadata contains auditing information for a run.
|
|
||||||
type RunMetadata struct {
|
|
||||||
RunID string
|
|
||||||
PromptID string
|
|
||||||
PromptVersion string
|
|
||||||
PromptHash string
|
|
||||||
RenderedPromptHash string
|
|
||||||
SelectedProfileID string
|
|
||||||
InputHashes map[string]string
|
|
||||||
ModelEndpoint string
|
|
||||||
ModelName string
|
|
||||||
Params ExecutionTarget
|
|
||||||
Timestamp time.Time
|
|
||||||
Duration time.Duration
|
|
||||||
Usage TokenUsage
|
|
||||||
ValidationMode ValidationMode
|
|
||||||
ValidationStatus ValidationStatus
|
|
||||||
RepairAttempts int
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
package domain
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPreparedRunJSONDoesNotIncludeSecretValues(t *testing.T) {
|
|
||||||
const envName = "SCRIPTORIUM_TEST_API_KEY"
|
|
||||||
const secret = "super-secret-value"
|
|
||||||
t.Setenv(envName, secret)
|
|
||||||
|
|
||||||
prepared := PreparedRun{
|
|
||||||
PromptID: "prompt.id",
|
|
||||||
PromptVersion: "v1",
|
|
||||||
PromptHash: "prompt-hash",
|
|
||||||
SelectedProfileID: "local-fast",
|
|
||||||
EffectiveModelParams: ExecutionTarget{
|
|
||||||
Endpoint: "http://llm/v1",
|
|
||||||
Model: "gpt-test",
|
|
||||||
APIKeyEnv: envName,
|
|
||||||
},
|
|
||||||
InputHashes: map[string]string{"transcript": "hash-1"},
|
|
||||||
RenderedPromptHash: "rendered-hash",
|
|
||||||
Messages: []RenderedMessage{
|
|
||||||
{Role: "system", Content: "You are helpful."},
|
|
||||||
{Role: "user", Content: "Summarize this."},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
b, err := json.Marshal(prepared)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("marshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
out := string(b)
|
|
||||||
if strings.Contains(out, secret) {
|
|
||||||
t.Fatalf("prepared run JSON unexpectedly contains secret value: %s", out)
|
|
||||||
}
|
|
||||||
if !strings.Contains(out, `"api_key_env":"`+envName+`"`) {
|
|
||||||
t.Fatalf("prepared run JSON should include api_key_env name: %s", out)
|
|
||||||
}
|
|
||||||
|
|
||||||
var top map[string]any
|
|
||||||
if err := json.Unmarshal(b, &top); err != nil {
|
|
||||||
t.Fatalf("unmarshal failed: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, forbidden := range []string{"raw_output", "validation", "artifact"} {
|
|
||||||
if _, ok := top[forbidden]; ok {
|
|
||||||
t.Fatalf("prepared run JSON should not include %q", forbidden)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,54 +0,0 @@
|
|||||||
package filecatalog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// FindYAMLFiles returns sorted full paths for .yaml and .yml files under root.
|
|
||||||
func FindYAMLFiles(ctx context.Context, root string) ([]string, error) {
|
|
||||||
var files []string
|
|
||||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
if d.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if !isYAMLFile(d.Name()) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
files = append(files, path)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
sort.Strings(files)
|
|
||||||
return files, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// RelativePath computes a clean relative path from root to path.
|
|
||||||
func RelativePath(root string, path string) string {
|
|
||||||
rel, err := filepath.Rel(root, path)
|
|
||||||
if err != nil {
|
|
||||||
return filepath.Clean(path)
|
|
||||||
}
|
|
||||||
return filepath.Clean(rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stem strips .yaml or .yml from a file name.
|
|
||||||
func Stem(name string) string {
|
|
||||||
name = strings.TrimSuffix(name, ".yaml")
|
|
||||||
name = strings.TrimSuffix(name, ".yml")
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
|
|
||||||
func isYAMLFile(name string) bool {
|
|
||||||
return strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")
|
|
||||||
}
|
|
||||||
@@ -1,84 +0,0 @@
|
|||||||
package filecatalog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFindYAMLFilesNestedSortedAndFiltered(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
mustWriteFile(t, filepath.Join(root, "z", "prompt.yml"), "id: z")
|
|
||||||
mustWriteFile(t, filepath.Join(root, "a", "profile.yaml"), "id: a")
|
|
||||||
mustWriteFile(t, filepath.Join(root, "a", "ignore.txt"), "not yaml")
|
|
||||||
mustWriteFile(t, filepath.Join(root, "b", "ignore.yaml.bak"), "not yaml")
|
|
||||||
|
|
||||||
got, err := FindYAMLFiles(context.Background(), root)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []string{
|
|
||||||
filepath.Join(root, "a", "profile.yaml"),
|
|
||||||
filepath.Join(root, "z", "prompt.yml"),
|
|
||||||
}
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Fatalf("expected sorted YAML files %v, got %v", want, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFindYAMLFilesHonorsContextCancellation(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
mustWriteFile(t, filepath.Join(root, "one.yaml"), "id: one")
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
_, err := FindYAMLFiles(ctx, root)
|
|
||||||
if !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatalf("expected context.Canceled, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRelativePathNested(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
path := filepath.Join(root, "nested", "profiles", "local.yaml")
|
|
||||||
got := RelativePath(root, path)
|
|
||||||
want := filepath.Join("nested", "profiles", "local.yaml")
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("expected relative path %q, got %q", want, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStemStripsYAMLExtensions(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
in string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{name: "yaml", in: "prompt.yaml", want: "prompt"},
|
|
||||||
{name: "yml", in: "profile.yml", want: "profile"},
|
|
||||||
{name: "other", in: "file.txt", want: "file.txt"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
if got := Stem(tc.in); got != tc.want {
|
|
||||||
t.Fatalf("expected %q, got %q", tc.want, got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustWriteFile(t *testing.T, path string, content string) {
|
|
||||||
t.Helper()
|
|
||||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
|
||||||
t.Fatalf("failed to create directory: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
|
|
||||||
t.Fatalf("failed to write file %q: %v", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
// Package format formats already-prepared domain data for adapters.
|
// Package format formats already-prepared public data for adapters.
|
||||||
package format
|
package format
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format")
|
var ErrUnknownPreparedRunFormat = errors.New("unknown prepared run format")
|
||||||
@@ -26,7 +26,7 @@ const (
|
|||||||
|
|
||||||
// PreparedRunFormatter serializes a prepared run without performing use case work.
|
// PreparedRunFormatter serializes a prepared run without performing use case work.
|
||||||
type PreparedRunFormatter interface {
|
type PreparedRunFormatter interface {
|
||||||
Format(prepared *domain.PreparedRun) ([]byte, error)
|
Format(prepared *promptkit.PreparedRun) ([]byte, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParsePreparedRunOutputFormat parses a format name.
|
// ParsePreparedRunOutputFormat parses a format name.
|
||||||
@@ -56,7 +56,7 @@ func FormatterForPreparedRun(outputFormat PreparedRunOutputFormat) (PreparedRunF
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FormatPreparedRun formats a prepared run using the selected format.
|
// FormatPreparedRun formats a prepared run using the selected format.
|
||||||
func FormatPreparedRun(prepared *domain.PreparedRun, outputFormat PreparedRunOutputFormat) ([]byte, error) {
|
func FormatPreparedRun(prepared *promptkit.PreparedRun, outputFormat PreparedRunOutputFormat) ([]byte, error) {
|
||||||
formatter, err := FormatterForPreparedRun(outputFormat)
|
formatter, err := FormatterForPreparedRun(outputFormat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -65,7 +65,7 @@ func FormatPreparedRun(prepared *domain.PreparedRun, outputFormat PreparedRunOut
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FormatPreparedRunByName parses a format name and formats a prepared run.
|
// FormatPreparedRunByName parses a format name and formats a prepared run.
|
||||||
func FormatPreparedRunByName(prepared *domain.PreparedRun, rawFormat string) ([]byte, error) {
|
func FormatPreparedRunByName(prepared *promptkit.PreparedRun, rawFormat string) ([]byte, error) {
|
||||||
outputFormat, err := ParsePreparedRunOutputFormat(rawFormat)
|
outputFormat, err := ParsePreparedRunOutputFormat(rawFormat)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -75,7 +75,7 @@ func FormatPreparedRunByName(prepared *domain.PreparedRun, rawFormat string) ([]
|
|||||||
|
|
||||||
type jsonPreparedRunFormatter struct{}
|
type jsonPreparedRunFormatter struct{}
|
||||||
|
|
||||||
func (jsonPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, error) {
|
func (jsonPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte, error) {
|
||||||
if prepared == nil {
|
if prepared == nil {
|
||||||
return nil, errors.New("prepared run is nil")
|
return nil, errors.New("prepared run is nil")
|
||||||
}
|
}
|
||||||
@@ -84,7 +84,7 @@ func (jsonPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
|
|
||||||
type textPreparedRunFormatter struct{}
|
type textPreparedRunFormatter struct{}
|
||||||
|
|
||||||
func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, error) {
|
func (textPreparedRunFormatter) Format(prepared *promptkit.PreparedRun) ([]byte, error) {
|
||||||
if prepared == nil {
|
if prepared == nil {
|
||||||
return nil, errors.New("prepared run is nil")
|
return nil, errors.New("prepared run is nil")
|
||||||
}
|
}
|
||||||
@@ -96,6 +96,9 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
if prepared.PromptHash != "" {
|
if prepared.PromptHash != "" {
|
||||||
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
fmt.Fprintf(&b, "prompt_hash: %s\n", prepared.PromptHash)
|
||||||
}
|
}
|
||||||
|
if prepared.SessionID != "" {
|
||||||
|
fmt.Fprintf(&b, "session_id: %s\n", prepared.SessionID)
|
||||||
|
}
|
||||||
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
|
fmt.Fprintf(&b, "rendered_prompt_hash: %s\n", prepared.RenderedPromptHash)
|
||||||
|
|
||||||
target := prepared.EffectiveModelParams
|
target := prepared.EffectiveModelParams
|
||||||
@@ -123,7 +126,11 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
}
|
}
|
||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
for _, k := range keys {
|
for _, k := range keys {
|
||||||
fmt.Fprintf(&b, " %s: %s\n", k, target.ExtraParams[k])
|
renderedValue, err := formatExtraParamTextValue(target.ExtraParams[k])
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to format extra_params.%s: %w", k, err)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, " %s: %s\n", k, renderedValue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -139,7 +146,7 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
|
|
||||||
fmt.Fprintln(&b, "messages:")
|
fmt.Fprintln(&b, "messages:")
|
||||||
roleOrder := make([]string, 0)
|
roleOrder := make([]string, 0)
|
||||||
byRole := make(map[string][]domain.RenderedMessage)
|
byRole := make(map[string][]promptkit.RenderedMessage)
|
||||||
for _, msg := range prepared.Messages {
|
for _, msg := range prepared.Messages {
|
||||||
if _, exists := byRole[msg.Role]; !exists {
|
if _, exists := byRole[msg.Role]; !exists {
|
||||||
roleOrder = append(roleOrder, msg.Role)
|
roleOrder = append(roleOrder, msg.Role)
|
||||||
@@ -151,6 +158,13 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
messages := byRole[role]
|
messages := byRole[role]
|
||||||
for i, msg := range messages {
|
for i, msg := range messages {
|
||||||
fmt.Fprintf(&b, " - message: %d\n", i+1)
|
fmt.Fprintf(&b, " - message: %d\n", i+1)
|
||||||
|
if msg.CacheControl != nil {
|
||||||
|
fmt.Fprintf(&b, " cache_control: %s", msg.CacheControl.Type)
|
||||||
|
if msg.CacheControl.TTL != "" {
|
||||||
|
fmt.Fprintf(&b, " ttl=%s", msg.CacheControl.TTL)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(&b)
|
||||||
|
}
|
||||||
fmt.Fprintln(&b, " content: |")
|
fmt.Fprintln(&b, " content: |")
|
||||||
content := msg.Content
|
content := msg.Content
|
||||||
if content == "" {
|
if content == "" {
|
||||||
@@ -165,3 +179,15 @@ func (textPreparedRunFormatter) Format(prepared *domain.PreparedRun) ([]byte, er
|
|||||||
|
|
||||||
return b.Bytes(), nil
|
return b.Bytes(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func formatExtraParamTextValue(value any) (string, error) {
|
||||||
|
if s, ok := value.(string); ok {
|
||||||
|
return s, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return string(b), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
||||||
@@ -49,6 +49,36 @@ func TestTextFormatterIncludesPreparedRunDetails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterRendersExtraParamsDeterministically(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.EffectiveModelParams.ExtraParams = map[string]any{
|
||||||
|
"z_string": "enabled",
|
||||||
|
"b_number": 42,
|
||||||
|
"a_object": map[string]any{
|
||||||
|
"nested": "value",
|
||||||
|
"count": 2,
|
||||||
|
},
|
||||||
|
"c_array": []any{"first", 3, false},
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
s := string(out)
|
||||||
|
|
||||||
|
want := strings.Join([]string{
|
||||||
|
" extra_params:",
|
||||||
|
" a_object: {\"count\":2,\"nested\":\"value\"}",
|
||||||
|
" b_number: 42",
|
||||||
|
" c_array: [\"first\",3,false]",
|
||||||
|
" z_string: enabled",
|
||||||
|
}, "\n")
|
||||||
|
if !strings.Contains(s, want) {
|
||||||
|
t.Fatalf("expected deterministic extra_params block %q, got:\n%s", want, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||||
const secret = "super-secret-api-key"
|
const secret = "super-secret-api-key"
|
||||||
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
||||||
@@ -62,8 +92,93 @@ func TestTextFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
||||||
|
const directKey = "direct-format-key"
|
||||||
|
// PreparedRun intentionally has no field for direct API keys.
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(out), directKey) {
|
||||||
|
t.Fatalf("text output should not include direct api key value: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterIncludesMessageCacheControlBeforeContent(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.Messages = []promptkit.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "System guidance.",
|
||||||
|
CacheControl: &promptkit.CacheControl{
|
||||||
|
Type: promptkit.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "Summarize the transcript."},
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
s := string(out)
|
||||||
|
if !strings.Contains(s, " system:\n - message: 1\n cache_control: ephemeral ttl=1h\n content: |") {
|
||||||
|
t.Fatalf("expected system message cache control before content, got:\n%s", s)
|
||||||
|
}
|
||||||
|
if strings.Count(s, "cache_control:") != 1 {
|
||||||
|
t.Fatalf("expected exactly one cache_control line, got:\n%s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterIncludesSessionIDWhenPresent(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.SessionID = "session-123"
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(out), "session_id: session-123\n") {
|
||||||
|
t.Fatalf("expected session_id in text output, got:\n%s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTextFormatterOmitsEmptyCacheControlTTL(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.Messages = []promptkit.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "System guidance.",
|
||||||
|
CacheControl: &promptkit.CacheControl{
|
||||||
|
Type: promptkit.CacheControlEphemeral,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
s := string(out)
|
||||||
|
if !strings.Contains(s, " cache_control: ephemeral\n") {
|
||||||
|
t.Fatalf("expected cache_control line without ttl, got:\n%s", s)
|
||||||
|
}
|
||||||
|
if strings.Contains(s, "ttl=") {
|
||||||
|
t.Fatalf("expected empty ttl to be omitted, got:\n%s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
||||||
prepared := samplePreparedRun()
|
prepared := samplePreparedRun()
|
||||||
|
prepared.SessionID = "session-123"
|
||||||
|
prepared.EffectiveModelParams.ExtraParams = map[string]any{
|
||||||
|
"number": 42,
|
||||||
|
"nested": map[string]any{
|
||||||
|
"enabled": true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -87,9 +202,24 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
|||||||
if decoded["rendered_prompt_hash"] != "rendered-hash" {
|
if decoded["rendered_prompt_hash"] != "rendered-hash" {
|
||||||
t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"])
|
t.Fatalf("expected rendered_prompt_hash in json output, got %#v", decoded["rendered_prompt_hash"])
|
||||||
}
|
}
|
||||||
if _, ok := decoded["effective_model_params"]; !ok {
|
if decoded["session_id"] != "session-123" {
|
||||||
|
t.Fatalf("expected session_id in json output, got %#v", decoded["session_id"])
|
||||||
|
}
|
||||||
|
modelParams, ok := decoded["effective_model_params"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
t.Fatalf("expected effective_model_params in json output, got %#v", decoded)
|
t.Fatalf("expected effective_model_params in json output, got %#v", decoded)
|
||||||
}
|
}
|
||||||
|
extraParams, ok := modelParams["extra_params"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected extra_params in json output, got %#v", modelParams["extra_params"])
|
||||||
|
}
|
||||||
|
if extraParams["number"] != float64(42) {
|
||||||
|
t.Fatalf("unexpected numeric extra param in json output: %#v", extraParams["number"])
|
||||||
|
}
|
||||||
|
nested, ok := extraParams["nested"].(map[string]any)
|
||||||
|
if !ok || nested["enabled"] != true {
|
||||||
|
t.Fatalf("unexpected nested extra param in json output: %#v", extraParams["nested"])
|
||||||
|
}
|
||||||
if _, ok := decoded["input_hashes"]; !ok {
|
if _, ok := decoded["input_hashes"]; !ok {
|
||||||
t.Fatalf("expected input_hashes in json output, got %#v", decoded)
|
t.Fatalf("expected input_hashes in json output, got %#v", decoded)
|
||||||
}
|
}
|
||||||
@@ -98,6 +228,47 @@ func TestJSONFormatterEmitsValidJSONAndIncludesPreparedRunFields(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestJSONFormatterIncludesMessageCacheControlOnlyWhenPresent(t *testing.T) {
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
prepared.Messages = []promptkit.RenderedMessage{
|
||||||
|
{
|
||||||
|
Role: "system",
|
||||||
|
Content: "System guidance.",
|
||||||
|
CacheControl: &promptkit.CacheControl{
|
||||||
|
Type: promptkit.CacheControlEphemeral,
|
||||||
|
TTL: "1h",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{Role: "user", Content: "Summarize the transcript."},
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded struct {
|
||||||
|
Messages []map[string]any `json:"messages"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(out, &decoded); err != nil {
|
||||||
|
t.Fatalf("expected valid json output, got %v", err)
|
||||||
|
}
|
||||||
|
if len(decoded.Messages) != 2 {
|
||||||
|
t.Fatalf("expected 2 messages, got %d", len(decoded.Messages))
|
||||||
|
}
|
||||||
|
|
||||||
|
cacheControl, ok := decoded.Messages[0]["cache_control"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected first message cache_control, got %#v", decoded.Messages[0])
|
||||||
|
}
|
||||||
|
if cacheControl["type"] != string(promptkit.CacheControlEphemeral) || cacheControl["ttl"] != "1h" {
|
||||||
|
t.Fatalf("unexpected cache_control payload: %#v", cacheControl)
|
||||||
|
}
|
||||||
|
if _, ok := decoded.Messages[1]["cache_control"]; ok {
|
||||||
|
t.Fatalf("expected second message to omit cache_control, got %#v", decoded.Messages[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
||||||
const secret = "super-secret-api-key"
|
const secret = "super-secret-api-key"
|
||||||
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
t.Setenv("SCRIPTORIUM_API_KEY", secret)
|
||||||
@@ -111,6 +282,19 @@ func TestJSONFormatterDoesNotIncludeResolvedAPIKeyValue(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestJSONFormatterDoesNotIncludeDirectAPIKeyValue(t *testing.T) {
|
||||||
|
const directKey = "direct-format-key"
|
||||||
|
// PreparedRun intentionally has no field for direct API keys.
|
||||||
|
prepared := samplePreparedRun()
|
||||||
|
out, err := FormatPreparedRun(prepared, PreparedRunFormatJSON)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(out), directKey) {
|
||||||
|
t.Fatalf("json output should not include direct api key value: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) {
|
func TestParsePreparedRunOutputFormatRecognizesSupportedNames(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -156,13 +340,13 @@ func TestFormatPreparedRunByNameUnknownFailsClearly(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func samplePreparedRun() *domain.PreparedRun {
|
func samplePreparedRun() *promptkit.PreparedRun {
|
||||||
return &domain.PreparedRun{
|
return &promptkit.PreparedRun{
|
||||||
PromptID: "prompt.id",
|
PromptID: "prompt.id",
|
||||||
PromptVersion: "v1",
|
PromptVersion: "v1",
|
||||||
PromptHash: "prompt-hash",
|
PromptHash: "prompt-hash",
|
||||||
SelectedProfileID: "local-fast",
|
SelectedProfileID: "local-fast",
|
||||||
EffectiveModelParams: domain.ExecutionTarget{
|
EffectiveModelParams: promptkit.ExecutionTarget{
|
||||||
Endpoint: "http://llm/v1",
|
Endpoint: "http://llm/v1",
|
||||||
Model: "gpt-test",
|
Model: "gpt-test",
|
||||||
Temperature: 0.4,
|
Temperature: 0.4,
|
||||||
@@ -178,7 +362,7 @@ func samplePreparedRun() *domain.PreparedRun {
|
|||||||
"glossary": "hash-glossary",
|
"glossary": "hash-glossary",
|
||||||
},
|
},
|
||||||
RenderedPromptHash: "rendered-hash",
|
RenderedPromptHash: "rendered-hash",
|
||||||
Messages: []domain.RenderedMessage{
|
Messages: []promptkit.RenderedMessage{
|
||||||
{Role: "system", Content: "System guidance."},
|
{Role: "system", Content: "System guidance."},
|
||||||
{Role: "user", Content: "Summarize the transcript.\nInclude key entities."},
|
{Role: "user", Content: "Summarize the transcript.\nInclude key entities."},
|
||||||
{Role: "user", Content: "Second user message."},
|
{Role: "user", Content: "Second user message."},
|
||||||
|
|||||||
@@ -1,11 +0,0 @@
|
|||||||
package llm
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Client executes a rendered prompt against an LLM endpoint.
|
|
||||||
type Client interface {
|
|
||||||
Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error)
|
|
||||||
}
|
|
||||||
@@ -1,266 +0,0 @@
|
|||||||
package llm
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrInvalidConfig = errors.New("invalid llm client configuration")
|
|
||||||
ErrInvalidRequest = errors.New("invalid generate request")
|
|
||||||
ErrRequestFailed = errors.New("llm request failed")
|
|
||||||
ErrUnexpectedStatus = errors.New("llm returned non-success status")
|
|
||||||
ErrMalformedResponse = errors.New("malformed llm response")
|
|
||||||
)
|
|
||||||
|
|
||||||
type OpenAICompatibleConfig struct {
|
|
||||||
BaseURL string
|
|
||||||
Model string
|
|
||||||
Timeout time.Duration
|
|
||||||
HTTPClient *http.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
type OpenAICompatibleClient struct {
|
|
||||||
baseURL string
|
|
||||||
defaultModel string
|
|
||||||
timeout time.Duration
|
|
||||||
httpClient *http.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleClient, error) {
|
|
||||||
baseURL := strings.TrimSpace(cfg.BaseURL)
|
|
||||||
if baseURL != "" {
|
|
||||||
if _, err := url.ParseRequestURI(baseURL); err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: invalid base URL: %v", ErrInvalidConfig, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
timeout := cfg.Timeout
|
|
||||||
if timeout <= 0 {
|
|
||||||
timeout = defaults.LLMRequestTimeoutDefault
|
|
||||||
}
|
|
||||||
|
|
||||||
var client *http.Client
|
|
||||||
if cfg.HTTPClient != nil {
|
|
||||||
client = cfg.HTTPClient
|
|
||||||
if client.Timeout == 0 {
|
|
||||||
client.Timeout = timeout
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
client = &http.Client{Timeout: timeout}
|
|
||||||
}
|
|
||||||
|
|
||||||
return &OpenAICompatibleClient{
|
|
||||||
baseURL: strings.TrimRight(baseURL, "/"),
|
|
||||||
defaultModel: cfg.Model,
|
|
||||||
timeout: timeout,
|
|
||||||
httpClient: client,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
|
||||||
if req.Target.TimeoutSeconds < 0 {
|
|
||||||
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest)
|
|
||||||
}
|
|
||||||
|
|
||||||
endpoint := strings.TrimSpace(req.Target.Endpoint)
|
|
||||||
if endpoint == "" {
|
|
||||||
endpoint = c.baseURL
|
|
||||||
}
|
|
||||||
if endpoint == "" {
|
|
||||||
return nil, fmt.Errorf("%w: endpoint is required", ErrInvalidRequest)
|
|
||||||
}
|
|
||||||
endpoint = strings.TrimRight(endpoint, "/") + defaults.OpenAIChatCompletionsPath
|
|
||||||
|
|
||||||
wireReq, err := openAIChatRequestFromGenerateRequest(req, c.defaultModel)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
payload, err := json.Marshal(wireReq)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
|
||||||
}
|
|
||||||
httpReq.Header.Set("Content-Type", "application/json")
|
|
||||||
if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
|
|
||||||
apiKey := strings.TrimSpace(os.Getenv(envName))
|
|
||||||
if apiKey == "" {
|
|
||||||
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
|
||||||
}
|
|
||||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
|
||||||
}
|
|
||||||
|
|
||||||
effectiveTimeout := c.timeout
|
|
||||||
if req.Target.TimeoutSeconds > 0 {
|
|
||||||
effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second
|
|
||||||
}
|
|
||||||
|
|
||||||
httpClient := c.httpClient
|
|
||||||
if httpClient == nil {
|
|
||||||
httpClient = &http.Client{Timeout: effectiveTimeout}
|
|
||||||
} else if httpClient.Timeout != effectiveTimeout {
|
|
||||||
cloned := *httpClient
|
|
||||||
cloned.Timeout = effectiveTimeout
|
|
||||||
httpClient = &cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
httpResp, err := httpClient.Do(httpReq)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %v", ErrRequestFailed, err)
|
|
||||||
}
|
|
||||||
defer httpResp.Body.Close()
|
|
||||||
|
|
||||||
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
|
||||||
body, _ := io.ReadAll(io.LimitReader(httpResp.Body, 4096))
|
|
||||||
return nil, fmt.Errorf("%w: status=%d body=%q", ErrUnexpectedStatus, httpResp.StatusCode, strings.TrimSpace(string(body)))
|
|
||||||
}
|
|
||||||
|
|
||||||
var wireResp openAIChatResponse
|
|
||||||
if err := json.NewDecoder(httpResp.Body).Decode(&wireResp); err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to decode response: %v", ErrMalformedResponse, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(wireResp.Choices) == 0 {
|
|
||||||
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
|
||||||
}
|
|
||||||
content := wireResp.Choices[0].Message.Content
|
|
||||||
if content == "" {
|
|
||||||
return nil, fmt.Errorf("%w: first choice has empty message content", ErrMalformedResponse)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &domain.GenerateResponse{
|
|
||||||
Content: content,
|
|
||||||
Usage: domain.TokenUsage{
|
|
||||||
PromptTokens: wireResp.Usage.PromptTokens,
|
|
||||||
CompletionTokens: wireResp.Usage.CompletionTokens,
|
|
||||||
TotalTokens: wireResp.Usage.TotalTokens,
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultModel string) (openAIChatRequest, error) {
|
|
||||||
model := strings.TrimSpace(req.Target.Model)
|
|
||||||
if model == "" {
|
|
||||||
model = strings.TrimSpace(defaultModel)
|
|
||||||
}
|
|
||||||
if model == "" {
|
|
||||||
return openAIChatRequest{}, errors.New("model is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
wireReq := openAIChatRequest{
|
|
||||||
Model: model,
|
|
||||||
}
|
|
||||||
|
|
||||||
wireReq.Messages = make([]openAIChatMessage, 0, len(req.Prompt.Messages))
|
|
||||||
for _, msg := range req.Prompt.Messages {
|
|
||||||
wireReq.Messages = append(wireReq.Messages, openAIChatMessage{
|
|
||||||
Role: msg.Role,
|
|
||||||
Content: msg.Content,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if req.Target.Temperature != 0 {
|
|
||||||
wireReq.Temperature = &req.Target.Temperature
|
|
||||||
}
|
|
||||||
if req.Target.MaxTokens != 0 {
|
|
||||||
wireReq.MaxTokens = &req.Target.MaxTokens
|
|
||||||
}
|
|
||||||
if req.Target.TopP != 0 {
|
|
||||||
wireReq.TopP = &req.Target.TopP
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(req.Target.ServiceTier) != "" {
|
|
||||||
wireReq.ServiceTier = req.Target.ServiceTier
|
|
||||||
}
|
|
||||||
if req.StructuredOutput != nil {
|
|
||||||
responseFormat, err := toOpenAIResponseFormat(req.StructuredOutput)
|
|
||||||
if err != nil {
|
|
||||||
return openAIChatRequest{}, err
|
|
||||||
}
|
|
||||||
wireReq.ResponseFormat = responseFormat
|
|
||||||
}
|
|
||||||
|
|
||||||
return wireReq, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAIChatRequest struct {
|
|
||||||
Model string `json:"model"`
|
|
||||||
Messages []openAIChatMessage `json:"messages"`
|
|
||||||
Temperature *float64 `json:"temperature,omitempty"`
|
|
||||||
MaxTokens *int `json:"max_tokens,omitempty"`
|
|
||||||
TopP *float64 `json:"top_p,omitempty"`
|
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
|
||||||
ResponseFormat *openAIResponseFormat `json:"response_format,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAIChatMessage struct {
|
|
||||||
Role string `json:"role"`
|
|
||||||
Content string `json:"content"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAIChatResponse struct {
|
|
||||||
Choices []struct {
|
|
||||||
Message openAIChatMessage `json:"message"`
|
|
||||||
} `json:"choices"`
|
|
||||||
Usage struct {
|
|
||||||
PromptTokens int `json:"prompt_tokens"`
|
|
||||||
CompletionTokens int `json:"completion_tokens"`
|
|
||||||
TotalTokens int `json:"total_tokens"`
|
|
||||||
} `json:"usage"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAIResponseFormat struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
JSONSchema *openAIJSONSchemaEnvelope `json:"json_schema,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type openAIJSONSchemaEnvelope struct {
|
|
||||||
Name string `json:"name"`
|
|
||||||
Strict bool `json:"strict"`
|
|
||||||
Schema any `json:"schema"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func toOpenAIResponseFormat(spec *domain.StructuredOutputSpec) (*openAIResponseFormat, error) {
|
|
||||||
if spec == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
switch spec.Type {
|
|
||||||
case domain.StructuredOutputJSONSchema:
|
|
||||||
if spec.JSONSchema == nil {
|
|
||||||
return nil, errors.New("json_schema structured output requires schema payload")
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(spec.JSONSchema.Name) == "" {
|
|
||||||
return nil, errors.New("json_schema structured output requires non-empty schema name")
|
|
||||||
}
|
|
||||||
if spec.JSONSchema.Schema == nil {
|
|
||||||
return nil, errors.New("json_schema structured output requires schema document")
|
|
||||||
}
|
|
||||||
return &openAIResponseFormat{
|
|
||||||
Type: "json_schema",
|
|
||||||
JSONSchema: &openAIJSONSchemaEnvelope{
|
|
||||||
Name: spec.JSONSchema.Name,
|
|
||||||
Strict: spec.JSONSchema.Strict,
|
|
||||||
Schema: spec.JSONSchema.Schema,
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("unsupported structured output type %q", spec.Type)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,520 +0,0 @@
|
|||||||
package llm
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
|
|
||||||
type observedRequest struct {
|
|
||||||
Authorization string
|
|
||||||
Body map[string]any
|
|
||||||
}
|
|
||||||
obs := &observedRequest{}
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
obs.Authorization = r.Header.Get("Authorization")
|
|
||||||
if r.URL.Path != "/v1/chat/completions" {
|
|
||||||
t.Fatalf("unexpected path: %s", r.URL.Path)
|
|
||||||
}
|
|
||||||
if ct := r.Header.Get("Content-Type"); ct != "application/json" {
|
|
||||||
t.Fatalf("unexpected content type: %s", ct)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer r.Body.Close()
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&obs.Body); err != nil {
|
|
||||||
t.Fatalf("failed to decode request body: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
_, _ = w.Write([]byte(`{
|
|
||||||
"choices": [{"message": {"role": "assistant", "content": "hello from model"}}],
|
|
||||||
"usage": {"prompt_tokens": 11, "completion_tokens": 22, "total_tokens": 33}
|
|
||||||
}`))
|
|
||||||
}))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
|
||||||
BaseURL: ts.URL + "/v1",
|
|
||||||
Timeout: 2 * time.Second,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected constructor error: %v", err)
|
|
||||||
}
|
|
||||||
t.Setenv("SCRIPTORIUM_TEST_API_KEY", "secret-key")
|
|
||||||
|
|
||||||
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
|
||||||
{Role: "system", Content: "You are helpful."},
|
|
||||||
{Role: "user", Content: "Say hello"},
|
|
||||||
}},
|
|
||||||
Target: domain.ExecutionTarget{
|
|
||||||
Model: "gpt-test",
|
|
||||||
Temperature: 0.4,
|
|
||||||
MaxTokens: 123,
|
|
||||||
TopP: 0.7,
|
|
||||||
ServiceTier: "priority",
|
|
||||||
APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY",
|
|
||||||
},
|
|
||||||
StructuredOutput: &domain.StructuredOutputSpec{
|
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
|
||||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
|
||||||
Name: "weather_schema",
|
|
||||||
Strict: true,
|
|
||||||
Schema: map[string]any{
|
|
||||||
"type": "object",
|
|
||||||
"properties": map[string]any{
|
|
||||||
"location": map[string]any{"type": "string"},
|
|
||||||
},
|
|
||||||
"required": []any{"location"},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if resp.Content != "hello from model" {
|
|
||||||
t.Fatalf("unexpected content: %q", resp.Content)
|
|
||||||
}
|
|
||||||
if resp.Usage.PromptTokens != 11 || resp.Usage.CompletionTokens != 22 || resp.Usage.TotalTokens != 33 {
|
|
||||||
t.Fatalf("unexpected usage: %+v", resp.Usage)
|
|
||||||
}
|
|
||||||
|
|
||||||
if obs.Authorization != "Bearer secret-key" {
|
|
||||||
t.Fatalf("unexpected Authorization header: %q", obs.Authorization)
|
|
||||||
}
|
|
||||||
if got, ok := obs.Body["model"].(string); !ok || got != "gpt-test" {
|
|
||||||
t.Fatalf("unexpected model payload: %#v", obs.Body["model"])
|
|
||||||
}
|
|
||||||
if got, ok := obs.Body["temperature"].(float64); !ok || got != 0.4 {
|
|
||||||
t.Fatalf("unexpected temperature payload: %#v", obs.Body["temperature"])
|
|
||||||
}
|
|
||||||
if got, ok := obs.Body["max_tokens"].(float64); !ok || got != 123 {
|
|
||||||
t.Fatalf("unexpected max_tokens payload: %#v", obs.Body["max_tokens"])
|
|
||||||
}
|
|
||||||
if got, ok := obs.Body["top_p"].(float64); !ok || got != 0.7 {
|
|
||||||
t.Fatalf("unexpected top_p payload: %#v", obs.Body["top_p"])
|
|
||||||
}
|
|
||||||
if got, ok := obs.Body["service_tier"].(string); !ok || got != "priority" {
|
|
||||||
t.Fatalf("unexpected service_tier payload: %#v", obs.Body["service_tier"])
|
|
||||||
}
|
|
||||||
|
|
||||||
msgs, ok := obs.Body["messages"].([]any)
|
|
||||||
if !ok || len(msgs) != 2 {
|
|
||||||
t.Fatalf("unexpected messages payload: %#v", obs.Body["messages"])
|
|
||||||
}
|
|
||||||
msg0 := msgs[0].(map[string]any)
|
|
||||||
if msg0["role"] != "system" || msg0["content"] != "You are helpful." {
|
|
||||||
t.Fatalf("unexpected first message: %#v", msg0)
|
|
||||||
}
|
|
||||||
msg1 := msgs[1].(map[string]any)
|
|
||||||
if msg1["role"] != "user" || msg1["content"] != "Say hello" {
|
|
||||||
t.Fatalf("unexpected second message: %#v", msg1)
|
|
||||||
}
|
|
||||||
|
|
||||||
responseFormat, ok := obs.Body["response_format"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected response_format payload, got %#v", obs.Body["response_format"])
|
|
||||||
}
|
|
||||||
if responseFormat["type"] != "json_schema" {
|
|
||||||
t.Fatalf("expected response_format.type=json_schema, got %#v", responseFormat["type"])
|
|
||||||
}
|
|
||||||
jsonSchema, ok := responseFormat["json_schema"].(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected response_format.json_schema map, got %#v", responseFormat["json_schema"])
|
|
||||||
}
|
|
||||||
if jsonSchema["name"] != "weather_schema" {
|
|
||||||
t.Fatalf("expected json_schema.name weather_schema, got %#v", jsonSchema["name"])
|
|
||||||
}
|
|
||||||
if jsonSchema["strict"] != true {
|
|
||||||
t.Fatalf("expected json_schema.strict=true, got %#v", jsonSchema["strict"])
|
|
||||||
}
|
|
||||||
if _, ok := jsonSchema["schema"].(map[string]any); !ok {
|
|
||||||
t.Fatalf("expected json_schema.schema object, got %#v", jsonSchema["schema"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientOmitsResponseFormatWhenNoStructuredOutput(t *testing.T) {
|
|
||||||
var observedBody map[string]any
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
defer r.Body.Close()
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
|
||||||
t.Fatalf("failed to decode request body: %v", err)
|
|
||||||
}
|
|
||||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
|
||||||
}))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if _, exists := observedBody["response_format"]; exists {
|
|
||||||
t.Fatalf("expected response_format omitted, got %#v", observedBody["response_format"])
|
|
||||||
}
|
|
||||||
if _, exists := observedBody["service_tier"]; exists {
|
|
||||||
t.Fatalf("expected service_tier omitted, got %#v", observedBody["service_tier"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientOmitsReasoningEffortAndExtraParams(t *testing.T) {
|
|
||||||
var observedBody map[string]any
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
defer r.Body.Close()
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&observedBody); err != nil {
|
|
||||||
t.Fatalf("failed to decode request body: %v", err)
|
|
||||||
}
|
|
||||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
|
||||||
}))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{
|
|
||||||
Model: "model",
|
|
||||||
ReasoningEffort: "high",
|
|
||||||
ExtraParams: map[string]string{
|
|
||||||
"provider_option": "on",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if _, exists := observedBody["reasoning_effort"]; exists {
|
|
||||||
t.Fatalf("expected reasoning_effort omitted, got %#v", observedBody["reasoning_effort"])
|
|
||||||
}
|
|
||||||
if _, exists := observedBody["extra_params"]; exists {
|
|
||||||
t.Fatalf("expected extra_params omitted, got %#v", observedBody["extra_params"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) {
|
|
||||||
hadAuth := false
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
hadAuth = r.Header.Get("Authorization") != ""
|
|
||||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
|
||||||
}))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{Model: "model"},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if hadAuth {
|
|
||||||
t.Fatal("did not expect Authorization header")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientAPIKeyEnvMissing(t *testing.T) {
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
|
||||||
}))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "model"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{APIKeyEnv: "SCRIPTORIUM_MISSING_KEY"},
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected missing API key env error")
|
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientModelFallbackFromConfig(t *testing.T) {
|
|
||||||
gotModel := ""
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
var body map[string]any
|
|
||||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
||||||
if m, ok := body["model"].(string); ok {
|
|
||||||
gotModel = m
|
|
||||||
}
|
|
||||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
|
||||||
}))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "default-model"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if gotModel != "default-model" {
|
|
||||||
t.Fatalf("expected default model, got %q", gotModel)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientEndpointOverride(t *testing.T) {
|
|
||||||
defaultHit := false
|
|
||||||
overrideHit := false
|
|
||||||
|
|
||||||
defaultServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
defaultHit = true
|
|
||||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"default"}}]}`))
|
|
||||||
}))
|
|
||||||
defer defaultServer.Close()
|
|
||||||
|
|
||||||
overrideServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
overrideHit = true
|
|
||||||
if r.URL.Path != "/v1/chat/completions" {
|
|
||||||
t.Fatalf("unexpected path: %s", r.URL.Path)
|
|
||||||
}
|
|
||||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"override"}}]}`))
|
|
||||||
}))
|
|
||||||
defer overrideServer.Close()
|
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: defaultServer.URL + "/v1", Model: "m"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{Endpoint: overrideServer.URL + "/v1"},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if resp.Content != "override" {
|
|
||||||
t.Fatalf("expected override response, got %q", resp.Content)
|
|
||||||
}
|
|
||||||
if defaultHit {
|
|
||||||
t.Fatal("default endpoint should not have been called")
|
|
||||||
}
|
|
||||||
if !overrideHit {
|
|
||||||
t.Fatal("override endpoint should have been called")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientNon2xxError(t *testing.T) {
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
|
||||||
_, _ = w.Write([]byte(`{"error":"bad request payload"}`))
|
|
||||||
}))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "m"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected non-2xx error")
|
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrUnexpectedStatus) {
|
|
||||||
t.Fatalf("expected ErrUnexpectedStatus, got %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "400") || !strings.Contains(err.Error(), "bad request payload") {
|
|
||||||
t.Fatalf("expected status/body details, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientMalformedResponseInvalidJSON(t *testing.T) {
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
_, _ = w.Write([]byte(`{not valid json`))
|
|
||||||
}))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "m"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected malformed response error")
|
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrMalformedResponse) {
|
|
||||||
t.Fatalf("expected ErrMalformedResponse, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientMalformedResponseMissingChoices(t *testing.T) {
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
_, _ = w.Write([]byte(`{"choices": []}`))
|
|
||||||
}))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "m"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected malformed response error")
|
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrMalformedResponse) {
|
|
||||||
t.Fatalf("expected ErrMalformedResponse, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientTimeout(t *testing.T) {
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
time.Sleep(250 * time.Millisecond)
|
|
||||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
|
||||||
}))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
|
||||||
BaseURL: ts.URL + "/v1",
|
|
||||||
Model: "m",
|
|
||||||
Timeout: 50 * time.Millisecond,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected timeout error")
|
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrRequestFailed) {
|
|
||||||
t.Fatalf("expected ErrRequestFailed, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientRequestTimeoutOverride(t *testing.T) {
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
time.Sleep(100 * time.Millisecond)
|
|
||||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
|
|
||||||
}))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
|
||||||
BaseURL: ts.URL + "/v1",
|
|
||||||
Model: "m",
|
|
||||||
Timeout: 50 * time.Millisecond,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{TimeoutSeconds: 1},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected request-level timeout override to succeed, got %v", err)
|
|
||||||
}
|
|
||||||
if resp.Content != "ok" {
|
|
||||||
t.Fatalf("expected response content ok, got %q", resp.Content)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientNegativeTimeoutRejected(t *testing.T) {
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
|
||||||
BaseURL: "http://example.com/v1",
|
|
||||||
Model: "m",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{TimeoutSeconds: -1},
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected invalid request error")
|
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientAllowsEmptyConfiguredBaseURL(t *testing.T) {
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
|
||||||
BaseURL: "",
|
|
||||||
Model: "m",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected empty configured base URL to be allowed, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{Endpoint: "http://localhost:9999/v1"},
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected request failure due to unreachable endpoint")
|
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrRequestFailed) {
|
|
||||||
t.Fatalf("expected ErrRequestFailed with request endpoint override, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOpenAICompatibleClientRequiresEndpointWhenUnsetEverywhere(t *testing.T) {
|
|
||||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
|
|
||||||
BaseURL: "",
|
|
||||||
Model: "m",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = client.Generate(context.Background(), domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
|
|
||||||
Target: domain.ExecutionTarget{},
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected endpoint-required error")
|
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,167 +0,0 @@
|
|||||||
package profile
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrProfileNotFound = errors.New("execution profile not found")
|
|
||||||
ErrInvalidYAML = errors.New("invalid YAML format")
|
|
||||||
ErrInvalidProfile = errors.New("invalid execution profile configuration")
|
|
||||||
ErrRawAPIKeyNotAllowed = errors.New("raw api_key is not allowed; use api_key_env")
|
|
||||||
)
|
|
||||||
|
|
||||||
type filesystemRepository struct {
|
|
||||||
dir string
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewFilesystemRepository(dir string) Repository {
|
|
||||||
return &filesystemRepository{dir: dir}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
|
||||||
if strings.TrimSpace(id) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read profile directory: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var matches []profileMatch
|
|
||||||
for _, fullPath := range files {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
relPath := filecatalog.RelativePath(r.dir, fullPath)
|
|
||||||
fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
|
|
||||||
data, err := os.ReadFile(fullPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
|
|
||||||
}
|
|
||||||
metadata := readProfileFileMetadata(data)
|
|
||||||
idMatch := fileMatch || metadata.id == id
|
|
||||||
if metadata.hasRawAPIKey {
|
|
||||||
if idMatch {
|
|
||||||
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
var prof domain.ExecutionProfile
|
|
||||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
|
||||||
decoder.KnownFields(true)
|
|
||||||
if err := decoder.Decode(&prof); err != nil {
|
|
||||||
if idMatch {
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if prof.ID != id {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := validateProfile(&prof); err != nil {
|
|
||||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
||||||
return nil, fmt.Errorf("%w: %s", err, relPath)
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
|
||||||
}
|
|
||||||
matches = append(matches, profileMatch{
|
|
||||||
profile: &prof,
|
|
||||||
path: relPath,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(matches) > 1 {
|
|
||||||
paths := make([]string, 0, len(matches))
|
|
||||||
for _, match := range matches {
|
|
||||||
paths = append(paths, match.path)
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("%w: duplicate execution profile id %q found in: %s", ErrInvalidProfile, id, strings.Join(paths, ", "))
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(matches) == 1 {
|
|
||||||
return matches[0].profile, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, ErrProfileNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
type profileMatch struct {
|
|
||||||
profile *domain.ExecutionProfile
|
|
||||||
path string
|
|
||||||
}
|
|
||||||
|
|
||||||
type profileFileMetadata struct {
|
|
||||||
id string
|
|
||||||
hasRawAPIKey bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func readProfileFileMetadata(data []byte) profileFileMetadata {
|
|
||||||
var node yaml.Node
|
|
||||||
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&node); err != nil {
|
|
||||||
return profileFileMetadata{}
|
|
||||||
}
|
|
||||||
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
|
|
||||||
return profileFileMetadata{}
|
|
||||||
}
|
|
||||||
mapping := node.Content[0]
|
|
||||||
if mapping.Kind != yaml.MappingNode {
|
|
||||||
return profileFileMetadata{}
|
|
||||||
}
|
|
||||||
|
|
||||||
var metadata profileFileMetadata
|
|
||||||
for i := 0; i+1 < len(mapping.Content); i += 2 {
|
|
||||||
key := mapping.Content[i]
|
|
||||||
value := mapping.Content[i+1]
|
|
||||||
switch key.Value {
|
|
||||||
case "id":
|
|
||||||
metadata.id = strings.TrimSpace(value.Value)
|
|
||||||
case "api_key":
|
|
||||||
metadata.hasRawAPIKey = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return metadata
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateProfile(p *domain.ExecutionProfile) error {
|
|
||||||
if strings.TrimSpace(p.ID) == "" {
|
|
||||||
return errors.New("id is required")
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(p.Endpoint) == "" {
|
|
||||||
return errors.New("endpoint is required")
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(p.Model) == "" {
|
|
||||||
return errors.New("model is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
if p.Temperature < 0 || p.Temperature > 2 {
|
|
||||||
return errors.New("temperature must be between 0 and 2")
|
|
||||||
}
|
|
||||||
if p.MaxTokens < 0 {
|
|
||||||
return errors.New("max_tokens must be greater than or equal to 0")
|
|
||||||
}
|
|
||||||
if p.TopP < 0 || p.TopP > 1 {
|
|
||||||
return errors.New("top_p must be between 0 and 1")
|
|
||||||
}
|
|
||||||
if p.TimeoutSeconds < 0 {
|
|
||||||
return errors.New("timeout_seconds must be greater than or equal to 0")
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
package profile
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Repository loads execution profiles.
|
|
||||||
type Repository interface {
|
|
||||||
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
|
|
||||||
}
|
|
||||||
@@ -1,205 +0,0 @@
|
|||||||
package profile
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFilesystemRepository_GetProfile(t *testing.T) {
|
|
||||||
tmpDir, err := os.MkdirTemp("", "execution_profile_test")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer os.RemoveAll(tmpDir)
|
|
||||||
|
|
||||||
files, err := os.ReadDir("testdata")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read testdata: %v", err)
|
|
||||||
}
|
|
||||||
for _, f := range files {
|
|
||||||
src := filepath.Join("testdata", f.Name())
|
|
||||||
dst := filepath.Join(tmpDir, f.Name())
|
|
||||||
data, err := os.ReadFile(src)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(dst, data, 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
repo := NewFilesystemRepository(tmpDir)
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
t.Run("valid local profile", func(t *testing.T) {
|
|
||||||
p, err := repo.GetProfile(ctx, "local-default")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.ID != "local-default" {
|
|
||||||
t.Fatalf("unexpected id: %q", p.ID)
|
|
||||||
}
|
|
||||||
if p.Endpoint == "" || p.Model == "" {
|
|
||||||
t.Fatalf("expected endpoint/model to be set: %+v", p)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
|
||||||
p, err := repo.GetProfile(ctx, "local-secure")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.APIKeyEnv != "SCRIPTORIUM_API_KEY" {
|
|
||||||
t.Fatalf("unexpected api_key_env: %q", p.APIKeyEnv)
|
|
||||||
}
|
|
||||||
if p.ReasoningEffort != "medium" {
|
|
||||||
t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort)
|
|
||||||
}
|
|
||||||
if p.ServiceTier != "priority" {
|
|
||||||
t.Fatalf("unexpected service_tier: %q", p.ServiceTier)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("valid nested profile", func(t *testing.T) {
|
|
||||||
nestedDir := filepath.Join(tmpDir, "local")
|
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
writeProfileTestFile(t, filepath.Join(nestedDir, "nested-local.yaml"), `
|
|
||||||
id: nested-local
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: nested-model
|
|
||||||
temperature: 0.1
|
|
||||||
`)
|
|
||||||
|
|
||||||
p, err := repo.GetProfile(ctx, "nested-local")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.Model != "nested-model" {
|
|
||||||
t.Fatalf("unexpected model: %q", p.Model)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) {
|
|
||||||
writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), `
|
|
||||||
id: duplicate-profile
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: first-model
|
|
||||||
`)
|
|
||||||
nestedDir := filepath.Join(tmpDir, "duplicates")
|
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
writeProfileTestFile(t, filepath.Join(nestedDir, "duplicate-profile-b.yaml"), `
|
|
||||||
id: duplicate-profile
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: second-model
|
|
||||||
`)
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(ctx, "duplicate-profile")
|
|
||||||
if !errors.Is(err, ErrInvalidProfile) {
|
|
||||||
t.Fatalf("expected duplicate profile to return ErrInvalidProfile, got %v", err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"duplicate execution profile id", "duplicate-profile-a.yaml", filepath.Join("duplicates", "duplicate-profile-b.yaml")} {
|
|
||||||
if !strings.Contains(err.Error(), want) {
|
|
||||||
t.Fatalf("expected error to contain %q, got %v", want, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("nested raw api_key rejected for likely target file", func(t *testing.T) {
|
|
||||||
nestedDir := filepath.Join(tmpDir, "secure")
|
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
writeProfileTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), `
|
|
||||||
id: nested_raw_api_key
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: m
|
|
||||||
api_key: secret
|
|
||||||
`)
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(ctx, "nested_raw_api_key")
|
|
||||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
||||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), filepath.Join("secure", "not_named_like_id.yaml")) {
|
|
||||||
t.Fatalf("expected nested path in error, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("raw api_key in non-target profile is ignored", func(t *testing.T) {
|
|
||||||
writeProfileTestFile(t, filepath.Join(tmpDir, "raw-api-key-non-target.yaml"), `
|
|
||||||
id: raw-api-key-non-target
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: m
|
|
||||||
api_key: secret
|
|
||||||
`)
|
|
||||||
|
|
||||||
_, err := repo.GetProfile(ctx, "does-not-exist-with-raw-key-nearby")
|
|
||||||
if !errors.Is(err, ErrProfileNotFound) {
|
|
||||||
t.Fatalf("expected ErrProfileNotFound for non-target raw api_key file, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("invalid yaml", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "invalid_yaml")
|
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
|
||||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing id", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "missing_id")
|
|
||||||
if !errors.Is(err, ErrProfileNotFound) {
|
|
||||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing endpoint", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "missing-endpoint")
|
|
||||||
if !errors.Is(err, ErrInvalidProfile) {
|
|
||||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("missing model", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "missing-model")
|
|
||||||
if !errors.Is(err, ErrInvalidProfile) {
|
|
||||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("unknown field", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "unknown_field")
|
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
|
||||||
t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("raw api_key rejected", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "raw_api_key")
|
|
||||||
if !errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
|
||||||
t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("profile not found", func(t *testing.T) {
|
|
||||||
_, err := repo.GetProfile(ctx, "does-not-exist")
|
|
||||||
if !errors.Is(err, ErrProfileNotFound) {
|
|
||||||
t.Fatalf("expected ErrProfileNotFound, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeProfileTestFile(t *testing.T, path string, content string) {
|
|
||||||
t.Helper()
|
|
||||||
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
|
|
||||||
t.Fatalf("failed to write profile test file %q: %v", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
3
internal/profile/testdata/invalid_yaml.yaml
vendored
3
internal/profile/testdata/invalid_yaml.yaml
vendored
@@ -1,3 +0,0 @@
|
|||||||
id: invalid_yaml
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: [broken
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
id: missing-endpoint
|
|
||||||
model: gpt-4o-mini
|
|
||||||
2
internal/profile/testdata/missing_id.yaml
vendored
2
internal/profile/testdata/missing_id.yaml
vendored
@@ -1,2 +0,0 @@
|
|||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: gpt-4o-mini
|
|
||||||
2
internal/profile/testdata/missing_model.yaml
vendored
2
internal/profile/testdata/missing_model.yaml
vendored
@@ -1,2 +0,0 @@
|
|||||||
id: missing-model
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
4
internal/profile/testdata/raw_api_key.yaml
vendored
4
internal/profile/testdata/raw_api_key.yaml
vendored
@@ -1,4 +0,0 @@
|
|||||||
id: raw-api-key
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: gpt-4o-mini
|
|
||||||
api_key: super-secret-should-not-be-here
|
|
||||||
4
internal/profile/testdata/unknown_field.yaml
vendored
4
internal/profile/testdata/unknown_field.yaml
vendored
@@ -1,4 +0,0 @@
|
|||||||
id: unknown-field
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: gpt-4o-mini
|
|
||||||
foo: bar
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
id: local-default
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: gpt-4o-mini
|
|
||||||
temperature: 0.2
|
|
||||||
max_tokens: 700
|
|
||||||
top_p: 1.0
|
|
||||||
timeout_seconds: 120
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: local-secure
|
|
||||||
endpoint: http://localhost:8000/v1
|
|
||||||
model: gpt-4o-mini
|
|
||||||
api_key_env: SCRIPTORIUM_API_KEY
|
|
||||||
service_tier: priority
|
|
||||||
reasoning_effort: medium
|
|
||||||
extra_params:
|
|
||||||
provider: local
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
package prompt
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
"text/template"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrMissingRequiredInput = errors.New("missing required input artifact")
|
|
||||||
ErrUnknownInput = errors.New("referenced unknown input artifact")
|
|
||||||
ErrInvalidTemplate = errors.New("invalid prompt template")
|
|
||||||
ErrRenderFailure = errors.New("prompt render failure")
|
|
||||||
ErrInvalidMessageRole = errors.New("invalid or empty message role")
|
|
||||||
)
|
|
||||||
|
|
||||||
type goRenderer struct{}
|
|
||||||
|
|
||||||
func NewGoRenderer() Renderer {
|
|
||||||
return &goRenderer{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) {
|
|
||||||
if definition == nil {
|
|
||||||
return nil, fmt.Errorf("%w: nil prompt definition", ErrRenderFailure)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. Verify required inputs
|
|
||||||
for _, in := range definition.Inputs {
|
|
||||||
if !in.Required {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
art, ok := inputs[in.Name]
|
|
||||||
if !ok || art == nil {
|
|
||||||
return nil, fmt.Errorf("%w: %s", ErrMissingRequiredInput, in.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Setup template functions
|
|
||||||
funcs := template.FuncMap{
|
|
||||||
"input": func(name string) (string, error) {
|
|
||||||
art, ok := inputs[name]
|
|
||||||
if !ok || art == nil {
|
|
||||||
return "", fmt.Errorf("%w: %s", ErrUnknownInput, name)
|
|
||||||
}
|
|
||||||
return string(art.Body), nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
var renderedMessages []domain.RenderedMessage
|
|
||||||
|
|
||||||
for i, tmplMsg := range definition.Templates {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
if tmplMsg.Role == "" {
|
|
||||||
return nil, fmt.Errorf("%w: message %d", ErrInvalidMessageRole, i)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse and execute template
|
|
||||||
tmpl, err := template.New(fmt.Sprintf("msg_%d", i)).Funcs(funcs).Option("missingkey=error").Parse(tmplMsg.Content)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidTemplate, i, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
if err := tmpl.Execute(&buf, vars); err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: message %d: %w", ErrRenderFailure, i, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
renderedMessages = append(renderedMessages, domain.RenderedMessage{
|
|
||||||
Role: tmplMsg.Role,
|
|
||||||
Content: buf.String(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return &domain.RenderedPrompt{
|
|
||||||
Messages: renderedMessages,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
package prompt
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Renderer renders prompt templates using named artifacts and variables.
|
|
||||||
type Renderer interface {
|
|
||||||
Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error)
|
|
||||||
}
|
|
||||||
@@ -1,212 +0,0 @@
|
|||||||
package prompt
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestGoRenderer_Render(t *testing.T) {
|
|
||||||
renderer := NewGoRenderer()
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
inputs := map[string]*domain.Artifact{
|
|
||||||
"transcript": {Body: []byte("The quick brown fox.")},
|
|
||||||
}
|
|
||||||
vars := map[string]string{
|
|
||||||
"role": "helpful assistant",
|
|
||||||
"tone": "concise",
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("rendering inline message content", func(t *testing.T) {
|
|
||||||
def := &domain.PromptDefinition{
|
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
|
||||||
Templates: []domain.PromptMessageTemplate{
|
|
||||||
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if len(res.Messages) != 1 {
|
|
||||||
t.Fatalf("expected 1 message, got %d", len(res.Messages))
|
|
||||||
}
|
|
||||||
if res.Messages[0].Content != "Analyze this: The quick brown fox." {
|
|
||||||
t.Fatalf("unexpected rendered content: %q", res.Messages[0].Content)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("rendering file-backed message content loaded into prompt definition", func(t *testing.T) {
|
|
||||||
def := &domain.PromptDefinition{
|
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
|
||||||
Templates: []domain.PromptMessageTemplate{
|
|
||||||
{Role: "user", Content: "From file: {{input \"transcript\"}}", ContentFile: "/tmp/user.tmpl"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if got := res.Messages[0].Content; got != "From file: The quick brown fox." {
|
|
||||||
t.Fatalf("unexpected file-backed render result: %q", got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("rendering system and user messages", func(t *testing.T) {
|
|
||||||
def := &domain.PromptDefinition{
|
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
|
||||||
Templates: []domain.PromptMessageTemplate{
|
|
||||||
{Role: "system", Content: "You are a {{.role}}."},
|
|
||||||
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if len(res.Messages) != 2 {
|
|
||||||
t.Fatalf("expected 2 messages, got %d", len(res.Messages))
|
|
||||||
}
|
|
||||||
if res.Messages[0].Role != "system" || res.Messages[1].Role != "user" {
|
|
||||||
t.Fatalf("unexpected roles: %#v", res.Messages)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("accessing vars", func(t *testing.T) {
|
|
||||||
def := &domain.PromptDefinition{
|
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
|
||||||
Templates: []domain.PromptMessageTemplate{
|
|
||||||
{Role: "system", Content: "Speak in a {{.tone}} tone."},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if res.Messages[0].Content != "Speak in a concise tone." {
|
|
||||||
t.Fatalf("unexpected vars rendering: %q", res.Messages[0].Content)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("inserting required input artifact", func(t *testing.T) {
|
|
||||||
def := &domain.PromptDefinition{
|
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
|
||||||
Templates: []domain.PromptMessageTemplate{
|
|
||||||
{Role: "user", Content: "{{input \"transcript\"}}"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if res.Messages[0].Content != "The quick brown fox." {
|
|
||||||
t.Fatalf("unexpected required input rendering: %q", res.Messages[0].Content)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("optional input absent and not referenced", func(t *testing.T) {
|
|
||||||
def := &domain.PromptDefinition{
|
|
||||||
Inputs: []domain.PromptInput{
|
|
||||||
{Name: "transcript", Required: true},
|
|
||||||
{Name: "glossary", Required: false},
|
|
||||||
},
|
|
||||||
Templates: []domain.PromptMessageTemplate{
|
|
||||||
{Role: "user", Content: "Transcript: {{input \"transcript\"}}"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := renderer.Render(ctx, def, inputs, vars)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if len(res.Messages) != 1 {
|
|
||||||
t.Fatalf("expected one rendered message, got %d", len(res.Messages))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("optional input absent but referenced, expecting failure", func(t *testing.T) {
|
|
||||||
def := &domain.PromptDefinition{
|
|
||||||
Inputs: []domain.PromptInput{
|
|
||||||
{Name: "transcript", Required: true},
|
|
||||||
{Name: "glossary", Required: false},
|
|
||||||
},
|
|
||||||
Templates: []domain.PromptMessageTemplate{
|
|
||||||
{Role: "user", Content: "Glossary: {{input \"glossary\"}}"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
|
||||||
if !errors.Is(err, ErrRenderFailure) {
|
|
||||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrUnknownInput) {
|
|
||||||
t.Fatalf("expected ErrUnknownInput, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("required input missing, expecting failure", func(t *testing.T) {
|
|
||||||
def := &domain.PromptDefinition{
|
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
|
||||||
Templates: []domain.PromptMessageTemplate{
|
|
||||||
{Role: "user", Content: "Analyze this: {{input \"transcript\"}}"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, map[string]*domain.Artifact{}, vars)
|
|
||||||
if !errors.Is(err, ErrMissingRequiredInput) {
|
|
||||||
t.Fatalf("expected ErrMissingRequiredInput, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("invalid template syntax", func(t *testing.T) {
|
|
||||||
def := &domain.PromptDefinition{
|
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
|
||||||
Templates: []domain.PromptMessageTemplate{
|
|
||||||
{Role: "user", Content: "Hello {{.unclosed"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
|
||||||
if !errors.Is(err, ErrInvalidTemplate) {
|
|
||||||
t.Fatalf("expected ErrInvalidTemplate, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("unknown input reference", func(t *testing.T) {
|
|
||||||
def := &domain.PromptDefinition{
|
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
|
||||||
Templates: []domain.PromptMessageTemplate{
|
|
||||||
{Role: "user", Content: "Hello {{input \"ghost\"}}"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
|
||||||
if !errors.Is(err, ErrRenderFailure) {
|
|
||||||
t.Fatalf("expected ErrRenderFailure, got %v", err)
|
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrUnknownInput) {
|
|
||||||
t.Fatalf("expected ErrUnknownInput, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("empty message role", func(t *testing.T) {
|
|
||||||
def := &domain.PromptDefinition{
|
|
||||||
Inputs: []domain.PromptInput{{Name: "transcript", Required: true}},
|
|
||||||
Templates: []domain.PromptMessageTemplate{
|
|
||||||
{Role: "", Content: "Hello"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
_, err := renderer.Render(ctx, def, inputs, vars)
|
|
||||||
if !errors.Is(err, ErrInvalidMessageRole) {
|
|
||||||
t.Fatalf("expected ErrInvalidMessageRole, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,293 +0,0 @@
|
|||||||
package promptdef
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/filecatalog"
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
|
|
||||||
ErrInvalidYAML = errors.New("invalid YAML format")
|
|
||||||
ErrInvalidPromptDefinition = errors.New("invalid prompt definition configuration")
|
|
||||||
)
|
|
||||||
|
|
||||||
type filesystemRepository struct {
|
|
||||||
dir string
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptDefinitionFile struct {
|
|
||||||
ID string `yaml:"id"`
|
|
||||||
Version string `yaml:"version"`
|
|
||||||
DefaultProfile *string `yaml:"default_profile"`
|
|
||||||
Description string `yaml:"description"`
|
|
||||||
Inputs []promptInputFile `yaml:"inputs"`
|
|
||||||
Messages []promptMessageFile `yaml:"messages"`
|
|
||||||
Output promptOutputContractFile `yaml:"output"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptInputFile struct {
|
|
||||||
Name string `yaml:"name"`
|
|
||||||
Required bool `yaml:"required"`
|
|
||||||
ContentType string `yaml:"content_type"`
|
|
||||||
Description string `yaml:"description"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptMessageFile struct {
|
|
||||||
Role string `yaml:"role"`
|
|
||||||
Content string `yaml:"content"`
|
|
||||||
ContentFile string `yaml:"content_file"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptOutputContractFile struct {
|
|
||||||
Format domain.OutputFormat `yaml:"format"`
|
|
||||||
ValidationMode domain.ValidationMode `yaml:"validation_mode"`
|
|
||||||
SchemaPath string `yaml:"schema_path"`
|
|
||||||
RepairAttempts int `yaml:"repair_attempts"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewFilesystemRepository(dir string) Repository {
|
|
||||||
return &filesystemRepository{dir: dir}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
|
|
||||||
if strings.TrimSpace(id) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := filecatalog.FindYAMLFiles(ctx, r.dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var matches []promptDefinitionMatch
|
|
||||||
for _, fullPath := range files {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
relPath := filecatalog.RelativePath(r.dir, fullPath)
|
|
||||||
fileMatch := filecatalog.Stem(filepath.Base(fullPath)) == id
|
|
||||||
|
|
||||||
raw, err := loadPromptDefinitionFile(fullPath)
|
|
||||||
if err != nil {
|
|
||||||
if fileMatch || promptDefinitionFileHasID(fullPath, id) {
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
def, err := normalizePromptDefinition(raw, fullPath)
|
|
||||||
if err != nil {
|
|
||||||
if fileMatch || strings.TrimSpace(raw.ID) == id {
|
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, relPath, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if def.ID != id {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if version != "" && def.Version != version {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
matches = append(matches, promptDefinitionMatch{
|
|
||||||
def: def,
|
|
||||||
path: relPath,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(matches) > 1 {
|
|
||||||
paths := make([]string, 0, len(matches))
|
|
||||||
for _, match := range matches {
|
|
||||||
paths = append(paths, match.path)
|
|
||||||
}
|
|
||||||
if version != "" {
|
|
||||||
return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", "))
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", "))
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(matches) == 1 {
|
|
||||||
return matches[0].def, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, ErrPromptDefinitionNotFound
|
|
||||||
}
|
|
||||||
|
|
||||||
type promptDefinitionMatch struct {
|
|
||||||
def *domain.PromptDefinition
|
|
||||||
path string
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadPromptDefinitionFile(path string) (*promptDefinitionFile, error) {
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read prompt definition file: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var raw promptDefinitionFile
|
|
||||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
|
||||||
decoder.KnownFields(true)
|
|
||||||
if err := decoder.Decode(&raw); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &raw, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func promptDefinitionFileHasID(path string, id string) bool {
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
var raw struct {
|
|
||||||
ID string `yaml:"id"`
|
|
||||||
}
|
|
||||||
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return strings.TrimSpace(raw.ID) == id
|
|
||||||
}
|
|
||||||
|
|
||||||
func normalizePromptDefinition(raw *promptDefinitionFile, sourcePath string) (*domain.PromptDefinition, error) {
|
|
||||||
if raw == nil {
|
|
||||||
return nil, errors.New("prompt definition is nil")
|
|
||||||
}
|
|
||||||
|
|
||||||
id := strings.TrimSpace(raw.ID)
|
|
||||||
if id == "" {
|
|
||||||
return nil, errors.New("id is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
version := strings.TrimSpace(raw.Version)
|
|
||||||
if version == "" {
|
|
||||||
return nil, errors.New("version is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(raw.Messages) == 0 {
|
|
||||||
return nil, errors.New("at least one message is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
inputs := make([]domain.PromptInput, 0, len(raw.Inputs))
|
|
||||||
seenInputNames := make(map[string]struct{}, len(raw.Inputs))
|
|
||||||
for i, in := range raw.Inputs {
|
|
||||||
name := strings.TrimSpace(in.Name)
|
|
||||||
if name == "" {
|
|
||||||
return nil, fmt.Errorf("input %d has empty name", i)
|
|
||||||
}
|
|
||||||
if _, exists := seenInputNames[name]; exists {
|
|
||||||
return nil, fmt.Errorf("duplicate input name %q", name)
|
|
||||||
}
|
|
||||||
seenInputNames[name] = struct{}{}
|
|
||||||
|
|
||||||
inputs = append(inputs, domain.PromptInput{
|
|
||||||
Name: name,
|
|
||||||
Required: in.Required,
|
|
||||||
ContentType: strings.TrimSpace(in.ContentType),
|
|
||||||
Description: strings.TrimSpace(in.Description),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
|
|
||||||
promptDir := filepath.Dir(sourcePath)
|
|
||||||
for i, msg := range raw.Messages {
|
|
||||||
role := strings.TrimSpace(msg.Role)
|
|
||||||
if role == "" {
|
|
||||||
return nil, fmt.Errorf("message %d role is required", i)
|
|
||||||
}
|
|
||||||
|
|
||||||
hasContent := strings.TrimSpace(msg.Content) != ""
|
|
||||||
hasContentFile := strings.TrimSpace(msg.ContentFile) != ""
|
|
||||||
if hasContent == hasContentFile {
|
|
||||||
return nil, fmt.Errorf("message %d (%s) must set exactly one of content or content_file", i, role)
|
|
||||||
}
|
|
||||||
|
|
||||||
templateContent := msg.Content
|
|
||||||
resolvedContentFile := ""
|
|
||||||
if hasContentFile {
|
|
||||||
resolvedPath := strings.TrimSpace(msg.ContentFile)
|
|
||||||
if !filepath.IsAbs(resolvedPath) {
|
|
||||||
resolvedPath = filepath.Join(promptDir, resolvedPath)
|
|
||||||
}
|
|
||||||
resolvedPath = filepath.Clean(resolvedPath)
|
|
||||||
|
|
||||||
body, err := os.ReadFile(resolvedPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("prompt %q message %d (%s): failed to read content_file %q: %w", id, i, role, msg.ContentFile, err)
|
|
||||||
}
|
|
||||||
templateContent = string(body)
|
|
||||||
resolvedContentFile = resolvedPath
|
|
||||||
}
|
|
||||||
|
|
||||||
templates = append(templates, domain.PromptMessageTemplate{
|
|
||||||
Role: role,
|
|
||||||
Content: templateContent,
|
|
||||||
ContentFile: resolvedContentFile,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if !isValidOutputFormat(raw.Output.Format) {
|
|
||||||
return nil, fmt.Errorf("invalid output format: %q", raw.Output.Format)
|
|
||||||
}
|
|
||||||
if !isValidValidationMode(raw.Output.ValidationMode) {
|
|
||||||
return nil, fmt.Errorf("invalid validation mode: %q", raw.Output.ValidationMode)
|
|
||||||
}
|
|
||||||
if raw.Output.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(raw.Output.SchemaPath) == "" {
|
|
||||||
return nil, errors.New("output.schema_path is required when output.validation_mode is json_schema")
|
|
||||||
}
|
|
||||||
if raw.Output.RepairAttempts < 0 {
|
|
||||||
return nil, errors.New("output.repair_attempts must be greater than or equal to 0")
|
|
||||||
}
|
|
||||||
|
|
||||||
defaultProfile := ""
|
|
||||||
if raw.DefaultProfile != nil {
|
|
||||||
defaultProfile = strings.TrimSpace(*raw.DefaultProfile)
|
|
||||||
if defaultProfile == "" {
|
|
||||||
return nil, errors.New("default_profile must be a non-empty string when set")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return &domain.PromptDefinition{
|
|
||||||
ID: id,
|
|
||||||
Version: version,
|
|
||||||
DefaultProfile: defaultProfile,
|
|
||||||
Description: strings.TrimSpace(raw.Description),
|
|
||||||
Inputs: inputs,
|
|
||||||
Templates: templates,
|
|
||||||
OutputFormat: raw.Output.Format,
|
|
||||||
Validation: domain.OutputContract{
|
|
||||||
Format: raw.Output.Format,
|
|
||||||
ValidationMode: raw.Output.ValidationMode,
|
|
||||||
SchemaPath: strings.TrimSpace(raw.Output.SchemaPath),
|
|
||||||
RepairAttempts: raw.Output.RepairAttempts,
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func isValidOutputFormat(f domain.OutputFormat) bool {
|
|
||||||
switch f {
|
|
||||||
case domain.FormatText, domain.FormatMarkdown, domain.FormatJSON:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func isValidValidationMode(m domain.ValidationMode) bool {
|
|
||||||
switch m {
|
|
||||||
case domain.ValidationNone, domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
package promptdef
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Repository loads prompt definitions.
|
|
||||||
type Repository interface {
|
|
||||||
GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error)
|
|
||||||
}
|
|
||||||
@@ -1,316 +0,0 @@
|
|||||||
package promptdef
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFilesystemRepository_GetPromptDefinition(t *testing.T) {
|
|
||||||
tmpDir := t.TempDir()
|
|
||||||
if err := copyTree("testdata", tmpDir); err != nil {
|
|
||||||
t.Fatalf("failed to copy testdata: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
repo := NewFilesystemRepository(tmpDir)
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
t.Run("valid inline prompt", func(t *testing.T) {
|
|
||||||
p, err := repo.GetPromptDefinition(ctx, "valid-inline", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.ID != "valid-inline" {
|
|
||||||
t.Fatalf("unexpected id: %q", p.ID)
|
|
||||||
}
|
|
||||||
if p.Version != "1.0.0" {
|
|
||||||
t.Fatalf("unexpected version: %q", p.Version)
|
|
||||||
}
|
|
||||||
if p.OutputFormat != domain.FormatMarkdown {
|
|
||||||
t.Fatalf("unexpected output format: %q", p.OutputFormat)
|
|
||||||
}
|
|
||||||
if p.Validation.ValidationMode != domain.ValidationBasic {
|
|
||||||
t.Fatalf("unexpected validation mode: %q", p.Validation.ValidationMode)
|
|
||||||
}
|
|
||||||
if len(p.Templates) != 2 {
|
|
||||||
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
|
|
||||||
}
|
|
||||||
if len(p.Inputs) != 1 {
|
|
||||||
t.Fatalf("expected 1 input, got %d", len(p.Inputs))
|
|
||||||
}
|
|
||||||
if p.Inputs[0].ContentType != "text/markdown" {
|
|
||||||
t.Fatalf("expected input content_type to be preserved, got %q", p.Inputs[0].ContentType)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("valid file-backed prompt", func(t *testing.T) {
|
|
||||||
p, err := repo.GetPromptDefinition(ctx, "valid-file-backed", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if len(p.Templates) != 2 {
|
|
||||||
t.Fatalf("expected 2 messages, got %d", len(p.Templates))
|
|
||||||
}
|
|
||||||
if !strings.Contains(p.Templates[1].Content, "{{input \"transcript\"}}") {
|
|
||||||
t.Fatalf("expected content_file template body to be loaded, got %q", p.Templates[1].Content)
|
|
||||||
}
|
|
||||||
if p.Templates[1].ContentFile == "" {
|
|
||||||
t.Fatal("expected ContentFile source metadata to be preserved")
|
|
||||||
}
|
|
||||||
if !filepath.IsAbs(p.Templates[1].ContentFile) {
|
|
||||||
t.Fatalf("expected resolved content_file path to be absolute, got %q", p.Templates[1].ContentFile)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) {
|
|
||||||
nestedDir := filepath.Join(tmpDir, "dnd", "recap")
|
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
writePromptTestFile(t, filepath.Join(nestedDir, "nested_recap.yaml"), `
|
|
||||||
id: nested-recap
|
|
||||||
version: "1.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content_file: ./nested_recap.user.tmpl
|
|
||||||
output:
|
|
||||||
format: markdown
|
|
||||||
validation_mode: basic
|
|
||||||
repair_attempts: 0
|
|
||||||
`)
|
|
||||||
writePromptTestFile(t, filepath.Join(nestedDir, "nested_recap.user.tmpl"), `Nested recap: {{input "transcript"}}`)
|
|
||||||
|
|
||||||
p, err := repo.GetPromptDefinition(ctx, "nested-recap", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if len(p.Templates) != 1 {
|
|
||||||
t.Fatalf("expected one template, got %d", len(p.Templates))
|
|
||||||
}
|
|
||||||
if !strings.Contains(p.Templates[0].Content, "Nested recap") {
|
|
||||||
t.Fatalf("expected nested content file body, got %q", p.Templates[0].Content)
|
|
||||||
}
|
|
||||||
if !strings.Contains(p.Templates[0].ContentFile, filepath.Join("dnd", "recap", "nested_recap.user.tmpl")) {
|
|
||||||
t.Fatalf("expected nested content file path, got %q", p.Templates[0].ContentFile)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("prompt with default_profile", func(t *testing.T) {
|
|
||||||
p, err := repo.GetPromptDefinition(ctx, "with-default-profile", "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if p.DefaultProfile != "local-default" {
|
|
||||||
t.Fatalf("unexpected default profile: %q", p.DefaultProfile)
|
|
||||||
}
|
|
||||||
if len(p.Inputs) != 1 {
|
|
||||||
t.Fatalf("expected one input, got %d", len(p.Inputs))
|
|
||||||
}
|
|
||||||
if p.Inputs[0].ContentType != "" {
|
|
||||||
t.Fatalf("expected missing content_type to remain empty, got %q", p.Inputs[0].ContentType)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("duplicate prompt IDs fail as ambiguous", func(t *testing.T) {
|
|
||||||
writePromptTestFile(t, filepath.Join(tmpDir, "duplicate_a.yaml"), `
|
|
||||||
id: duplicate-prompt
|
|
||||||
version: "1.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: First duplicate.
|
|
||||||
output:
|
|
||||||
format: markdown
|
|
||||||
validation_mode: basic
|
|
||||||
repair_attempts: 0
|
|
||||||
`)
|
|
||||||
nestedDir := filepath.Join(tmpDir, "nested")
|
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
writePromptTestFile(t, filepath.Join(nestedDir, "duplicate_b.yaml"), `
|
|
||||||
id: duplicate-prompt
|
|
||||||
version: "2.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: Second duplicate.
|
|
||||||
output:
|
|
||||||
format: markdown
|
|
||||||
validation_mode: basic
|
|
||||||
repair_attempts: 0
|
|
||||||
`)
|
|
||||||
|
|
||||||
_, err := repo.GetPromptDefinition(ctx, "duplicate-prompt", "")
|
|
||||||
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
|
||||||
t.Fatalf("expected duplicate prompt to return ErrInvalidPromptDefinition, got %v", err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"duplicate prompt definition id", "duplicate_a.yaml", filepath.Join("nested", "duplicate_b.yaml")} {
|
|
||||||
if !strings.Contains(err.Error(), want) {
|
|
||||||
t.Fatalf("expected error to contain %q, got %v", want, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("duplicate prompt ID and requested version fails as ambiguous", func(t *testing.T) {
|
|
||||||
writePromptTestFile(t, filepath.Join(tmpDir, "version_duplicate_a.yaml"), `
|
|
||||||
id: duplicate-version-prompt
|
|
||||||
version: "1.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: First duplicate version.
|
|
||||||
output:
|
|
||||||
format: markdown
|
|
||||||
validation_mode: basic
|
|
||||||
repair_attempts: 0
|
|
||||||
`)
|
|
||||||
nestedDir := filepath.Join(tmpDir, "versioned")
|
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
writePromptTestFile(t, filepath.Join(nestedDir, "version_duplicate_b.yaml"), `
|
|
||||||
id: duplicate-version-prompt
|
|
||||||
version: "1.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: Second duplicate version.
|
|
||||||
output:
|
|
||||||
format: markdown
|
|
||||||
validation_mode: basic
|
|
||||||
repair_attempts: 0
|
|
||||||
`)
|
|
||||||
|
|
||||||
_, err := repo.GetPromptDefinition(ctx, "duplicate-version-prompt", "1.0.0")
|
|
||||||
if !errors.Is(err, ErrInvalidPromptDefinition) {
|
|
||||||
t.Fatalf("expected duplicate prompt version to return ErrInvalidPromptDefinition, got %v", err)
|
|
||||||
}
|
|
||||||
for _, want := range []string{"duplicate prompt definition id", "version \"1.0.0\"", "version_duplicate_a.yaml", filepath.Join("versioned", "version_duplicate_b.yaml")} {
|
|
||||||
if !strings.Contains(err.Error(), want) {
|
|
||||||
t.Fatalf("expected error to contain %q, got %v", want, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("non-matching malformed nested prompt is ignored for not found lookup", func(t *testing.T) {
|
|
||||||
nestedDir := filepath.Join(tmpDir, "broken")
|
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
writePromptTestFile(t, filepath.Join(nestedDir, "unrelated.yaml"), "id: [")
|
|
||||||
|
|
||||||
_, err := repo.GetPromptDefinition(ctx, "does-not-exist-even-with-broken-nested-file", "")
|
|
||||||
if !errors.Is(err, ErrPromptDefinitionNotFound) {
|
|
||||||
t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("strict decode failure in nested prompt matches by YAML ID", func(t *testing.T) {
|
|
||||||
nestedDir := filepath.Join(tmpDir, "strict")
|
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
writePromptTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), `
|
|
||||||
id: nested-strict-error
|
|
||||||
version: "1.0.0"
|
|
||||||
unknown_field: true
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: Invalid because of unknown field.
|
|
||||||
output:
|
|
||||||
format: markdown
|
|
||||||
validation_mode: basic
|
|
||||||
repair_attempts: 0
|
|
||||||
`)
|
|
||||||
|
|
||||||
_, err := repo.GetPromptDefinition(ctx, "nested-strict-error", "")
|
|
||||||
if !errors.Is(err, ErrInvalidYAML) {
|
|
||||||
t.Fatalf("expected ErrInvalidYAML, got %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), filepath.Join("strict", "not_named_like_id.yaml")) {
|
|
||||||
t.Fatalf("expected nested path in error, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("version lookup", func(t *testing.T) {
|
|
||||||
_, err := repo.GetPromptDefinition(ctx, "valid-inline", "9.9.9")
|
|
||||||
if !errors.Is(err, ErrPromptDefinitionNotFound) {
|
|
||||||
t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
id string
|
|
||||||
targetErr error
|
|
||||||
errSubstrs []string
|
|
||||||
}{
|
|
||||||
{name: "invalid YAML", id: "invalid_yaml", targetErr: ErrInvalidYAML},
|
|
||||||
{name: "missing id", id: "missing_id", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"id is required"}},
|
|
||||||
{name: "no messages", id: "no_messages", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"at least one message is required"}},
|
|
||||||
{name: "both content and content_file", id: "both_content_and_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}},
|
|
||||||
{name: "neither content nor content_file", id: "neither_content_nor_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}},
|
|
||||||
{name: "missing content_file", id: "missing_content_file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"failed to read content_file"}},
|
|
||||||
{name: "duplicate input names", id: "duplicate_input_names", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"duplicate input name"}},
|
|
||||||
{name: "invalid validation mode", id: "invalid_validation_mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}},
|
|
||||||
{name: "json_schema without schema_path", id: "json_schema_without_schema_path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}},
|
|
||||||
{name: "unknown input field", id: "unknown_input_field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
_, err := repo.GetPromptDefinition(ctx, tc.id, "")
|
|
||||||
if !errors.Is(err, tc.targetErr) {
|
|
||||||
t.Fatalf("expected %v, got %v", tc.targetErr, err)
|
|
||||||
}
|
|
||||||
for _, sub := range tc.errSubstrs {
|
|
||||||
if !strings.Contains(err.Error(), sub) {
|
|
||||||
t.Fatalf("expected error to contain %q, got %v", sub, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("prompt definition not found", func(t *testing.T) {
|
|
||||||
_, err := repo.GetPromptDefinition(ctx, "does-not-exist", "")
|
|
||||||
if !errors.Is(err, ErrPromptDefinitionNotFound) {
|
|
||||||
t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func writePromptTestFile(t *testing.T, path string, content string) {
|
|
||||||
t.Helper()
|
|
||||||
if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil {
|
|
||||||
t.Fatalf("failed to write prompt test file %q: %v", path, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyTree(src, dst string) error {
|
|
||||||
return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
rel, err := filepath.Rel(src, path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if rel == "." {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
target := filepath.Join(dst, rel)
|
|
||||||
if d.IsDir() {
|
|
||||||
return os.MkdirAll(target, 0o755)
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return os.WriteFile(target, data, 0o644)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
id: both-content-and-content-file
|
|
||||||
version: "1.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: "Hi"
|
|
||||||
content_file: ./messages/user_prompt.tmpl
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: none
|
|
||||||
repair_attempts: 0
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
id: duplicate-input-names
|
|
||||||
version: "1.0.0"
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
- name: transcript
|
|
||||||
required: false
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: "Hi"
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: none
|
|
||||||
repair_attempts: 0
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: invalid-validation-mode
|
|
||||||
version: "1.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: "Hi"
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: nope
|
|
||||||
repair_attempts: 0
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: invalid-yaml
|
|
||||||
version: "1.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: [broken
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: none
|
|
||||||
repair_attempts: 0
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: json-schema-without-schema-path
|
|
||||||
version: "1.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: "Return JSON"
|
|
||||||
output:
|
|
||||||
format: json
|
|
||||||
validation_mode: json_schema
|
|
||||||
repair_attempts: 0
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
Use transcript:
|
|
||||||
{{input "transcript"}}
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
id: missing-content-file
|
|
||||||
version: "1.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content_file: ./messages/does_not_exist.tmpl
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: none
|
|
||||||
repair_attempts: 0
|
|
||||||
8
internal/promptdef/testdata/missing_id.yaml
vendored
8
internal/promptdef/testdata/missing_id.yaml
vendored
@@ -1,8 +0,0 @@
|
|||||||
version: "1.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: "Hi"
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: none
|
|
||||||
repair_attempts: 0
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
id: neither-content-nor-content-file
|
|
||||||
version: "1.0.0"
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: none
|
|
||||||
repair_attempts: 0
|
|
||||||
6
internal/promptdef/testdata/no_messages.yaml
vendored
6
internal/promptdef/testdata/no_messages.yaml
vendored
@@ -1,6 +0,0 @@
|
|||||||
id: no-messages
|
|
||||||
version: "1.0.0"
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: none
|
|
||||||
repair_attempts: 0
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
id: unknown-input-field
|
|
||||||
version: "1.0.0"
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
unknown_input_setting: true
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: "Hi"
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: none
|
|
||||||
repair_attempts: 0
|
|
||||||
@@ -1,14 +0,0 @@
|
|||||||
id: valid-file-backed
|
|
||||||
version: "1.0.0"
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
messages:
|
|
||||||
- role: system
|
|
||||||
content: "Return markdown."
|
|
||||||
- role: user
|
|
||||||
content_file: ./messages/user_prompt.tmpl
|
|
||||||
output:
|
|
||||||
format: markdown
|
|
||||||
validation_mode: basic
|
|
||||||
repair_attempts: 0
|
|
||||||
18
internal/promptdef/testdata/valid_inline.yaml
vendored
18
internal/promptdef/testdata/valid_inline.yaml
vendored
@@ -1,18 +0,0 @@
|
|||||||
id: valid-inline
|
|
||||||
version: "1.0.0"
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
content_type: text/markdown
|
|
||||||
description: Transcript content
|
|
||||||
messages:
|
|
||||||
- role: system
|
|
||||||
content: "You are concise."
|
|
||||||
- role: user
|
|
||||||
content: |
|
|
||||||
Summarize:
|
|
||||||
{{input "transcript"}}
|
|
||||||
output:
|
|
||||||
format: markdown
|
|
||||||
validation_mode: basic
|
|
||||||
repair_attempts: 0
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
id: with-default-profile
|
|
||||||
version: "1.0.0"
|
|
||||||
default_profile: local-default
|
|
||||||
inputs:
|
|
||||||
- name: transcript
|
|
||||||
required: true
|
|
||||||
messages:
|
|
||||||
- role: user
|
|
||||||
content: "Write output"
|
|
||||||
output:
|
|
||||||
format: text
|
|
||||||
validation_mode: none
|
|
||||||
repair_attempts: 0
|
|
||||||
@@ -1,127 +0,0 @@
|
|||||||
package usecase
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
|
||||||
)
|
|
||||||
|
|
||||||
type integrationLLM struct{}
|
|
||||||
|
|
||||||
func (f *integrationLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
|
||||||
lastIntegrationRequest = req
|
|
||||||
return &domain.GenerateResponse{
|
|
||||||
Content: `{"summary":"Party discovered a captive scout beneath the tower.","events":[{"title":"Scout found in cellar","type":"discovery","notes":"Scout requested rescue from goblin raiders."}]}`,
|
|
||||||
Usage: domain.TokenUsage{
|
|
||||||
PromptTokens: 42,
|
|
||||||
CompletionTokens: 36,
|
|
||||||
TotalTokens: 78,
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var lastIntegrationRequest domain.GenerateRequest
|
|
||||||
|
|
||||||
func TestRunnerIntegrationWithPromptAndProfileFixturesAndValidation(t *testing.T) {
|
|
||||||
root, err := filepath.Abs(filepath.Join("..", ".."))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to resolve repo root: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
promptsDir := filepath.Join(root, "examples", "prompts")
|
|
||||||
profilesDir := filepath.Join(root, "examples", "profiles")
|
|
||||||
schemasDir := filepath.Join(root, "examples", "schemas")
|
|
||||||
fixturesDir := filepath.Join(root, "examples", "fixtures")
|
|
||||||
t.Setenv("SCRIPTORIUM_API_KEY", "test-key")
|
|
||||||
|
|
||||||
runner := NewRunner(
|
|
||||||
promptdef.NewFilesystemRepository(promptsDir),
|
|
||||||
profile.NewFilesystemRepository(profilesDir),
|
|
||||||
artifact.NewCompositeReader(),
|
|
||||||
prompt.NewGoRenderer(),
|
|
||||||
&integrationLLM{},
|
|
||||||
validate.NewStandardValidator(schemasDir),
|
|
||||||
)
|
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
|
||||||
PromptID: "generic.structured_events",
|
|
||||||
Inputs: map[string]domain.ArtifactRef{
|
|
||||||
"transcript": {
|
|
||||||
Type: domain.ArtifactRefFile,
|
|
||||||
URI: filepath.Join(fixturesDir, "transcript.md"),
|
|
||||||
},
|
|
||||||
"glossary": {
|
|
||||||
Type: domain.ArtifactRefFile,
|
|
||||||
URI: filepath.Join(fixturesDir, "glossary.yml"),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if res.PromptID != "generic.structured_events" {
|
|
||||||
t.Fatalf("unexpected prompt id: %q", res.PromptID)
|
|
||||||
}
|
|
||||||
if res.SelectedProfileID != "local-quality" {
|
|
||||||
t.Fatalf("expected selected profile local-quality from prompt default, got %q", res.SelectedProfileID)
|
|
||||||
}
|
|
||||||
if res.RunID == "" {
|
|
||||||
t.Fatal("expected run id")
|
|
||||||
}
|
|
||||||
if res.PromptHash == "" {
|
|
||||||
t.Fatal("expected prompt hash")
|
|
||||||
}
|
|
||||||
if res.PromptVersion != "1.0.0" {
|
|
||||||
t.Fatalf("unexpected prompt version: %q", res.PromptVersion)
|
|
||||||
}
|
|
||||||
if res.Validation.Status != domain.ValidationPassed {
|
|
||||||
t.Fatalf("expected passed validation, got %q", res.Validation.Status)
|
|
||||||
}
|
|
||||||
if res.Validation.Mode != domain.ValidationJSONSchema {
|
|
||||||
t.Fatalf("expected json_schema mode, got %q", res.Validation.Mode)
|
|
||||||
}
|
|
||||||
if lastIntegrationRequest.StructuredOutput == nil {
|
|
||||||
t.Fatal("expected provider-level structured output request for json_schema prompt")
|
|
||||||
}
|
|
||||||
if lastIntegrationRequest.StructuredOutput.Type != domain.StructuredOutputJSONSchema {
|
|
||||||
t.Fatalf("expected structured output type json_schema, got %q", lastIntegrationRequest.StructuredOutput.Type)
|
|
||||||
}
|
|
||||||
if lastIntegrationRequest.StructuredOutput.JSONSchema == nil || lastIntegrationRequest.StructuredOutput.JSONSchema.Schema == nil {
|
|
||||||
t.Fatalf("expected structured output json_schema payload, got %+v", lastIntegrationRequest.StructuredOutput.JSONSchema)
|
|
||||||
}
|
|
||||||
if res.Artifact.ContentType != "application/json" {
|
|
||||||
t.Fatalf("expected application/json output, got %q", res.Artifact.ContentType)
|
|
||||||
}
|
|
||||||
if len(res.RawOutput) == 0 {
|
|
||||||
t.Fatal("expected raw output to be preserved")
|
|
||||||
}
|
|
||||||
if res.PromptHash == "" {
|
|
||||||
t.Fatal("expected non-empty prompt hash")
|
|
||||||
}
|
|
||||||
if len(res.InputHashes) != 2 {
|
|
||||||
t.Fatalf("expected two input hashes, got %d", len(res.InputHashes))
|
|
||||||
}
|
|
||||||
if res.InputHashes["transcript"] == "" || res.InputHashes["glossary"] == "" {
|
|
||||||
t.Fatalf("expected both input hashes to be set, got %#v", res.InputHashes)
|
|
||||||
}
|
|
||||||
if res.Usage.TotalTokens != 78 {
|
|
||||||
t.Fatalf("expected usage from fake llm, got %+v", res.Usage)
|
|
||||||
}
|
|
||||||
if res.StartTime.IsZero() || res.EndTime.IsZero() {
|
|
||||||
t.Fatal("expected start/end timestamps")
|
|
||||||
}
|
|
||||||
if res.EndTime.Before(res.StartTime) {
|
|
||||||
t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime)
|
|
||||||
}
|
|
||||||
if res.Duration < 0 {
|
|
||||||
t.Fatalf("expected non-negative duration, got %s", res.Duration)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,76 +0,0 @@
|
|||||||
package usecase
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
|
||||||
)
|
|
||||||
|
|
||||||
type OutputRepairer interface {
|
|
||||||
Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
type RepairRequest struct {
|
|
||||||
PreviousOutput string
|
|
||||||
ValidationErrors []string
|
|
||||||
Target domain.ExecutionTarget
|
|
||||||
StructuredOutput *domain.StructuredOutputSpec
|
|
||||||
Attempt int
|
|
||||||
MaxAttempts int
|
|
||||||
Mode domain.ValidationMode
|
|
||||||
}
|
|
||||||
|
|
||||||
type defaultOutputRepairer struct {
|
|
||||||
llm llm.Client
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewDefaultOutputRepairer(llmClient llm.Client) OutputRepairer {
|
|
||||||
return &defaultOutputRepairer{llm: llmClient}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) {
|
|
||||||
if r.llm == nil {
|
|
||||||
return nil, errors.New("llm client is required for repair")
|
|
||||||
}
|
|
||||||
|
|
||||||
errs := "(none provided)"
|
|
||||||
if len(req.ValidationErrors) > 0 {
|
|
||||||
errs = strings.Join(req.ValidationErrors, "\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
prompt := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
|
||||||
{
|
|
||||||
Role: "system",
|
|
||||||
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Role: "user",
|
|
||||||
Content: fmt.Sprintf(
|
|
||||||
"Repair attempt %d of %d for validation mode %s.\n\nValidation errors:\n%s\n\nPrevious output:\n%s\n\nReturn only corrected JSON.",
|
|
||||||
req.Attempt,
|
|
||||||
req.MaxAttempts,
|
|
||||||
req.Mode,
|
|
||||||
errs,
|
|
||||||
req.PreviousOutput,
|
|
||||||
),
|
|
||||||
},
|
|
||||||
}}
|
|
||||||
|
|
||||||
resp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
|
||||||
Prompt: prompt,
|
|
||||||
Target: req.Target,
|
|
||||||
StructuredOutput: req.StructuredOutput,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if resp == nil {
|
|
||||||
return nil, errors.New("repair llm returned nil response")
|
|
||||||
}
|
|
||||||
|
|
||||||
return resp, nil
|
|
||||||
}
|
|
||||||
@@ -1,484 +0,0 @@
|
|||||||
package usecase
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
"unicode"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/artifact"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/defaults"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/llm"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/profile"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/prompt"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/promptdef"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/validate"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
ErrInvalidRequest = errors.New("invalid run request")
|
|
||||||
ErrProfileRequired = errors.New("profile selection is required")
|
|
||||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
|
||||||
ErrProfileLoad = errors.New("failed to load prompt definition")
|
|
||||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
|
||||||
ErrPromptRender = errors.New("failed to render prompt")
|
|
||||||
ErrLLMGenerate = errors.New("failed to generate output")
|
|
||||||
ErrValidation = errors.New("failed to validate output")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Runner executes the Scriptorium core use case.
|
|
||||||
type Runner struct {
|
|
||||||
promptDefs promptdef.Repository
|
|
||||||
profiles profile.Repository
|
|
||||||
artifacts artifact.Reader
|
|
||||||
renderer prompt.Renderer
|
|
||||||
llm llm.Client
|
|
||||||
validator validate.Validator
|
|
||||||
repairer OutputRepairer
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewRunner(
|
|
||||||
promptDefs promptdef.Repository,
|
|
||||||
profiles profile.Repository,
|
|
||||||
artifacts artifact.Reader,
|
|
||||||
renderer prompt.Renderer,
|
|
||||||
llmClient llm.Client,
|
|
||||||
validator validate.Validator,
|
|
||||||
) *Runner {
|
|
||||||
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewRunnerWithRepairer(
|
|
||||||
promptDefs promptdef.Repository,
|
|
||||||
profiles profile.Repository,
|
|
||||||
artifacts artifact.Reader,
|
|
||||||
renderer prompt.Renderer,
|
|
||||||
llmClient llm.Client,
|
|
||||||
validator validate.Validator,
|
|
||||||
repairer OutputRepairer,
|
|
||||||
) *Runner {
|
|
||||||
return &Runner{
|
|
||||||
promptDefs: promptDefs,
|
|
||||||
profiles: profiles,
|
|
||||||
artifacts: artifacts,
|
|
||||||
renderer: renderer,
|
|
||||||
llm: llmClient,
|
|
||||||
validator: validator,
|
|
||||||
repairer: repairer,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) {
|
|
||||||
runID, err := newRunID()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to create run id: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
start := time.Now().UTC()
|
|
||||||
|
|
||||||
prepared, err := r.Prepare(ctx, req)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
|
||||||
Prompt: domain.RenderedPrompt{Messages: prepared.Messages},
|
|
||||||
Target: prepared.EffectiveModelParams,
|
|
||||||
StructuredOutput: prepared.StructuredOutput,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
|
||||||
validationResult, err := r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, 0)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.shouldAttemptRepair(prepared.OutputContract, validationResult) {
|
|
||||||
attemptsUsed := 0
|
|
||||||
for attemptsUsed < prepared.OutputContract.RepairAttempts && validationResult.Status == domain.ValidationFailed {
|
|
||||||
attemptsUsed++
|
|
||||||
|
|
||||||
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
|
|
||||||
PreviousOutput: genResp.Content,
|
|
||||||
ValidationErrors: validationResult.Errors,
|
|
||||||
Target: prepared.EffectiveModelParams,
|
|
||||||
StructuredOutput: prepared.StructuredOutput,
|
|
||||||
Attempt: attemptsUsed,
|
|
||||||
MaxAttempts: prepared.OutputContract.RepairAttempts,
|
|
||||||
Mode: prepared.OutputContract.ValidationMode,
|
|
||||||
})
|
|
||||||
if repairErr != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, repairErr)
|
|
||||||
}
|
|
||||||
if repairResp == nil {
|
|
||||||
return nil, fmt.Errorf("%w: repairer returned nil response", ErrValidation)
|
|
||||||
}
|
|
||||||
|
|
||||||
genResp = repairResp
|
|
||||||
outputArtifact = buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
|
||||||
|
|
||||||
validationResult, err = r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, attemptsUsed)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
end := time.Now().UTC()
|
|
||||||
|
|
||||||
return &domain.RunResult{
|
|
||||||
RunID: runID,
|
|
||||||
Artifact: outputArtifact,
|
|
||||||
RawOutput: genResp.Content,
|
|
||||||
Validation: validationResult,
|
|
||||||
PromptID: prepared.PromptID,
|
|
||||||
PromptVersion: prepared.PromptVersion,
|
|
||||||
PromptHash: prepared.PromptHash,
|
|
||||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
|
||||||
ModelName: prepared.EffectiveModelParams.Model,
|
|
||||||
Endpoint: prepared.EffectiveModelParams.Endpoint,
|
|
||||||
EffectiveModelParams: prepared.EffectiveModelParams,
|
|
||||||
InputHashes: prepared.InputHashes,
|
|
||||||
Usage: genResp.Usage,
|
|
||||||
StartTime: start,
|
|
||||||
EndTime: end,
|
|
||||||
Duration: end.Sub(start),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.PreparedRun, error) {
|
|
||||||
if strings.TrimSpace(req.PromptID) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
|
||||||
}
|
|
||||||
|
|
||||||
start := time.Now().UTC()
|
|
||||||
|
|
||||||
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
|
||||||
}
|
|
||||||
promptDefinitionHash, err := hashPromptDefinition(def)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrProfileLoad, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
selectedProfileID := strings.TrimSpace(req.ProfileID)
|
|
||||||
if selectedProfileID == "" {
|
|
||||||
selectedProfileID = strings.TrimSpace(def.DefaultProfile)
|
|
||||||
}
|
|
||||||
if selectedProfileID == "" {
|
|
||||||
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
|
|
||||||
}
|
|
||||||
|
|
||||||
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
effectiveModel := resolveExecutionTarget(execProfile, req.Execution)
|
|
||||||
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(effectiveModel.Model) == "" {
|
|
||||||
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
|
|
||||||
}
|
|
||||||
if err := validateAPIKeyEnv(effectiveModel.APIKeyEnv); err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
effectiveContract := resolveOutputContract(def, req.Validation)
|
|
||||||
structuredOutput, err := r.resolveStructuredOutput(ctx, def, effectiveContract)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
|
|
||||||
inputHashes := make(map[string]string, len(req.Inputs))
|
|
||||||
for name, ref := range req.Inputs {
|
|
||||||
art, readErr := r.artifacts.Read(ctx, ref)
|
|
||||||
if readErr != nil {
|
|
||||||
return nil, fmt.Errorf("%w: input %q: %w", ErrArtifactLoad, name, readErr)
|
|
||||||
}
|
|
||||||
if art.Name == "" {
|
|
||||||
art.Name = name
|
|
||||||
}
|
|
||||||
resolvedInputs[name] = art
|
|
||||||
inputHashes[name] = art.Hash
|
|
||||||
}
|
|
||||||
|
|
||||||
renderedPrompt, err := r.renderer.Render(ctx, def, resolvedInputs, req.Vars)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
end := time.Now().UTC()
|
|
||||||
return &domain.PreparedRun{
|
|
||||||
PromptID: def.ID,
|
|
||||||
PromptVersion: def.Version,
|
|
||||||
PromptHash: promptDefinitionHash,
|
|
||||||
SelectedProfileID: selectedProfileID,
|
|
||||||
EffectiveModelParams: effectiveModel,
|
|
||||||
OutputContract: effectiveContract,
|
|
||||||
StructuredOutput: structuredOutput,
|
|
||||||
InputHashes: inputHashes,
|
|
||||||
RenderedPromptHash: hashRenderedPrompt(*renderedPrompt),
|
|
||||||
Messages: renderedPrompt.Messages,
|
|
||||||
StartTime: start,
|
|
||||||
EndTime: end,
|
|
||||||
DurationMS: end.Sub(start).Milliseconds(),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) {
|
|
||||||
if contract.ValidationMode != domain.ValidationJSONSchema {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
loader, ok := r.validator.(validate.SchemaDocumentLoader)
|
|
||||||
if !ok || loader == nil {
|
|
||||||
return nil, fmt.Errorf("%w: json_schema output requires schema document loader", ErrValidation)
|
|
||||||
}
|
|
||||||
|
|
||||||
schemaDoc, err := loader.LoadSchemaDocument(ctx, contract.SchemaPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &domain.StructuredOutputSpec{
|
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
|
||||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
|
||||||
Name: deriveStructuredSchemaName(def.ID, def.Version),
|
|
||||||
Strict: true,
|
|
||||||
Schema: schemaDoc,
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func deriveStructuredSchemaName(promptID string, promptVersion string) string {
|
|
||||||
raw := strings.TrimSpace(promptID)
|
|
||||||
if v := strings.TrimSpace(promptVersion); v != "" {
|
|
||||||
if raw == "" {
|
|
||||||
raw = v
|
|
||||||
} else {
|
|
||||||
raw = raw + "_" + v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var b strings.Builder
|
|
||||||
for _, r := range raw {
|
|
||||||
if unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == '-' {
|
|
||||||
b.WriteRune(r)
|
|
||||||
} else {
|
|
||||||
b.WriteRune('_')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
name := strings.Trim(b.String(), "_-")
|
|
||||||
if name == "" {
|
|
||||||
return "scriptorium_schema"
|
|
||||||
}
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) validateOutput(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, attemptsUsed int) (domain.ValidationResult, error) {
|
|
||||||
if r.validator == nil || contract.ValidationMode == domain.ValidationNone {
|
|
||||||
return domain.ValidationResult{
|
|
||||||
Status: domain.ValidationSkipped,
|
|
||||||
Mode: contract.ValidationMode,
|
|
||||||
SchemaPath: contract.SchemaPath,
|
|
||||||
RepairAttempts: attemptsUsed,
|
|
||||||
IsValid: true,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := r.validator.Validate(ctx, artifact, contract)
|
|
||||||
if err != nil {
|
|
||||||
return domain.ValidationResult{}, err
|
|
||||||
}
|
|
||||||
res.RepairAttempts = attemptsUsed
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationResult domain.ValidationResult) bool {
|
|
||||||
if r.repairer == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if contract.RepairAttempts <= 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if validationResult.Status != domain.ValidationFailed {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return contract.ValidationMode == domain.ValidationJSON || contract.ValidationMode == domain.ValidationJSONSchema
|
|
||||||
}
|
|
||||||
|
|
||||||
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
|
|
||||||
out := base
|
|
||||||
if override.Endpoint != "" {
|
|
||||||
out.Endpoint = override.Endpoint
|
|
||||||
}
|
|
||||||
if override.Model != "" {
|
|
||||||
out.Model = override.Model
|
|
||||||
}
|
|
||||||
if override.Temperature != 0 {
|
|
||||||
out.Temperature = override.Temperature
|
|
||||||
}
|
|
||||||
if override.MaxTokens != 0 {
|
|
||||||
out.MaxTokens = override.MaxTokens
|
|
||||||
}
|
|
||||||
if override.TopP != 0 {
|
|
||||||
out.TopP = override.TopP
|
|
||||||
}
|
|
||||||
if override.TimeoutSeconds != 0 {
|
|
||||||
out.TimeoutSeconds = override.TimeoutSeconds
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(override.ServiceTier) != "" {
|
|
||||||
out.ServiceTier = override.ServiceTier
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(override.ReasoningEffort) != "" {
|
|
||||||
out.ReasoningEffort = override.ReasoningEffort
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
|
||||||
out.APIKeyEnv = override.APIKeyEnv
|
|
||||||
}
|
|
||||||
if len(override.ExtraParams) > 0 {
|
|
||||||
cp := make(map[string]string, len(override.ExtraParams))
|
|
||||||
for k, v := range override.ExtraParams {
|
|
||||||
cp[k] = v
|
|
||||||
}
|
|
||||||
out.ExtraParams = cp
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTarget) domain.ExecutionTarget {
|
|
||||||
out := defaults.ExecutionTargetDefault()
|
|
||||||
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
|
||||||
if override != nil {
|
|
||||||
out = mergeExecutionTarget(out, *override)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateAPIKeyEnv(apiKeyEnv string) error {
|
|
||||||
envName := strings.TrimSpace(apiKeyEnv)
|
|
||||||
if envName == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(os.Getenv(envName)) == "" {
|
|
||||||
return fmt.Errorf("%w: api key environment variable %q is not set", ErrAPIKeyEnvMissing, envName)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget {
|
|
||||||
if p == nil {
|
|
||||||
return domain.ExecutionTarget{}
|
|
||||||
}
|
|
||||||
cp := map[string]string(nil)
|
|
||||||
if len(p.ExtraParams) > 0 {
|
|
||||||
cp = make(map[string]string, len(p.ExtraParams))
|
|
||||||
for k, v := range p.ExtraParams {
|
|
||||||
cp[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return domain.ExecutionTarget{
|
|
||||||
Endpoint: p.Endpoint,
|
|
||||||
Model: p.Model,
|
|
||||||
Temperature: p.Temperature,
|
|
||||||
MaxTokens: p.MaxTokens,
|
|
||||||
TopP: p.TopP,
|
|
||||||
TimeoutSeconds: p.TimeoutSeconds,
|
|
||||||
ServiceTier: p.ServiceTier,
|
|
||||||
ReasoningEffort: p.ReasoningEffort,
|
|
||||||
APIKeyEnv: p.APIKeyEnv,
|
|
||||||
ExtraParams: cp,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract {
|
|
||||||
contract := def.Validation
|
|
||||||
if contract.Format == "" {
|
|
||||||
contract.Format = def.OutputFormat
|
|
||||||
}
|
|
||||||
if override != nil {
|
|
||||||
contract = *override
|
|
||||||
}
|
|
||||||
if contract.Format == "" {
|
|
||||||
contract.Format = domain.FormatText
|
|
||||||
}
|
|
||||||
return contract
|
|
||||||
}
|
|
||||||
|
|
||||||
func hashRenderedPrompt(p domain.RenderedPrompt) string {
|
|
||||||
var b strings.Builder
|
|
||||||
for _, msg := range p.Messages {
|
|
||||||
b.WriteString(msg.Role)
|
|
||||||
b.WriteByte('\n')
|
|
||||||
b.WriteString(msg.Content)
|
|
||||||
b.WriteString("\n---\n")
|
|
||||||
}
|
|
||||||
h := sha256.Sum256([]byte(b.String()))
|
|
||||||
return hex.EncodeToString(h[:])
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildOutputArtifact(content string, format domain.OutputFormat) domain.Artifact {
|
|
||||||
body := []byte(content)
|
|
||||||
hash := sha256.Sum256(body)
|
|
||||||
|
|
||||||
contentType := defaults.ContentTypeTextPlain
|
|
||||||
switch format {
|
|
||||||
case domain.FormatMarkdown:
|
|
||||||
contentType = defaults.ContentTypeTextMarkdown
|
|
||||||
case domain.FormatJSON:
|
|
||||||
contentType = defaults.ContentTypeApplicationJSON
|
|
||||||
}
|
|
||||||
|
|
||||||
return domain.Artifact{
|
|
||||||
Name: defaults.OutputArtifactName,
|
|
||||||
ContentType: contentType,
|
|
||||||
Body: body,
|
|
||||||
Size: int64(len(body)),
|
|
||||||
Hash: hex.EncodeToString(hash[:]),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func hashPromptDefinition(def *domain.PromptDefinition) (string, error) {
|
|
||||||
b, err := json.Marshal(def)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
sum := sha256.Sum256(b)
|
|
||||||
return hex.EncodeToString(sum[:]), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func newRunID() (string, error) {
|
|
||||||
var b [16]byte
|
|
||||||
if _, err := rand.Read(b[:]); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
// UUID v4 (RFC 4122 variant).
|
|
||||||
b[6] = (b[6] & 0x0f) | 0x40
|
|
||||||
b[8] = (b[8] & 0x3f) | 0x80
|
|
||||||
|
|
||||||
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
|
|
||||||
b[0:4],
|
|
||||||
b[4:6],
|
|
||||||
b[6:8],
|
|
||||||
b[8:10],
|
|
||||||
b[10:16],
|
|
||||||
), nil
|
|
||||||
}
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,151 +0,0 @@
|
|||||||
package validate
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
|
||||||
)
|
|
||||||
|
|
||||||
// StandardValidator provides basic, JSON, and JSON Schema output validation.
|
|
||||||
type StandardValidator struct {
|
|
||||||
schemaBaseDir string
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewStandardValidator(schemaBaseDir string) Validator {
|
|
||||||
return &StandardValidator{schemaBaseDir: schemaBaseDir}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *StandardValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return domain.ValidationResult{}, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
res := domain.ValidationResult{
|
|
||||||
Mode: contract.ValidationMode,
|
|
||||||
SchemaPath: contract.SchemaPath,
|
|
||||||
RepairAttempts: contract.RepairAttempts,
|
|
||||||
}
|
|
||||||
|
|
||||||
if artifact == nil {
|
|
||||||
return domain.ValidationResult{}, errors.New("artifact is required for validation")
|
|
||||||
}
|
|
||||||
|
|
||||||
switch contract.ValidationMode {
|
|
||||||
case domain.ValidationNone:
|
|
||||||
res.Status = domain.ValidationSkipped
|
|
||||||
res.IsValid = true
|
|
||||||
return res, nil
|
|
||||||
case domain.ValidationBasic:
|
|
||||||
if strings.TrimSpace(string(artifact.Body)) == "" {
|
|
||||||
res.Status = domain.ValidationFailed
|
|
||||||
res.IsValid = false
|
|
||||||
res.Errors = []string{"output is empty"}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
res.Status = domain.ValidationPassed
|
|
||||||
res.IsValid = true
|
|
||||||
return res, nil
|
|
||||||
case domain.ValidationJSON:
|
|
||||||
_, jsonErr := parseJSON(artifact.Body)
|
|
||||||
if jsonErr != nil {
|
|
||||||
res.Status = domain.ValidationFailed
|
|
||||||
res.IsValid = false
|
|
||||||
res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
res.Status = domain.ValidationPassed
|
|
||||||
res.IsValid = true
|
|
||||||
return res, nil
|
|
||||||
case domain.ValidationJSONSchema:
|
|
||||||
instance, jsonErr := parseJSON(artifact.Body)
|
|
||||||
if jsonErr != nil {
|
|
||||||
res.Status = domain.ValidationFailed
|
|
||||||
res.IsValid = false
|
|
||||||
res.Errors = []string{fmt.Sprintf("invalid JSON: %v", jsonErr)}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
schemaPath, err := v.resolveSchemaPath(contract.SchemaPath)
|
|
||||||
if err != nil {
|
|
||||||
return domain.ValidationResult{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
compiler := jsonschema.NewCompiler()
|
|
||||||
schema, err := compiler.Compile(schemaPath)
|
|
||||||
if err != nil {
|
|
||||||
return domain.ValidationResult{}, fmt.Errorf("failed to compile JSON schema %q: %w", schemaPath, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := schema.Validate(instance); err != nil {
|
|
||||||
res.Status = domain.ValidationFailed
|
|
||||||
res.IsValid = false
|
|
||||||
res.Errors = []string{fmt.Sprintf("json schema validation failed: %v", err)}
|
|
||||||
return res, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
res.Status = domain.ValidationPassed
|
|
||||||
res.IsValid = true
|
|
||||||
return res, nil
|
|
||||||
default:
|
|
||||||
return domain.ValidationResult{}, fmt.Errorf("unsupported validation mode: %q", contract.ValidationMode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseJSON(body []byte) (any, error) {
|
|
||||||
var v any
|
|
||||||
if err := json.Unmarshal(body, &v); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return v, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error) {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
resolved, err := v.resolveSchemaPath(schemaPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
raw, err := os.ReadFile(resolved)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read schema file %q: %w", resolved, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var doc any
|
|
||||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
|
||||||
}
|
|
||||||
return doc, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error) {
|
|
||||||
if strings.TrimSpace(schemaPath) == "" {
|
|
||||||
return "", errors.New("schema path is required for json_schema validation")
|
|
||||||
}
|
|
||||||
|
|
||||||
resolved := schemaPath
|
|
||||||
if !filepath.IsAbs(schemaPath) {
|
|
||||||
resolved = filepath.Join(v.schemaBaseDir, schemaPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
resolved = filepath.Clean(resolved)
|
|
||||||
if _, err := os.Stat(resolved); err != nil {
|
|
||||||
return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return resolved, nil
|
|
||||||
}
|
|
||||||
@@ -1,252 +0,0 @@
|
|||||||
package validate
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestStandardValidatorNoneSkipped(t *testing.T) {
|
|
||||||
v := NewStandardValidator("")
|
|
||||||
|
|
||||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte("ignored")}, domain.OutputContract{
|
|
||||||
ValidationMode: domain.ValidationNone,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if res.Status != domain.ValidationSkipped {
|
|
||||||
t.Fatalf("expected skipped, got %q", res.Status)
|
|
||||||
}
|
|
||||||
if !res.IsValid {
|
|
||||||
t.Fatal("expected valid=true for skipped")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorBasicSuccess(t *testing.T) {
|
|
||||||
v := NewStandardValidator("")
|
|
||||||
|
|
||||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte("hello")}, domain.OutputContract{
|
|
||||||
ValidationMode: domain.ValidationBasic,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if res.Status != domain.ValidationPassed || !res.IsValid {
|
|
||||||
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorBasicFailureEmpty(t *testing.T) {
|
|
||||||
v := NewStandardValidator("")
|
|
||||||
|
|
||||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(" \n\t ")}, domain.OutputContract{
|
|
||||||
ValidationMode: domain.ValidationBasic,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if res.Status != domain.ValidationFailed || res.IsValid {
|
|
||||||
t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid)
|
|
||||||
}
|
|
||||||
if len(res.Errors) == 0 {
|
|
||||||
t.Fatal("expected validation errors")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorJSONSuccess(t *testing.T) {
|
|
||||||
v := NewStandardValidator("")
|
|
||||||
|
|
||||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":true}`)}, domain.OutputContract{
|
|
||||||
ValidationMode: domain.ValidationJSON,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if res.Status != domain.ValidationPassed || !res.IsValid {
|
|
||||||
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorJSONFailure(t *testing.T) {
|
|
||||||
v := NewStandardValidator("")
|
|
||||||
|
|
||||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"ok":`)}, domain.OutputContract{
|
|
||||||
ValidationMode: domain.ValidationJSON,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if res.Status != domain.ValidationFailed || res.IsValid {
|
|
||||||
t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid)
|
|
||||||
}
|
|
||||||
if len(res.Errors) == 0 {
|
|
||||||
t.Fatal("expected parse errors")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorJSONSchemaSuccess(t *testing.T) {
|
|
||||||
tmp := t.TempDir()
|
|
||||||
schemaPath := filepath.Join(tmp, "schema.json")
|
|
||||||
if err := os.WriteFile(schemaPath, []byte(`{
|
|
||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
||||||
"type": "object",
|
|
||||||
"required": ["name"],
|
|
||||||
"properties": {
|
|
||||||
"name": {"type": "string"}
|
|
||||||
}
|
|
||||||
}`), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
v := NewStandardValidator(tmp)
|
|
||||||
|
|
||||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{
|
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
|
||||||
SchemaPath: "schema.json",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if res.Status != domain.ValidationPassed || !res.IsValid {
|
|
||||||
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) {
|
|
||||||
tmp := t.TempDir()
|
|
||||||
nestedDir := filepath.Join(tmp, "dnd")
|
|
||||||
if err := os.MkdirAll(nestedDir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(nestedDir, "schema.json"), []byte(`{
|
|
||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
||||||
"type": "object",
|
|
||||||
"required": ["name"],
|
|
||||||
"properties": {
|
|
||||||
"name": {"type": "string"}
|
|
||||||
}
|
|
||||||
}`), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
v := NewStandardValidator(tmp)
|
|
||||||
|
|
||||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{
|
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
|
||||||
SchemaPath: filepath.Join("dnd", "schema.json"),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if res.Status != domain.ValidationPassed || !res.IsValid {
|
|
||||||
t.Fatalf("expected passed/valid, got status=%q valid=%v", res.Status, res.IsValid)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorJSONSchemaNestedSchemaPathMissing(t *testing.T) {
|
|
||||||
v := NewStandardValidator(t.TempDir())
|
|
||||||
|
|
||||||
_, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{
|
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
|
||||||
SchemaPath: filepath.Join("dnd", "missing.json"),
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected nested schema load error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorJSONSchemaFailure(t *testing.T) {
|
|
||||||
tmp := t.TempDir()
|
|
||||||
schemaPath := filepath.Join(tmp, "schema.json")
|
|
||||||
if err := os.WriteFile(schemaPath, []byte(`{
|
|
||||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
||||||
"type": "object",
|
|
||||||
"required": ["name"],
|
|
||||||
"properties": {
|
|
||||||
"name": {"type": "string"}
|
|
||||||
}
|
|
||||||
}`), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
v := NewStandardValidator(tmp)
|
|
||||||
|
|
||||||
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"count":1}`)}, domain.OutputContract{
|
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
|
||||||
SchemaPath: "schema.json",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
if res.Status != domain.ValidationFailed || res.IsValid {
|
|
||||||
t.Fatalf("expected failed/invalid, got status=%q valid=%v", res.Status, res.IsValid)
|
|
||||||
}
|
|
||||||
if len(res.Errors) == 0 {
|
|
||||||
t.Fatal("expected schema errors")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorJSONSchemaSchemaLoadError(t *testing.T) {
|
|
||||||
v := NewStandardValidator(t.TempDir())
|
|
||||||
|
|
||||||
_, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"name":"eris"}`)}, domain.OutputContract{
|
|
||||||
ValidationMode: domain.ValidationJSONSchema,
|
|
||||||
SchemaPath: "missing.json",
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected schema load error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorLoadSchemaDocumentSuccess(t *testing.T) {
|
|
||||||
tmp := t.TempDir()
|
|
||||||
if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{
|
|
||||||
"type": "object",
|
|
||||||
"properties": {
|
|
||||||
"name": {"type": "string"}
|
|
||||||
}
|
|
||||||
}`), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
v := NewStandardValidator(tmp)
|
|
||||||
loader, ok := v.(SchemaDocumentLoader)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("standard validator must implement SchemaDocumentLoader")
|
|
||||||
}
|
|
||||||
|
|
||||||
doc, err := loader.LoadSchemaDocument(context.Background(), "schema.json")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("expected no error, got %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
obj, ok := doc.(map[string]any)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("expected object document, got %#v", doc)
|
|
||||||
}
|
|
||||||
if obj["type"] != "object" {
|
|
||||||
t.Fatalf("expected schema type=object, got %#v", obj["type"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStandardValidatorLoadSchemaDocumentInvalidJSON(t *testing.T) {
|
|
||||||
tmp := t.TempDir()
|
|
||||||
if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{`), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
v := NewStandardValidator(tmp)
|
|
||||||
loader, ok := v.(SchemaDocumentLoader)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("standard validator must implement SchemaDocumentLoader")
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := loader.LoadSchemaDocument(context.Background(), "schema.json")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected decode error")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
package validate
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"gitea.maximumdirect.net/eric/scriptorium/internal/domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Validator validates the generated artifact based on the output contract.
|
|
||||||
type Validator interface {
|
|
||||||
Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// SchemaDocumentLoader loads JSON schema documents using validator path semantics.
|
|
||||||
type SchemaDocumentLoader interface {
|
|
||||||
LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error)
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user