Compare commits
99 Commits
v0.1.0
...
6db2dc8d2a
| Author | SHA1 | Date | |
|---|---|---|---|
| 6db2dc8d2a | |||
| 98b03a4629 | |||
| 610bdb4fea | |||
| 68ec69f2e4 | |||
| 451f6c0bb9 | |||
| 3011dd91ca | |||
| ae65b95374 | |||
| a5bbfea9b9 | |||
| ae9c2e1d5e | |||
| 1d3a444df8 | |||
| f044c00a7c | |||
| 7d89c2702b | |||
| 93653cccb8 | |||
| a024492dbf | |||
| 304c68f9fc | |||
| c5f2b14ff4 | |||
| fc8e03f98c | |||
| 16de4b6437 | |||
| 0f30888b00 | |||
| 3e67be6ac3 | |||
| 5ef027b6f0 | |||
| 666b4bf801 | |||
| d593bfee0a | |||
| b7ad66f0e0 | |||
| 249e49c928 | |||
| e54e74ed88 | |||
| 582c5dceed | |||
| a9d8505cdb | |||
| 7c95791e94 | |||
| aa14faa3cb | |||
| cc6b050367 | |||
| bcedf19a08 | |||
| c05ecb58d8 | |||
| 9e3f8809b3 | |||
| 4f057b99ac | |||
| aec807fcb0 | |||
| 79a585d17e | |||
| 671ff6d132 | |||
| 9b2d0297b7 | |||
| f91e643932 | |||
| 35fe405448 | |||
| 524f2ffb8e | |||
| 68b426cdb0 | |||
| 223f3751e8 | |||
| 7861d040df | |||
| 47cf7e76ec | |||
| 8cafa64174 | |||
| ecba0ad725 | |||
| b3757dcf7b | |||
| 3e456ec4d4 | |||
| 5b1efc89f6 | |||
| aee48d011e | |||
| 3217bb3e12 | |||
| 3df686f474 | |||
| 7d4c027d09 | |||
| 31d70a2dd7 | |||
| c9fbb331e2 | |||
| f6224dcbee | |||
| de6689bc1d | |||
| 0fc740470f | |||
| 49d94cc2e9 | |||
| 291298cf7b | |||
| 9532ae8121 | |||
| 7601731a2c | |||
| 3aa88ab9d3 | |||
| c1ba94192d | |||
| 22032dfd6d | |||
| 4cafde2502 | |||
| 8c623b7ad8 | |||
| 43dc954440 | |||
| 51053d390d | |||
| 9278797aa9 | |||
| 39e49d7f77 | |||
| 84c4c06712 | |||
| eab640aa21 | |||
| a516944086 | |||
| be6803ffa1 | |||
| ef4bdd4f9f | |||
| 2f97895732 | |||
| 9e89b88efc | |||
| a57c6397e3 | |||
| 39e071f5ca | |||
| 70d733edaf | |||
| 1c31f56af1 | |||
| f9999a73df | |||
| 11d8187052 | |||
| 86bff552c1 | |||
| d3f790095e | |||
| 95218218e2 | |||
| e700df82d8 | |||
| e19cc02c4d | |||
| 8a5419448f | |||
| c8217549a8 | |||
| 2130414899 | |||
| 7f83a20fa6 | |||
| 317ab0472d | |||
| e5eb0ba5c8 | |||
| b95af4f87d | |||
| 11073b613c |
7
.gitignore
vendored
7
.gitignore
vendored
@@ -1,3 +1,7 @@
|
||||
# build and testing artifacts
|
||||
notarius
|
||||
notarius-output
|
||||
|
||||
# ---> Go
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
@@ -47,7 +51,8 @@ go.work.sum
|
||||
.LSOverride
|
||||
|
||||
# Icon must end with two \r
|
||||
Icon
|
||||
Icon
|
||||
|
||||
|
||||
# Thumbnails
|
||||
._*
|
||||
|
||||
@@ -1,3 +1,2 @@
|
||||
Please carefully review the documents in `docs/policy` before making any changes to this repository.
|
||||
- `architecture.md` provides the canonical high-level architecture policy for this repository.
|
||||
- `documentation.md` provides the canonical documentation policy for this repository.
|
||||
Please review `docs/development.md` for initial orientation in this repository
|
||||
and follow its task-specific reading guide.
|
||||
|
||||
18
README.md
18
README.md
@@ -4,33 +4,29 @@ Notarius is a Go CLI for extracting structured artifacts from source material
|
||||
with explicit, configurable pipeline modules.
|
||||
|
||||
The current implementation reads Seriatim transcript JSON, chunks the source
|
||||
units, extracts D&D spell-cast artifacts with an OpenAI-compatible LLM, and
|
||||
writes JSON output plus diagnostics for each run.
|
||||
units, extracts D&D spell-cast artifacts with a Scriptorium-backed LLM runtime,
|
||||
and writes JSON output plus diagnostics for each run.
|
||||
|
||||
```sh
|
||||
NOTARIUS_LLM_DEFAULT_BASE_URL=http://127.0.0.1:8080/v1 \
|
||||
NOTARIUS_LLM_DEFAULT_MODEL=your-model \
|
||||
OPENROUTER_API_KEY=... \
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json
|
||||
```
|
||||
|
||||
If the provider requires authentication, set
|
||||
`NOTARIUS_LLM_DEFAULT_API_KEY` in the environment before running the command.
|
||||
Outputs are written under `./notarius-output/<run-id>/` unless `--output-dir`
|
||||
is provided.
|
||||
The maintained example uses Scriptorium's built-in `mistral-small-3` profile,
|
||||
which reads `OPENROUTER_API_KEY`. Outputs are written under
|
||||
`./notarius-output/<run-id>/` unless `--output-dir` is provided.
|
||||
|
||||
Useful references:
|
||||
|
||||
- [CLI reference](docs/cli.md)
|
||||
- [Configuration reference](docs/config.md)
|
||||
- [Operations](docs/operations.md)
|
||||
- [Troubleshooting](docs/troubleshooting.md)
|
||||
- [Seriatim input contract](docs/integrations/seriatim.md)
|
||||
- [OpenAI-compatible provider contract](docs/integrations/openai-compatible.md)
|
||||
- [JSON output contract](docs/integrations/json-output.md)
|
||||
- [D&D spell artifact contract](docs/integrations/dnd-spell-artifacts.md)
|
||||
- [Developer workflow](docs/policy/development.md)
|
||||
- [Developer guide](docs/development.md)
|
||||
- [Internal architecture docs](docs/internal/overview.md)
|
||||
- [Maintained example config](examples/dnd-spells.config.yml)
|
||||
- [Maintained example input](examples/seriatim-minimal-transcript.json)
|
||||
|
||||
23
docs/adr/0001-record-architecture-decisions.md
Normal file
23
docs/adr/0001-record-architecture-decisions.md
Normal file
@@ -0,0 +1,23 @@
|
||||
# ADR-0001: Record architecture decisions as ADRs
|
||||
|
||||
**Status:** Accepted
|
||||
**Date:** 2026-07-13
|
||||
|
||||
## Context
|
||||
Architectural reasoning made during design (pattern choices, rejected
|
||||
alternatives, trigger conditions for revisiting) is lost if only the final
|
||||
state is documented.
|
||||
|
||||
## Decision
|
||||
We keep a living overview in docs/policy/architecture.md describing current
|
||||
intended state, and immutable, numbered ADRs (Nygard format) in docs/adr/
|
||||
recording each significant decision, its alternatives, and its consequences.
|
||||
Changed decisions get a new ADR that marks the old one Superseded.
|
||||
|
||||
## Alternatives considered
|
||||
- Overview doc only: loses the "why" and the rejected options.
|
||||
- arc42 / RFC-style design docs: heavier than warranted for a solo repo.
|
||||
|
||||
## Consequences
|
||||
Small ongoing writing cost; durable reasoning trail; cheap onboarding for
|
||||
future contributors (including future-us).
|
||||
161
docs/cli.md
161
docs/cli.md
@@ -6,21 +6,22 @@ interface.
|
||||
## Quick Run
|
||||
|
||||
```sh
|
||||
NOTARIUS_LLM_DEFAULT_BASE_URL=http://127.0.0.1:8080/v1 \
|
||||
NOTARIUS_LLM_DEFAULT_MODEL=your-model \
|
||||
OPENROUTER_API_KEY=... \
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json
|
||||
```
|
||||
|
||||
Set `NOTARIUS_LLM_DEFAULT_API_KEY` if the OpenAI-compatible provider requires
|
||||
a bearer token.
|
||||
The maintained example uses prompt defaults and Scriptorium's built-in
|
||||
`mistral-small-3` profile, which reads `OPENROUTER_API_KEY`. To use another
|
||||
endpoint or model, configure a Scriptorium profile source and select its profile
|
||||
ID in config or with `--llm-profile`.
|
||||
|
||||
## Commands
|
||||
|
||||
```text
|
||||
notarius help
|
||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b]
|
||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
|
||||
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||
notarius pipelines list --config path/to/config.yml [--json]
|
||||
```
|
||||
@@ -40,23 +41,125 @@ Flags:
|
||||
`NOTARIUS_CONFIG`, then `/usr/local/etc/notarius/config.yml`.
|
||||
- `--only lane-a,lane-b`: run only the named artifact lanes. Values are
|
||||
comma-separated and must be non-empty.
|
||||
- `--resume`: reuse valid workspace checkpoints for this invocation. Requires
|
||||
an effective workspace directory and `workspace.resume.enabled: true`.
|
||||
- `--output-dir path`: output root. The run writes to `<path>/<run-id>/`.
|
||||
Defaults to `./notarius-output`.
|
||||
- `--diagnostics-dir path`: diagnostics work directory override for this
|
||||
invocation.
|
||||
- `--llm-profile id`: override every effective module binding to use one LLM
|
||||
profile.
|
||||
invocation. It does not change the workspace directory.
|
||||
- `--llm-profile id`: override every effective LLM-capable pipeline module
|
||||
binding to use one Scriptorium profile ID. Validator-specific profiles are
|
||||
not overridden. Configured LLM-backed validators with explicit profiles are
|
||||
validated against the configured Scriptorium profile source.
|
||||
- `--session-id id`: pass a stable prompt session identifier through LLM-backed
|
||||
module calls.
|
||||
- `--reference selector=path`: bind a reference path to a chunk, extractor,
|
||||
merger, or normalizer reference slot. Repeatable.
|
||||
- `--without-reference selector`: remove a configured optional reference binding.
|
||||
Repeatable. It accepts the same selector forms as `--reference`, without
|
||||
`=path`.
|
||||
|
||||
On success, the command prints the completed pipeline ID, approved and rejected
|
||||
artifact counts, and the output directory. If the run completes with warnings,
|
||||
On success, the command prints the completed pipeline ID, normalized output and
|
||||
rejected output counts, and the output directory. If the run completes with warnings,
|
||||
the warning count is printed to stderr.
|
||||
|
||||
Reference flags are resolved against selected chunk, extractor, merger, and normalizer
|
||||
targets before the run starts. Flat slot names are accepted only when exactly
|
||||
one selected target declares that slot. Bound reference files are read before
|
||||
pipeline work starts, validated as UTF-8 text, and recorded as provenance for
|
||||
the target that declares the slot. Runtime reference content is passed to the
|
||||
chunker, extractor, merger, or normalizer target that declares the slot. Notarius infers
|
||||
reference media types from file extensions for provenance and for optional slot
|
||||
checks. Reference content is not written to diagnostics, logs, errors, or
|
||||
manifests.
|
||||
|
||||
Reference binding precedence is:
|
||||
|
||||
1. pipeline-level config `references`;
|
||||
2. target-local config references, including legacy lane-level extractor
|
||||
`references`;
|
||||
3. `--reference` run flags;
|
||||
4. `--without-reference` run flags.
|
||||
|
||||
`--reference` binds or replaces one slot for one selected target. Selectors are:
|
||||
|
||||
- `slot=path`: valid when exactly one selected target declares `slot`;
|
||||
- `chunk.slot=path`: target the chunker;
|
||||
- `merge.slot=path`: valid when exactly one selected merger declares `slot`;
|
||||
- `lane.slot=path`: valid when exactly one selected extractor, merger, or
|
||||
normalizer in that lane declares `slot`;
|
||||
- `lane.extract.slot=path`: target a lane extractor;
|
||||
- `lane.merge.slot=path`: target a lane merger;
|
||||
- `lane.normalize.slot=path`: target a lane normalizer.
|
||||
|
||||
Use `slot=path` when the selected targets declare the slot unambiguously:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--reference roster=./campaign-roster.txt
|
||||
```
|
||||
|
||||
Use an explicit selector when multiple selected targets declare the same slot or
|
||||
when you want to target a specific target:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--reference spells.extract.glossary=./campaign-glossary.txt
|
||||
```
|
||||
|
||||
The same grammar can target chunk, merge, and normalize slots when the configured
|
||||
modules declare them:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config path/to/config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--reference chunk.scene_guide=./campaign-scenes.txt \
|
||||
--reference spells.merge.merge_notes=./merge-notes.txt \
|
||||
--reference spells.normalize.normalization_notes=./normalization-notes.txt
|
||||
```
|
||||
|
||||
Use `--without-reference` to remove a configured optional binding for a run:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--without-reference glossary
|
||||
```
|
||||
|
||||
Use `--session-id` when an external orchestrator needs all prompt calls from one
|
||||
run to share an identifier:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--session-id campaign-17-session-04
|
||||
```
|
||||
|
||||
Use `--resume` to reuse valid checkpoints from a previous compatible
|
||||
invocation:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--resume
|
||||
```
|
||||
|
||||
Plain `run` does not skip completed work. It executes the pipeline normally and
|
||||
refreshes checkpoints when checkpointing is enabled. `--resume` verifies each
|
||||
checkpoint before reuse and executes any missing, corrupt, or incompatible step
|
||||
normally.
|
||||
|
||||
For durable output, diagnostics, retention, and failure inspection, see
|
||||
[Operations](operations.md).
|
||||
|
||||
The current `run` command requires the resolved pipeline to use exactly one
|
||||
distinct LLM profile after defaults and overrides are applied.
|
||||
|
||||
## `config validate`
|
||||
|
||||
`notarius config validate` loads and validates configuration.
|
||||
@@ -117,13 +220,37 @@ go run ./cmd/notarius pipelines list \
|
||||
The production CLI currently registers these module keys:
|
||||
|
||||
- input: `seriatim`
|
||||
- chunk: `generic`
|
||||
- chunk: `generic`, `dnd/scenes`
|
||||
- extract: `dnd/spells`
|
||||
- merge: `appendorder`
|
||||
- normalize: `noop`
|
||||
- output: `json`
|
||||
|
||||
The production CLI does not currently register validator modules.
|
||||
## Implemented Production Validators
|
||||
|
||||
For YAML structure, defaults, environment overrides, and module binding syntax,
|
||||
see [Configuration](config.md).
|
||||
The production CLI currently registers these validator keys:
|
||||
|
||||
- `generic/always_accept`
|
||||
- `generic/always_reject`
|
||||
- `generic/valid_json`
|
||||
- `generic/valid_json_schema`
|
||||
- `extract/dnd/spells/shape`
|
||||
- `extract/dnd/spells/source_refs`
|
||||
- `extract/dnd/spells/source_relatedness`
|
||||
|
||||
The production default chain for the `dnd/spells` extractor is:
|
||||
|
||||
1. `generic/valid_json`
|
||||
2. `generic/valid_json_schema`
|
||||
3. `extract/dnd/spells/shape`
|
||||
4. `extract/dnd/spells/source_refs`
|
||||
5. `extract/dnd/spells/source_relatedness`
|
||||
|
||||
Validator chain overrides are configured on `chunk`, lane `extract`, lane
|
||||
`merge`, and lane `normalize` bindings. Omitted overrides use production
|
||||
defaults, `validators: []` disables validation for that binding, and non-empty
|
||||
lists replace the default chain in configured order. Validator keys are resolved
|
||||
against the registered validator catalog.
|
||||
|
||||
For YAML structure, Scriptorium profile sources, environment overrides, and
|
||||
module binding syntax, see [Configuration](config.md).
|
||||
|
||||
364
docs/config.md
364
docs/config.md
@@ -2,7 +2,7 @@
|
||||
|
||||
This is the canonical reference for implemented Notarius configuration.
|
||||
|
||||
Notarius reads YAML config files with `version: 1`. File config is applied over
|
||||
Notarius reads YAML config files with `version: 2`. File config is applied over
|
||||
built-in defaults, then environment overrides are applied.
|
||||
|
||||
## Discovery
|
||||
@@ -18,15 +18,13 @@ If none is available, the command fails with a config file not found error.
|
||||
## Minimal Example
|
||||
|
||||
```yaml
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
provider: openai-compatible
|
||||
base_url: http://127.0.0.1:8080/v1
|
||||
model: your-model
|
||||
version: 2
|
||||
pipelines:
|
||||
dnd-session:
|
||||
input: seriatim
|
||||
references:
|
||||
party: ./dnd-spells-roster.txt
|
||||
glossary: ./dnd-spells-glossary.txt
|
||||
chunk:
|
||||
module: generic
|
||||
options:
|
||||
@@ -40,71 +38,96 @@ The maintained fixture is [examples/dnd-spells.config.yml](../examples/dnd-spell
|
||||
|
||||
## Top-Level Fields
|
||||
|
||||
- `version`: required. The only supported value is `1`.
|
||||
- `llm_profiles`: optional map of LLM profile IDs to profile settings.
|
||||
- `version`: required. The only supported value is `2`.
|
||||
- `scriptorium`: optional Scriptorium profile source settings.
|
||||
- `pipelines`: optional map of pipeline IDs to pipeline definitions.
|
||||
- `concurrency`: optional global concurrency settings.
|
||||
- `workspace`: optional workspace settings for Notarius-owned local state.
|
||||
- `diagnostics`: optional diagnostics settings.
|
||||
|
||||
Unknown YAML fields are rejected.
|
||||
Unknown YAML fields are rejected. The removed top-level `llm_profiles` field is
|
||||
rejected; execution profiles now come from Scriptorium.
|
||||
|
||||
## Defaults
|
||||
|
||||
Built-in defaults:
|
||||
|
||||
```yaml
|
||||
llm_profiles:
|
||||
default:
|
||||
provider: openai-compatible
|
||||
timeout: 600
|
||||
max_retries: 3
|
||||
max_concurrency: 1
|
||||
concurrency:
|
||||
total_llm: 1
|
||||
diagnostics:
|
||||
work_dir: /tmp/notarius
|
||||
retention: auto
|
||||
workspace:
|
||||
diagnostics:
|
||||
enabled: true
|
||||
resume:
|
||||
enabled: false
|
||||
debug:
|
||||
enabled: false
|
||||
```
|
||||
|
||||
`workspace.directory` is unset by default. Without a workspace directory,
|
||||
diagnostics continue to use `/tmp/notarius`, and checkpoint and debug workspace
|
||||
features have no storage root.
|
||||
|
||||
No pipelines are built in. A run requires a configured pipeline.
|
||||
|
||||
## LLM Profiles
|
||||
If `scriptorium` is omitted, Notarius uses Scriptorium's built-in profile
|
||||
catalog. Prompt definitions may also name default profile IDs. The current D&D
|
||||
scene and spell prompts use Scriptorium prompt defaults when a module binding
|
||||
does not set `llm_profile`.
|
||||
|
||||
Each `llm_profiles` entry may contain:
|
||||
## Scriptorium Profiles
|
||||
|
||||
- `provider`: optional provider key. Empty means `openai-compatible`; any other
|
||||
non-empty value must be `openai-compatible`.
|
||||
- `base_url`: provider base URL. Required for actual LLM calls.
|
||||
- `model`: provider model name. Required for actual LLM calls.
|
||||
- `api_key_env`: environment variable name to read for the API key.
|
||||
- `timeout`: request timeout as whole seconds or a Go-style duration string such
|
||||
as `10m`.
|
||||
- `max_retries`: retry count for provider calls. Must be zero or greater.
|
||||
- `max_concurrency`: per-profile LLM concurrency. Must be zero or greater; when
|
||||
zero, Notarius uses `concurrency.total_llm`.
|
||||
`scriptorium` fields:
|
||||
|
||||
Raw API keys are not accepted as file config fields. Use `api_key_env` or an
|
||||
environment override.
|
||||
- `profile_dir`: optional directory containing Scriptorium profile YAML files.
|
||||
- `profile_file`: optional Scriptorium profile YAML file.
|
||||
|
||||
`profile_dir` and `profile_file` are mutually exclusive. Custom profiles
|
||||
overlay Scriptorium built-in profiles by profile ID.
|
||||
|
||||
Scriptorium profile files use Scriptorium's profile schema. A minimal profile
|
||||
looks like:
|
||||
|
||||
```yaml
|
||||
id: local-fast
|
||||
endpoint: http://127.0.0.1:8080/v1
|
||||
model: your-model
|
||||
api_key_env: SCRIPTORIUM_API_KEY
|
||||
timeout_seconds: 180
|
||||
```
|
||||
|
||||
Notarius does not accept raw API keys in Notarius config. For file-backed
|
||||
Scriptorium profiles, store the environment variable name in `api_key_env` and
|
||||
set that variable in the run environment. Scriptorium rejects raw `api_key`
|
||||
fields in profile YAML.
|
||||
|
||||
## Environment Overrides
|
||||
|
||||
These environment variables are applied after the config file:
|
||||
|
||||
- `NOTARIUS_CONFIG`: config discovery path.
|
||||
- `NOTARIUS_LLM_DEFAULT_API_KEY`: API key for the `default` LLM profile.
|
||||
- `NOTARIUS_LLM_DEFAULT_BASE_URL`: base URL for the `default` LLM profile.
|
||||
- `NOTARIUS_LLM_DEFAULT_MODEL`: model for the `default` LLM profile.
|
||||
- `NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS`: integer timeout seconds for the
|
||||
`default` LLM profile.
|
||||
- `NOTARIUS_LLM_DEFAULT_MAX_RETRIES`: integer retry count for the `default` LLM
|
||||
profile.
|
||||
- `NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY`: integer max concurrency for the
|
||||
`default` LLM profile.
|
||||
- `NOTARIUS_TOTAL_LLM_CONCURRENCY`: integer global LLM concurrency.
|
||||
- `NOTARIUS_WORK_DIR`: diagnostics work directory.
|
||||
- `NOTARIUS_DIAGNOSTICS_RETENTION`: diagnostics retention mode.
|
||||
- `NOTARIUS_WORKSPACE_DIR`: workspace directory.
|
||||
- `NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED`: boolean diagnostics enablement.
|
||||
- `NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION`: workspace diagnostics retention
|
||||
mode.
|
||||
- `NOTARIUS_WORKSPACE_RESUME_ENABLED`: boolean resume checkpointing
|
||||
enablement.
|
||||
- `NOTARIUS_WORKSPACE_DEBUG_ENABLED`: boolean debug artifact enablement.
|
||||
- `NOTARIUS_WORK_DIR`: deprecated diagnostics work directory compatibility
|
||||
override.
|
||||
- `NOTARIUS_DIAGNOSTICS_RETENTION`: deprecated diagnostics retention
|
||||
compatibility override.
|
||||
|
||||
Integer environment values must parse as base-10 integers.
|
||||
Integer environment values must parse as base-10 integers. Boolean environment
|
||||
values must parse as Go booleans such as `true`, `false`, `1`, or `0`.
|
||||
|
||||
The removed `NOTARIUS_LLM_DEFAULT_*` variables are not read. Configure provider
|
||||
endpoint, model, and credential environment variable names through Scriptorium
|
||||
profiles.
|
||||
|
||||
## Pipelines
|
||||
|
||||
@@ -121,19 +144,92 @@ Pipeline fields:
|
||||
- `artifacts`: required for pipeline resolution. It maps artifact lane IDs to
|
||||
lane definitions.
|
||||
- `output`: optional module binding. Default module is `json`.
|
||||
- `references`: optional map of reference slot names to reference paths. These
|
||||
bindings are defaults for eligible pipeline targets that declare the matching
|
||||
slot.
|
||||
|
||||
Artifact lane fields:
|
||||
|
||||
- `extract`: required module binding.
|
||||
- `merge`: optional module binding. Default module is `appendorder`.
|
||||
- `normalize`: optional module binding. Default module is `noop`.
|
||||
- `validators`: optional list of module bindings. The production CLI currently
|
||||
does not register validator modules.
|
||||
- `validators`: deprecated lane-level validator list. Non-empty lists are
|
||||
rejected; use `extract.validators`, `merge.validators`, or
|
||||
`normalize.validators`.
|
||||
- `references`: optional compatibility alias for extractor reference bindings.
|
||||
Lane bindings override pipeline-level bindings for the same slot.
|
||||
|
||||
`notarius run` and `notarius config validate --pipeline` resolve the pipeline
|
||||
against the production module catalog and fail fast for unknown or incompatible
|
||||
module keys.
|
||||
|
||||
Reference bindings are validated against reference slots declared by eligible
|
||||
chunk, extract, merge, and normalize targets during pipeline resolution. Required slots
|
||||
must be bound after config defaults, target-local references, lane-level
|
||||
compatibility bindings, and run-time `--reference` or `--without-reference`
|
||||
overrides are applied. Config-relative paths are resolved relative to the config
|
||||
file; CLI reference paths are resolved relative to the current working
|
||||
directory. Materialized bound files must be UTF-8 text. Materialized reference
|
||||
provenance is recorded for chunk, extractor, merger, and normalizer targets, and runtime
|
||||
reference content is passed to the target that declares the slot. Reference
|
||||
media types are inferred from file extensions, recorded as canonical base media
|
||||
types, and checked only when a module declares `AcceptedMediaTypes`; unknown
|
||||
extensions are recorded as `application/octet-stream`. Reference content is not
|
||||
written to diagnostics, logs, errors, or manifests.
|
||||
|
||||
Pipeline-level `references` are defaults. They are valid when at least one
|
||||
eligible target in the full configured pipeline declares the slot, including
|
||||
chunk, extractor, merger, and normalizer targets. During a run, they apply only
|
||||
to the selected targets that declare the slot:
|
||||
|
||||
```yaml
|
||||
pipelines:
|
||||
dnd-session:
|
||||
input: seriatim
|
||||
references:
|
||||
players: ./campaign/players.txt
|
||||
party: ./campaign/party-roster.txt
|
||||
glossary: ./campaign/glossary.txt
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
```
|
||||
|
||||
Extractor binding `references` are the canonical lane-local location. The
|
||||
legacy lane-level `references` field remains supported as an alias; when both
|
||||
bind the same slot, `extract.references` wins:
|
||||
|
||||
```yaml
|
||||
pipelines:
|
||||
dnd-session:
|
||||
input: seriatim
|
||||
references:
|
||||
glossary: ./campaign/glossary.txt
|
||||
artifacts:
|
||||
spells:
|
||||
references:
|
||||
roster: ./campaign/legacy-roster.txt
|
||||
extract:
|
||||
module: dnd/spells
|
||||
references:
|
||||
party: ./campaign/session-party.txt
|
||||
```
|
||||
|
||||
`chunk.references`, `merge.references`, and `normalize.references` are accepted
|
||||
in object-form bindings. They override pipeline-level defaults for slots
|
||||
declared by that target module. Extractor-local references apply only to the
|
||||
extractor, merger-local references apply only to the merger, and
|
||||
normalizer-local references apply only to the normalizer.
|
||||
|
||||
Target-local reference fields use the same map shape at:
|
||||
|
||||
- `pipelines.<id>.chunk.references`
|
||||
- `pipelines.<id>.artifacts.<lane>.extract.references`
|
||||
- `pipelines.<id>.artifacts.<lane>.merge.references`
|
||||
- `pipelines.<id>.artifacts.<lane>.normalize.references`
|
||||
|
||||
Each binding is valid only when that target module declares the slot.
|
||||
|
||||
## Module Bindings
|
||||
|
||||
Every module binding may use shorthand:
|
||||
@@ -146,20 +242,47 @@ or object form:
|
||||
|
||||
```yaml
|
||||
chunk:
|
||||
module: generic
|
||||
llm_profile: default
|
||||
options:
|
||||
max_units: 50
|
||||
module: dnd/scenes
|
||||
llm_profile: local-fast
|
||||
```
|
||||
|
||||
Binding fields:
|
||||
|
||||
- `module`: module key.
|
||||
- `llm_profile`: optional LLM profile ID. Empty means `default`.
|
||||
- `llm_profile`: optional Scriptorium profile ID. Empty or omitted lets the
|
||||
Scriptorium prompt default select the profile.
|
||||
- `retries`: non-negative retry count for extra runtime attempts after the
|
||||
first attempt. The runner applies retries to `chunk`, `extract`, `merge`, and
|
||||
`normalize` bindings.
|
||||
- `options`: optional module-specific settings.
|
||||
- `references`: optional reference bindings. Supported only for `chunk`,
|
||||
`extract`, `merge`, and `normalize` bindings. `input` and `output` bindings
|
||||
reject this field during validation.
|
||||
- `validators`: optional stage-local validator chain override. Supported only
|
||||
for `chunk`, `extract`, `merge`, and `normalize` bindings. Omit the field to
|
||||
use the production default chain; set `validators: []` to force an empty
|
||||
chain; set a non-empty list to use exactly those validators in configured
|
||||
order.
|
||||
|
||||
The `--llm-profile` run flag overrides every effective module binding to use
|
||||
one configured profile.
|
||||
Validator bindings use the same shorthand or object module-binding form, but
|
||||
only these fields are supported:
|
||||
|
||||
- `module`: validator key.
|
||||
- `llm_profile`: optional Scriptorium profile ID for LLM-backed validators.
|
||||
- `options`: optional validator-specific settings.
|
||||
|
||||
Validator bindings reject `references`, `retries`, and nested `validators`.
|
||||
During resolution, deterministic validators reject explicit `llm_profile`
|
||||
values.
|
||||
|
||||
The `--llm-profile` run flag overrides every effective LLM-capable module
|
||||
binding to use one Scriptorium profile ID: chunk, every selected lane extract,
|
||||
merge, and normalize binding. It does not override validator-specific
|
||||
`llm_profile` values.
|
||||
|
||||
Configured LLM-backed validators with explicit `llm_profile` values are
|
||||
validated against the configured Scriptorium profile source. Deterministic
|
||||
production validators do not call the LLM and must not set `llm_profile`.
|
||||
|
||||
## Implemented Production Modules
|
||||
|
||||
@@ -167,10 +290,37 @@ one configured profile.
|
||||
| --- | --- | --- |
|
||||
| input | `seriatim` | Reads Seriatim transcript JSON. |
|
||||
| chunk | `generic` | Splits source units into ordered chunks. |
|
||||
| extract | `dnd/spells` | Extracts `dnd.spell_cast` artifacts. |
|
||||
| merge | `appendorder` | Keeps candidates in append order. |
|
||||
| normalize | `noop` | Passes merged artifacts through unchanged. |
|
||||
| output | `json` | Produces JSON output files. |
|
||||
| chunk | `dnd/scenes` | Uses an LLM to split transcript source units into D&D scenes. |
|
||||
| extract | `dnd/spells` | Extracts D&D spell raw outputs. |
|
||||
| merge | `appendorder` | Merges JSON raw extract outputs in chunk order. |
|
||||
| normalize | `noop` | Passes merged raw outputs through unchanged. |
|
||||
| output | `json` | Produces JSON output files for normalized `application/json` lanes. |
|
||||
|
||||
## Implemented Production Validators
|
||||
|
||||
| Key | Execution | Notes |
|
||||
| --- | --- | --- |
|
||||
| `generic/always_accept` | deterministic | Accepts returned module output. |
|
||||
| `generic/always_reject` | deterministic | Rejects returned module output with reason `always_reject`. |
|
||||
| `generic/valid_json` | deterministic | Rejects payloads that are not syntactically valid JSON. |
|
||||
| `generic/valid_json_schema` | deterministic | Rejects invalid JSON or JSON that does not conform to the module response schema. |
|
||||
| `extract/dnd/spells/shape` | deterministic | Rejects malformed D&D spell-cast JSON payloads. |
|
||||
| `extract/dnd/spells/source_refs` | deterministic | Rejects missing or invalid D&D spell source references. |
|
||||
| `extract/dnd/spells/source_relatedness` | deterministic | Emits warnings when a spell name is not found near its cited source text. |
|
||||
|
||||
The production default chain for the `dnd/spells` extractor is:
|
||||
|
||||
```yaml
|
||||
validators:
|
||||
- generic/valid_json
|
||||
- generic/valid_json_schema
|
||||
- extract/dnd/spells/shape
|
||||
- extract/dnd/spells/source_refs
|
||||
- extract/dnd/spells/source_relatedness
|
||||
```
|
||||
|
||||
No other production module currently has a default validator chain. Empty
|
||||
chains approve output by default.
|
||||
|
||||
The `generic` chunker accepts:
|
||||
|
||||
@@ -178,30 +328,113 @@ The `generic` chunker accepts:
|
||||
- `overlap_units`: non-negative integer, default `0`, and must be less than
|
||||
`max_units`.
|
||||
|
||||
The `dnd/scenes` chunker requires transcript source capabilities, calls the
|
||||
configured structured LLM runtime, and does not accept module options. It
|
||||
declares optional `players`, `party`, and `glossary` references for scene
|
||||
disambiguation, and accepts `roster` as a deprecated compatibility alias for
|
||||
`party`.
|
||||
|
||||
The `dnd/spells` extractor declares optional reference slots:
|
||||
|
||||
- `players`
|
||||
- `party`
|
||||
- `glossary`
|
||||
- `roster` as a deprecated compatibility alias for `party`
|
||||
|
||||
Both modules accept UTF-8 plain text, Markdown, YAML, or JSON reference files.
|
||||
The extractor uses references only as supporting disambiguation material; spell
|
||||
casts still must be present in the source transcript.
|
||||
|
||||
## Workspace
|
||||
|
||||
`workspace` fields:
|
||||
|
||||
- `directory`: optional workspace root for Notarius-owned local state.
|
||||
- `diagnostics.enabled`: set to `false` to skip diagnostics run directories and
|
||||
diagnostics artifact writes. Default: `true`.
|
||||
- `diagnostics.retention`: `auto`, `always`, or `never`.
|
||||
- `resume.enabled`: boolean resume checkpointing setting. Default: `false`.
|
||||
- `debug.enabled`: boolean debug artifact setting. Default: `false`.
|
||||
|
||||
Use `/var/lib/notarius` as the standard production workspace directory. For
|
||||
local development, prefer a project-local ignored path such as
|
||||
`./.notarius/workspace`.
|
||||
|
||||
```yaml
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
enabled: true
|
||||
retention: auto
|
||||
resume:
|
||||
enabled: false
|
||||
debug:
|
||||
enabled: false
|
||||
```
|
||||
|
||||
When `workspace.directory` is set, diagnostics are written under
|
||||
`<workspace.directory>/diagnostics/`.
|
||||
|
||||
When both `workspace.directory` and `workspace.resume.enabled` are set, runs
|
||||
write stage-owned checkpoint artifacts under
|
||||
`<workspace.directory>/checkpoints/`. `notarius run --resume` can reuse valid
|
||||
checkpoints from a compatible invocation. Checkpoints may contain source text,
|
||||
intermediate raw outputs, rejected outputs, metadata, and warnings. Protect the
|
||||
workspace as sensitive local state.
|
||||
|
||||
When both `workspace.directory` and `workspace.debug.enabled` are set, runs
|
||||
write per-invocation debug artifacts under
|
||||
`<workspace.directory>/debug/<run-id>/`. Debug artifacts may contain source
|
||||
material, reference material, prompt inputs, model outputs, validation payloads,
|
||||
and other sensitive content. Debug is disabled by default.
|
||||
|
||||
`workspace.resume.enabled` and `workspace.debug.enabled` are independent.
|
||||
Enabling one does not enable the other.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Preferred workspace diagnostics fields:
|
||||
|
||||
- `workspace.directory`: workspace root for Notarius-owned local state.
|
||||
- `workspace.diagnostics.enabled`: set to `false` to skip creating diagnostics
|
||||
run directories and diagnostics artifacts. Default: `true`.
|
||||
- `workspace.diagnostics.retention`: `auto`, `always`, or `never`.
|
||||
|
||||
When `workspace.directory` is set, diagnostics use
|
||||
`<workspace.directory>/diagnostics` as their work directory.
|
||||
`workspace.diagnostics.retention` overrides legacy diagnostics retention when
|
||||
set.
|
||||
|
||||
`diagnostics` fields:
|
||||
|
||||
- `work_dir`: directory for per-run diagnostics. Default: `/tmp/notarius`.
|
||||
- `retention`: `auto`, `always`, or `never`. Empty uses `auto`.
|
||||
- `work_dir`: deprecated compatibility directory for per-run diagnostics.
|
||||
Default: `/tmp/notarius`.
|
||||
- `retention`: deprecated compatibility retention mode. `auto`, `always`, or
|
||||
`never`. Empty uses `auto`.
|
||||
|
||||
Existing `diagnostics.work_dir`, `diagnostics.retention`, `NOTARIUS_WORK_DIR`,
|
||||
and `NOTARIUS_DIAGNOSTICS_RETENTION` inputs remain supported for compatibility.
|
||||
New configuration should use `workspace.directory` and
|
||||
`workspace.diagnostics.retention` instead.
|
||||
|
||||
`auto` retains diagnostics for failed runs and successful runs with warnings.
|
||||
`always` retains diagnostics for every run. `never` removes diagnostics for
|
||||
successful runs without regard to warnings; failed runs are retained.
|
||||
|
||||
The `--diagnostics-dir` run flag overrides `diagnostics.work_dir` for that
|
||||
invocation.
|
||||
The `--diagnostics-dir` run flag overrides the effective diagnostics work
|
||||
directory for that invocation. It affects diagnostics only and does not change
|
||||
the workspace directory.
|
||||
|
||||
## Validation
|
||||
|
||||
Configuration validation checks:
|
||||
|
||||
- supported config version and known YAML fields;
|
||||
- mutually exclusive `scriptorium.profile_dir` and `scriptorium.profile_file`;
|
||||
- non-empty, non-duplicated IDs after trimming;
|
||||
- supported LLM provider and non-negative profile limits;
|
||||
- positive global LLM concurrency;
|
||||
- supported diagnostics retention and non-empty work directory;
|
||||
- module binding LLM profiles refer to configured profiles.
|
||||
- stale removed fields such as `llm_profiles`.
|
||||
|
||||
Pipeline resolution additionally checks:
|
||||
|
||||
@@ -210,4 +443,11 @@ Pipeline resolution additionally checks:
|
||||
- selected lanes exist when `--only` is used;
|
||||
- required module keys are present;
|
||||
- module keys are registered for the expected slot;
|
||||
- module capability requirements are satisfied.
|
||||
- module capability requirements are satisfied;
|
||||
- non-empty validator overrides reference registered validator keys;
|
||||
- deterministic validators do not set `llm_profile`;
|
||||
- LLM-backed validators with explicit `llm_profile` values reference configured
|
||||
Scriptorium profile IDs;
|
||||
- bound reference slots are declared by selected chunk, extractor, merger, or
|
||||
normalizer targets;
|
||||
- required reference slots are bound for selected targets.
|
||||
|
||||
75
docs/development.md
Normal file
75
docs/development.md
Normal file
@@ -0,0 +1,75 @@
|
||||
# Development
|
||||
|
||||
This is the first-read landing page for people and LLM coding agents working on
|
||||
Notarius. It provides a concise repository orientation and routes each kind of
|
||||
change to its canonical documentation.
|
||||
|
||||
## Orientation
|
||||
|
||||
Notarius is a small Go application for extracting structured data from source
|
||||
material. It is a general extraction platform with an initial Seriatim and D&D
|
||||
implementation. Configured modules run through a fixed workflow:
|
||||
|
||||
```text
|
||||
input -> chunk -> extract -> merge -> normalize -> output
|
||||
```
|
||||
|
||||
The CLI is the application boundary. Core packages own deterministic models and
|
||||
policy, framework packages own reusable contracts and orchestration, and module
|
||||
and validator packages own concrete behavior.
|
||||
|
||||
## Repository Map
|
||||
|
||||
- `cmd/notarius`: executable entry point.
|
||||
- `internal/cli`: CLI behavior and production composition.
|
||||
- `internal/core`: deterministic source, config, artifact, diagnostics, and
|
||||
workspace packages.
|
||||
- `internal/framework`: contracts, pipeline orchestration, validation helpers,
|
||||
checkpoints, debug recording, and LLM runtime plumbing.
|
||||
- `internal/modules`: concrete implementations of the six pipeline stages.
|
||||
- `internal/validators`: concrete output validators.
|
||||
- `docs`: canonical policy, reference, integration, internal, ADR, and roadmap
|
||||
documentation.
|
||||
- `examples`: maintained, secret-free example inputs and configuration.
|
||||
|
||||
See [Internal Overview](internal/overview.md) for the implemented component
|
||||
map and links to focused internal documentation.
|
||||
|
||||
## What To Read
|
||||
|
||||
| When working on | Read | Why |
|
||||
| --- | --- | --- |
|
||||
| Application shape, package boundaries, contracts, dependency direction, runtime guarantees, or safety properties | [Architecture](policy/architecture.md) and relevant [ADRs](adr/) | Architecture defines the intended system and its invariants; ADRs preserve significant decision rationale. |
|
||||
| Any documentation addition or revision | [Documentation Policy](policy/documentation.md) | It defines canonical homes, audiences, current-behavior rules, and maintenance requirements. |
|
||||
| Pipeline resolution or execution | [Pipeline Internals](internal/pipeline.md) | It documents profiles, references, validation, retries, checkpoints, and runner behavior. |
|
||||
| Production modules or validators | [Module Internals](internal/modules.md) | It documents implemented module contracts, capabilities, assets, and registration. |
|
||||
| LLM clients, prompts, schemas, profiles, or scheduling | [LLM Runtime](internal/llm.md) | It documents the transport boundary and Scriptorium integration. |
|
||||
| Diagnostics, workspace state, resume, or debug artifacts | [Diagnostics Internals](internal/diagnostics.md), [Operations](operations.md), and [Configuration](config.md) | These separate implementation details, operator behavior, and configuration contracts. |
|
||||
| CLI or user-visible configuration behavior | [CLI Reference](cli.md) and [Configuration](config.md) | These are the canonical user and operator references. |
|
||||
| External input formats, artifact schemas, or durable output files | [Integration Contracts](integrations/) | Integration documents define external and durable data contracts. |
|
||||
| Proposed or unimplemented behavior | [Roadmap](roadmap/) | Future work belongs only in roadmap documentation until implemented. |
|
||||
|
||||
For an existing subsystem, also inspect its focused tests and the package-local
|
||||
types and contracts before changing behavior.
|
||||
|
||||
## Validation
|
||||
|
||||
Use focused package tests while iterating. Run the repository-wide checks when
|
||||
a change affects shared contracts, application behavior, or maintained
|
||||
documentation examples:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go vet ./...
|
||||
go build ./cmd/notarius
|
||||
```
|
||||
|
||||
## Universal Reminders
|
||||
|
||||
- Keep non-roadmap documentation limited to implemented behavior.
|
||||
- Update affected documentation and maintained examples in the same change as
|
||||
behavior.
|
||||
- Use fakes, fixtures, or local test servers instead of real external services
|
||||
in tests.
|
||||
- Do not expose secrets in code, errors, logs, diagnostics, manifests,
|
||||
documentation, or examples.
|
||||
@@ -1,83 +1,29 @@
|
||||
# D&D Spell-Cast Artifacts
|
||||
# D&D Spell Raw Output
|
||||
|
||||
This document is the durable artifact contract for approved
|
||||
`dnd.spell_cast` artifacts produced by the implemented `dnd/spells` extractor.
|
||||
This document is the durable raw output contract for the implemented
|
||||
`dnd/spells` extractor.
|
||||
|
||||
## Artifact Identity
|
||||
## Identity
|
||||
|
||||
- Extractor key: `dnd/spells`
|
||||
- Artifact type: `dnd.spell_cast`
|
||||
- Schema version: `v1`
|
||||
- Prompt ID: `dnd.spells`
|
||||
- Response schema key: `dnd_spells`
|
||||
- Response schema ID: `notarius.dnd.spells`
|
||||
- Response schema name: `notarius_dnd_spells_v1`
|
||||
- Response schema version: `v1`
|
||||
- Media type: `application/json`
|
||||
|
||||
The extractor requires source chunks and transcript source capability. It
|
||||
returns generic artifact candidates that are serialized by the JSON output
|
||||
module.
|
||||
returns canonical spell-cast JSON derived from the structured LLM response. The
|
||||
extractor assigns source IDs deterministically and keeps source-unit ranges as
|
||||
model-authored evidence locations. The default `appendorder` merger passes a
|
||||
single chunk output through and concatenates multiple
|
||||
`spell_casts` arrays in chunk order. The default `noop` normalizer passes the
|
||||
merge output through unchanged.
|
||||
|
||||
## Artifact Envelope
|
||||
## Output Shape
|
||||
|
||||
Approved artifacts use the generic artifact envelope documented in
|
||||
[JSON Output](json-output.md#artifact-files):
|
||||
|
||||
```json
|
||||
{
|
||||
"extractor_key": "dnd/spells",
|
||||
"artifact_type": "dnd.spell_cast",
|
||||
"schema_version": "v1",
|
||||
"payload": {
|
||||
"caster": "Aria",
|
||||
"spell": "Cure Wounds",
|
||||
"effect": "heals an injured ally",
|
||||
"narrative_description": "Aria raises her holy symbol and casts Cure Wounds."
|
||||
},
|
||||
"source_refs": [
|
||||
{
|
||||
"source_id": "session-alpha",
|
||||
"start_unit_id": "seg-001",
|
||||
"end_unit_id": "seg-001"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Payload Fields
|
||||
|
||||
The `payload` object contains:
|
||||
|
||||
- `caster`: in-world character or creature casting the spell;
|
||||
- `spell`: spell name;
|
||||
- `effect`: concise spell effect in the scene;
|
||||
- `narrative_description`: short description of the spell cast in context.
|
||||
|
||||
All payload fields are strings and must be non-empty after trimming.
|
||||
|
||||
`caster` is the in-world caster, not the transcript speaker.
|
||||
|
||||
## Source References
|
||||
|
||||
Source references live on the artifact envelope as `source_refs`; they are not
|
||||
duplicated inside the `payload`.
|
||||
|
||||
Each source reference uses the generic source-reference shape:
|
||||
|
||||
- `source_id`
|
||||
- `start_unit_id`
|
||||
- `end_unit_id`
|
||||
|
||||
Validation requires:
|
||||
|
||||
- at least one source reference;
|
||||
- non-empty source ID and unit IDs;
|
||||
- source ID matching the source document ID;
|
||||
- start and end unit IDs existing in the source document;
|
||||
- start unit appearing before or at the same position as end unit.
|
||||
|
||||
## Structured LLM Response Shape
|
||||
|
||||
The extractor asks the LLM for this top-level response shape:
|
||||
For a single chunk, `lanes/spells.json` has this shape:
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -90,8 +36,8 @@ The extractor asks the LLM for this top-level response shape:
|
||||
"source_refs": [
|
||||
{
|
||||
"source_id": "session-alpha",
|
||||
"start_unit_id": "seg-001",
|
||||
"end_unit_id": "seg-001"
|
||||
"start_unit_id": 1,
|
||||
"end_unit_id": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -101,26 +47,66 @@ The extractor asks the LLM for this top-level response shape:
|
||||
|
||||
`spell_casts` must be present. It may be empty when no spell casts are found.
|
||||
|
||||
The response schema asset is embedded at
|
||||
`internal/modules/extract/dnd/spells/assets/schemas/dnd_spells.v1.json`.
|
||||
For multiple chunks with the default merger, the lane output keeps the same
|
||||
top-level shape and concatenates `spell_casts` in chunk order:
|
||||
|
||||
## Validators
|
||||
```json
|
||||
{
|
||||
"spell_casts": [
|
||||
{
|
||||
"caster": "Aria",
|
||||
"spell": "Cure Wounds",
|
||||
"effect": "heals an injured ally",
|
||||
"narrative_description": "Aria raises her holy symbol and casts Cure Wounds.",
|
||||
"source_refs": [
|
||||
{
|
||||
"source_id": "session-alpha",
|
||||
"start_unit_id": 1,
|
||||
"end_unit_id": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The extractor supplies two deterministic validators by default:
|
||||
## Spell-Cast Fields
|
||||
|
||||
- `dnd/spells/shape`
|
||||
- `dnd/spells/source_refs`
|
||||
Each spell cast contains:
|
||||
|
||||
Rejection reason codes:
|
||||
- `caster`: in-world character or creature casting the spell;
|
||||
- `spell`: spell name;
|
||||
- `effect`: concise spell effect in the scene;
|
||||
- `narrative_description`: short description of the spell cast in context;
|
||||
- `source_refs`: transcript source references with extractor-assigned source
|
||||
IDs and model-supplied unit ranges.
|
||||
|
||||
- `invalid_payload`: payload JSON cannot be decoded as a spell-cast payload.
|
||||
- `missing_required_field`: `caster`, `spell`, `effect`, or
|
||||
`narrative_description` is blank.
|
||||
- `missing_source_ref`: candidate has no source references.
|
||||
- `invalid_source_ref`: at least one source reference fails generic source
|
||||
reference validation.
|
||||
`caster` is the in-world caster, not the transcript speaker.
|
||||
|
||||
Rejected candidates are written to `rejected.json` by the JSON output module.
|
||||
## Source References
|
||||
|
||||
Each source reference uses the generic source-reference shape:
|
||||
|
||||
- `source_id`
|
||||
- `start_unit_id`
|
||||
- `end_unit_id`
|
||||
|
||||
The LLM-facing prompt schema asks only for integer `start_unit_id` and
|
||||
`end_unit_id` values matching source-unit IDs. `source_id` is assigned by the
|
||||
extractor from the source document ID before validation and output, and is
|
||||
required in this durable output contract.
|
||||
|
||||
## References
|
||||
|
||||
The extractor accepts optional UTF-8 text references:
|
||||
|
||||
- `players`
|
||||
- `party`
|
||||
- `glossary`
|
||||
- `roster`, a deprecated compatibility alias for `party`
|
||||
|
||||
References are supporting disambiguation material only. They are not source
|
||||
evidence and are not addressable through `source_refs`.
|
||||
|
||||
## Manifest Metadata
|
||||
|
||||
@@ -145,8 +131,3 @@ manifest metadata:
|
||||
```
|
||||
|
||||
Raw prompt and schema content are not included in manifest metadata.
|
||||
|
||||
## Compatibility Limit
|
||||
|
||||
This contract covers only `dnd.spell_cast` artifacts produced by the
|
||||
implemented spell-cast extractor.
|
||||
|
||||
@@ -20,11 +20,11 @@ The `json` output module writes:
|
||||
|
||||
- `index.json`
|
||||
- `manifest.json`
|
||||
- `artifacts/<artifact-type>.json`, one file per approved artifact type
|
||||
- `lanes/<lane-id>.json`, one file per normalized raw lane output
|
||||
- `rejected.json`
|
||||
- `warnings.json`
|
||||
|
||||
Files are pretty-printed JSON with a trailing newline.
|
||||
Files are pretty-printed JSON with a trailing newline when the payload is JSON.
|
||||
|
||||
## `index.json`
|
||||
|
||||
@@ -33,10 +33,15 @@ Shape:
|
||||
```json
|
||||
{
|
||||
"manifest_file": "manifest.json",
|
||||
"artifact_files": [
|
||||
"output_files": [
|
||||
{
|
||||
"artifact_type": "dnd.spell_cast",
|
||||
"file": "artifacts/dnd.spell_cast.json"
|
||||
"lane_id": "spells",
|
||||
"media_type": "application/json",
|
||||
"file": "lanes/spells.json",
|
||||
"module_key": "noop",
|
||||
"schema_id": "notarius.dnd.spells",
|
||||
"schema_name": "notarius_dnd_spells_v1",
|
||||
"schema_version": "v1"
|
||||
}
|
||||
],
|
||||
"rejected_file": "rejected.json",
|
||||
@@ -44,8 +49,14 @@ Shape:
|
||||
}
|
||||
```
|
||||
|
||||
`artifact_files` is sorted by artifact type. It is empty when no artifacts are
|
||||
approved.
|
||||
`output_files` is sorted by lane ID. Output file names are produced by
|
||||
sanitizing the lane ID:
|
||||
|
||||
- characters outside `A-Z`, `a-z`, `0-9`, `.`, `_`, and `-` become `_`;
|
||||
- repeated `..` sequences are replaced;
|
||||
- leading and trailing `.`, `_`, and `-` are trimmed;
|
||||
- empty sanitized names are rejected;
|
||||
- two lanes that sanitize to the same output file are rejected.
|
||||
|
||||
## `manifest.json`
|
||||
|
||||
@@ -57,7 +68,7 @@ approved.
|
||||
"pipeline_id": "dnd-session",
|
||||
"pipeline_digest": "sha256:...",
|
||||
"input_module": "seriatim",
|
||||
"chunker": "generic",
|
||||
"chunker": "dnd/scenes",
|
||||
"source_digests": ["sha256:..."],
|
||||
"extractors": ["dnd/spells"],
|
||||
"merger": "appendorder",
|
||||
@@ -71,11 +82,33 @@ approved.
|
||||
"normalizer": "noop"
|
||||
}
|
||||
],
|
||||
"llm_profiles": [
|
||||
"validator_chains": [
|
||||
{
|
||||
"id": "default",
|
||||
"provider": "openai-compatible",
|
||||
"model": "configured-model"
|
||||
"stage": "extract",
|
||||
"lane_id": "spells",
|
||||
"module_key": "dnd/spells",
|
||||
"validators": [
|
||||
{
|
||||
"key": "generic/valid_json",
|
||||
"execution_class": "deterministic"
|
||||
},
|
||||
{
|
||||
"key": "generic/valid_json_schema",
|
||||
"execution_class": "deterministic"
|
||||
},
|
||||
{
|
||||
"key": "extract/dnd/spells/shape",
|
||||
"execution_class": "deterministic"
|
||||
},
|
||||
{
|
||||
"key": "extract/dnd/spells/source_refs",
|
||||
"execution_class": "deterministic"
|
||||
},
|
||||
{
|
||||
"key": "extract/dnd/spells/source_relatedness",
|
||||
"execution_class": "deterministic"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"validation_status": "approved",
|
||||
@@ -86,27 +119,54 @@ approved.
|
||||
|
||||
Fields with empty values may be omitted by JSON encoding.
|
||||
|
||||
`validation_status` is `approved` when no candidates were rejected and
|
||||
`rejected` when one or more candidates were rejected.
|
||||
`source_digests` contains source document digests only. Bound references are
|
||||
recorded separately under `references`, which contains provenance only: target
|
||||
stage, lane ID when present, slot name, origin type and URI, digest, media
|
||||
type, byte size, and binding source. Reference content is not written to
|
||||
durable output.
|
||||
|
||||
## Artifact Files
|
||||
Reference `stage` is `chunk`, `extract`, `merge`, or `normalize`. `lane_id` is
|
||||
omitted for chunk references and present for extract, merge, and normalize
|
||||
references.
|
||||
|
||||
Each artifact file has this shape:
|
||||
`validation_status` is `approved` when no raw outputs were rejected and
|
||||
`rejected` when one or more raw outputs were rejected.
|
||||
|
||||
`validator_chains` records the resolved validator chain for each validation
|
||||
point. Entries include stage, lane ID when applicable, module key, and validators
|
||||
with key and execution class. Empty chains are recorded with an empty
|
||||
`validators` array, including chains resolved from explicit empty config
|
||||
overrides.
|
||||
|
||||
`normalized_outputs` summarizes each normalized lane output without embedding
|
||||
payload bytes. Entries include lane ID, normalizer module key, source ID, media
|
||||
type, and response schema provenance where available.
|
||||
|
||||
`rejected_outputs` summarizes rejected module outputs without embedding raw
|
||||
payload bytes. Entries include stage, lane, module, chunk, validator or reason,
|
||||
message, attempt count, and optional diagnostic artifact path.
|
||||
|
||||
## Output Payload Files
|
||||
|
||||
Each normalized raw output is written to `lanes/<sanitized-lane-id>.json`.
|
||||
The JSON output encoder accepts only `application/json` normalized outputs. The
|
||||
file contains the raw JSON payload pretty-printed.
|
||||
|
||||
For the current D&D spell extractor, `lanes/spells.json` has this shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"artifact_type": "dnd.spell_cast",
|
||||
"artifacts": [
|
||||
"spell_casts": [
|
||||
{
|
||||
"extractor_key": "dnd/spells",
|
||||
"artifact_type": "dnd.spell_cast",
|
||||
"schema_version": "v1",
|
||||
"payload": {},
|
||||
"caster": "Aria",
|
||||
"spell": "Cure Wounds",
|
||||
"effect": "heals an injured ally",
|
||||
"narrative_description": "Aria raises her holy symbol and casts Cure Wounds.",
|
||||
"source_refs": [
|
||||
{
|
||||
"source_id": "session-alpha",
|
||||
"start_unit_id": "seg-001",
|
||||
"end_unit_id": "seg-001"
|
||||
"start_unit_id": 1,
|
||||
"end_unit_id": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -114,50 +174,20 @@ Each artifact file has this shape:
|
||||
}
|
||||
```
|
||||
|
||||
Artifact envelope fields:
|
||||
|
||||
- `extractor_key`: extractor module key.
|
||||
- `artifact_type`: artifact type.
|
||||
- `schema_version`: artifact schema version.
|
||||
- `payload`: artifact-type-specific JSON payload.
|
||||
- `source_refs`: optional generic source references.
|
||||
- `metadata`: optional artifact metadata.
|
||||
|
||||
Artifact file names are produced by sanitizing the artifact type:
|
||||
|
||||
- characters outside `A-Z`, `a-z`, `0-9`, `.`, `_`, and `-` become `_`;
|
||||
- repeated `..` sequences are replaced;
|
||||
- leading and trailing `.`, `_`, and `-` are trimmed;
|
||||
- empty sanitized names are rejected.
|
||||
|
||||
For current D&D spell-cast artifacts, the file is
|
||||
`artifacts/dnd.spell_cast.json`.
|
||||
|
||||
## `rejected.json`
|
||||
|
||||
Shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"rejected": [
|
||||
{
|
||||
"candidate": {
|
||||
"index": 0,
|
||||
"extractor_key": "dnd/spells",
|
||||
"artifact_type": "dnd.spell_cast",
|
||||
"schema_version": "v1",
|
||||
"payload": {},
|
||||
"source_refs": []
|
||||
},
|
||||
"validator_name": "dnd/spells/source_refs",
|
||||
"reason_code": "missing_source_ref",
|
||||
"message": "spell cast candidate must include at least one source ref"
|
||||
}
|
||||
]
|
||||
"rejected": []
|
||||
}
|
||||
```
|
||||
|
||||
`rejected` is an empty array when no candidates are rejected.
|
||||
When raw output validation rejects an output, entries use the
|
||||
`contracts.RejectedOutput` shape, including stage, lane ID, module key,
|
||||
validator name, reason code, message, attempt count, and optional diagnostic
|
||||
artifact path.
|
||||
|
||||
## `warnings.json`
|
||||
|
||||
@@ -167,26 +197,12 @@ Shape:
|
||||
{
|
||||
"warnings": [
|
||||
{
|
||||
"scope": "output",
|
||||
"reason_code": "example_warning",
|
||||
"message": "warning message"
|
||||
"scope": "extract",
|
||||
"reason_code": "example",
|
||||
"message": "human-readable warning"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
`warnings` is an empty array when no warnings are reported.
|
||||
|
||||
## Path Safety
|
||||
|
||||
The output module returns slash-separated logical paths. The CLI also validates
|
||||
logical output names before writing:
|
||||
|
||||
- names must be non-empty;
|
||||
- names must be relative;
|
||||
- names must be clean;
|
||||
- names must use `/`, not `\`;
|
||||
- names must not contain `..`;
|
||||
- resolved paths must stay under the run output directory.
|
||||
|
||||
Durable writes are atomic per file.
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
# OpenAI-Compatible Structured Output
|
||||
|
||||
This document describes the external LLM provider contract implemented by the
|
||||
production Notarius LLM client.
|
||||
|
||||
## Provider
|
||||
|
||||
- Provider key: `openai-compatible`
|
||||
- HTTP method: `POST`
|
||||
- Endpoint: `<base_url>/chat/completions`
|
||||
- Request body: JSON
|
||||
- Response mode: chat completions with structured JSON schema output
|
||||
|
||||
`base_url` is trimmed of trailing slashes before `/chat/completions` is
|
||||
appended. Configure provider settings in [Configuration](../config.md).
|
||||
|
||||
## Request
|
||||
|
||||
The client sends a JSON object with:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "configured-model",
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "..."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "..."
|
||||
}
|
||||
],
|
||||
"response_format": {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "schema_name",
|
||||
"strict": true,
|
||||
"schema": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Implemented request behavior:
|
||||
|
||||
- `model` comes from the structured completion request when set, otherwise from
|
||||
the configured LLM profile.
|
||||
- `messages` must be non-empty; each role and content must be non-empty after
|
||||
trimming.
|
||||
- `response_format.type` is always `json_schema`.
|
||||
- `response_format.json_schema.strict` is always `true`.
|
||||
- `response_format.json_schema.name` and `schema` come from the extractor or
|
||||
validator making the call.
|
||||
|
||||
If an API key is configured, the client sends:
|
||||
|
||||
```text
|
||||
Authorization: Bearer <api-key>
|
||||
```
|
||||
|
||||
The client always sends `Content-Type: application/json`.
|
||||
|
||||
## Response
|
||||
|
||||
The client expects a JSON response with at least one choice:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "provider-model",
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": "{\"field\":\"value\"}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"total_tokens": 15
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`choices[0].message.content` may be either:
|
||||
|
||||
- a JSON string whose contents are valid JSON; or
|
||||
- raw JSON.
|
||||
|
||||
The decoded content is unmarshaled into the caller-provided structured output
|
||||
target. If `usage` is present, prompt, completion, and total token counts are
|
||||
copied into the completion response.
|
||||
|
||||
## Errors And Retries
|
||||
|
||||
The client validates base URL, model, response schema name, response schema
|
||||
JSON, messages, and output target before or during the call.
|
||||
|
||||
Retryable failures:
|
||||
|
||||
- HTTP request failure;
|
||||
- response body read failure;
|
||||
- HTTP `429`;
|
||||
- HTTP `5xx`;
|
||||
- malformed provider response envelope;
|
||||
- missing choices;
|
||||
- missing, empty, or invalid assistant JSON content;
|
||||
- structured-output decode failure.
|
||||
|
||||
Non-retryable provider status codes include non-`429` `4xx` responses.
|
||||
|
||||
Provider error bodies are parsed for `error.message` or `message` when present.
|
||||
Configured API key values and bearer-token values are redacted from returned
|
||||
provider errors.
|
||||
|
||||
## Timeouts And Concurrency
|
||||
|
||||
The configured profile timeout is applied per provider request when greater
|
||||
than zero. Context cancellation is respected.
|
||||
|
||||
The production CLI wraps the provider client with the LLM scheduler. Effective
|
||||
concurrency is described in [LLM runtime internals](../internal/llm.md).
|
||||
|
||||
## Limits
|
||||
|
||||
This contract documents only the fields the implemented client sends and reads.
|
||||
Provider-specific extensions are ignored unless they affect those fields.
|
||||
@@ -28,7 +28,7 @@ output that provides the same required segment fields.
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": "seg-001",
|
||||
"id": 1,
|
||||
"start": 0,
|
||||
"end": 4,
|
||||
"speaker": "Aria",
|
||||
@@ -56,10 +56,9 @@ The adapter rejects:
|
||||
- missing, null, or non-object `metadata`;
|
||||
- missing, null, non-array, or empty `segments`;
|
||||
- segment values that are not objects;
|
||||
- segment `id` values that are neither strings nor numbers;
|
||||
- segment `id` values that are not positive integer JSON numbers or numeric
|
||||
strings;
|
||||
- non-string `speaker` or `text`;
|
||||
- empty segment IDs;
|
||||
- segment IDs with leading or trailing whitespace;
|
||||
- duplicate segment IDs;
|
||||
- missing or empty `speaker`;
|
||||
- missing, empty, invalid, non-finite, or negative `start`;
|
||||
@@ -87,8 +86,7 @@ The adapter maps input to `SourceDocument`:
|
||||
|
||||
Each segment becomes one `SourceUnit`:
|
||||
|
||||
- `segment.id` becomes `SourceUnit.ID`; numeric IDs are converted to their JSON
|
||||
number text, so `1` becomes `"1"`;
|
||||
- `segment.id` becomes integer `SourceUnit.ID`;
|
||||
- `segment.text` becomes `SourceUnit.Text`;
|
||||
- `SourceUnit.Kind` is `transcript_segment`;
|
||||
- `speaker`, `start`, and `end` are stored in source-unit metadata.
|
||||
|
||||
@@ -22,6 +22,11 @@ Diagnostics must not expose secrets.
|
||||
If `workDir` is empty, it defaults to `/tmp/notarius`. Empty retention defaults
|
||||
to `auto`.
|
||||
|
||||
The CLI passes the effective diagnostics root from workspace configuration.
|
||||
When `workspace.directory` is set and diagnostics are enabled, that root is
|
||||
`<workspace.directory>/diagnostics`. The legacy diagnostics work directory and
|
||||
`--diagnostics-dir` still pass a diagnostics-only root to this constructor.
|
||||
|
||||
The writer makes the work directory if needed, then attempts to create a unique
|
||||
run directory. It retries run ID creation a bounded number of times if a
|
||||
collision occurs.
|
||||
@@ -33,6 +38,8 @@ Implemented artifact names:
|
||||
- `invocation.json`
|
||||
- `effective-config.json`
|
||||
- `resolved-pipeline.json`
|
||||
- `resolved-references.json`
|
||||
- `checkpoint-events.json`
|
||||
- `source-document.json`
|
||||
- `run-manifest.json`
|
||||
- `run-report.json`
|
||||
@@ -68,8 +75,13 @@ Retention is decided by `ShouldRetainRunDirectory`.
|
||||
|
||||
## CLI Failure Behavior
|
||||
|
||||
The CLI creates the diagnostics run directory after config loading and before
|
||||
pipeline resolution. Failures before that point do not have diagnostics.
|
||||
When diagnostics are enabled, the CLI creates the diagnostics run directory
|
||||
after config loading and before pipeline resolution. Failures before that point
|
||||
do not have diagnostics.
|
||||
|
||||
When workspace diagnostics are explicitly disabled, the CLI does not create a
|
||||
diagnostics run directory and skips diagnostics artifact writes. Failures are
|
||||
still printed to stderr.
|
||||
|
||||
After diagnostics creation, run failures call `WriteErrorLog` and apply
|
||||
retention with `RunSucceeded: false`, so the run directory remains available.
|
||||
@@ -86,3 +98,5 @@ manifest before logging the failure.
|
||||
information needed for recovery.
|
||||
- Durable output file contracts belong to output modules and integration docs,
|
||||
not to diagnostics.
|
||||
- Checkpoint and debug workspace files are separate framework-owned artifacts,
|
||||
not diagnostics artifacts.
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# LLM Runtime
|
||||
|
||||
The implemented LLM runtime lives in `internal/framework/llm`. It provides
|
||||
transport-neutral structured completion contracts, an OpenAI-compatible HTTP
|
||||
adapter, concurrency scheduling, schema registry helpers, retry behavior, and
|
||||
secret redaction.
|
||||
transport-neutral structured completion contracts, a Scriptorium-backed
|
||||
production client, concurrency scheduling, prompt/schema asset registration,
|
||||
schema registry helpers, and secret redaction.
|
||||
|
||||
## Contract
|
||||
|
||||
@@ -13,68 +13,66 @@ Modules depend on `contracts.StructuredLLMClient`:
|
||||
CompleteStructured(ctx, request, out) (response, error)
|
||||
```
|
||||
|
||||
The request contains messages, optional model override, response schema name,
|
||||
and response schema JSON. The caller supplies a pointer target for decoded
|
||||
structured output.
|
||||
The request contains prompt ID/version, profile ID, session ID, prompt input
|
||||
materials, and variables. The caller supplies a pointer target for decoded
|
||||
structured output. The response also carries the raw structured output bytes
|
||||
returned by the runtime so modules can preserve raw payloads in pipeline stage
|
||||
outputs.
|
||||
|
||||
Extractors own prompts and schemas. Provider adapters should not contain
|
||||
domain-specific prompt logic.
|
||||
Modules that call the LLM own their prompts, schemas, prompt IDs, and
|
||||
domain-specific interpretation. Validator packages own approve/reject policy,
|
||||
and central catalog mappings decide which validators run by default. Provider
|
||||
adapters should not contain domain-specific prompt logic.
|
||||
|
||||
Prompt input materials carry source or reference bytes with optional origin
|
||||
metadata. The Scriptorium-backed runtime receives them as named artifacts rather
|
||||
than rendered prompt strings owned by Notarius modules.
|
||||
|
||||
## Production Client Construction
|
||||
|
||||
`internal/cli` builds the production LLM client from the effective config:
|
||||
|
||||
1. find the effective LLM profile;
|
||||
2. build `OpenAICompatibleClientConfig`;
|
||||
3. create an OpenAI-compatible client;
|
||||
4. create a scheduler from profile or global concurrency;
|
||||
5. wrap the client with `NewScheduledClient`;
|
||||
6. return non-secret LLM profile manifest metadata.
|
||||
1. collect production Scriptorium prompt and schema assets from module packages;
|
||||
2. create a Scriptorium-backed structured client using effective Scriptorium
|
||||
profile source settings from `scriptorium.profile_dir` or
|
||||
`scriptorium.profile_file`;
|
||||
3. create a scheduler from global LLM concurrency;
|
||||
4. wrap the client with `NewScheduledClient`;
|
||||
5. let the runtime report non-secret profile manifest metadata after calls.
|
||||
|
||||
The current run command requires exactly one distinct effective LLM profile for
|
||||
the resolved pipeline.
|
||||
The runtime records the actual selected Scriptorium profile, provider, and model
|
||||
used during execution. Manifest population does not rely on a precomputed
|
||||
profile ID before pipeline execution.
|
||||
|
||||
## OpenAI-Compatible Adapter
|
||||
Explicit profile validation applies to LLM-capable pipeline stages: chunk,
|
||||
extract, merge, normalize, and LLM-backed validators with explicit
|
||||
`llm_profile` values. Input, output, and deterministic validators do not call
|
||||
the LLM. The `--llm-profile` run flag overrides effective chunk, extract, merge,
|
||||
and normalize bindings; it does not override validator-specific profiles.
|
||||
|
||||
`OpenAICompatibleClient` posts JSON to:
|
||||
## Scriptorium Adapter
|
||||
|
||||
```text
|
||||
<base_url>/chat/completions
|
||||
```
|
||||
`ScriptoriumClient` implements `contracts.StructuredLLMClient` by converting
|
||||
Notarius prompt requests into Scriptorium `RunRequest` values. It:
|
||||
|
||||
It sends:
|
||||
- validates the caller output target and prompt ID;
|
||||
- converts `LLMInputMaterial` values into inline Scriptorium artifacts, using a
|
||||
single space for empty material so optional blank references remain explicit;
|
||||
- passes `session_id` through Scriptorium variables and request metadata when
|
||||
present;
|
||||
- sends explicit profile IDs only when the request supplies one;
|
||||
- lets Scriptorium render prompts, call the configured provider, and validate
|
||||
structured output;
|
||||
- unmarshals successful JSON into the caller-provided target;
|
||||
- returns the validated raw structured output bytes to the caller;
|
||||
- maps token usage and selected profile/model metadata into the Notarius
|
||||
response and manifest profile recorder.
|
||||
|
||||
- `model`
|
||||
- `messages`
|
||||
- `response_format.type = "json_schema"`
|
||||
- `response_format.json_schema.name`
|
||||
- `response_format.json_schema.strict = true`
|
||||
- `response_format.json_schema.schema`
|
||||
|
||||
If an API key is configured, the adapter sends an `Authorization: Bearer ...`
|
||||
header.
|
||||
|
||||
The adapter accepts assistant content either as a JSON string containing JSON or
|
||||
as raw JSON content. It then unmarshals that content into the caller-provided
|
||||
target.
|
||||
|
||||
External wire-contract details belong in the
|
||||
[OpenAI-compatible integration doc](../integrations/openai-compatible.md).
|
||||
|
||||
## Retries And Timeouts
|
||||
|
||||
The adapter retries:
|
||||
|
||||
- provider request failures;
|
||||
- response read failures;
|
||||
- HTTP `429`;
|
||||
- HTTP `5xx`;
|
||||
- malformed provider envelopes;
|
||||
- malformed assistant JSON;
|
||||
- structured-output decode failures.
|
||||
|
||||
Non-retryable `4xx` responses are returned without retry. Request timeout comes
|
||||
from the effective LLM profile. Context cancellation is respected.
|
||||
Generated-output validation failures are returned as Notarius errors. Provider
|
||||
and runtime errors are wrapped with prompt context and bearer tokens are
|
||||
redacted from error strings. Prompt text, raw source input, reference content,
|
||||
schema JSON, API keys, and bearer tokens are not added to default diagnostics or
|
||||
run manifests.
|
||||
|
||||
## Scheduler
|
||||
|
||||
@@ -87,9 +85,8 @@ inside the scheduler.
|
||||
|
||||
Effective concurrency is:
|
||||
|
||||
1. `llm_profiles.<id>.max_concurrency`, when greater than zero;
|
||||
2. `concurrency.total_llm`, when greater than zero;
|
||||
3. `1`.
|
||||
1. `concurrency.total_llm`, when greater than zero;
|
||||
2. `1`.
|
||||
|
||||
## Schema Registry
|
||||
|
||||
@@ -104,13 +101,14 @@ helpers for caller-owned schemas:
|
||||
`DiagnosticsMap` omits raw schema content and includes metadata such as key,
|
||||
ID, version, name, and SHA-256.
|
||||
|
||||
The D&D spell extractor owns and loads its own embedded response schema.
|
||||
Production modules own and register their Scriptorium prompt and schema assets.
|
||||
Framework packages may collect those files but must not contain D&D-specific
|
||||
prompt content.
|
||||
|
||||
## Secret Redaction
|
||||
|
||||
Provider errors are passed through `ErrorWithSecretsRedacted` with the API key
|
||||
and bearer-token value. Config diagnostics use redacted effective config
|
||||
payloads.
|
||||
Provider errors are redacted before surfacing through the Scriptorium-backed
|
||||
client. Config diagnostics use redacted effective config payloads.
|
||||
|
||||
Do not add raw provider request bodies, response bodies, API keys, or prompt
|
||||
payloads to diagnostics by default.
|
||||
|
||||
@@ -5,6 +5,9 @@ contract from `internal/framework/contracts`, exposes a `ModuleSpec`, and
|
||||
registers itself with the matching pipeline registry.
|
||||
|
||||
The CLI production catalog currently registers only the modules listed here.
|
||||
Validator implementations live under `internal/validators` and are registered
|
||||
separately from modules. Production default validator chains are central CLI
|
||||
catalog policy; module packages do not own their default validation chains.
|
||||
|
||||
## Contract Pattern
|
||||
|
||||
@@ -20,6 +23,50 @@ A production module package should provide:
|
||||
Module specs should describe capabilities accurately. Resolution uses specs to
|
||||
reject incompatible pipelines before execution.
|
||||
|
||||
Chunk, extract, merge, and normalize modules that accept auxiliary reference material
|
||||
must declare slots through both `ReferenceSlots()` and
|
||||
`ModuleSpec().ReferenceSlots`. The runtime slot list and registry metadata
|
||||
should match so config validation can inspect slots without constructing module
|
||||
instances. A slot declaration names the slot, whether it is required, accepted
|
||||
media types, whether multiple items are allowed, and any byte limit. Empty
|
||||
`AcceptedMediaTypes` means any inferred media type is accepted, though the file
|
||||
must still be UTF-8 text. When a slot declares accepted media types, Notarius
|
||||
compares the canonical base media type inferred from the file extension,
|
||||
case-insensitively and without parameters.
|
||||
|
||||
The resolver materializes reference content for chunk, extractor, merger, and
|
||||
normalizer targets. Runtime delivery uses `contracts.ChunkRequest.References`,
|
||||
`contracts.ExtractionRequest.References`, `contracts.MergeRequest.References`,
|
||||
and `contracts.NormalizeRequest.References`. Reference material is not source
|
||||
evidence and must not be converted into `SourceRef` values. If a module prompt
|
||||
uses references, pass them as prompt input materials through the structured LLM
|
||||
request. Prompt metadata hashes remain based on prompt asset source, not
|
||||
rendered reference bytes.
|
||||
|
||||
LLM-backed modules own Scriptorium prompt definitions and response schemas in
|
||||
their embedded assets. Module-owned prompts live under each module's shallow
|
||||
`assets/prompts` tree and schemas live under `assets/schemas`. Generic shared
|
||||
prompt filesystem composition lives under `internal/modules/sharedassets`.
|
||||
Common D&D prompt fragments, reference slot helpers, prompt input assembly, and
|
||||
reference rendering live under `internal/modules/sharedassets/dnd`. Module
|
||||
contracts should expose prompt IDs, versions, input material names, and
|
||||
non-secret prompt/schema hashes through manifest metadata; they should not
|
||||
expose Scriptorium public types through chunk, extract, merge, or normalize contracts.
|
||||
|
||||
Chunk modules receive the structured LLM client, configured Scriptorium profile
|
||||
ID, prompt session ID, and raw source input material through
|
||||
`contracts.ChunkRequest` when they need model-backed chunking. The pipeline
|
||||
runner validates generic chunk result invariants before extraction; module-owned
|
||||
policies may be stricter but must stay within the module package.
|
||||
|
||||
Normalize modules receive the structured LLM client, configured Scriptorium
|
||||
profile ID, prompt session ID, and reference material through
|
||||
`contracts.NormalizeRequest` when they need model-backed reconciliation.
|
||||
|
||||
Merge modules receive the structured LLM client, configured Scriptorium profile
|
||||
ID, prompt session ID, raw source input material, and reference material through
|
||||
`contracts.MergeRequest` when they need model-backed merge behavior.
|
||||
|
||||
## `seriatim` Input
|
||||
|
||||
Package: `internal/modules/input/seriatim`
|
||||
@@ -44,6 +91,11 @@ The `generic` chunker splits source units into ordered chunks. It validates the
|
||||
source document, clones source units, assigns chunk IDs such as `chunk-000001`,
|
||||
and records chunk metadata for start unit, end unit, and unit count.
|
||||
|
||||
The pipeline runner canonicalizes chunk units from the source document by
|
||||
integer ID before extractors and mergers run. Chunkers also populate chunk
|
||||
start and end unit IDs, content bytes, and media type. Chunker-owned context
|
||||
should stay in `SourceChunk.Metadata`.
|
||||
|
||||
Options:
|
||||
|
||||
- `max_units`: positive integer, default `50`;
|
||||
@@ -54,14 +106,63 @@ Provides:
|
||||
|
||||
- `chunks`
|
||||
|
||||
## `dnd/scenes` Chunker
|
||||
|
||||
Package: `internal/modules/chunk/dnd/scenes`
|
||||
|
||||
The `dnd/scenes` chunker uses the structured LLM client to divide transcript
|
||||
source units into coherent D&D scenes. It supplies the embedded Scriptorium
|
||||
prompt ID, prompt version, transcript input material, response schema, and
|
||||
session ID to the runtime; validates model-authored source-unit boundaries; and
|
||||
converts each scene into a deterministic source chunk.
|
||||
|
||||
Its prompt definition lives under `assets/prompts` and its schema under
|
||||
`assets/schemas`. Shared reusable D&D prompt fragments are provided by
|
||||
`internal/modules/sharedassets/dnd` and referenced from prompt definitions under
|
||||
`./sharedassets/`.
|
||||
|
||||
Requires:
|
||||
|
||||
- `source.transcript`
|
||||
|
||||
Provides:
|
||||
|
||||
- `chunks`
|
||||
- `chunks.scenes`
|
||||
|
||||
Options: none. Non-empty options are rejected.
|
||||
|
||||
The chunker enforces full source-unit coverage from the first source unit to the
|
||||
last, sequential contiguous scenes, and no overlap. Its LLM-facing schema uses
|
||||
integer `start_unit_id` and `end_unit_id` values matching source-unit IDs. It
|
||||
assigns chunk IDs such as `scene-000001`, emits JSON chunk content, and stores
|
||||
scene metadata including title, primary mode, participants, summary, boundary
|
||||
note, confidence, boundary unit IDs, and unit count. Boundary caveats become
|
||||
warnings with reason code
|
||||
`scene_boundary_caveat`. Whitespace-only caveats are treated as malformed
|
||||
structured output rather than silently dropped.
|
||||
|
||||
Malformed model output fails explicitly rather than falling back to another
|
||||
chunker. The chunker exposes prompt and response-schema provenance through
|
||||
top-level `module_metadata.chunker` without raw prompts, raw schemas, source
|
||||
text, or secrets.
|
||||
|
||||
## `dnd/spells` Extractor
|
||||
|
||||
Package: `internal/modules/extract/dnd/spells`
|
||||
|
||||
The `dnd/spells` extractor owns D&D spell-cast artifact semantics. It renders
|
||||
embedded prompts, loads the embedded structured response schema, calls the
|
||||
structured LLM client, converts spell-cast responses into artifact candidates,
|
||||
and supplies deterministic validators.
|
||||
The `dnd/spells` extractor owns D&D spell-cast extraction semantics. It
|
||||
supplies the embedded Scriptorium prompt ID, prompt version, chunk-scoped
|
||||
transcript input material, reference input materials, response schema, and
|
||||
session ID to the runtime; then returns the structured LLM `spell_casts`
|
||||
response as raw JSON.
|
||||
Its LLM-facing source-reference schema uses integer `start_unit_id` and
|
||||
`end_unit_id` values matching source-unit IDs.
|
||||
|
||||
Its prompt definition lives under `assets/prompts` and its schema under
|
||||
`assets/schemas`. Shared reusable D&D prompt fragments are provided by
|
||||
`internal/modules/sharedassets/dnd` and referenced from prompt definitions under
|
||||
`./sharedassets/`.
|
||||
|
||||
Requires:
|
||||
|
||||
@@ -72,38 +173,39 @@ Provides:
|
||||
|
||||
- `dnd.spell_casts`
|
||||
|
||||
Artifact type and schema version:
|
||||
Response schema identity:
|
||||
|
||||
- artifact type: `dnd.spell_cast`
|
||||
- schema ID: `notarius.dnd.spells`
|
||||
- schema name: `notarius_dnd_spells_v1`
|
||||
- schema version: `v1`
|
||||
|
||||
The extractor adds prompt and response-schema provenance to lane manifest
|
||||
metadata. Durable artifact payload details belong in the
|
||||
[D&D spell artifact contract](../integrations/dnd-spell-artifacts.md).
|
||||
metadata under `artifact_lanes[].metadata.extractor`. Durable raw output
|
||||
details belong in the
|
||||
[D&D spell raw output contract](../integrations/dnd-spell-artifacts.md).
|
||||
|
||||
## D&D Spell Validators
|
||||
The production catalog validates `dnd/spells` raw extract output with generic
|
||||
JSON validators followed by D&D spell validators under
|
||||
`internal/validators/extract/dnd/spells`. The extractor itself remains
|
||||
responsible for prompt, schema, and raw output production rather than
|
||||
approve/reject policy.
|
||||
|
||||
The spell extractor returns two built-in validators:
|
||||
|
||||
- `dnd/spells/shape`: rejects malformed payloads and missing required fields.
|
||||
- `dnd/spells/source_refs`: rejects candidates without valid source references.
|
||||
|
||||
Reason codes include:
|
||||
|
||||
- `invalid_payload`
|
||||
- `missing_required_field`
|
||||
- `missing_source_ref`
|
||||
- `invalid_source_ref`
|
||||
|
||||
These validators are supplied by the extractor when no validators are configured
|
||||
for the lane.
|
||||
The `dnd/scenes` chunker and `dnd/spells` extractor declare optional `players`,
|
||||
`party`, and `glossary` reference slots accepting UTF-8 plain text, Markdown,
|
||||
YAML, or JSON. They also accept `roster` as a deprecated compatibility alias for
|
||||
`party`. Their prompts frame references as supporting disambiguation material
|
||||
only; spell-cast artifacts must still be grounded in the source transcript.
|
||||
|
||||
## `appendorder` Merger
|
||||
|
||||
Package: `internal/modules/merge/appendorder`
|
||||
|
||||
The `appendorder` merger clones and appends candidates in chunk order. It does
|
||||
not deduplicate or reconcile candidates.
|
||||
The `appendorder` merger preserves chunk order for raw extract outputs. A
|
||||
single JSON extract output is passed through as the merge output. Multiple JSON
|
||||
object outputs with one common top-level array field are merged by concatenating
|
||||
that array field in chunk order. Other valid JSON shapes are merged as a JSON
|
||||
array of decoded values in chunk order. Non-JSON media types and invalid JSON
|
||||
are rejected.
|
||||
|
||||
Provides:
|
||||
|
||||
@@ -113,7 +215,7 @@ Provides:
|
||||
|
||||
Package: `internal/modules/normalize/noop`
|
||||
|
||||
The `noop` normalizer clones merged candidates and returns them unchanged.
|
||||
The `noop` normalizer clones the raw merge output and returns it unchanged.
|
||||
|
||||
Requires:
|
||||
|
||||
@@ -127,9 +229,10 @@ Provides:
|
||||
|
||||
Package: `internal/modules/output/json`
|
||||
|
||||
The `json` output encoder converts approved artifacts, rejected artifacts,
|
||||
warnings, and the run manifest into logical JSON output files. It groups
|
||||
approved artifacts by artifact type and sanitizes artifact-type file names.
|
||||
The `json` output encoder converts normalized raw outputs, rejected raw outputs,
|
||||
warnings, and the run manifest into logical JSON output files. It writes one
|
||||
payload file per lane under `lanes/` and sanitizes lane IDs for file names.
|
||||
Normalized output payloads must be valid `application/json`.
|
||||
|
||||
Requires:
|
||||
|
||||
@@ -157,9 +260,9 @@ When adding a module, keep source-format and extraction-domain boundaries clear:
|
||||
|
||||
- input modules may know external source formats;
|
||||
- extract modules may know artifact semantics and prompt/schema assets;
|
||||
- merge and normalize modules own candidate combination and reconciliation;
|
||||
- merge and normalize modules own raw output combination and reconciliation;
|
||||
- output modules own serialization, not diagnostics or CLI reporting.
|
||||
|
||||
Update [Development](../policy/development.md), [Configuration](../config.md),
|
||||
Update [Development](../development.md), [Configuration](../config.md),
|
||||
internal docs, integration docs, and examples when the new module becomes
|
||||
implemented production behavior.
|
||||
|
||||
@@ -1,86 +1,127 @@
|
||||
# Internal Overview
|
||||
|
||||
This directory documents implemented Notarius internals for developers and LLM
|
||||
coding agents. It complements [Architecture](../policy/architecture.md), which
|
||||
is the durable policy for boundaries and invariants.
|
||||
This document maps the implemented Notarius components and their ownership. It
|
||||
complements the durable invariants in [Architecture](../policy/architecture.md)
|
||||
and links to focused internal documentation for deeper behavior.
|
||||
|
||||
## Executable And CLI
|
||||
## Execution Path
|
||||
|
||||
`cmd/notarius` calls the CLI package. `internal/cli` owns:
|
||||
The executable delegates to the CLI, which resolves configuration and wires the
|
||||
production application around the framework runner:
|
||||
|
||||
- command parsing and usage;
|
||||
- config discovery and loading;
|
||||
- production module catalog and registry wiring;
|
||||
- production LLM client construction;
|
||||
- run directory creation;
|
||||
- durable output writes;
|
||||
- user-facing stdout, stderr, and exit codes.
|
||||
```text
|
||||
cmd/notarius
|
||||
-> internal/cli
|
||||
-> config resolution + production registries + LLM client
|
||||
-> input -> chunk -> extract -> merge -> normalize -> output
|
||||
-> durable output writes
|
||||
|
||||
The CLI should stay thin around framework contracts. Domain extraction behavior
|
||||
belongs in modules, not in command handlers.
|
||||
Pipeline side channels:
|
||||
diagnostics checkpoints debug artifacts
|
||||
```
|
||||
|
||||
Pipeline execution is serial. Configuration selects modules for the fixed stage
|
||||
shape; registries construct them after profile, capability, validator, and
|
||||
reference resolution.
|
||||
|
||||
## Application Boundary
|
||||
|
||||
`cmd/notarius` contains the executable entry point and delegates process exit
|
||||
behavior to `internal/cli`.
|
||||
|
||||
`internal/cli` owns command parsing, configuration discovery, production module
|
||||
and validator registration, prompt asset collection, production LLM client
|
||||
construction, reference preparation, workspace recorder setup, durable output
|
||||
writes, and user-facing stdout, stderr, and exit codes. It is the composition
|
||||
root for concrete production packages.
|
||||
|
||||
## Core Packages
|
||||
|
||||
- `internal/core/artifacts`: artifact candidates, approved artifacts, rejected
|
||||
artifacts, validation decisions, and run manifests.
|
||||
- `internal/core/config`: defaults, YAML config parsing, environment overrides,
|
||||
validation, redaction, and resolved pipeline config.
|
||||
- `internal/core/diagnostics`: per-run diagnostics directory creation,
|
||||
diagnostics artifact writers, atomic writes, and retention decisions.
|
||||
- `internal/core/source`: source documents, source units, source references, and
|
||||
validation.
|
||||
| Package | Implemented responsibility |
|
||||
| --- | --- |
|
||||
| `internal/core/artifacts` | Run manifests and artifact serialization shapes. |
|
||||
| `internal/core/config` | Defaults, YAML parsing, environment overrides, validation, redaction, and effective pipeline configuration. |
|
||||
| `internal/core/diagnostics` | Diagnostics run directories, artifact writers, atomic writes, and retention decisions. |
|
||||
| `internal/core/source` | Generic source documents, units, references, and validation. |
|
||||
| `internal/core/workspace` | Workspace settings, safe paths and writes, checkpoint identities, and checkpoint manifest types. |
|
||||
|
||||
Core packages should remain deterministic and concrete. They should not import
|
||||
production modules.
|
||||
These packages provide concrete, deterministic models and policy. Production
|
||||
module registration occurs at the CLI boundary rather than in core packages.
|
||||
|
||||
## Framework Packages
|
||||
|
||||
- `internal/framework/contracts`: interfaces and request/result structs for
|
||||
input adapters, chunkers, extractors, mergers, normalizers, validators, output
|
||||
encoders, and structured LLM clients.
|
||||
- `internal/framework/pipeline`: module registries, module specs, profile
|
||||
resolution, capability checks, run orchestration, warnings, validation, and
|
||||
manifest population.
|
||||
- `internal/framework/llm`: OpenAI-compatible structured-output client,
|
||||
scheduler, schema registry, retries, and secret redaction.
|
||||
- `internal/framework/prompt`: embedded prompt registry and template rendering.
|
||||
- `internal/framework/validate`: validator decision helpers and cardinality
|
||||
enforcement.
|
||||
| Package | Implemented responsibility |
|
||||
| --- | --- |
|
||||
| `internal/framework/contracts` | Stage, validator, reference, output, and structured LLM interfaces and request/result types. |
|
||||
| `internal/framework/pipeline` | Module registries, profile resolution, capability checks, reference materialization, validation chains, retries, orchestration, warnings, and manifest population. |
|
||||
| `internal/framework/validate` | Shared validator decision and cardinality helpers. |
|
||||
| `internal/framework/llm` | Scriptorium-backed structured completions, prompt/schema asset registration, scheduling, profile recording, and secret redaction. |
|
||||
| `internal/framework/checkpoint` | Workspace-backed checkpoint loading, recording, and payload envelopes. |
|
||||
| `internal/framework/debug` | Workspace-backed framework and LLM debug artifacts. |
|
||||
|
||||
Framework code should stay source-agnostic and domain-agnostic.
|
||||
Framework contracts carry raw stage outputs between modules. The runner owns
|
||||
provenance, validation sequencing, rejection handling, checkpoint boundaries,
|
||||
debug boundaries, and final manifest assembly.
|
||||
|
||||
## Module Packages
|
||||
## Production Modules
|
||||
|
||||
Production module packages live under `internal/modules`:
|
||||
Production implementations live under `internal/modules` and register through
|
||||
the CLI catalog.
|
||||
|
||||
- `input/seriatim`
|
||||
- `chunk/generic`
|
||||
- `extract/dnd/spells`
|
||||
- `merge/appendorder`
|
||||
- `normalize/noop`
|
||||
- `output/json`
|
||||
| Stage | Module key | Package | Role |
|
||||
| --- | --- | --- | --- |
|
||||
| Input | `seriatim` | `internal/modules/input/seriatim` | Converts Seriatim transcript JSON into the generic source model. |
|
||||
| Chunk | `generic` | `internal/modules/chunk/generic` | Splits ordered source units by configured unit counts and overlap. |
|
||||
| Chunk | `dnd/scenes` | `internal/modules/chunk/dnd/scenes` | Uses structured LLM output to create contiguous D&D scene chunks. |
|
||||
| Extract | `dnd/spells` | `internal/modules/extract/dnd/spells` | Extracts source-grounded D&D spell-cast artifacts. |
|
||||
| Merge | `appendorder` | `internal/modules/merge/appendorder` | Combines accepted extract outputs in chunk order. |
|
||||
| Normalize | `noop` | `internal/modules/normalize/noop` | Preserves accepted merged output unchanged. |
|
||||
| Output | `json` | `internal/modules/output/json` | Encodes manifests, indexes, warnings, rejections, and accepted lane payloads as logical JSON files. |
|
||||
|
||||
Each module package owns its contract implementation, module spec,
|
||||
registration, options, focused tests, and module-specific errors.
|
||||
`internal/modules/sharedassets` composes shared prompt filesystems.
|
||||
`internal/modules/sharedassets/dnd` owns shared D&D prompt fragments, reference
|
||||
slots, prompt input assembly, and source-unit reference helpers.
|
||||
|
||||
## Fixtures And Tests
|
||||
## Validators
|
||||
|
||||
The repository uses focused package tests plus a fixture-driven CLI workflow.
|
||||
Concrete validators live under `internal/validators` and register separately
|
||||
from stage modules. Generic validators cover unconditional test decisions, JSON
|
||||
syntax, and JSON Schema. D&D spell validators cover artifact shape, source
|
||||
reference validity, and source relatedness.
|
||||
|
||||
- CLI acceptance tests cover maintained examples under `examples/`.
|
||||
- Pipeline tests cover registry composition and end-to-end framework behavior
|
||||
with fakes.
|
||||
- Module tests cover implemented module contracts without requiring real
|
||||
provider calls.
|
||||
- LLM tests use local test servers and fakes.
|
||||
The production default chain for `dnd/spells` extract output is registered
|
||||
centrally in `internal/cli`; module packages produce raw output but do not own
|
||||
the production approve/reject policy.
|
||||
|
||||
Do not use real external services in tests. Use fakes, fixtures, or local test
|
||||
servers.
|
||||
## Files And Run State
|
||||
|
||||
## Boundary Reminders
|
||||
Notarius keeps distinct output and inspection surfaces:
|
||||
|
||||
- Source-format details stay in input modules and integration docs.
|
||||
- Extraction-domain details stay in extract modules and artifact docs.
|
||||
- Provider wire details stay in the LLM runtime and provider integration docs.
|
||||
- Durable output contracts belong in integration docs.
|
||||
- Operator procedures belong in `docs/operations.md`, not internal docs.
|
||||
| Surface | Owner | Purpose |
|
||||
| --- | --- | --- |
|
||||
| Durable output | Output module and CLI writer | User-consumable run files. |
|
||||
| Diagnostics | `internal/core/diagnostics` and CLI | Redacted run inspection, reports, warnings, and failures. |
|
||||
| Checkpoints | `internal/framework/checkpoint` | Validated stage reuse for explicit resume. |
|
||||
| Debug artifacts | `internal/framework/debug` and pipeline instrumentation | Sensitive framework-boundary and LLM call inspection. |
|
||||
|
||||
Workspace settings determine whether and where diagnostics, checkpoints, and
|
||||
debug artifacts are written. Concrete stage modules do not receive workspace
|
||||
paths.
|
||||
|
||||
## Focused Internal Documentation
|
||||
|
||||
- [Pipeline Internals](pipeline.md): resolution, execution, validators,
|
||||
references, retries, checkpoints, outputs, and manifests.
|
||||
- [Module Internals](modules.md): production module contracts, capabilities,
|
||||
options, prompts, schemas, and registration.
|
||||
- [LLM Runtime](llm.md): structured completion contracts, Scriptorium adapter,
|
||||
assets, scheduling, profile recording, and redaction.
|
||||
- [Diagnostics Internals](diagnostics.md): diagnostics files, retention,
|
||||
failure behavior, and path safety.
|
||||
|
||||
## Test Surfaces
|
||||
|
||||
The repository uses focused package tests, registry and pipeline composition
|
||||
tests, a fake-backed walking skeleton, fixture-driven CLI coverage, and local
|
||||
test servers for LLM integration behavior. Tests do not require real provider
|
||||
calls.
|
||||
|
||||
@@ -23,18 +23,86 @@ before execution:
|
||||
- merge: `appendorder`
|
||||
- normalize: `noop`
|
||||
- output: `json`
|
||||
- LLM profile: `default`
|
||||
- LLM profile: empty, which lets Scriptorium prompt defaults choose a
|
||||
profile.
|
||||
4. The module catalog is checked for each bound module key.
|
||||
5. Module capabilities are checked in workflow order.
|
||||
6. A digest is calculated from the resolved pipeline without the digest field.
|
||||
|
||||
The CLI writes the resolved pipeline and digest to diagnostics.
|
||||
|
||||
Pipeline profiles and artifact lanes may include reference binding maps keyed by
|
||||
reference slot name. During resolution, pipeline-level bindings act as defaults
|
||||
for selected chunk, extractor, merger, and normalizer targets that declare the
|
||||
slot; target-local bindings override or add bindings for that target. Runtime
|
||||
`--reference` requests override target config bindings, and runtime unbinds
|
||||
remove optional target bindings. Flat runtime slot names are resolved only when
|
||||
exactly one selected target declares the slot; otherwise the CLI requires a more
|
||||
specific selector such as `chunk.slot`, `lane.extract.slot`,
|
||||
`lane.merge.slot`, or `lane.normalize.slot`. Resolution validates bindings
|
||||
against the declaring target specs and stores the bindings in target-aware
|
||||
resolved reference holders. It does not read reference files or include
|
||||
reference bytes in source digests.
|
||||
|
||||
During run preparation, resolved file references for chunk, extractor, merger,
|
||||
and normalizer targets are materialized before any LLM-backed pipeline work. Config
|
||||
bindings resolve relative to the config file, and CLI bindings resolve relative
|
||||
to the current working directory. Materialization accepts UTF-8 text files,
|
||||
computes `sha256:` content digests, records file origins, infers canonical base
|
||||
media types from file extensions, enforces declared byte limits, and warns for
|
||||
empty bound files. Media-type acceptance is checked only when a slot declares
|
||||
`AcceptedMediaTypes`; unknown extensions are recorded as
|
||||
`application/octet-stream`. Reference content is omitted from diagnostics and
|
||||
manifests. The CLI writes provenance-only resolved reference diagnostics, and
|
||||
the run manifest records target-stage reference provenance separately from
|
||||
source digests. Runtime reference content is passed to the matching chunker,
|
||||
extractor, merger, or normalizer request. LLM-backed modules pass that material
|
||||
onward as named Scriptorium prompt inputs.
|
||||
|
||||
The CLI carries raw input bytes into `pipeline.RunInput`. Input adapters parse
|
||||
those bytes into the source document. Chunk, merge, and normalize requests
|
||||
receive the original source material as `SourceInput`; extraction requests
|
||||
receive chunk-scoped source material built from the current `SourceChunk`
|
||||
content, media type, and origin metadata. The raw input payload is not written
|
||||
to manifests or default diagnostics.
|
||||
|
||||
The CLI also carries an optional run `session_id`. The runner makes it available
|
||||
to chunk, extract, merge, and normalize requests; LLM-backed modules forward it
|
||||
through their structured completion requests so Scriptorium can include it in
|
||||
prompt execution metadata.
|
||||
|
||||
When workspace resume checkpointing is enabled, the CLI constructs a checkpoint
|
||||
recorder after pipeline resolution and reference materialization and passes it
|
||||
through `pipeline.RunInput`. The runner records source, chunk, extract, merge,
|
||||
and normalize outcomes through that interface. Concrete modules do not receive
|
||||
workspace paths and do not write checkpoint files directly.
|
||||
|
||||
For `run --resume`, the CLI also passes a checkpoint loader. The runner consults
|
||||
the loader in workflow order and reuses only checkpoints whose manifest schema,
|
||||
status, identity digest, dependency fingerprints, payload files, and payload
|
||||
digests validate for the current invocation. The identity includes the resolved
|
||||
pipeline, selected lanes, source/input digest, runtime overrides that affect
|
||||
execution, and materialized reference digests. Missing or invalid checkpoints
|
||||
fall back to normal execution and are refreshed by the recorder.
|
||||
|
||||
When workspace debug output is enabled, the CLI passes a debug recorder for the
|
||||
current run ID. The runner writes framework-boundary inputs, outputs,
|
||||
structured LLM calls, validator calls, timing, and retry attempt metadata
|
||||
through that interface. Each retry or validator attempt records any LLM calls
|
||||
made within that attempt in an `llm_calls` array and writes paired
|
||||
`prompt-000N.json` and `response-000N.json` metadata files under the attempt
|
||||
directory. LLM response bodies are written as sibling `response-content-000N.*`
|
||||
files, using pretty-printed JSON when the content is valid JSON and raw text
|
||||
otherwise. Debug output is not used for resume and can contain sensitive source,
|
||||
reference, prompt, and model-output material. Concrete modules still do not
|
||||
receive workspace paths.
|
||||
|
||||
## Registries And Module Specs
|
||||
|
||||
`pipeline.Registries` holds concrete constructors for execution. A
|
||||
`pipeline.ModuleCatalog` exposes module specs for config validation and
|
||||
resolution.
|
||||
resolution. The catalog also exposes validator specs and central default
|
||||
validator-chain mappings without constructing modules or validators.
|
||||
|
||||
Every production module registers a `ModuleSpec` with:
|
||||
|
||||
@@ -44,8 +112,20 @@ Every production module registers a `ModuleSpec` with:
|
||||
- `Provides`: capabilities added after that module runs;
|
||||
- `Requires`: capabilities that must already be available.
|
||||
|
||||
Chunk, extract, merge, and normalize specs may also declare reference slots. Slot
|
||||
declarations are available from registry metadata without constructing module
|
||||
instances. Input, validate, and output specs must not declare reference slots.
|
||||
|
||||
Capability checks prevent incompatible pipeline composition before a run starts.
|
||||
|
||||
Every production validator registers a `ValidatorSpec` with:
|
||||
|
||||
- `Key`: validator key used in config and manifests;
|
||||
- `ExecutionClass`: `deterministic` or `llm_backed`.
|
||||
|
||||
Default validator chains are keyed by workflow stage and module key. Production
|
||||
currently registers a default chain for `extract` module `dnd/spells` only.
|
||||
|
||||
## Runner Input And Output
|
||||
|
||||
`pipeline.RunInput` carries:
|
||||
@@ -58,8 +138,8 @@ Capability checks prevent incompatible pipeline composition before a run starts.
|
||||
`pipeline.RunOutput` carries:
|
||||
|
||||
- run manifest;
|
||||
- approved artifacts;
|
||||
- rejected artifacts;
|
||||
- normalized raw outputs;
|
||||
- rejected raw outputs;
|
||||
- warnings;
|
||||
- logical output files returned by the output encoder.
|
||||
|
||||
@@ -72,41 +152,109 @@ The runner:
|
||||
1. validates run input and registries;
|
||||
2. builds the input adapter and parses the raw input into a source document;
|
||||
3. validates the source document;
|
||||
4. builds the chunker and produces source chunks;
|
||||
5. runs each selected artifact lane in sorted resolved order;
|
||||
6. builds the output encoder and validates logical output file names.
|
||||
4. builds the chunker and produces source chunks, retrying when configured;
|
||||
5. validates source chunks against framework invariants and the resolved chunk
|
||||
validator chain;
|
||||
6. runs each selected artifact lane in sorted resolved order;
|
||||
7. builds the output encoder and validates logical output file names.
|
||||
8. passes accepted normalized raw outputs, rejected output records, warnings,
|
||||
and the manifest to the output encoder.
|
||||
|
||||
## Chunk Results
|
||||
|
||||
Chunkers implement `contracts.Chunker` and receive a `contracts.ChunkRequest`
|
||||
with the validated source document, reference set, structured LLM client, the
|
||||
configured LLM profile, module options, and run metadata. Deterministic and
|
||||
LLM-backed chunkers use the same contract; provider construction stays outside
|
||||
chunk modules.
|
||||
|
||||
When chunking succeeds, the runner validates generic chunk invariants before
|
||||
running extractors:
|
||||
|
||||
- chunk IDs must be non-empty and unique in the chunk result;
|
||||
- each chunk `SourceID` must match the source document ID;
|
||||
- each chunk `Index` must match its zero-based returned order;
|
||||
- each chunk start and end unit ID must exist in the source document, with the
|
||||
start unit at or before the end unit;
|
||||
- each chunk must include non-empty extraction content and media type;
|
||||
- each chunk must contain at least one source unit;
|
||||
- a chunk must not repeat a source unit;
|
||||
- every chunk source unit must exist in the source document;
|
||||
- source units inside each chunk must appear in source-document order.
|
||||
|
||||
After validation, the runner rebuilds each chunk from source-document units by
|
||||
integer ID, preserving chunk boundaries, content bytes, media type, and cloned
|
||||
chunk metadata. Extractors and downstream stages therefore see canonical source
|
||||
units, while `SourceChunk.Metadata` remains the supported place for
|
||||
chunker-owned context.
|
||||
|
||||
If chunk validation rejects the chunk result after configured retries, the runner
|
||||
records a rejected raw output and skips downstream lane execution. Framework-level
|
||||
chunking or validation errors that remain after configured retries fail the run.
|
||||
|
||||
The framework does not require complete source-unit coverage and does not reject
|
||||
overlap between different chunks. Stricter policies, such as full coverage or
|
||||
non-overlap, belong to individual chunk modules when they are part of that
|
||||
module's contract.
|
||||
|
||||
Within an artifact lane, the runner:
|
||||
|
||||
1. builds the extractor, merger, and normalizer;
|
||||
2. records module manifest metadata when modules provide it;
|
||||
3. extracts candidates from each chunk;
|
||||
4. normalizes candidate envelope fields such as index, extractor key, artifact
|
||||
type, and schema version;
|
||||
5. merges candidates;
|
||||
6. normalizes merged candidates;
|
||||
7. validates candidate envelope consistency;
|
||||
8. runs validators;
|
||||
9. converts approved candidates to artifacts.
|
||||
3. extracts one raw `ExtractOutput` from each accepted chunk, retrying when
|
||||
configured;
|
||||
4. fills runner-owned provenance on each extract output, including lane ID,
|
||||
extractor key, source ID, chunk ID, and chunk index;
|
||||
5. validates raw extract outputs and omits rejected outputs from merge input;
|
||||
6. merges ordered accepted extract outputs into one raw `MergeOutput`, retrying
|
||||
when configured;
|
||||
7. validates raw merge output and skips normalization for rejected merge output;
|
||||
8. normalizes the accepted merge output into one raw `NormalizeOutput`,
|
||||
retrying when configured;
|
||||
9. validates raw normalize output and appends accepted normalized raw output to
|
||||
`RunOutput.NormalizeOutputs`.
|
||||
|
||||
## Validators
|
||||
|
||||
If a lane declares validators in config, the runner builds those validators from
|
||||
the validator registry. Otherwise it uses validators returned by the extractor.
|
||||
The runner handoff is raw-output based. Chunkers, extractors, mergers, and
|
||||
normalizers do not advertise validator chains through their module interfaces.
|
||||
Resolved validation chains receive the raw module output plus stage, lane,
|
||||
module, source, chunk, schema, session, reference, LLM client/profile, binding
|
||||
option, and run metadata context. Chunk validators receive the chunk result
|
||||
collection, merge validators receive the ordered extract outputs used by the
|
||||
merge, and normalize validators receive the accepted merge output. Empty chains
|
||||
approve output by default. Response-schema provenance may include in-memory JSON
|
||||
schema bytes for validators. Those bytes are omitted from manifests,
|
||||
diagnostics, and encoded output files.
|
||||
|
||||
Each validator must return exactly one decision for each eligible candidate. The
|
||||
runner enforces decision cardinality with `internal/framework/validate`.
|
||||
Rejected candidates are removed before the next validator runs. Approved
|
||||
candidates continue through the chain.
|
||||
Resolved validator chains come from central default mappings unless a
|
||||
stage-local config override is set on `chunk`, lane `extract`, lane `merge`, or
|
||||
lane `normalize`. Explicit empty overrides are valid and are recorded as empty
|
||||
chains in manifests. Explicit non-empty overrides replace the default chain and
|
||||
preserve configured order.
|
||||
|
||||
The production CLI currently registers no standalone validator modules. The
|
||||
current D&D spell extractor supplies deterministic shape and source-reference
|
||||
validators.
|
||||
The production default chain for `extract` module `dnd/spells` is:
|
||||
|
||||
1. `generic/valid_json`
|
||||
2. `generic/valid_json_schema`
|
||||
3. `extract/dnd/spells/shape`
|
||||
4. `extract/dnd/spells/source_refs`
|
||||
5. `extract/dnd/spells/source_relatedness`
|
||||
|
||||
No other production module currently has a default validator chain.
|
||||
|
||||
Validator rejection is a non-fatal run outcome: the rejected output is recorded
|
||||
in `RunOutput.Rejected` and does not pass to the next stage. Validator execution
|
||||
errors are framework-level errors and retry according to the relevant binding.
|
||||
Warning-only validators return approved results with warnings; those warnings
|
||||
are promoted only from successful attempts whose outputs are used.
|
||||
|
||||
## Warnings And Failures
|
||||
|
||||
Warnings from chunking, extraction, merging, normalization, validation, and
|
||||
output encoding are accumulated in `RunOutput.Warnings`.
|
||||
Warnings from the successful chunking, extraction, merging, and normalization
|
||||
attempts whose outputs are used are accumulated in `RunOutput.Warnings`, along
|
||||
with output encoder warnings. Warnings from discarded retry attempts are not
|
||||
promoted to final warnings.
|
||||
|
||||
Errors wrap the operation and module key or lane context. If execution fails
|
||||
after a manifest exists, the returned manifest is marked `failed` and receives a
|
||||
@@ -114,14 +262,34 @@ completion timestamp.
|
||||
|
||||
On successful execution, the manifest validation status is:
|
||||
|
||||
- `approved` when no candidates were rejected;
|
||||
- `rejected` when at least one candidate was rejected.
|
||||
- `approved` when no raw outputs were rejected;
|
||||
- `rejected` when at least one raw output was rejected.
|
||||
|
||||
## Manifest Population
|
||||
|
||||
The manifest records run ID, pipeline ID, pipeline digest, module keys, artifact
|
||||
lanes, LLM profile metadata, source digest, validation status, and timing.
|
||||
The manifest records run ID, pipeline ID, pipeline digest, module keys, top-level
|
||||
module metadata, artifact lanes, LLM profile metadata, source digest,
|
||||
reference provenance, normalized raw output summaries, rejected output
|
||||
summaries, validation status, and timing. Raw output summaries include lane ID,
|
||||
normalizer module key, media type, source ID, and response-schema provenance
|
||||
when present. Rejected output summaries include stage, lane, module, chunk,
|
||||
validator or reason, message, attempt count, and optional diagnostic artifact
|
||||
path. The manifest does not include raw output payload bytes.
|
||||
|
||||
Modules can add non-secret manifest metadata by implementing
|
||||
`contracts.ManifestMetadataProvider`. The D&D spell extractor uses this for
|
||||
Singleton pipeline modules may add non-secret metadata by implementing
|
||||
`contracts.ManifestMetadataProvider`. The runner records that metadata under
|
||||
`module_metadata` with stable keys for `input`, `chunker`, and `output`.
|
||||
|
||||
Lane-owned modules may add non-secret metadata through
|
||||
`artifact_lanes[].metadata`. The runner records extractor, merger, and
|
||||
normalizer metadata there. The D&D spell extractor uses lane metadata for
|
||||
prompt and response-schema provenance.
|
||||
|
||||
## JSON Output
|
||||
|
||||
The production JSON output encoder writes `manifest.json`, `index.json`,
|
||||
`warnings.json`, `rejected.json`, and one pretty-printed JSON file per accepted
|
||||
normalized lane output under `lanes/`. It accepts only normalized outputs with
|
||||
valid `application/json` payloads. Unsupported media types, invalid JSON, unsafe
|
||||
logical paths, and duplicate sanitized lane file names fail the run before
|
||||
durable output files are written.
|
||||
|
||||
@@ -5,8 +5,8 @@ This is the canonical reference for operating implemented Notarius runs.
|
||||
## Normal Run
|
||||
|
||||
A run reads one source file, resolves one configured pipeline, calls the
|
||||
configured OpenAI-compatible LLM profile, writes durable JSON output, and writes
|
||||
diagnostics for inspection.
|
||||
configured Scriptorium-backed LLM runtime, writes durable JSON output, and
|
||||
writes diagnostics for inspection.
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
@@ -16,8 +16,12 @@ go run ./cmd/notarius run dnd-session \
|
||||
--diagnostics-dir /tmp/notarius
|
||||
```
|
||||
|
||||
The command prints a success line with the pipeline ID, approved and rejected
|
||||
artifact counts, and the output path.
|
||||
The command prints a success line with the pipeline ID, normalized output count,
|
||||
rejected output count, and the output path.
|
||||
|
||||
For production, configure a workspace such as `/var/lib/notarius` and ensure the
|
||||
Notarius process can create files below it. For local development, prefer an
|
||||
ignored project-local workspace such as `./.notarius/workspace`.
|
||||
|
||||
## Output Directory
|
||||
|
||||
@@ -32,14 +36,14 @@ different root.
|
||||
|
||||
The `json` output module writes these files:
|
||||
|
||||
- `index.json`: file index with paths to the manifest, artifact files,
|
||||
rejected artifacts, and warnings.
|
||||
- `manifest.json`: run manifest with resolved pipeline provenance, module keys,
|
||||
validation status, and timing.
|
||||
- `artifacts/<artifact-type>.json`: approved artifacts grouped by artifact
|
||||
type. For the current D&D spell extractor, this includes
|
||||
`artifacts/dnd.spell_cast.json` when spell-cast artifacts are approved.
|
||||
- `rejected.json`: rejected candidates and validator decisions.
|
||||
- `index.json`: file index with paths to the manifest, lane output files,
|
||||
rejected outputs, and warnings.
|
||||
- `manifest.json`: run manifest with resolved pipeline provenance, top-level
|
||||
module metadata, module keys, reference provenance, validation status, and
|
||||
timing.
|
||||
- `lanes/<lane-id>.json`: normalized raw JSON output payloads, one file per
|
||||
lane. For the current D&D spell extractor, this includes `lanes/spells.json`.
|
||||
- `rejected.json`: rejected raw output records.
|
||||
- `warnings.json`: warnings reported by pipeline modules or the output encoder.
|
||||
|
||||
Output writes are atomic per file. Logical output file names must be clean,
|
||||
@@ -54,16 +58,30 @@ Diagnostics are written under:
|
||||
```
|
||||
|
||||
The default diagnostics work directory is `/tmp/notarius`. It can be set with
|
||||
`diagnostics.work_dir`, `NOTARIUS_WORK_DIR`, or `--diagnostics-dir`.
|
||||
`workspace.directory`, `NOTARIUS_WORKSPACE_DIR`, legacy
|
||||
`diagnostics.work_dir`, legacy `NOTARIUS_WORK_DIR`, or `--diagnostics-dir`.
|
||||
When a workspace directory is set, diagnostics are written under
|
||||
`<workspace.directory>/diagnostics/<run-id>/` unless `--diagnostics-dir`
|
||||
overrides the diagnostics work directory for that invocation.
|
||||
|
||||
Set `workspace.diagnostics.enabled: false` or
|
||||
`NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED=false` to skip diagnostics directory
|
||||
creation and diagnostics artifact writes. Concise failures are still printed to
|
||||
stderr.
|
||||
|
||||
Implemented diagnostics artifacts:
|
||||
|
||||
- `invocation.json`: command metadata such as operation, config path, input
|
||||
path, selected lanes, run ID, and pipeline digest when available.
|
||||
- `effective-config.json`: resolved config with API keys redacted.
|
||||
- `effective-config.json`: resolved config without raw API keys.
|
||||
- `resolved-pipeline.json`: resolved module bindings and pipeline digest.
|
||||
- `resolved-references.json`: resolved reference provenance, including target
|
||||
stage, lane ID when present, origin, digest, media type, byte size, and
|
||||
binding source, without reference content.
|
||||
- `checkpoint-events.json`: checkpoint steps that were reused or executed
|
||||
during an explicit resume invocation.
|
||||
- `run-manifest.json`: the same run manifest written to durable output when it
|
||||
is available.
|
||||
is available, including top-level module metadata when present.
|
||||
- `warnings.json`: warning list.
|
||||
- `run-report.json`: counts, status, output path, diagnostics path, and run ID.
|
||||
- `error.log`: failure message, written after diagnostics directory creation
|
||||
@@ -72,10 +90,71 @@ Implemented diagnostics artifacts:
|
||||
`source-document.json` is supported by the diagnostics writer but is not written
|
||||
by the current CLI run workflow.
|
||||
|
||||
## Checkpoints
|
||||
|
||||
When `workspace.resume.enabled: true` and `workspace.directory` is set, runs
|
||||
write checkpoints under:
|
||||
|
||||
```text
|
||||
<workspace.directory>/checkpoints/<pipeline-id>/<input-key>-<source-or-input-digest>/<pipeline-digest>/<identity-digest>/
|
||||
```
|
||||
|
||||
Each workflow step owns its own manifest and payload files. There is no
|
||||
root-level checkpoint summary. Ordinary `notarius run` invocations execute the
|
||||
pipeline normally and refresh checkpoints. `notarius run --resume` reuses valid
|
||||
checkpoints and executes any missing, invalid, or incompatible step normally.
|
||||
|
||||
Checkpoint payloads preserve byte content with base64 envelopes, media type,
|
||||
metadata, warnings, and content digests where applicable. Checkpoints do not
|
||||
include raw prompts, raw reference contents, raw LLM request payloads, or debug
|
||||
traces. They can still contain source text, intermediate extracted content,
|
||||
rejected outputs, metadata, and warnings. Treat checkpoint directories as
|
||||
sensitive local state.
|
||||
|
||||
A checkpoint is reused only when its workspace schema version, checkpoint
|
||||
identity digest, step status, dependency fingerprints, payload files, and
|
||||
payload digests match the current invocation. Changes to input bytes, resolved
|
||||
pipeline digest, selected lanes, runtime LLM profile override, materialized
|
||||
reference digests, or other identity material invalidate reuse and use a
|
||||
separate checkpoint directory.
|
||||
|
||||
Plain `notarius run` does not reuse checkpoints. It executes the workflow and
|
||||
refreshes checkpoint files when checkpointing is enabled. `notarius run
|
||||
--resume` is the explicit reuse path.
|
||||
|
||||
## Debug
|
||||
|
||||
When `workspace.debug.enabled: true` and `workspace.directory` is set, runs
|
||||
write debug artifacts under:
|
||||
|
||||
```text
|
||||
<workspace.directory>/debug/<run-id>/
|
||||
```
|
||||
|
||||
Debug output is per invocation. It is independent of checkpointing and is not
|
||||
used for resume. Enabling debug does not write checkpoints, and enabling resume
|
||||
checkpointing does not write debug output.
|
||||
|
||||
Debug artifacts include framework-boundary inputs and outputs for source,
|
||||
chunk, extract, merge, normalize, and output work, structured LLM request and
|
||||
response data from Notarius contracts, validator requests and results, timing,
|
||||
and retry attempt metadata. LLM calls made inside a retry or validator attempt
|
||||
write `prompt-000N.json`, `response-000N.json`, and
|
||||
`response-content-000N.*` files under that attempt directory and are linked from
|
||||
the attempt `llm_calls` array. Prompt content is written inline in the prompt
|
||||
artifact. Response metadata is written to `response-000N.json`, while the
|
||||
response body is written separately as pretty-printed JSON when possible or as
|
||||
raw text otherwise. Debug artifacts may contain source material, reference
|
||||
material, prompt inputs, model outputs, and other sensitive data. API keys are
|
||||
not written, and obvious credential-shaped values and sensitive map keys are
|
||||
redacted in framework envelopes, but debug directories should still be protected
|
||||
as sensitive local state.
|
||||
|
||||
## Retention
|
||||
|
||||
Diagnostics retention is configured with `diagnostics.retention`,
|
||||
`NOTARIUS_DIAGNOSTICS_RETENTION`, or the default `auto`.
|
||||
Diagnostics retention is configured with `workspace.diagnostics.retention`,
|
||||
`NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION`, legacy `diagnostics.retention`,
|
||||
legacy `NOTARIUS_DIAGNOSTICS_RETENTION`, or the default `auto`.
|
||||
|
||||
- `auto`: keep failed runs and successful runs with warnings; remove successful
|
||||
warning-free runs.
|
||||
@@ -104,9 +183,12 @@ retained for inspection and may include `run-manifest.json`, `warnings.json`,
|
||||
A successful run with warnings exits with code `0`, prints a warning count to
|
||||
stderr, and writes warnings to durable output and diagnostics when retained.
|
||||
|
||||
The run manifest `validation_status` indicates whether final artifacts were
|
||||
The run manifest `validation_status` indicates whether raw outputs were
|
||||
approved or rejected after validation.
|
||||
|
||||
Reference-related warnings include empty bound reference files. Empty references
|
||||
are still passed to extractors so optional slots can be intentionally blank.
|
||||
|
||||
## Cleanup
|
||||
|
||||
It is safe to remove specific old run directories after their output and
|
||||
@@ -117,16 +199,26 @@ rm -rf /tmp/notarius/run-1234567890
|
||||
rm -rf ./notarius-output/run-1234567890
|
||||
```
|
||||
|
||||
Workspace checkpoint and debug directories can also be removed when no longer
|
||||
needed. Remove exact identity or run directories, for example:
|
||||
|
||||
```sh
|
||||
rm -rf /var/lib/notarius/checkpoints/dnd-session/seriatim-abcdef123456/7890abcd1234/identityabcd1234
|
||||
rm -rf /var/lib/notarius/debug/run-1234567890
|
||||
```
|
||||
|
||||
Use exact run-directory paths. Avoid broad cleanup commands against parent
|
||||
directories unless they are part of your own operational policy.
|
||||
|
||||
## Operational Limits
|
||||
|
||||
There is no command to resume a failed run. Re-run `notarius run` after fixing
|
||||
the cause.
|
||||
If `--resume` cannot reuse a checkpoint, Notarius executes that step and writes
|
||||
a fresh checkpoint when checkpointing is enabled.
|
||||
|
||||
Provider retries are limited to the OpenAI-compatible client retry behavior
|
||||
configured by the effective LLM profile. There is no separate CLI retry command.
|
||||
Provider retries and timeouts are handled by Scriptorium according to the
|
||||
selected execution profile. Pipeline module retries are controlled by module
|
||||
binding `retries` values in config for chunk, extract, merge, and normalize.
|
||||
There is no separate CLI retry command.
|
||||
|
||||
Notarius writes local files only. Remote storage and archive management are not
|
||||
part of the implemented CLI.
|
||||
|
||||
@@ -1,210 +1,179 @@
|
||||
# Architecture
|
||||
|
||||
This document defines Notarius development policy. It is inward-facing:
|
||||
developers and LLM coding agents should use it to preserve the project's shape,
|
||||
boundaries, and invariants as the code evolves.
|
||||
This document defines the intended high-level architecture of Notarius and the
|
||||
invariants that changes must preserve. Implemented component details belong in
|
||||
[Internal Overview](../internal/overview.md) and its linked documents. The
|
||||
reasoning behind significant architectural choices belongs in
|
||||
[ADRs](../adr/).
|
||||
|
||||
Keep this document concise. It should describe durable architectural rules, not
|
||||
CLI syntax, configuration reference material, module catalogs, or roadmap items.
|
||||
## System Shape
|
||||
|
||||
## Project Shape
|
||||
Notarius is a small, dependency-light Go application for extracting structured
|
||||
artifacts from source material. It is a general extraction platform whose
|
||||
source formats, extraction domains, validation policies, LLM providers, and
|
||||
output formats are isolated behind explicit boundaries.
|
||||
|
||||
Notarius is a small, explicit, dependency-light Go application for extracting
|
||||
structured artifacts from source material using modular pipeline stages.
|
||||
|
||||
The application is contract-first but not abstraction-heavy. Add interfaces and
|
||||
extension points when they protect a real boundary:
|
||||
|
||||
- external source formats;
|
||||
- pipeline stage modules;
|
||||
- validators;
|
||||
- LLM providers and runtime plumbing;
|
||||
- output schemas and embedded assets.
|
||||
|
||||
Avoid abstractions that only anticipate hypothetical complexity. Prefer narrow
|
||||
contracts that can be exercised by tests and real modules.
|
||||
|
||||
## Core Invariants
|
||||
|
||||
The framework must remain source-agnostic and domain-agnostic.
|
||||
|
||||
Source-format details belong in input modules. Transcript-specific concepts such
|
||||
as segments, speakers, timestamps, and transcript schemas must not spread into
|
||||
runner, extractor, validator, or LLM framework code.
|
||||
|
||||
Extraction-domain details belong in domain modules. D&D-specific concepts such
|
||||
as spells, NPCs, items, combat turns, and encounters must not spread into core
|
||||
source, runner, or LLM framework packages.
|
||||
|
||||
Extracted facts should be grounded with source references. Source references
|
||||
should point to generic source units, not transcript-only structures. Framework
|
||||
code should preserve source-reference ranges exactly and should not merge or
|
||||
rewrite overlapping ranges unless a module explicitly owns that behavior.
|
||||
|
||||
The application workflow is fixed:
|
||||
The application has one fixed pipeline shape:
|
||||
|
||||
```text
|
||||
input -> chunk -> extract -> merge -> normalize -> output
|
||||
```
|
||||
|
||||
These stages should remain explicit in the architecture. Chunking, merging, and
|
||||
normalization must not be hidden inside domain extractors when they represent
|
||||
general pipeline behavior.
|
||||
Pipelines are configured compositions of this shape. They are not arbitrary
|
||||
DAGs or a general workflow language. Every stage remains explicit; general
|
||||
chunking, merging, or normalization behavior must not be hidden inside an
|
||||
extractor.
|
||||
|
||||
Pipelines are fixed-shape templates for this workflow, not arbitrary DAGs or a
|
||||
general workflow language. Module selection should be configuration- and
|
||||
registry-driven, not scattered through conditionals.
|
||||
Input and chunking are pipeline-wide. Each selected artifact lane owns its
|
||||
extract, merge, and normalize stages, and the output stage aggregates the run's
|
||||
lane outcomes.
|
||||
|
||||
## Package Boundaries
|
||||
Notarius is contract-first without being abstraction-heavy. Interfaces and
|
||||
extension points should protect demonstrated boundaries. New abstraction is not
|
||||
itself an architectural goal.
|
||||
|
||||
Prefer fewer, larger framework packages until a boundary proves itself through
|
||||
import direction, ownership, test seams, or substantial file size.
|
||||
## Package Layout And Dependency Direction
|
||||
|
||||
Core packages should contain deterministic models and policy. Framework
|
||||
packages should contain reusable orchestration and provider plumbing. Concrete
|
||||
business logic should live under stage-oriented module packages:
|
||||
| Area | Ownership |
|
||||
| --- | --- |
|
||||
| `cmd/notarius` | Executable entry point; delegates to the CLI. |
|
||||
| `internal/cli` | Application boundary, production composition, runtime setup, durable writes, and user-facing results. |
|
||||
| `internal/core` | Generic deterministic models and policy for source material, configuration, manifests, diagnostics, and workspace identity. |
|
||||
| `internal/framework` | Reusable contracts, registries, pipeline orchestration, validation mechanics, checkpoints, debug boundaries, and LLM runtime plumbing. |
|
||||
| `internal/modules` | Concrete pipeline stage behavior. |
|
||||
| `internal/validators` | Concrete approve/reject policies. |
|
||||
|
||||
```text
|
||||
internal/modules/input/...
|
||||
internal/modules/chunk/...
|
||||
internal/modules/extract/...
|
||||
internal/modules/merge/...
|
||||
internal/modules/normalize/...
|
||||
internal/modules/output/...
|
||||
```
|
||||
The CLI is the composition root and may import concrete implementations. Core
|
||||
and framework packages cooperate as generic application layers; neither may
|
||||
depend on production modules or validators. Concrete implementations may depend
|
||||
on core models and framework contracts.
|
||||
|
||||
Use short, lowercase, idiomatic Go package names. Avoid package names that repeat
|
||||
parent-stage context.
|
||||
The following dependency boundaries are mandatory:
|
||||
|
||||
Input modules translate external source formats into the core source model.
|
||||
They may know about external schema details, source-specific metadata, and
|
||||
format-specific validation rules. They should not own extraction-domain
|
||||
decisions.
|
||||
- extractors and validators do not depend on concrete input adapters;
|
||||
- provider-specific types do not cross the LLM runtime boundary;
|
||||
- external dependency types do not leak across internal package boundaries
|
||||
unless that dependency is the package's explicit contract.
|
||||
|
||||
Extract modules own artifact semantics, prompt usage, structured response schema
|
||||
selection, validator defaults, and domain-specific interpretation. They should
|
||||
depend on framework contracts and core source/artifact types, not concrete input
|
||||
module packages.
|
||||
Shared helpers may support demonstrated common needs, but must not move
|
||||
source-format or extraction-domain knowledge into generic framework packages.
|
||||
External dependencies require a clear correctness, security, interoperability,
|
||||
or complexity benefit.
|
||||
|
||||
Merge modules combine extracted candidates. Normalize modules reconcile merged
|
||||
candidates for semantic consistency. Generic behavior may exist for simple
|
||||
artifact types, but domain-specific behavior belongs in modules for the relevant
|
||||
stage.
|
||||
## Source And Domain Boundaries
|
||||
|
||||
Output modules serialize final artifacts and may report warnings out of band.
|
||||
CLI, diagnostics, and reporting layers are responsible for surfacing those
|
||||
warnings.
|
||||
Input modules translate external source formats into the generic source model.
|
||||
Format-specific schemas, fields, and validation remain with the input module
|
||||
and its integration contract.
|
||||
|
||||
Framework stages operate on source documents, source units, and source
|
||||
references rather than format-specific structures. A source reference identifies
|
||||
an ordered range of generic source units. Framework code preserves those ranges
|
||||
and does not merge or rewrite them unless a stage module explicitly owns that
|
||||
behavior.
|
||||
|
||||
Extract modules own artifact semantics, prompt use, response schemas, and
|
||||
domain interpretation. Domain-specific concepts remain in the relevant module,
|
||||
validator, shared domain helper, and artifact contract.
|
||||
|
||||
Auxiliary references provide context or disambiguation. They are not source
|
||||
evidence and must not be converted into source references.
|
||||
|
||||
## Pipeline Composition And Ownership
|
||||
|
||||
Module selection is configuration- and registry-driven. The framework resolves
|
||||
named pipeline definitions, applies explicit defaults and runtime overrides,
|
||||
and verifies module availability and capabilities before execution. Structural
|
||||
pipeline choices must not be scattered through conditionals or hidden behind
|
||||
ad hoc command flags.
|
||||
|
||||
Stage ownership is explicit:
|
||||
|
||||
- input modules convert external material into the generic source model;
|
||||
- chunk modules partition source material for extraction;
|
||||
- extract modules produce domain artifacts from chunks;
|
||||
- merge modules combine accepted extraction outputs;
|
||||
- normalize modules reconcile merged output;
|
||||
- output modules encode accepted results and run outcomes into logical files.
|
||||
|
||||
The framework owns orchestration and handoff provenance. Modules return logical
|
||||
results and warnings; they do not own CLI reporting, workspace paths, durable
|
||||
file placement, checkpoints, or diagnostics.
|
||||
|
||||
## Validation
|
||||
|
||||
Validators should be independently testable and composable.
|
||||
Validation is a framework-managed boundary around raw outputs from chunk,
|
||||
extract, merge, and normalize stages. Validators receive immutable stage output
|
||||
and make an explicit whole-output decision: approve, approve with warnings, or
|
||||
reject.
|
||||
|
||||
Deterministic validators should run before LLM-backed validators when both are
|
||||
present. Validator decision semantics should be explicit: each candidate
|
||||
artifact evaluated by a validator should receive exactly one decision from that
|
||||
validator.
|
||||
Rejection is a recorded pipeline outcome, not a framework execution error.
|
||||
Validator execution failures are framework errors. Rejected output does not
|
||||
advance to the next stage.
|
||||
|
||||
LLM-backed review belongs in module-owned validator chains, not in an implicit
|
||||
global review phase. Extract and normalize modules may both use deterministic
|
||||
and LLM-backed validators.
|
||||
Default validator chains are production composition policy and are registered
|
||||
centrally by stage and module. Configuration may replace a stage-local default,
|
||||
including with an explicitly empty chain. Configured validator order is
|
||||
authoritative; the framework must not silently reorder it.
|
||||
|
||||
Shared validator runtime mechanics belong in framework code. Concrete validator
|
||||
behavior belongs in module or validator implementation packages.
|
||||
## LLM Boundary
|
||||
|
||||
## LLM Runtime
|
||||
Modules and validators use transport-neutral structured completion contracts.
|
||||
Provider request and response types, authentication, transport behavior, and
|
||||
provider error adaptation remain inside the LLM runtime.
|
||||
|
||||
LLM provider details belong behind transport-neutral framework contracts.
|
||||
The caller of the LLM owns prompt selection, prompt inputs, response schema,
|
||||
and interpretation of structured output. Provider adapters do not own source-
|
||||
or domain-specific prompt logic.
|
||||
|
||||
Provider-specific HTTP request and response types should stay inside the LLM
|
||||
runtime package. Prompt construction should stay in extractors, validators, or
|
||||
shared prompt helpers; provider adapters should not own domain prompt logic.
|
||||
LLM calls and other external operations accept cancellation and respect
|
||||
timeouts. Concurrency control belongs in shared runtime plumbing rather than in
|
||||
individual modules.
|
||||
|
||||
Errors, diagnostics, reports, manifests, and redacted configuration must not
|
||||
expose secrets.
|
||||
## Configuration And Provenance
|
||||
|
||||
## Configuration
|
||||
Configuration loading, precedence, defaults, environment overrides, redaction,
|
||||
and validation are centralized. Named pipeline definitions make structural
|
||||
composition explicit and discoverable. Operational overrides are permitted
|
||||
when they do not obscure the configured pipeline structure.
|
||||
|
||||
Configuration should make pipeline composition explicit and discoverable.
|
||||
Run preparation fails before stage execution when statically discoverable
|
||||
modules, capabilities, reference bindings, or explicitly selected profiles are
|
||||
invalid or incompatible.
|
||||
|
||||
Centralize configuration loading, precedence, defaults, and validation. Structural
|
||||
pipeline choices should come from named pipeline definitions, not ad hoc command
|
||||
flags. Operational overrides may be handled separately when they do not obscure
|
||||
the configured pipeline structure.
|
||||
Run manifests record enough resolved pipeline, module, source, reference, and
|
||||
LLM provenance to make a run auditable after configuration changes. Manifests
|
||||
record identities and summaries rather than secret or large payload content.
|
||||
|
||||
Module registries should expose module metadata and capabilities without
|
||||
requiring module construction. Configuration validation should fail fast when a
|
||||
pipeline binds incompatible or unknown modules.
|
||||
## State, Output, And Safety
|
||||
|
||||
Run manifests should record enough resolved pipeline provenance to make a run
|
||||
auditable after named configuration changes over time.
|
||||
Durable output, diagnostics, checkpoints, and debug artifacts are separate
|
||||
surfaces with separate ownership:
|
||||
|
||||
## Dependencies
|
||||
- output modules define logical durable output; the application boundary owns
|
||||
filesystem placement;
|
||||
- diagnostics provide redacted run inspection and are not the durable output
|
||||
contract;
|
||||
- checkpoints support validated stage reuse and are not diagnostics;
|
||||
- debug artifacts are opt-in inspection data and may contain sensitive source,
|
||||
prompt, reference, and model-output content.
|
||||
|
||||
Prefer the Go standard library where practical.
|
||||
Writes of durable state are atomic where practical. Paths for writes, moves,
|
||||
overwrites, and deletion must be narrow and explicit. Cleanup that can lose data
|
||||
is opt-in.
|
||||
|
||||
Use external dependencies only when justified by correctness, security,
|
||||
interoperability, or substantial complexity reduction. Good reasons include
|
||||
widely used file formats, complex validation behavior, or secure transport
|
||||
handling.
|
||||
Secrets must not appear in errors, logs, diagnostics, manifests,
|
||||
documentation, examples, or redacted configuration. Default logs and
|
||||
diagnostics must not include large source, prompt, reference, or artifact
|
||||
payloads.
|
||||
|
||||
Avoid dependencies for small conveniences. Do not let external dependency types
|
||||
leak across internal package boundaries unless the dependency is itself the
|
||||
explicit contract of that package.
|
||||
## Architectural Non-Goals
|
||||
|
||||
## State, Files, and Safety
|
||||
Notarius does not aim to provide:
|
||||
|
||||
If the application writes durable state, writes should be atomic where
|
||||
practical. Multi-step workflows should preserve enough diagnostics to support
|
||||
inspection after failure.
|
||||
|
||||
Code that deletes, moves, or overwrites files must use narrow, explicit paths.
|
||||
Avoid broad parent-directory operations. Cleanup that can cause data loss must
|
||||
be opt-in.
|
||||
|
||||
## Errors and Logging
|
||||
|
||||
Errors should be actionable and preserve context. Wrap errors with operation and
|
||||
path or resource context. CLI code should convert internal errors into concise
|
||||
user-facing messages.
|
||||
|
||||
Errors and logs must not expose secrets. Logs should describe operations,
|
||||
external calls, retries, and failure causes, but should not include large source
|
||||
or artifact payloads by default.
|
||||
|
||||
Long-running operations should accept `context.Context`. External calls,
|
||||
subprocesses, HTTP requests, storage operations, LLM calls, and multi-stage
|
||||
workflows should respect cancellation and timeouts.
|
||||
|
||||
## Testing
|
||||
|
||||
Core logic should be testable without real external services. Use fakes,
|
||||
fixtures, or local test doubles for input modules, extract modules, validators,
|
||||
and LLM clients where practical.
|
||||
|
||||
Contract-first work should include fake implementations that prove interfaces
|
||||
compose before real modules depend on them.
|
||||
|
||||
Maintain a fixture-driven walking skeleton that exercises the full pipeline with
|
||||
fake modules and fake external clients. This protects stage composition as real
|
||||
modules evolve.
|
||||
|
||||
Important CLI and configuration workflows should have tests. Adapter, extractor,
|
||||
validator, and stage contracts should have focused tests that do not require
|
||||
running the full application unless end-to-end coverage is intentional.
|
||||
|
||||
## Documentation
|
||||
|
||||
Documentation should follow the project documentation policy. Keep user docs
|
||||
focused on implemented behavior. Put future, planned, or aspirational work only
|
||||
under `docs/roadmap/`.
|
||||
|
||||
Core documentation should use generic terms such as source document, source
|
||||
unit, source reference, input adapter, extractor, chunker, merger, normalizer,
|
||||
artifact, validator, and run manifest.
|
||||
|
||||
Source-format details belong in input module or integration docs.
|
||||
Domain-specific extraction details belong in extract module or artifact docs.
|
||||
|
||||
When changing architecture, config, CLI behavior, stage modules, extractor
|
||||
contracts, validator contracts, LLM runtime behavior, or artifact schemas, update
|
||||
the relevant docs and examples in the same change.
|
||||
- an arbitrary workflow graph or general workflow language;
|
||||
- source-format or extraction-domain behavior in generic framework packages;
|
||||
- provider-specific contracts exposed to modules;
|
||||
- structural pipeline composition through ad hoc CLI flags;
|
||||
- implicit cross-stage behavior that bypasses the fixed pipeline;
|
||||
- abstractions introduced solely for hypothetical future complexity.
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
# Development
|
||||
|
||||
This document defines contributor workflow for Notarius. For architectural
|
||||
invariants and package boundaries, read [Architecture](architecture.md) first.
|
||||
|
||||
## Required Reading
|
||||
|
||||
Before changing the repository, review:
|
||||
|
||||
- [Architecture](architecture.md)
|
||||
- [Documentation Policy](documentation.md)
|
||||
|
||||
Keep current-behavior documentation limited to implemented behavior. Put planned
|
||||
or deferred behavior under `docs/roadmap/`.
|
||||
|
||||
## Repository Layout
|
||||
|
||||
- `cmd/notarius`: executable entry point.
|
||||
- `internal/cli`: CLI parsing, production catalog wiring, config loading, run
|
||||
command orchestration, output writes, and user-facing errors.
|
||||
- `internal/core`: deterministic models and policy for artifacts, source
|
||||
documents, config, and diagnostics.
|
||||
- `internal/framework`: reusable contracts, pipeline orchestration, prompt
|
||||
helpers, validation helpers, and LLM runtime plumbing.
|
||||
- `internal/modules`: concrete input, chunk, extract, merge, normalize, and
|
||||
output modules.
|
||||
- `docs`: policy, user/operator docs, internal docs, integration docs, and
|
||||
roadmap files.
|
||||
- `examples`: maintained, secret-free examples covered by tests where practical.
|
||||
|
||||
## Validation Commands
|
||||
|
||||
Run focused tests for the area changed, then run the broader checks when the
|
||||
change affects shared contracts, CLI behavior, or documentation examples.
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go vet ./...
|
||||
go build ./cmd/notarius
|
||||
```
|
||||
|
||||
Useful focused checks:
|
||||
|
||||
```sh
|
||||
go test ./internal/cli
|
||||
go test ./internal/core/config
|
||||
go test ./internal/framework/pipeline
|
||||
go test ./internal/framework/llm
|
||||
go test ./internal/modules/input/seriatim
|
||||
go test ./internal/modules/extract/dnd/spells
|
||||
go test ./internal/modules/output/json
|
||||
```
|
||||
|
||||
## Go Conventions
|
||||
|
||||
- Prefer the standard library unless a dependency is justified by correctness,
|
||||
security, interoperability, or substantial complexity reduction.
|
||||
- Keep package names short, lowercase, and idiomatic.
|
||||
- Preserve import direction: framework and core code must not depend on concrete
|
||||
production modules.
|
||||
- Use `context.Context` for long-running operations and external calls.
|
||||
- Return contextual errors that name the operation and relevant module, path, or
|
||||
resource.
|
||||
- Do not include secrets in errors, logs, diagnostics, manifests, or docs.
|
||||
|
||||
## Adding Config Fields
|
||||
|
||||
Config behavior is centralized under `internal/core/config`.
|
||||
|
||||
When adding a file config field:
|
||||
|
||||
1. Update file config structs and YAML parsing in `file_config.go`.
|
||||
2. Apply the field over defaults in config application code.
|
||||
3. Add validation in `validation.go` when the field has constraints.
|
||||
4. Add environment override support in `env.go` only for operational overrides.
|
||||
5. Update redaction if the field can contain secrets.
|
||||
6. Add focused config tests.
|
||||
7. Update [Configuration](../config.md) and maintained examples when behavior
|
||||
changes.
|
||||
|
||||
Pipeline composition should remain config-driven. Do not add command flags that
|
||||
silently replace structural pipeline definitions.
|
||||
|
||||
## Adding CLI Flags Or Commands
|
||||
|
||||
CLI behavior lives in `internal/cli`.
|
||||
|
||||
When adding CLI surface:
|
||||
|
||||
1. Keep syntax explicit and update usage text.
|
||||
2. Validate arguments before running expensive work.
|
||||
3. Convert internal errors into concise user-facing messages.
|
||||
4. Add CLI tests for success, syntax errors, and failure modes.
|
||||
5. Update [CLI Reference](../cli.md), and update
|
||||
[Operations](../operations.md) or [Troubleshooting](../troubleshooting.md)
|
||||
if run behavior changes.
|
||||
|
||||
## Adding Modules Or Adapters
|
||||
|
||||
Concrete modules live under `internal/modules/<kind>/...` and implement the
|
||||
interfaces in `internal/framework/contracts`.
|
||||
|
||||
For a new production module:
|
||||
|
||||
1. Implement the relevant contract.
|
||||
2. Expose a `ModuleSpec` with the correct module key, module kind, provided
|
||||
capabilities, and required capabilities.
|
||||
3. Expose a `Register` function that registers the module with its registry.
|
||||
4. Add focused module tests for contract behavior, registration, options,
|
||||
validation, and errors.
|
||||
5. Register the module in `internal/cli/catalog.go` only when it is production
|
||||
ready.
|
||||
6. Update internal docs and user-facing docs only for implemented behavior.
|
||||
|
||||
Source-format behavior belongs in input modules and integration docs.
|
||||
Extraction-domain behavior belongs in extract modules and artifact docs.
|
||||
|
||||
## Updating Examples
|
||||
|
||||
Examples must be valid, secret-free, and small.
|
||||
|
||||
- Prefer environment-based secret configuration.
|
||||
- Keep `examples/dnd-spells.config.yml` loadable by CLI tests.
|
||||
- Keep `examples/seriatim-minimal-transcript.json` compatible with the Seriatim
|
||||
adapter.
|
||||
- Do not add expected-output fixtures unless they are validated or have a clear
|
||||
regeneration procedure.
|
||||
|
||||
## Documentation Updates
|
||||
|
||||
Update docs in the same change when behavior changes.
|
||||
|
||||
- CLI syntax: `docs/cli.md`
|
||||
- Config fields and defaults: `docs/config.md`
|
||||
- Output, diagnostics, retention, or recovery: `docs/operations.md`
|
||||
- Common user-facing failures: `docs/troubleshooting.md`
|
||||
- Internal architecture and contracts: `docs/internal/`
|
||||
- External file formats and durable integration contracts: `docs/integrations/`
|
||||
- Future or planned work only: `docs/roadmap/`
|
||||
@@ -1,446 +1,143 @@
|
||||
# Go Project Documentation Policy
|
||||
# Documentation Policy
|
||||
|
||||
## Purpose
|
||||
|
||||
Project documentation must help five audiences:
|
||||
|
||||
1. users who need to run the application;
|
||||
2. administrators/operators who need to configure and operate it;
|
||||
3. developers who need to understand and change it safely;
|
||||
4. LLM coding agents that need clear scope, boundaries, and invariants;
|
||||
5. developers and LLM coding agents integrating this project from another codebase.
|
||||
|
||||
Docs should be accurate, concise, task-oriented, and organized by audience. Prefer links to canonical docs over repetition.
|
||||
This policy assigns each documentation topic to one canonical owner. Its goal is
|
||||
to keep Notarius documentation accurate, concise, discoverable, and resistant
|
||||
to drift for users, operators, developers, integrators, and LLM coding agents.
|
||||
|
||||
## Core Rules
|
||||
|
||||
### 1. Keep docs concise
|
||||
### One Canonical Owner
|
||||
|
||||
Each document should cover a defined scope and only the essentials for that scope.
|
||||
Each authoritative fact belongs in one document. A non-owning document may give
|
||||
a short, stable summary for orientation, but it must link to the canonical owner
|
||||
instead of repeating volatile details.
|
||||
|
||||
Avoid:
|
||||
- long background explanations;
|
||||
- repeated reference material;
|
||||
- implementation detail in user-facing docs;
|
||||
- aspirational language outside roadmap docs;
|
||||
- verbose examples where one minimal example is clearer.
|
||||
Volatile details include commands, flags, configuration fields and defaults,
|
||||
module keys, schemas, file names, paths, status codes, retry behavior, and
|
||||
runtime guarantees. If readers could reasonably treat a statement as a
|
||||
contract, maintain it only in the owning document.
|
||||
|
||||
### 2. Document only implemented behavior outside roadmap files
|
||||
### Current And Future Behavior
|
||||
|
||||
Unimplemented, planned, aspirational, experimental, or future work may be described only under:
|
||||
Outside `docs/roadmap/`, documentation describes implemented behavior only.
|
||||
Partial features may be described only to their implemented boundary.
|
||||
|
||||
ADRs are the narrow exception: an ADR may record an accepted architectural
|
||||
decision before implementation, but acceptance must not be presented as proof
|
||||
that the behavior exists. The roadmap owns implementation status and sequencing
|
||||
until the decision is implemented. Current architecture, user, operator,
|
||||
integration, and internal documentation are updated when the behavior lands.
|
||||
|
||||
- `docs/roadmap/`
|
||||
### Audience And Detail
|
||||
|
||||
No other documentation file, including `README.md`, should describe code, features, modules, stages, commands, config fields, or behaviors that do not currently exist.
|
||||
Write for the document's stated audience and include only the detail needed for
|
||||
its owned topic. User and operator docs should not expose implementation detail.
|
||||
Developer docs should link to user-facing and external contracts rather than
|
||||
restate them.
|
||||
|
||||
If a feature is partial, non-roadmap docs may describe only the implemented portion and its current boundary.
|
||||
### Examples
|
||||
|
||||
### 3. Use canonical homes
|
||||
|
||||
Each type of information should have one canonical location.
|
||||
|
||||
Canonical homes:
|
||||
|
||||
- project purpose and quickstart: `README.md`
|
||||
- development principles: `docs/policy/architecture.md`
|
||||
- public HTTP API reference: `docs/api.md`
|
||||
- configuration reference: `docs/config.md`
|
||||
- CLI reference: `docs/cli.md`
|
||||
- operations and recovery: `docs/operations.md`
|
||||
- troubleshooting: `docs/troubleshooting.md`
|
||||
- public API/package consumer guidance: `docs/consumers/`
|
||||
- implemented internals: `docs/internal/`
|
||||
- external protocol, service, and file-format contracts: `docs/integrations/`
|
||||
- future work: `docs/roadmap/`
|
||||
- contributor workflow: `docs/policy/development.md`
|
||||
- copyable examples: `examples/`
|
||||
|
||||
Other files should summarize briefly and link to the canonical source.
|
||||
|
||||
### 4. Keep examples real
|
||||
|
||||
Examples should be valid, maintained, and free of secrets.
|
||||
|
||||
Where practical:
|
||||
- example configs should load successfully;
|
||||
- example commands should match real CLI syntax;
|
||||
- important examples should be covered by tests.
|
||||
|
||||
## Documentation Profiles
|
||||
|
||||
All projects require:
|
||||
|
||||
- `README.md`
|
||||
- `docs/policy/architecture.md`
|
||||
|
||||
Additional docs depend on the project.
|
||||
|
||||
### Small library
|
||||
|
||||
Recommended:
|
||||
- `docs/policy/development.md`, if contributor conventions are non-obvious
|
||||
|
||||
### Simple CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Config-driven CLI
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`
|
||||
- `docs/config.md`
|
||||
|
||||
Recommended:
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Stateful or operator-facing application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `examples/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
### Modular, service-oriented, or orchestration application
|
||||
|
||||
Required:
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Public HTTP API service
|
||||
|
||||
Required:
|
||||
- `docs/api.md`
|
||||
- `docs/cli.md`, if CLI-based
|
||||
- `docs/config.md`, if config-driven
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/`
|
||||
- `docs/policy/development.md`
|
||||
|
||||
Recommended:
|
||||
- `docs/troubleshooting.md`
|
||||
- `docs/consumers/`, for task-oriented client integration guides
|
||||
- `docs/integrations/`, for upstream/downstream service contracts
|
||||
- validated examples under `examples/`
|
||||
|
||||
### Project with public packages or consumer APIs
|
||||
|
||||
Required:
|
||||
- `docs/consumers/api.md`
|
||||
- one `docs/consumers/pkg-<name>.md` file per public package, if public packages exist
|
||||
|
||||
Recommended:
|
||||
- copyable consumer examples under `examples/`, if practical
|
||||
|
||||
## Required Documents
|
||||
|
||||
### README.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
The README is the outward-facing project orientation page.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. concise description;
|
||||
2. elevator pitch;
|
||||
3. shortest useful command or usage example;
|
||||
4. links to targeted docs.
|
||||
|
||||
The README should be short. It is not a manual.
|
||||
|
||||
The “shortest useful command” means the simplest command that performs the project’s core use case. (It does not mean `app --help`.)
|
||||
|
||||
### docs/policy/architecture.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
`docs/policy/architecture.md` is required for every project.
|
||||
|
||||
It is an inward-facing development policy document. It should describe how the project is intended to be built and changed.
|
||||
|
||||
It should include:
|
||||
|
||||
- project shape;
|
||||
- core design principles;
|
||||
- package and boundary philosophy;
|
||||
- state/persistence philosophy, if applicable;
|
||||
- external integration philosophy, if applicable;
|
||||
- error-handling and logging principles;
|
||||
- testing expectations;
|
||||
- documentation expectations;
|
||||
- architectural invariants;
|
||||
- explicit non-goals, if useful.
|
||||
|
||||
Notably, this file should prescribe a core development *policy* that should remain unchanged as the application evolves. It is not a place for details (e.g., CLI flags) that could change over time.
|
||||
|
||||
The contents of `architecture.md` should be trim and concise. LLMs may be directed to review it routinely via AGENTS.md, CLAUDE.md, or similar.
|
||||
|
||||
### docs/api.md
|
||||
|
||||
**Audience:** external HTTP API consumers, developers, LLM coding agents integrating by HTTP
|
||||
|
||||
Required for projects whose primary public interface is HTTP.
|
||||
|
||||
`docs/api.md` is the canonical public HTTP API contract. It should be normative for external consumers and should not be duplicated by README, operations docs, consumer guides, or integration docs.
|
||||
|
||||
It should include:
|
||||
|
||||
1. base URL conventions;
|
||||
2. authentication and authorization behavior, if implemented;
|
||||
3. response envelope;
|
||||
4. supported media types and content negotiation behavior;
|
||||
5. shared query parameters;
|
||||
6. endpoint reference grouped by route family;
|
||||
7. request parameters and validation rules;
|
||||
8. response fields, units, nullability, and optionality;
|
||||
9. error response shape and status codes;
|
||||
10. pagination, caching, rate-limit, idempotency, and retry behavior, if implemented;
|
||||
11. compact request and response examples.
|
||||
|
||||
It must document only implemented endpoints and behavior. Planned endpoints, proposed fields, future filters, and experimental response shapes belong only under `docs/roadmap/`.
|
||||
|
||||
For HTTP API projects, `docs/consumers/` may provide task-oriented client integration guides, but those guides should link to `docs/api.md` for the authoritative endpoint contract.
|
||||
|
||||
### docs/policy/development.md
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects maintained by humans and LLM coding agents.
|
||||
|
||||
It should include:
|
||||
|
||||
- repository layout;
|
||||
- build/test commands;
|
||||
- coding conventions;
|
||||
- dependency policy;
|
||||
- how to add config fields;
|
||||
- how to add CLI flags;
|
||||
- how to add modules or adapters, if applicable;
|
||||
- how to update examples;
|
||||
- documentation update expectations.
|
||||
|
||||
### docs/config.md
|
||||
|
||||
**Audience:** administrators, operators, advanced users
|
||||
|
||||
Required for applications with configuration files.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. config file locations and discovery precedence;
|
||||
2. minimal working config;
|
||||
3. production-oriented config;
|
||||
4. full configuration reference;
|
||||
5. secrets handling, if applicable;
|
||||
6. links to maintained examples.
|
||||
|
||||
The full configuration reference should be canonical.
|
||||
|
||||
### docs/cli.md
|
||||
|
||||
**Audience:** users, administrators, operators
|
||||
|
||||
Required for CLI applications.
|
||||
|
||||
It should include, in order:
|
||||
|
||||
1. shortest useful command;
|
||||
2. command overview;
|
||||
3. complete flag reference;
|
||||
4. common workflows;
|
||||
5. diagnostic or recovery commands, if applicable.
|
||||
|
||||
Explain when commands are useful, not just their syntax.
|
||||
|
||||
### docs/operations.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Required for applications that maintain state, support resume behavior, run multi-step workflows, write durable artifacts, use remote storage, or require recovery procedures.
|
||||
|
||||
It should cover:
|
||||
|
||||
- normal workflow;
|
||||
- filesystem layout;
|
||||
- remote storage layout, if applicable;
|
||||
- logs and manifests;
|
||||
- resume/retry behavior;
|
||||
- cleanup behavior;
|
||||
- archive/backup behavior;
|
||||
- safe recovery procedures;
|
||||
- operational caveats.
|
||||
|
||||
### docs/troubleshooting.md
|
||||
|
||||
**Audience:** administrators, operators
|
||||
|
||||
Recommended once recurring failure modes exist.
|
||||
|
||||
Each entry should include:
|
||||
|
||||
- symptom;
|
||||
- likely cause;
|
||||
- diagnostic command or inspection step;
|
||||
- safe fix;
|
||||
- relevant links.
|
||||
|
||||
### docs/consumers/
|
||||
|
||||
**Audience:** developers and LLM coding agents integrating this project from another codebase
|
||||
|
||||
Required for projects with public packages, SDKs, client APIs, plugin APIs, or other application-facing integration surfaces.
|
||||
|
||||
This directory describes how an external codebase should consume the project's public API. It should be task-oriented and copyable where useful. It is not the place for internal implementation details or operator procedures.
|
||||
|
||||
For projects whose public API is HTTP, `docs/consumers/` is not required, and it should not duplicate the endpoint reference in `docs/api.md`. If present, it may provide practical integration workflows, client-specific examples, or migration notes that link back to `docs/api.md`.
|
||||
|
||||
`docs/consumers/api.md` should provide the consumer-facing overview and primary implementation workflow. It should include:
|
||||
|
||||
1. intended consumer audience and use cases;
|
||||
2. required inputs supplied by operators or deployment configuration;
|
||||
3. recommended public package or API workflow;
|
||||
4. minimal copyable example;
|
||||
5. consumer responsibilities and boundaries;
|
||||
6. retry, idempotency, or status behavior, if applicable;
|
||||
7. links to package-specific docs and canonical integration contracts.
|
||||
|
||||
Package-specific docs should be named `pkg-<name>.md` and should include:
|
||||
|
||||
1. import path;
|
||||
2. intended use cases;
|
||||
3. primary types and functions needed by consumers;
|
||||
4. minimal examples;
|
||||
5. validation, error, retry, and boundary behavior;
|
||||
6. links to canonical file-format or wire-protocol contracts.
|
||||
|
||||
### docs/internal/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for modular, service-oriented, or orchestration projects.
|
||||
|
||||
This directory describes implemented internal components. It is not the roadmap.
|
||||
|
||||
Use one file per major component where useful.
|
||||
|
||||
Each component doc should include:
|
||||
|
||||
1. purpose;
|
||||
2. inputs and outputs;
|
||||
3. boundaries;
|
||||
4. config fields used;
|
||||
5. external adapters used;
|
||||
6. state or manifest behavior, if applicable;
|
||||
7. skip/resume behavior, if applicable;
|
||||
8. failure behavior;
|
||||
9. tests to inspect before changing;
|
||||
10. architectural invariants.
|
||||
|
||||
### docs/roadmap/
|
||||
|
||||
**Audience:** maintainers, developers, LLM coding agents
|
||||
|
||||
This is the only place for planned, future, aspirational, experimental, or unimplemented work.
|
||||
|
||||
Roadmap docs should clearly distinguish:
|
||||
|
||||
- proposed work;
|
||||
- accepted plans;
|
||||
- deferred ideas;
|
||||
- rejected ideas;
|
||||
- implementation prompts or task breakdowns, if useful.
|
||||
|
||||
Roadmap docs should not be confused with current behavior.
|
||||
|
||||
### docs/integrations/
|
||||
|
||||
**Audience:** developers, LLM coding agents
|
||||
|
||||
Required for projects that depend on external CLIs, APIs, services, protocols, or file formats where the integration contract is important to maintain.
|
||||
|
||||
This directory contains concise, versioned reference notes for external integration contracts. It should document only the parts of the external system that this project actually uses or exposes.
|
||||
|
||||
For public HTTP API services, `docs/integrations/` should document upstream, downstream, storage, protocol, or runtime contracts that the service depends on or bridges. It should not become a second copy of the public HTTP endpoint reference; that belongs in `docs/api.md`.
|
||||
|
||||
Use one file per integration where useful.
|
||||
|
||||
## Examples Directory
|
||||
|
||||
Projects with non-trivial configuration or workflows should include `examples/`.
|
||||
|
||||
Useful examples include:
|
||||
|
||||
- minimal working config;
|
||||
- production-oriented config;
|
||||
- full annotated config;
|
||||
- local development config;
|
||||
- remote/object-storage config;
|
||||
- minimal session/input file.
|
||||
|
||||
Examples should be valid, maintained, tested when practical, and linked from relevant docs.
|
||||
|
||||
## Security and Privacy
|
||||
|
||||
Docs and examples must not include:
|
||||
|
||||
- real API keys;
|
||||
- tokens;
|
||||
- passwords;
|
||||
- private keys;
|
||||
- private environment dumps;
|
||||
- sensitive user data;
|
||||
- raw private transcripts;
|
||||
- private infrastructure details unless intentionally public.
|
||||
|
||||
Document secret-handling mechanisms, not actual secret values.
|
||||
|
||||
## Maintenance Rules
|
||||
|
||||
When docs change, verify the affected behavior.
|
||||
|
||||
Where practical:
|
||||
|
||||
- load example config files in tests;
|
||||
- test CLI examples or command parser behavior;
|
||||
- validate documented flags against real flags;
|
||||
- remove stale references;
|
||||
- update links after renames;
|
||||
- keep roadmap content out of non-roadmap docs.
|
||||
|
||||
If documentation and code disagree, fix the documentation and/or open a roadmap item; do not leave aspirational behavior in current-behavior docs.
|
||||
|
||||
Documentation is complete only when it matches the current code.
|
||||
|
||||
## Documentation Change Checklist
|
||||
|
||||
Before merging documentation changes, verify:
|
||||
|
||||
- README is concise and orientation-focused.
|
||||
- `docs/policy/architecture.md` describes development principles.
|
||||
- `docs/api.md` is the canonical HTTP contract for HTTP API services.
|
||||
- Future work appears only under `docs/roadmap/`.
|
||||
- User-facing docs avoid unnecessary internals.
|
||||
- Consumer-facing docs explain public APIs without duplicating HTTP endpoint or integration contracts.
|
||||
- Developer-facing docs preserve boundaries and invariants.
|
||||
- Config examples match the schema.
|
||||
- CLI examples match real commands and flags.
|
||||
- Defaults appear in the canonical config reference.
|
||||
- No secrets or private data are included.
|
||||
- Links are accurate.
|
||||
Complete copyable files belong in `examples/`. Documentation may use the
|
||||
smallest illustrative snippet needed to explain its owned topic, but should link
|
||||
to maintained examples instead of embedding a second complete copy.
|
||||
|
||||
Examples must be valid, secret-free, and tested where practical. Commands and
|
||||
configuration used in documentation should match the application.
|
||||
|
||||
### Security And Privacy
|
||||
|
||||
Documentation and examples must not contain real credentials, private keys,
|
||||
private environment dumps, sensitive source material, or private infrastructure
|
||||
details unless intentionally public. Document secret-handling mechanisms, not
|
||||
secret values.
|
||||
|
||||
## Canonical Ownership
|
||||
|
||||
| Topic | Canonical owner | Owned content | Content owned elsewhere |
|
||||
| --- | --- | --- | --- |
|
||||
| Product orientation and minimal end-to-end quickstart | `README.md` | What Notarius is, why it is useful, one shortest successful invocation, and links onward. | Complete command reference, configuration reference, operational procedures, implementation detail. |
|
||||
| Contributor entry point | `docs/development.md` | Task-oriented reading guide, minimal contributor orientation, baseline validation commands, and links to canonical docs. | Package inventory, architecture rules, subsystem behavior, detailed change recipes. |
|
||||
| Current application architecture | `docs/policy/architecture.md` | System shape, normative ownership, dependency direction, architectural boundaries, invariants, safety properties, and non-goals. | Concrete package inventory, implementation mechanics, contributor procedures, decision history, future work. |
|
||||
| Documentation organization | `docs/policy/documentation.md` | Documentation ownership, audience boundaries, maintenance rules, and ADR/document lifecycle. | Application architecture or product behavior. |
|
||||
| CLI contract | `docs/cli.md` | Commands, arguments, flags, invocation semantics, and exit codes. | End-to-end operating procedures, configuration field definitions, runtime filesystem layout, module implementation details. |
|
||||
| Configuration contract | `docs/config.md` | Discovery and precedence, file schema, fields, defaults, environment overrides, validation rules, and user-selectable module or validator keys. | Complete example files, CLI syntax, runtime state lifecycle, module implementation details. |
|
||||
| Operations | `docs/operations.md` | Runtime workflows, physical filesystem and state layout, diagnostics use, retention, resume, cleanup, permissions, recovery, and operational limits. | CLI flag syntax, configuration field definitions, logical output schemas, implementation mechanics. |
|
||||
| Public HTTP contract, if introduced | `docs/api.md` | Routes, authentication, media types, request and response schemas, status codes, pagination, caching, idempotency, rate limits, and HTTP retry semantics. | Client walkthroughs, upstream or downstream integration internals, implementation detail. |
|
||||
| Consumer guidance, if a public package or API is introduced | `docs/consumers/` | Task-oriented use of the public interface, minimal client examples, and consumer responsibilities. | HTTP wire semantics, external protocol contracts, internal implementation detail. |
|
||||
| External and durable integration contracts | `docs/integrations/` | External file formats and protocols, upstream and downstream contracts, logical output bundle paths and schemas, media types, and compatibility behavior. | Physical runtime placement and lifecycle, internal transformations, CLI syntax, configuration defaults. |
|
||||
| Implemented component inventory | `docs/internal/overview.md` | Current packages and components, their implemented responsibilities, and links to focused internal docs. | Normative architecture, contributor reading policy, external contracts. |
|
||||
| Internal component behavior | Other files under `docs/internal/` | Implementation flow, internal collaborators and state transitions, package-local guarantees and failures, and relevant tests. | Global architecture invariants, configuration definitions and defaults, external schemas, operator procedures. |
|
||||
| Architectural decision history | `docs/adr/` | Significant decisions, context, alternatives, rationale, consequences, and supersession history. | Current behavior reference, implementation status, task sequencing. |
|
||||
| Future work and implementation status | `docs/roadmap/` | Proposed, accepted, deferred, or rejected work; implementation status; sequencing; and task breakdowns. | Implemented behavior reference and architectural decision rationale. |
|
||||
| Complete copyable artifacts | `examples/` | Maintained configuration, inputs, and other files intended to be copied or run. | Field-by-field reference, command reference, prose explanation. |
|
||||
|
||||
Documents that do not exist are required only when the corresponding interface
|
||||
or responsibility exists. Do not create placeholder API, consumer, integration,
|
||||
or operations documents for behavior the application does not have.
|
||||
|
||||
## Boundary Rules
|
||||
|
||||
### Orientation
|
||||
|
||||
The README owns product orientation. The developer guide routes contributors.
|
||||
Architecture owns normative structure. Internal overview owns the current
|
||||
concrete component map. These documents may link to one another but should not
|
||||
maintain parallel package or behavior descriptions.
|
||||
|
||||
### Commands, Configuration, And Operations
|
||||
|
||||
CLI documentation answers how to invoke the application. Configuration
|
||||
documentation answers what settings mean. Operations answers what happens to
|
||||
runtime state and how to operate or recover the application. When a workflow
|
||||
crosses these topics, choose the document that owns the task and link to the
|
||||
other contracts.
|
||||
|
||||
### Contracts And Implementation
|
||||
|
||||
Integration and API documents define externally observable shapes and
|
||||
semantics. Internal documents explain how Notarius implements or consumes those
|
||||
contracts. Internal docs may name a field, file, or protocol to identify a
|
||||
dependency, but must link to its canonical contract for the definition.
|
||||
|
||||
### Security Topics
|
||||
|
||||
This policy owns what documentation and examples may contain. Architecture owns
|
||||
application security invariants. Configuration owns credential-supply
|
||||
mechanisms. Operations owns permissions and handling of sensitive runtime
|
||||
artifacts. Internal docs own implementation mechanisms only.
|
||||
|
||||
## Architecture Decision Records
|
||||
|
||||
Use sequentially numbered ADR filenames such as
|
||||
`0001-record-architecture-decisions.md`. Follow the lightweight Nygard format:
|
||||
|
||||
1. title;
|
||||
2. status;
|
||||
3. date;
|
||||
4. context;
|
||||
5. decision;
|
||||
6. alternatives considered;
|
||||
7. consequences.
|
||||
|
||||
Treat the decision content of an accepted ADR as immutable. When a decision
|
||||
changes, create a new ADR and update the earlier ADR's status to superseded.
|
||||
Rejected architectural alternatives belong in the ADR; rejected product ideas
|
||||
belong in the roadmap.
|
||||
|
||||
## Maintenance
|
||||
|
||||
When behavior changes, update its canonical owner in the same change. If
|
||||
ownership moves, remove the old definition and replace it with a link where
|
||||
navigation remains useful.
|
||||
|
||||
Before completing documentation work:
|
||||
|
||||
- verify affected behavior and examples;
|
||||
- check commands, flags, fields, defaults, schemas, and paths against their
|
||||
implementation;
|
||||
- keep unimplemented behavior in the roadmap, subject to the ADR exception;
|
||||
- remove stale references and validate links;
|
||||
- confirm that non-owning documents summarize and link rather than redefine;
|
||||
- confirm that no secrets or sensitive private data were added.
|
||||
|
||||
130
docs/roadmap/documentation.md
Normal file
130
docs/roadmap/documentation.md
Normal file
@@ -0,0 +1,130 @@
|
||||
# Documentation Policy Migration
|
||||
|
||||
The documentation ownership policy has been revised to assign each topic to one
|
||||
canonical document. This roadmap organizes the likely migration work into three
|
||||
ordered passes.
|
||||
|
||||
This is a planning inventory, not a review of the current documents. Each item
|
||||
must be verified before content is moved, removed, or rewritten. Complete the
|
||||
passes in order so canonical content is established before non-owning copies are
|
||||
removed.
|
||||
|
||||
## Pass 1: User And Operator Contracts
|
||||
|
||||
Establish the canonical externally observable facts first. These documents will
|
||||
become the targets linked from developer and internal documentation in the next
|
||||
pass.
|
||||
|
||||
### Product And CLI
|
||||
|
||||
- Audit `README.md` so it owns product orientation and the single minimal
|
||||
end-to-end quickstart without becoming a command or configuration reference.
|
||||
- Audit `docs/cli.md` for material owned by the README, configuration reference,
|
||||
or operations guide.
|
||||
- Keep commands, arguments, flags, invocation semantics, and exit codes
|
||||
canonical in `docs/cli.md`.
|
||||
|
||||
### Configuration And Examples
|
||||
|
||||
- Audit `docs/config.md` for complete example files, CLI syntax, runtime state
|
||||
lifecycle, or implementation detail.
|
||||
- Keep fields, defaults, precedence, environment overrides, validation, and
|
||||
user-selectable module and validator keys canonical in `docs/config.md`.
|
||||
- Audit minimal and production-oriented configuration examples, moving complete
|
||||
copyable files under `examples/` and leaving only small illustrative fragments
|
||||
in reference documentation.
|
||||
- Inventory complete configuration, input, command, and output examples embedded
|
||||
in prose documents.
|
||||
- Select one maintained copy for each complete artifact under `examples/` where
|
||||
practical, and verify important examples through automated tests.
|
||||
|
||||
### Operations And Integrations
|
||||
|
||||
- Audit `docs/operations.md` for duplicated CLI syntax, configuration field
|
||||
definitions, logical output schemas, or implementation mechanics.
|
||||
- Keep runtime workflows, physical state, retention, recovery, permissions, and
|
||||
operational limits canonical in `docs/operations.md`.
|
||||
- Audit `docs/integrations/` so external formats, protocols, logical output
|
||||
paths, schemas, media types, and compatibility rules have one canonical home.
|
||||
- Separate logical output bundle contracts from physical runtime placement and
|
||||
lifecycle.
|
||||
|
||||
### Pass 1 Completion
|
||||
|
||||
- Confirm that README, CLI, configuration, operations, integrations, and
|
||||
examples have non-overlapping ownership.
|
||||
- Validate user-facing commands, fields, defaults, keys, schemas, paths, and
|
||||
maintained examples against implemented behavior.
|
||||
- Validate links among the user and operator documents.
|
||||
|
||||
## Pass 2: Developer And Internal Documentation
|
||||
|
||||
Use the canonical contracts established in Pass 1 to remove duplicated facts
|
||||
from contributor and implementation documentation.
|
||||
|
||||
### Orientation And Architecture
|
||||
|
||||
- Audit `docs/development.md` so it routes contributors without maintaining a
|
||||
parallel package inventory or architectural description.
|
||||
- Audit `docs/internal/overview.md` so it owns the implemented component map
|
||||
without restating normative architecture.
|
||||
- Audit `docs/policy/architecture.md` so it contains current normative
|
||||
architecture without implementation inventory, decision history, or future
|
||||
behavior.
|
||||
|
||||
### Internal Components
|
||||
|
||||
- Audit internal component documents for repeated configuration definitions,
|
||||
external input or output schemas, operator procedures, and global
|
||||
architectural invariants.
|
||||
- Replace duplicated external schemas or field definitions with links to the
|
||||
canonical configuration or integration contracts from Pass 1.
|
||||
- Keep implementation flow, internal collaborators, state transitions,
|
||||
package-local guarantees, failures, and relevant tests in focused internal
|
||||
documents.
|
||||
- Identify task-specific contributor recipes that need a focused internal home
|
||||
rather than the developer landing page.
|
||||
- Retain production module and validator implementation details in module
|
||||
internals while linking user-selectable keys to `docs/config.md`.
|
||||
|
||||
### Pass 2 Completion
|
||||
|
||||
- Confirm that development routes, architecture governs, internal overview
|
||||
inventories, and focused internal docs explain implementation.
|
||||
- Confirm that developer documents link to configuration, operations, and
|
||||
integration contracts rather than redefining them.
|
||||
- Validate developer-facing links and relevant focused tests.
|
||||
|
||||
## Pass 3: Lifecycle And Final Deduplication
|
||||
|
||||
Finish the migration by reconciling historical and future documentation, then
|
||||
perform a repository-wide ownership audit.
|
||||
|
||||
### ADR And Roadmap Lifecycle
|
||||
|
||||
- Audit roadmap files for implemented behavior or completed status summaries
|
||||
that should be removed or replaced with links to current canonical docs.
|
||||
- Ensure ADRs own architectural rationale and supersession history without
|
||||
becoming current behavior references or implementation trackers.
|
||||
- Ensure accepted but unimplemented ADR decisions link to roadmap-owned
|
||||
implementation status where appropriate.
|
||||
- Distinguish rejected architectural alternatives in ADRs from rejected product
|
||||
ideas in roadmap files.
|
||||
|
||||
### Repository-Wide Audit
|
||||
|
||||
- Inventory repeated commands, flags, defaults, module keys, validator keys,
|
||||
file names, paths, schemas, retry semantics, and runtime guarantees.
|
||||
- Assign each repeated fact to the canonical owner defined by the policy.
|
||||
- Replace non-owning copies with short summaries and links where navigation is
|
||||
useful.
|
||||
- Retain only minimal illustrative snippets in prose and link them to maintained
|
||||
examples.
|
||||
- Validate local links and remove references to deleted or relocated material.
|
||||
|
||||
### Pass 3 Completion
|
||||
|
||||
- Confirm that every contractual or volatile fact has one canonical owner.
|
||||
- Confirm that roadmap files contain future work and implementation status, ADRs
|
||||
contain decision rationale, and current docs contain implemented behavior.
|
||||
- Run final link, example, and documentation consistency checks.
|
||||
49
docs/roadmap/future.md
Normal file
49
docs/roadmap/future.md
Normal file
@@ -0,0 +1,49 @@
|
||||
# Future Work
|
||||
|
||||
Current Notarius behavior is documented in the canonical README, CLI,
|
||||
configuration, operations, internal, and integration docs. This roadmap records
|
||||
future work only.
|
||||
|
||||
## Candidate Product Work
|
||||
|
||||
- Additional input adapters, such as Markdown or note-export formats.
|
||||
- Additional D&D extractors beyond spell casts.
|
||||
- Add non-file reference producers, such as prior-run artifacts, derived
|
||||
summaries, or entity registries, without changing module-facing reference
|
||||
item contracts.
|
||||
- Add token budgeting and model context-window management for reference content.
|
||||
- Add per-slot or per-chunk inclusion policies so modules can avoid repeating
|
||||
large reference content in every prompt when that becomes important.
|
||||
- Add structured or parsed references, such as typed roster schemas, when a
|
||||
module has a clear need for more than opaque UTF-8 text.
|
||||
- Add reference caching, preprocessing, summarization, embedding, or retrieval
|
||||
if references become large enough to require preprocessing.
|
||||
- Cross-lane entity normalization.
|
||||
- Cross-chunk semantic deduplication.
|
||||
- Additional validator packages and production default chains for future
|
||||
modules.
|
||||
- Production LLM-backed validators when there is a concrete review policy that
|
||||
benefits from model judgment.
|
||||
- Validator diagnostics and timing summaries if operators need more detail than
|
||||
`manifest.json`, `rejected.json`, and `warnings.json` provide.
|
||||
- Media-type validators for non-JSON module outputs when such modules are
|
||||
introduced.
|
||||
- Validator compatibility metadata if real deployments need config-time
|
||||
enforcement that a validator is suitable for a specific stage or module.
|
||||
- Batching or context-window controls for LLM-backed validators if validator
|
||||
inputs become large enough to require them.
|
||||
- Parallel execution where it preserves deterministic manifests and diagnostics.
|
||||
- Additional output encoders.
|
||||
|
||||
## Candidate Operational Work
|
||||
|
||||
- Packaged release artifacts for alpha distribution.
|
||||
- A documented versioning and release process.
|
||||
- Optional generated example output fixtures with a regeneration procedure.
|
||||
- Additional diagnostics or reporting views if operator workflows need them.
|
||||
|
||||
## Non-Goals To Revisit Deliberately
|
||||
|
||||
- A general workflow language.
|
||||
- Structural module selection through ad hoc run flags.
|
||||
- Storing secrets in config files, diagnostics, manifests, or examples.
|
||||
25
docs/roadmap/implementation.md
Normal file
25
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,25 @@
|
||||
# Workspace Implementation Status
|
||||
|
||||
The workspace implementation described by this roadmap has landed. Current
|
||||
behavior is documented in the canonical current-behavior docs:
|
||||
|
||||
- [Configuration](../config.md)
|
||||
- [CLI Reference](../cli.md)
|
||||
- [Operations](../operations.md)
|
||||
- [Diagnostics Internals](../internal/diagnostics.md)
|
||||
- [Pipeline Internals](../internal/pipeline.md)
|
||||
|
||||
Implemented behavior includes:
|
||||
|
||||
- `workspace.directory` as the root for Notarius-owned local state;
|
||||
- workspace diagnostics under `<workspace.directory>/diagnostics/<run-id>/`;
|
||||
- compatibility for legacy `diagnostics.work_dir`, `diagnostics.retention`,
|
||||
`NOTARIUS_WORK_DIR`, and `NOTARIUS_DIAGNOSTICS_RETENTION`;
|
||||
- checkpoint writes under `<workspace.directory>/checkpoints/` when resume
|
||||
checkpointing is enabled;
|
||||
- explicit checkpoint reuse through `notarius run --resume`;
|
||||
- debug artifacts under `<workspace.directory>/debug/<run-id>/` when debug
|
||||
output is enabled;
|
||||
- independent resume and debug settings.
|
||||
|
||||
Deferred workspace ideas remain in [Workspace Roadmap](workspace.md).
|
||||
@@ -1,29 +0,0 @@
|
||||
# Future Work
|
||||
|
||||
Current Notarius behavior is documented in the canonical README, CLI,
|
||||
configuration, operations, internal, and integration docs. This roadmap records
|
||||
future work only.
|
||||
|
||||
## Candidate Product Work
|
||||
|
||||
- Additional input adapters, such as Markdown or note-export formats.
|
||||
- Additional D&D extractors beyond spell casts.
|
||||
- Cross-lane entity normalization.
|
||||
- Cross-chunk semantic deduplication.
|
||||
- Configurable validator chains with production validator modules.
|
||||
- Multiple effective LLM profiles in one run.
|
||||
- Parallel execution where it preserves deterministic manifests and diagnostics.
|
||||
- Additional output encoders.
|
||||
|
||||
## Candidate Operational Work
|
||||
|
||||
- Packaged release artifacts for alpha distribution.
|
||||
- A documented versioning and release process.
|
||||
- Optional generated example output fixtures with a regeneration procedure.
|
||||
- Additional diagnostics or reporting views if operator workflows need them.
|
||||
|
||||
## Non-Goals To Revisit Deliberately
|
||||
|
||||
- A general workflow language.
|
||||
- Structural module selection through ad hoc run flags.
|
||||
- Storing secrets in config files, diagnostics, manifests, or examples.
|
||||
26
docs/roadmap/workspace.md
Normal file
26
docs/roadmap/workspace.md
Normal file
@@ -0,0 +1,26 @@
|
||||
# Workspace Roadmap Status
|
||||
|
||||
The local workspace feature has been implemented. Current behavior is documented
|
||||
in [Configuration](../config.md), [CLI Reference](../cli.md),
|
||||
[Operations](../operations.md), and the relevant internal docs.
|
||||
|
||||
The implemented workspace provides one configurable root for Notarius-owned
|
||||
local state:
|
||||
|
||||
```text
|
||||
<workspace.directory>/
|
||||
diagnostics/
|
||||
checkpoints/
|
||||
debug/
|
||||
```
|
||||
|
||||
Implemented behavior includes workspace-backed diagnostics, checkpoint writing,
|
||||
explicit checkpoint reuse through `notarius run --resume`, workspace debug
|
||||
artifacts, safe workspace-relative writes, and compatibility for legacy
|
||||
diagnostics configuration.
|
||||
|
||||
## Deferred Work
|
||||
|
||||
Default-idempotent `run` behavior with a force override, remote workspace
|
||||
storage, workspace garbage collection, archival policy, and cross-machine resume
|
||||
remain deferred.
|
||||
@@ -1,215 +0,0 @@
|
||||
# Troubleshooting
|
||||
|
||||
This guide maps common implemented failure modes to inspection steps and fixes.
|
||||
For command syntax, see [CLI Reference](cli.md). For YAML fields and
|
||||
environment overrides, see [Configuration](config.md). For output and
|
||||
diagnostics layout, see [Operations](operations.md).
|
||||
|
||||
## Config File Not Found
|
||||
|
||||
Symptom:
|
||||
|
||||
```text
|
||||
notarius: config file not found; pass --config or set NOTARIUS_CONFIG
|
||||
```
|
||||
|
||||
Fix:
|
||||
|
||||
- Pass `--config path/to/config.yml`.
|
||||
- Or set `NOTARIUS_CONFIG` to a readable file.
|
||||
- Or install a config at `/usr/local/etc/notarius/config.yml`.
|
||||
|
||||
If the message says the config path is a directory or is not available, correct
|
||||
the path or file permissions.
|
||||
|
||||
## Unsupported Or Invalid Config
|
||||
|
||||
Symptoms include:
|
||||
|
||||
- `unsupported config version`
|
||||
- `config version is required`
|
||||
- `field <name> not found`
|
||||
- `total LLM concurrency must be greater than zero`
|
||||
- `diagnostics retention "<value>" is not supported`
|
||||
|
||||
Fix:
|
||||
|
||||
- Use `version: 1`.
|
||||
- Remove unknown YAML fields.
|
||||
- Validate with:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius config validate --config path/to/config.yml
|
||||
```
|
||||
|
||||
## Unknown Pipeline
|
||||
|
||||
Symptom:
|
||||
|
||||
```text
|
||||
notarius: pipeline "..." is not configured
|
||||
```
|
||||
|
||||
Fix:
|
||||
|
||||
- List configured pipeline IDs:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius pipelines list --config path/to/config.yml
|
||||
```
|
||||
|
||||
- Use one of those IDs in `notarius run <pipeline-id>`.
|
||||
- Check indentation under the top-level `pipelines` map.
|
||||
|
||||
## Unknown Or Incompatible Module
|
||||
|
||||
Symptoms mention a module key, pipeline slot, lane, capability, or `not
|
||||
registered`.
|
||||
|
||||
Fix:
|
||||
|
||||
- Validate the pipeline against the production module catalog:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius config validate \
|
||||
--config path/to/config.yml \
|
||||
--pipeline dnd-session
|
||||
```
|
||||
|
||||
- Use only implemented production module keys listed in
|
||||
[Configuration](config.md#implemented-production-modules).
|
||||
- Check that artifact lanes include an `extract` binding.
|
||||
|
||||
## Invalid `--only`
|
||||
|
||||
Symptoms include:
|
||||
|
||||
- `--only must contain comma-separated non-empty artifact lane IDs`
|
||||
- `--only requires --pipeline`
|
||||
- `selected artifact lane`
|
||||
|
||||
Fix:
|
||||
|
||||
- Use comma-separated lane IDs with no empty entries:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config path/to/config.yml \
|
||||
--input path/to/input.json \
|
||||
--only spells
|
||||
```
|
||||
|
||||
- For `config validate`, include `--pipeline` when using `--only`.
|
||||
- Confirm the lane ID exists under `pipelines.<id>.artifacts`.
|
||||
|
||||
## Seriatim Input Validation Failure
|
||||
|
||||
Symptoms include `seriatim input`, `parse JSON`, `segments must not be empty`,
|
||||
or validation errors naming a segment field.
|
||||
|
||||
Fix:
|
||||
|
||||
- Compare the input to
|
||||
[examples/seriatim-minimal-transcript.json](../examples/seriatim-minimal-transcript.json).
|
||||
- Ensure the JSON has a `metadata` object and a non-empty `segments` array.
|
||||
- Each segment needs a non-empty `id`, non-empty `speaker`, non-empty `text`,
|
||||
non-negative numeric `start`, and non-negative numeric `end`.
|
||||
- Segment IDs must be unique and must not contain leading or trailing
|
||||
whitespace.
|
||||
- `end` must be greater than or equal to `start`.
|
||||
|
||||
## Missing LLM Base URL Or Model
|
||||
|
||||
Symptoms include:
|
||||
|
||||
- `LLM profile "default" base URL must not be empty`
|
||||
- `LLM profile "default" model must not be empty`
|
||||
- `base URL must be valid`
|
||||
|
||||
Fix:
|
||||
|
||||
- Set `base_url` and `model` in `llm_profiles.default`.
|
||||
- Or set `NOTARIUS_LLM_DEFAULT_BASE_URL` and
|
||||
`NOTARIUS_LLM_DEFAULT_MODEL`.
|
||||
- If a profile needs authentication, set `api_key_env` in YAML or set
|
||||
`NOTARIUS_LLM_DEFAULT_API_KEY`.
|
||||
|
||||
## LLM Profile Override Failure
|
||||
|
||||
Symptom:
|
||||
|
||||
```text
|
||||
notarius: LLM profile override "..." is not configured
|
||||
```
|
||||
|
||||
Fix:
|
||||
|
||||
- Add the profile under `llm_profiles`.
|
||||
- Or use an existing profile ID with `--llm-profile`.
|
||||
|
||||
Current runs require exactly one distinct effective LLM profile. If a pipeline
|
||||
uses several profiles, run with `--llm-profile <id>` or align the bindings in
|
||||
configuration.
|
||||
|
||||
## Provider HTTP Or Response Failure
|
||||
|
||||
Symptoms include:
|
||||
|
||||
- `provider request failed`
|
||||
- `provider returned status 400`
|
||||
- `provider returned status 403`
|
||||
- `provider response missing choices`
|
||||
- `provider response assistant message content is not valid JSON`
|
||||
- `decode structured output`
|
||||
|
||||
Fix:
|
||||
|
||||
- Confirm the `base_url` points to an OpenAI-compatible endpoint root. Notarius
|
||||
posts to `<base_url>/chat/completions`.
|
||||
- Check `model` and provider credentials.
|
||||
- Inspect the retained diagnostics `error.log`.
|
||||
- For 400 and 403 responses, fix the request configuration or credentials.
|
||||
- For 429 and 5xx responses, the client retries according to `max_retries`; if
|
||||
the failure persists, inspect the provider response and adjust capacity,
|
||||
credentials, or model settings.
|
||||
- The assistant message content must decode as JSON matching the extractor's
|
||||
structured response schema.
|
||||
|
||||
Provider error messages are redacted for configured API key values.
|
||||
|
||||
## Output Write Failure
|
||||
|
||||
Symptoms include:
|
||||
|
||||
- `create output directory`
|
||||
- `write output file`
|
||||
- `output file name must`
|
||||
|
||||
Fix:
|
||||
|
||||
- Ensure `--output-dir` points to a directory path or a path that can be
|
||||
created.
|
||||
- Check filesystem permissions and available disk space.
|
||||
- If diagnostics were retained, inspect `run-report.json`, `run-manifest.json`,
|
||||
and `error.log`.
|
||||
|
||||
The CLI rejects unsafe logical output paths before writing files.
|
||||
|
||||
## Diagnostics Directory Surprise
|
||||
|
||||
Symptom: the diagnostics directory is missing after a successful run.
|
||||
|
||||
Fix:
|
||||
|
||||
- Check `diagnostics.retention`.
|
||||
- With `auto`, successful runs without warnings are removed.
|
||||
- Use `diagnostics.retention: always` when every diagnostics run directory
|
||||
should be kept.
|
||||
- Use `--diagnostics-dir` to override the configured work directory for a run.
|
||||
|
||||
Symptom: diagnostics exist even with `retention: never`.
|
||||
|
||||
Explanation:
|
||||
|
||||
- Failed runs are retained so that `error.log` and available context can be
|
||||
inspected.
|
||||
2
examples/dnd-spells-glossary.txt
Normal file
2
examples/dnd-spells-glossary.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
Cure Wounds: healing spell cast by touch.
|
||||
Shield: defensive reaction spell.
|
||||
3
examples/dnd-spells-roster.txt
Normal file
3
examples/dnd-spells-roster.txt
Normal file
@@ -0,0 +1,3 @@
|
||||
Aria: party cleric and recurring healer.
|
||||
Borin: fighter ally.
|
||||
Bandit mage: hostile spellcaster.
|
||||
@@ -1,12 +1,25 @@
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
provider: openai-compatible
|
||||
base_url: http://127.0.0.1:1
|
||||
model: fake-model
|
||||
version: 2
|
||||
# For production runs, use a writable application-owned workspace such as:
|
||||
#
|
||||
# workspace:
|
||||
# directory: /var/lib/notarius
|
||||
# diagnostics:
|
||||
# retention: auto
|
||||
# resume:
|
||||
# enabled: false
|
||||
# debug:
|
||||
# enabled: false
|
||||
#
|
||||
# For local development, use a project-local ignored path such as:
|
||||
#
|
||||
# workspace:
|
||||
# directory: ./.notarius/workspace
|
||||
pipelines:
|
||||
dnd-session:
|
||||
input: seriatim
|
||||
references:
|
||||
party: ./dnd-spells-roster.txt
|
||||
glossary: ./dnd-spells-glossary.txt
|
||||
chunk:
|
||||
module: generic
|
||||
options:
|
||||
|
||||
@@ -5,14 +5,14 @@
|
||||
},
|
||||
"segments": [
|
||||
{
|
||||
"id": "seg-001",
|
||||
"id": 1,
|
||||
"start": 0,
|
||||
"end": 4,
|
||||
"speaker": "Aria",
|
||||
"text": "Aria raises her holy symbol and casts Cure Wounds."
|
||||
},
|
||||
{
|
||||
"id": "seg-002",
|
||||
"id": 2,
|
||||
"start": 4,
|
||||
"end": 8,
|
||||
"speaker": "DM",
|
||||
|
||||
10
go.mod
10
go.mod
@@ -1,5 +1,11 @@
|
||||
module gitea.maximumdirect.net/eric/notarius
|
||||
|
||||
go 1.24.0
|
||||
go 1.25.5
|
||||
|
||||
require gopkg.in/yaml.v3 v3.0.1
|
||||
require (
|
||||
gitea.maximumdirect.net/eric/scriptorium v0.11.0
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require golang.org/x/text v0.14.0 // indirect
|
||||
|
||||
8
go.sum
8
go.sum
@@ -1,3 +1,11 @@
|
||||
gitea.maximumdirect.net/eric/scriptorium v0.11.0 h1:rjvbt9FTaWHxYlHq7QlUzmMVUt3QdbTmeCkmH81N//o=
|
||||
gitea.maximumdirect.net/eric/scriptorium v0.11.0/go.mod h1:FQ5lEuNxmrQyNgIomkpZdxvfTC0jWjbXYuq3tbJWF64=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -3,30 +3,38 @@ package cli
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/dnd/scenes"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/chunk/generic"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/extract/dnd/spells"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/input/seriatim"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/merge/appendorder"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/normalize/noop"
|
||||
jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/output/json"
|
||||
spellshape "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/shape"
|
||||
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/source_refs"
|
||||
spellrelatedness "gitea.maximumdirect.net/eric/notarius/internal/validators/extract/dnd/spells/source_relatedness"
|
||||
alwaysaccept "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/always_accept"
|
||||
alwaysreject "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/always_reject"
|
||||
validjson "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/valid_json"
|
||||
validjsonschema "gitea.maximumdirect.net/eric/notarius/internal/validators/generic/valid_json_schema"
|
||||
)
|
||||
|
||||
func productionRegistries() (pipeline.Registries, error) {
|
||||
registries := pipeline.Registries{
|
||||
Inputs: pipeline.NewInputAdapterRegistry(),
|
||||
Chunkers: pipeline.NewChunkerRegistry(),
|
||||
Extractors: pipeline.NewExtractorRegistry(),
|
||||
Mergers: pipeline.NewMergerRegistry(),
|
||||
Normalizers: pipeline.NewNormalizerRegistry(),
|
||||
Validators: pipeline.NewValidatorRegistry(),
|
||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||
Inputs: pipeline.NewInputAdapterRegistry(),
|
||||
Chunkers: pipeline.NewChunkerRegistry(),
|
||||
Extractors: pipeline.NewExtractorRegistry(),
|
||||
Mergers: pipeline.NewMergerRegistry(),
|
||||
Normalizers: pipeline.NewNormalizerRegistry(),
|
||||
Validators: pipeline.NewValidatorRegistry(),
|
||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: pipeline.NewOutputEncoderRegistry(),
|
||||
}
|
||||
if err := seriatim.Register(registries.Inputs); err != nil {
|
||||
return pipeline.Registries{}, fmt.Errorf("register seriatim input: %w", err)
|
||||
@@ -34,6 +42,9 @@ func productionRegistries() (pipeline.Registries, error) {
|
||||
if err := generic.Register(registries.Chunkers); err != nil {
|
||||
return pipeline.Registries{}, fmt.Errorf("register generic chunker: %w", err)
|
||||
}
|
||||
if err := scenes.Register(registries.Chunkers); err != nil {
|
||||
return pipeline.Registries{}, fmt.Errorf("register dnd scenes chunker: %w", err)
|
||||
}
|
||||
if err := spells.Register(registries.Extractors); err != nil {
|
||||
return pipeline.Registries{}, fmt.Errorf("register dnd spells extractor: %w", err)
|
||||
}
|
||||
@@ -43,12 +54,56 @@ func productionRegistries() (pipeline.Registries, error) {
|
||||
if err := noop.Register(registries.Normalizers); err != nil {
|
||||
return pipeline.Registries{}, fmt.Errorf("register noop normalizer: %w", err)
|
||||
}
|
||||
if err := registerProductionValidators(registries.Validators); err != nil {
|
||||
return pipeline.Registries{}, err
|
||||
}
|
||||
if err := registerProductionValidatorChains(registries.ValidatorChains); err != nil {
|
||||
return pipeline.Registries{}, err
|
||||
}
|
||||
if err := jsonoutput.Register(registries.Outputs); err != nil {
|
||||
return pipeline.Registries{}, fmt.Errorf("register json output encoder: %w", err)
|
||||
}
|
||||
return registries, nil
|
||||
}
|
||||
|
||||
func registerProductionValidators(registry *pipeline.ValidatorRegistry) error {
|
||||
registrations := []struct {
|
||||
name string
|
||||
register func(*pipeline.ValidatorRegistry) error
|
||||
}{
|
||||
{name: "generic always accept validator", register: alwaysaccept.Register},
|
||||
{name: "generic always reject validator", register: alwaysreject.Register},
|
||||
{name: "generic valid json validator", register: validjson.Register},
|
||||
{name: "generic valid json schema validator", register: validjsonschema.Register},
|
||||
{name: "dnd spell shape validator", register: spellshape.Register},
|
||||
{name: "dnd spell source references validator", register: spellsourcerefs.Register},
|
||||
{name: "dnd spell source relatedness validator", register: spellrelatedness.Register},
|
||||
}
|
||||
for _, registration := range registrations {
|
||||
if err := registration.register(registry); err != nil {
|
||||
return fmt.Errorf("register %s: %w", registration.name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func registerProductionValidatorChains(registry *pipeline.ValidatorChainRegistry) error {
|
||||
if err := registry.Register(pipeline.ValidatorChainMapping{
|
||||
Stage: pipeline.StageExtract,
|
||||
Module: spells.Key,
|
||||
Validators: []pipeline.ModuleBinding{
|
||||
pipeline.Binding(validjson.Key),
|
||||
pipeline.Binding(validjsonschema.Key),
|
||||
pipeline.Binding(spellshape.Key),
|
||||
pipeline.Binding(spellsourcerefs.Key),
|
||||
pipeline.Binding(spellrelatedness.Key),
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("register dnd spells validator chain: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func productionCatalog() (pipeline.ModuleCatalog, error) {
|
||||
registries, err := productionRegistries()
|
||||
if err != nil {
|
||||
@@ -57,6 +112,17 @@ func productionCatalog() (pipeline.ModuleCatalog, error) {
|
||||
return catalogFromRegistries(registries), nil
|
||||
}
|
||||
|
||||
func productionPromptAssets() (*llm.AssetRegistry, error) {
|
||||
registry := llm.NewAssetRegistry()
|
||||
if err := scenes.RegisterPromptAssets(registry); err != nil {
|
||||
return nil, fmt.Errorf("register dnd scenes prompt assets: %w", err)
|
||||
}
|
||||
if err := spells.RegisterPromptAssets(registry); err != nil {
|
||||
return nil, fmt.Errorf("register dnd spells prompt assets: %w", err)
|
||||
}
|
||||
return registry, nil
|
||||
}
|
||||
|
||||
func effectiveCatalog(opts Options) (pipeline.ModuleCatalog, error) {
|
||||
if !isEmptyCatalog(opts.Catalog) {
|
||||
return opts.Catalog, nil
|
||||
@@ -79,25 +145,27 @@ func effectiveRegistries(opts Options) (pipeline.Registries, error) {
|
||||
|
||||
func catalogFromRegistries(registries pipeline.Registries) pipeline.ModuleCatalog {
|
||||
return pipeline.ModuleCatalog{
|
||||
Inputs: registries.Inputs,
|
||||
Chunkers: registries.Chunkers,
|
||||
Extractors: registries.Extractors,
|
||||
Mergers: registries.Mergers,
|
||||
Normalizers: registries.Normalizers,
|
||||
Validators: registries.Validators,
|
||||
Outputs: registries.Outputs,
|
||||
Inputs: registries.Inputs,
|
||||
Chunkers: registries.Chunkers,
|
||||
Extractors: registries.Extractors,
|
||||
Mergers: registries.Mergers,
|
||||
Normalizers: registries.Normalizers,
|
||||
Validators: registries.Validators,
|
||||
ValidatorChains: registries.ValidatorChains,
|
||||
Outputs: registries.Outputs,
|
||||
}
|
||||
}
|
||||
|
||||
func registriesFromCatalog(catalog pipeline.ModuleCatalog) pipeline.Registries {
|
||||
return pipeline.Registries{
|
||||
Inputs: catalog.Inputs,
|
||||
Chunkers: catalog.Chunkers,
|
||||
Extractors: catalog.Extractors,
|
||||
Mergers: catalog.Mergers,
|
||||
Normalizers: catalog.Normalizers,
|
||||
Validators: catalog.Validators,
|
||||
Outputs: catalog.Outputs,
|
||||
Inputs: catalog.Inputs,
|
||||
Chunkers: catalog.Chunkers,
|
||||
Extractors: catalog.Extractors,
|
||||
Mergers: catalog.Mergers,
|
||||
Normalizers: catalog.Normalizers,
|
||||
Validators: catalog.Validators,
|
||||
ValidatorChains: catalog.ValidatorChains,
|
||||
Outputs: catalog.Outputs,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,6 +176,7 @@ func isEmptyCatalog(catalog pipeline.ModuleCatalog) bool {
|
||||
catalog.Mergers == nil &&
|
||||
catalog.Normalizers == nil &&
|
||||
catalog.Validators == nil &&
|
||||
catalog.ValidatorChains == nil &&
|
||||
catalog.Outputs == nil
|
||||
}
|
||||
|
||||
@@ -118,6 +187,7 @@ func isEmptyRegistries(registries pipeline.Registries) bool {
|
||||
registries.Mergers == nil &&
|
||||
registries.Normalizers == nil &&
|
||||
registries.Validators == nil &&
|
||||
registries.ValidatorChains == nil &&
|
||||
registries.Outputs == nil
|
||||
}
|
||||
|
||||
@@ -125,48 +195,23 @@ func productionLLMClientFactory(ctx context.Context, cfg config.Config, profileI
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
trimmedID := strings.TrimSpace(profileID)
|
||||
if trimmedID == "" {
|
||||
trimmedID = pipeline.DefaultLLMProfile
|
||||
}
|
||||
|
||||
profile, ok := cfg.LLMProfile(trimmedID)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("LLM profile %q is not configured", trimmedID)
|
||||
}
|
||||
clientCfg, err := cfg.OpenAICompatibleClientConfig(trimmedID)
|
||||
assets, err := productionPromptAssets()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
client, err := llm.NewOpenAICompatibleClient(clientCfg)
|
||||
recorder := llm.NewLLMProfileRecorder()
|
||||
client, err := llm.NewScriptoriumClient(llm.ScriptoriumClientConfig{
|
||||
ProfileDir: cfg.Scriptorium.ProfileDir,
|
||||
ProfileFile: cfg.Scriptorium.ProfileFile,
|
||||
Assets: assets,
|
||||
Recorder: recorder,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create LLM client for profile %q: %w", trimmedID, err)
|
||||
return nil, nil, fmt.Errorf("create Scriptorium-backed LLM client: %w", err)
|
||||
}
|
||||
|
||||
scheduler, err := llm.NewScheduler(effectiveLLMConcurrency(cfg, profile))
|
||||
scheduler, err := llm.NewScheduler(cfg.Concurrency.TotalLLM)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create LLM scheduler for profile %q: %w", trimmedID, err)
|
||||
return nil, nil, fmt.Errorf("create LLM scheduler: %w", err)
|
||||
}
|
||||
provider := strings.TrimSpace(profile.Provider)
|
||||
if provider == "" {
|
||||
provider = "openai-compatible"
|
||||
}
|
||||
metadata := []artifacts.LLMProfileManifest{
|
||||
{
|
||||
ID: trimmedID,
|
||||
Provider: provider,
|
||||
Model: strings.TrimSpace(profile.Model),
|
||||
},
|
||||
}
|
||||
return llm.NewScheduledClient(client, scheduler), metadata, nil
|
||||
}
|
||||
|
||||
func effectiveLLMConcurrency(cfg config.Config, profile config.LLMProfile) int {
|
||||
if profile.MaxConcurrency > 0 {
|
||||
return profile.MaxConcurrency
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM > 0 {
|
||||
return cfg.Concurrency.TotalLLM
|
||||
}
|
||||
return 1
|
||||
return llm.NewScheduledClient(client, scheduler), nil, nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
@@ -16,7 +18,10 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
@@ -25,7 +30,7 @@ const defaultOutputRoot = "./notarius-output"
|
||||
|
||||
const usage = `Usage:
|
||||
notarius help
|
||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b]
|
||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
|
||||
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||
notarius pipelines list --config path/to/config.yml [--json]
|
||||
`
|
||||
@@ -95,6 +100,17 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
outputDir := fs.String("output-dir", "", "output directory")
|
||||
diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory")
|
||||
llmProfile := fs.String("llm-profile", "", "LLM profile override")
|
||||
resume := fs.Bool("resume", false, "reuse valid workspace checkpoints")
|
||||
sessionID := sessionIDFlag{}
|
||||
referenceFlags := stringListFlag{}
|
||||
withoutReferenceFlags := stringListFlag{}
|
||||
fs.Var(&sessionID, "session-id", "prompt session identifier")
|
||||
fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
|
||||
fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
|
||||
if err := validateRunFlagValues(args); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
if err := fs.Parse(reorderRunArgs(args)); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 2
|
||||
@@ -116,26 +132,47 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
fmt.Fprintln(stderr, "notarius: run requires --input")
|
||||
return 2
|
||||
}
|
||||
if sessionID.set && strings.TrimSpace(sessionID.value) == "" {
|
||||
fmt.Fprintln(stderr, "notarius: --session-id must not be empty")
|
||||
return 2
|
||||
}
|
||||
only, err := parseOnly(*onlyRaw)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
referenceRequests, err := parseReferenceFlags(referenceFlags)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
referenceUnbindRequests, err := parseReferenceUnbindFlags(withoutReferenceFlags)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
|
||||
cfg, loadedConfigPath, err := loadConfig(*configPath, opts)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
workspaceSettings := workspace.FromConfig(cfg)
|
||||
if dir := strings.TrimSpace(*diagnosticsDir); dir != "" {
|
||||
cfg.Diagnostics.WorkDir = dir
|
||||
workspaceSettings.DiagnosticsRoot = dir
|
||||
}
|
||||
|
||||
startedAt := opts.Now().UTC()
|
||||
runDir, err := diagnostics.NewRunDirectory(cfg.Diagnostics.WorkDir, cfg.Diagnostics.Retention)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
runID := fmt.Sprintf("run-%d", startedAt.UnixNano())
|
||||
var runDir *diagnostics.RunDirectory
|
||||
if workspaceSettings.DiagnosticsEnabled {
|
||||
var err error
|
||||
runDir, err = diagnostics.NewRunDirectory(workspaceSettings.DiagnosticsRoot, cfg.Diagnostics.Retention)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
runID = runDir.RunID()
|
||||
}
|
||||
invocation := diagnostics.InvocationMetadata{
|
||||
Operation: "run",
|
||||
@@ -144,40 +181,70 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
ConfigPath: loadedConfigPath,
|
||||
ConfigSource: configSource(*configPath),
|
||||
OnlyLanes: append([]string(nil), only...),
|
||||
RunID: runDir.RunID(),
|
||||
Resume: *resume,
|
||||
RunID: runID,
|
||||
StartedAt: startedAt,
|
||||
}
|
||||
if err := runDir.WriteInvocationMetadata(invocation); err != nil {
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteInvocationMetadata(invocation) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
|
||||
}
|
||||
if *resume && !workspaceSettings.ResumeEnabled {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("--resume requires workspace.resume.enabled: true"))
|
||||
}
|
||||
debugRecorder, err := frameworkdebug.NewWorkspaceRecorder(workspaceSettings, runID)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create debug recorder: %w", err))
|
||||
}
|
||||
|
||||
catalog, err := effectiveCatalog(opts)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||
}
|
||||
referenceOverrides, referenceUnbinds, err := resolveCLIReferenceRequests(cfg, pipelineID, only, catalog, referenceRequests, referenceUnbindRequests)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||
}
|
||||
effective, err := cfg.Resolve(config.ResolveInput{
|
||||
PipelineID: pipelineID,
|
||||
Only: only,
|
||||
Catalog: catalog,
|
||||
LLMProfileOverride: *llmProfile,
|
||||
ReferenceOverrides: referenceOverrides,
|
||||
ReferenceUnbinds: referenceUnbinds,
|
||||
})
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||
}
|
||||
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
|
||||
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, profileIDs); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||
}
|
||||
workingDir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("resolve working directory: %w", err))
|
||||
}
|
||||
materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
|
||||
ConfigPath: loadedConfigPath,
|
||||
WorkingDir: workingDir,
|
||||
})
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||
}
|
||||
effective.ResolvedPipeline = materialized
|
||||
invocation.PipelineDigest = effective.ResolvedPipeline.Digest
|
||||
if err := runDir.WriteInvocationMetadata(invocation); err != nil {
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteInvocationMetadata(invocation) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
|
||||
}
|
||||
if err := runDir.WriteRedactedEffectiveConfig(effective); err != nil {
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteRedactedEffectiveConfig(effective) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics effective config: %w", err))
|
||||
}
|
||||
if err := runDir.WriteResolvedPipeline(effective.ResolvedPipeline); err != nil {
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteResolvedPipeline(effective.ResolvedPipeline) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved pipeline: %w", err))
|
||||
}
|
||||
|
||||
profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
|
||||
if len(profileIDs) != 1 {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("pipeline %q uses %d distinct LLM profiles; current runs require exactly one: %s", pipelineID, len(profileIDs), strings.Join(profileIDs, ", ")))
|
||||
if err := writeDiagnostics(runDir, func() error {
|
||||
return runDir.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline))
|
||||
}); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved references: %w", err))
|
||||
}
|
||||
|
||||
rawInput, err := os.ReadFile(strings.TrimSpace(*inputPath))
|
||||
@@ -191,9 +258,17 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, profileIDs[0])
|
||||
factoryProfileID := ""
|
||||
if len(profileIDs) == 1 {
|
||||
factoryProfileID = profileIDs[0]
|
||||
}
|
||||
llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create LLM client for profile %q: %w", profileIDs[0], err))
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
|
||||
}
|
||||
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(workspaceSettings, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||
}
|
||||
|
||||
output, err := pipeline.New(registries).Run(ctx, pipeline.RunInput{
|
||||
@@ -201,49 +276,62 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
Path: strings.TrimSpace(*inputPath),
|
||||
RawInput: rawInput,
|
||||
LLMClient: llmClient,
|
||||
RunID: runDir.RunID(),
|
||||
SessionID: strings.TrimSpace(sessionID.value),
|
||||
RunID: runID,
|
||||
StartedAt: startedAt,
|
||||
LLMProfiles: llmProfiles,
|
||||
Metadata: runMetadata(*outputDir, *diagnosticsDir),
|
||||
Warnings: referenceWarnings,
|
||||
Checkpoints: checkpointRecorder,
|
||||
Checkpoint: checkpointLoader,
|
||||
Debug: debugRecorder,
|
||||
})
|
||||
if err != nil {
|
||||
if output.Manifest.PipelineID != "" {
|
||||
if output.Manifest.PipelineID != "" && runDir != nil {
|
||||
_ = runDir.WriteRunManifest(output.Manifest)
|
||||
_ = runDir.WriteCheckpointEvents(output.CheckpointEvents)
|
||||
}
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("run pipeline %q: %w", pipelineID, err))
|
||||
}
|
||||
|
||||
runOutputDir := filepath.Join(outputRoot(*outputDir), runDir.RunID())
|
||||
if err := runDir.WriteRunManifest(output.Manifest); err != nil {
|
||||
runOutputDir := filepath.Join(outputRoot(*outputDir), runID)
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteRunManifest(output.Manifest) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics run manifest: %w", err))
|
||||
}
|
||||
if err := runDir.WriteWarnings(output.Warnings); err != nil {
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteWarnings(output.Warnings) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics warnings: %w", err))
|
||||
}
|
||||
if err := runDir.WriteRunReport(runReport{
|
||||
RunID: runDir.RunID(),
|
||||
PipelineID: effective.PipelineID,
|
||||
OutputPath: runOutputDir,
|
||||
DiagnosticsPath: runDir.Path(),
|
||||
ApprovedCount: len(output.Approved),
|
||||
RejectedCount: len(output.Rejected),
|
||||
WarningCount: len(output.Warnings),
|
||||
ValidationStatus: output.Manifest.ValidationStatus,
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteCheckpointEvents(output.CheckpointEvents) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics checkpoint events: %w", err))
|
||||
}
|
||||
if err := writeDiagnostics(runDir, func() error {
|
||||
return runDir.WriteRunReport(runReport{
|
||||
RunID: runDir.RunID(),
|
||||
PipelineID: effective.PipelineID,
|
||||
OutputPath: runOutputDir,
|
||||
DiagnosticsPath: runDir.Path(),
|
||||
OutputCount: len(output.NormalizeOutputs),
|
||||
RejectedCount: len(output.Rejected),
|
||||
WarningCount: len(output.Warnings),
|
||||
ValidationStatus: output.Manifest.ValidationStatus,
|
||||
})
|
||||
}); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics run report: %w", err))
|
||||
}
|
||||
if err := writeOutputFiles(runOutputDir, output.OutputFiles); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||
}
|
||||
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RetentionMode: cfg.Diagnostics.Retention,
|
||||
RunSucceeded: true,
|
||||
HasWarnings: len(output.Warnings) > 0,
|
||||
if err := writeDiagnostics(runDir, func() error {
|
||||
return runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RetentionMode: cfg.Diagnostics.Retention,
|
||||
RunSucceeded: true,
|
||||
HasWarnings: len(output.Warnings) > 0,
|
||||
})
|
||||
}); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("apply diagnostics retention: %w", err))
|
||||
}
|
||||
|
||||
fmt.Fprintf(stdout, "pipeline %q complete: approved=%d rejected=%d output=%s\n", effective.PipelineID, len(output.Approved), len(output.Rejected), runOutputDir)
|
||||
fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir)
|
||||
if len(output.Warnings) > 0 {
|
||||
fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", len(output.Warnings))
|
||||
}
|
||||
@@ -255,7 +343,7 @@ type runReport struct {
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
OutputPath string `json:"output_path"`
|
||||
DiagnosticsPath string `json:"diagnostics_path,omitempty"`
|
||||
ApprovedCount int `json:"approved_count"`
|
||||
OutputCount int `json:"output_count"`
|
||||
RejectedCount int `json:"rejected_count"`
|
||||
WarningCount int `json:"warning_count"`
|
||||
ValidationStatus string `json:"validation_status,omitempty"`
|
||||
@@ -277,6 +365,83 @@ func failPipelineCommand(stderr io.Writer, runDir *diagnostics.RunDirectory, ret
|
||||
return 1
|
||||
}
|
||||
|
||||
func writeDiagnostics(runDir *diagnostics.RunDirectory, write func() error) error {
|
||||
if runDir == nil {
|
||||
return nil
|
||||
}
|
||||
return write()
|
||||
}
|
||||
|
||||
func checkpointHandlersForRun(
|
||||
settings workspace.Settings,
|
||||
resolved pipeline.ResolvedPipeline,
|
||||
rawInput []byte,
|
||||
only []string,
|
||||
llmProfiles []artifacts.LLMProfileManifest,
|
||||
llmProfileOverride string,
|
||||
sessionID string,
|
||||
resume bool,
|
||||
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
|
||||
identity, err := workspace.NewCheckpointIdentity(workspace.CheckpointIdentityInput{
|
||||
Pipeline: resolved,
|
||||
InputKey: resolved.Input.Module,
|
||||
RawInputDigest: rawInputDigest(rawInput),
|
||||
SelectedLanes: only,
|
||||
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID),
|
||||
References: pipeline.ReferenceProvenance(resolved),
|
||||
ProvenanceFingerprints: llmProfileFingerprints(llmProfiles),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
|
||||
}
|
||||
recorder, err := checkpoint.NewWorkspaceRecorder(settings, identity)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create checkpoint recorder: %w", err)
|
||||
}
|
||||
loader := pipeline.NoopCheckpointLoader()
|
||||
if resume {
|
||||
loader, err = checkpoint.NewWorkspaceLoader(settings, identity)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create checkpoint loader: %w", err)
|
||||
}
|
||||
}
|
||||
return recorder, loader, nil
|
||||
}
|
||||
|
||||
func rawInputDigest(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string) []workspace.Fingerprint {
|
||||
var values []workspace.Fingerprint
|
||||
if strings.TrimSpace(llmProfileOverride) != "" {
|
||||
values = append(values, workspace.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)})
|
||||
}
|
||||
if strings.TrimSpace(sessionID) != "" {
|
||||
values = append(values, workspace.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)})
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func llmProfileFingerprints(profiles []artifacts.LLMProfileManifest) []workspace.Fingerprint {
|
||||
if len(profiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
values := make([]workspace.Fingerprint, 0, len(profiles))
|
||||
for _, profile := range profiles {
|
||||
id := strings.TrimSpace(profile.ID)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
values = append(values, workspace.Fingerprint{
|
||||
Name: "llm_profile:" + id,
|
||||
Value: strings.TrimSpace(profile.Provider) + ":" + strings.TrimSpace(profile.Model),
|
||||
})
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func configSource(configPath string) string {
|
||||
if strings.TrimSpace(configPath) != "" {
|
||||
return "flag"
|
||||
@@ -412,13 +577,25 @@ func reorderRunArgs(args []string) []string {
|
||||
|
||||
func runFlagTakesValue(arg string) bool {
|
||||
switch arg {
|
||||
case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile":
|
||||
case "--config", "--input", "--only", "--output-dir", "--diagnostics-dir", "--llm-profile", "--session-id", "--reference", "--without-reference":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func validateRunFlagValues(args []string) error {
|
||||
for i, arg := range args {
|
||||
if arg != "--session-id" {
|
||||
continue
|
||||
}
|
||||
if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") {
|
||||
return fmt.Errorf("flag needs an argument: --session-id")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
|
||||
seen := make(map[string]struct{})
|
||||
add := func(binding pipeline.ModuleBinding) {
|
||||
@@ -427,15 +604,17 @@ func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
add(resolved.Input)
|
||||
add(resolved.Chunk)
|
||||
add(resolved.Output)
|
||||
for _, lane := range resolved.ArtifactLanes {
|
||||
add(lane.Extract)
|
||||
add(lane.Merge)
|
||||
add(lane.Normalize)
|
||||
for _, validator := range lane.Validators {
|
||||
add(validator)
|
||||
}
|
||||
for _, chain := range resolved.ValidatorChains {
|
||||
for _, validator := range chain.Validators {
|
||||
if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
|
||||
add(validator.Binding)
|
||||
}
|
||||
}
|
||||
}
|
||||
ids := make([]string, 0, len(seen))
|
||||
@@ -512,11 +691,16 @@ func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) in
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if _, err := cfg.Resolve(config.ResolveInput{
|
||||
effective, err := cfg.Resolve(config.ResolveInput{
|
||||
PipelineID: *pipelineID,
|
||||
Only: only,
|
||||
Catalog: catalog,
|
||||
}); err != nil {
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
if err := validateExplicitScriptoriumProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline)); err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
@@ -661,6 +845,487 @@ func parseOnly(raw string) ([]string, error) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type stringListFlag []string
|
||||
|
||||
func (flag *stringListFlag) String() string {
|
||||
if flag == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(*flag, ",")
|
||||
}
|
||||
|
||||
func (flag *stringListFlag) Set(value string) error {
|
||||
*flag = append(*flag, value)
|
||||
return nil
|
||||
}
|
||||
|
||||
type sessionIDFlag struct {
|
||||
value string
|
||||
set bool
|
||||
}
|
||||
|
||||
func (flag *sessionIDFlag) String() string {
|
||||
if flag == nil {
|
||||
return ""
|
||||
}
|
||||
return flag.value
|
||||
}
|
||||
|
||||
func (flag *sessionIDFlag) Set(value string) error {
|
||||
flag.value = value
|
||||
flag.set = true
|
||||
return nil
|
||||
}
|
||||
|
||||
type cliReferenceRequest struct {
|
||||
Selector cliReferenceSelector
|
||||
Source string
|
||||
}
|
||||
|
||||
type cliReferenceUnbindRequest struct {
|
||||
Selector cliReferenceSelector
|
||||
}
|
||||
|
||||
type cliReferenceSelector struct {
|
||||
LaneID string
|
||||
Stage pipeline.ModuleStage
|
||||
SlotName string
|
||||
}
|
||||
|
||||
func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
|
||||
if len(values) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
requests := make([]cliReferenceRequest, 0, len(values))
|
||||
for _, raw := range values {
|
||||
name, source, ok := strings.Cut(raw, "=")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("--reference must use slot=path or lane.slot=path")
|
||||
}
|
||||
if strings.TrimSpace(source) == "" {
|
||||
return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
|
||||
}
|
||||
selector, err := parseReferenceSelector(name, "--reference")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
requests = append(requests, cliReferenceRequest{
|
||||
Selector: selector,
|
||||
Source: strings.TrimSpace(source),
|
||||
})
|
||||
}
|
||||
return requests, nil
|
||||
}
|
||||
|
||||
func parseReferenceUnbindFlags(values []string) ([]cliReferenceUnbindRequest, error) {
|
||||
if len(values) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
requests := make([]cliReferenceUnbindRequest, 0, len(values))
|
||||
for _, raw := range values {
|
||||
if strings.Contains(raw, "=") {
|
||||
return nil, fmt.Errorf("--without-reference must use a reference selector without =path")
|
||||
}
|
||||
selector, err := parseReferenceSelector(raw, "--without-reference")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
requests = append(requests, cliReferenceUnbindRequest{
|
||||
Selector: selector,
|
||||
})
|
||||
}
|
||||
return requests, nil
|
||||
}
|
||||
|
||||
func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector, error) {
|
||||
selector := strings.TrimSpace(raw)
|
||||
if selector == "" {
|
||||
return cliReferenceSelector{}, fmt.Errorf("%s reference slot must not be empty", flagName)
|
||||
}
|
||||
parts := strings.Split(selector, ".")
|
||||
for _, part := range parts {
|
||||
if strings.TrimSpace(part) == "" {
|
||||
return cliReferenceSelector{}, fmt.Errorf("%s must use non-empty reference selector values", flagName)
|
||||
}
|
||||
}
|
||||
switch len(parts) {
|
||||
case 1:
|
||||
return cliReferenceSelector{SlotName: strings.TrimSpace(parts[0])}, nil
|
||||
case 2:
|
||||
first := strings.TrimSpace(parts[0])
|
||||
slotName := strings.TrimSpace(parts[1])
|
||||
if first == string(pipeline.StageChunk) {
|
||||
return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil
|
||||
}
|
||||
if first == string(pipeline.StageMerge) {
|
||||
return cliReferenceSelector{Stage: pipeline.StageMerge, SlotName: slotName}, nil
|
||||
}
|
||||
return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil
|
||||
case 3:
|
||||
laneID := strings.TrimSpace(parts[0])
|
||||
stage := pipeline.ModuleStage(strings.TrimSpace(parts[1]))
|
||||
slotName := strings.TrimSpace(parts[2])
|
||||
if stage != pipeline.StageExtract && stage != pipeline.StageMerge && stage != pipeline.StageNormalize {
|
||||
return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
|
||||
}
|
||||
return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil
|
||||
default:
|
||||
return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, merge.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
|
||||
}
|
||||
}
|
||||
|
||||
func resolveCLIReferenceRequests(
|
||||
cfg config.Config,
|
||||
pipelineID string,
|
||||
only []string,
|
||||
catalog pipeline.ModuleCatalog,
|
||||
referenceRequests []cliReferenceRequest,
|
||||
unbindRequests []cliReferenceUnbindRequest,
|
||||
) ([]pipeline.ReferenceBinding, []pipeline.ReferenceUnbind, error) {
|
||||
if len(referenceRequests) == 0 && len(unbindRequests) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
targets, err := selectedReferenceTargets(cfg, pipelineID, only, catalog)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests))
|
||||
for _, request := range referenceRequests {
|
||||
target, err := resolveCLIReferenceTarget(targets, request.Selector)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
overrides = append(overrides, pipeline.ReferenceBinding{
|
||||
Stage: target.stage,
|
||||
LaneID: target.laneID,
|
||||
SlotName: request.Selector.SlotName,
|
||||
Source: request.Source,
|
||||
BindingSource: contracts.ReferenceBindingSourceCLI,
|
||||
})
|
||||
}
|
||||
|
||||
unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
|
||||
for _, request := range unbindRequests {
|
||||
target, err := resolveCLIReferenceTarget(targets, request.Selector)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
unbinds = append(unbinds, pipeline.ReferenceUnbind{
|
||||
Stage: target.stage,
|
||||
LaneID: target.laneID,
|
||||
SlotName: request.Selector.SlotName,
|
||||
})
|
||||
}
|
||||
|
||||
return overrides, unbinds, nil
|
||||
}
|
||||
|
||||
type selectedReferenceTarget struct {
|
||||
laneID string
|
||||
stage pipeline.ModuleStage
|
||||
module string
|
||||
slots map[string]struct{}
|
||||
}
|
||||
|
||||
func selectedReferenceTargets(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceTarget, error) {
|
||||
profile, ok := lookupCLIReferencePipeline(cfg.Pipelines, pipelineID)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("pipeline %q is not configured", strings.TrimSpace(pipelineID))
|
||||
}
|
||||
|
||||
lanesByID := make(map[string]pipeline.ArtifactLaneProfile, len(profile.Artifacts))
|
||||
for rawLaneID, lane := range profile.Artifacts {
|
||||
laneID := strings.TrimSpace(rawLaneID)
|
||||
if laneID == "" {
|
||||
return nil, fmt.Errorf("pipeline %q artifact lane id must not be empty", strings.TrimSpace(pipelineID))
|
||||
}
|
||||
if _, ok := lanesByID[laneID]; ok {
|
||||
return nil, fmt.Errorf("pipeline %q artifact lane %q is duplicated after trimming", strings.TrimSpace(pipelineID), laneID)
|
||||
}
|
||||
lanesByID[laneID] = lane
|
||||
}
|
||||
|
||||
selectedIDs := make([]string, 0, len(lanesByID))
|
||||
if len(only) == 0 {
|
||||
for laneID := range lanesByID {
|
||||
selectedIDs = append(selectedIDs, laneID)
|
||||
}
|
||||
} else {
|
||||
seen := make(map[string]struct{}, len(only))
|
||||
for _, rawLaneID := range only {
|
||||
laneID := strings.TrimSpace(rawLaneID)
|
||||
if laneID == "" {
|
||||
return nil, fmt.Errorf("pipeline %q selected artifact lane id must not be empty", strings.TrimSpace(pipelineID))
|
||||
}
|
||||
if _, ok := lanesByID[laneID]; !ok {
|
||||
return nil, fmt.Errorf("pipeline %q selected artifact lane %q is not declared", strings.TrimSpace(pipelineID), laneID)
|
||||
}
|
||||
if _, ok := seen[laneID]; !ok {
|
||||
selectedIDs = append(selectedIDs, laneID)
|
||||
seen[laneID] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(selectedIDs)
|
||||
|
||||
targets := make([]selectedReferenceTarget, 0, 1+len(selectedIDs)*3)
|
||||
chunk := pipeline.Binding(profile.Chunk.Module)
|
||||
chunk.Module = strings.TrimSpace(profile.Chunk.Module)
|
||||
if chunk.Module == "" {
|
||||
chunk.Module = pipeline.DefaultChunkModule
|
||||
}
|
||||
chunkSpec, err := cliReferenceChunkerSpec(catalog, chunk.Module)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pipeline %q chunk module %q: %w", strings.TrimSpace(pipelineID), chunk.Module, err)
|
||||
}
|
||||
targets = append(targets, selectedReferenceTarget{
|
||||
stage: pipeline.StageChunk,
|
||||
module: chunk.Module,
|
||||
slots: referenceSlotSet(chunkSpec.ReferenceSlots),
|
||||
})
|
||||
|
||||
for _, laneID := range selectedIDs {
|
||||
lane := lanesByID[laneID]
|
||||
extractModule := strings.TrimSpace(lane.Extract.Module)
|
||||
if extractModule == "" {
|
||||
return nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", strings.TrimSpace(pipelineID), laneID)
|
||||
}
|
||||
extractSpec, err := cliReferenceExtractorSpec(catalog, extractModule)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pipeline %q lane %q extract module %q: %w", strings.TrimSpace(pipelineID), laneID, extractModule, err)
|
||||
}
|
||||
targets = append(targets, selectedReferenceTarget{
|
||||
laneID: laneID,
|
||||
stage: pipeline.StageExtract,
|
||||
module: extractModule,
|
||||
slots: referenceSlotSet(extractSpec.ReferenceSlots),
|
||||
})
|
||||
|
||||
mergeModule := strings.TrimSpace(lane.Merge.Module)
|
||||
if mergeModule == "" {
|
||||
mergeModule = pipeline.DefaultMergeModule
|
||||
}
|
||||
mergeSpec, err := cliReferenceMergerSpec(catalog, mergeModule)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pipeline %q lane %q merge module %q: %w", strings.TrimSpace(pipelineID), laneID, mergeModule, err)
|
||||
}
|
||||
targets = append(targets, selectedReferenceTarget{
|
||||
laneID: laneID,
|
||||
stage: pipeline.StageMerge,
|
||||
module: mergeModule,
|
||||
slots: referenceSlotSet(mergeSpec.ReferenceSlots),
|
||||
})
|
||||
|
||||
normalizeModule := strings.TrimSpace(lane.Normalize.Module)
|
||||
if normalizeModule == "" {
|
||||
normalizeModule = pipeline.DefaultNormalizeModule
|
||||
}
|
||||
normalizeSpec, err := cliReferenceNormalizerSpec(catalog, normalizeModule)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pipeline %q lane %q normalize module %q: %w", strings.TrimSpace(pipelineID), laneID, normalizeModule, err)
|
||||
}
|
||||
targets = append(targets, selectedReferenceTarget{
|
||||
laneID: laneID,
|
||||
stage: pipeline.StageNormalize,
|
||||
module: normalizeModule,
|
||||
slots: referenceSlotSet(normalizeSpec.ReferenceSlots),
|
||||
})
|
||||
}
|
||||
return targets, nil
|
||||
}
|
||||
|
||||
func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) {
|
||||
pipelineID = strings.TrimSpace(pipelineID)
|
||||
for rawID, profile := range profiles {
|
||||
if strings.TrimSpace(rawID) == pipelineID {
|
||||
return profile, true
|
||||
}
|
||||
}
|
||||
return pipeline.PipelineProfile{}, false
|
||||
}
|
||||
|
||||
func cliReferenceChunkerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
|
||||
if catalog.Chunkers == nil {
|
||||
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||
}
|
||||
spec, ok := catalog.Chunkers.Spec(module)
|
||||
if !ok {
|
||||
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||
}
|
||||
return spec, nil
|
||||
}
|
||||
|
||||
func cliReferenceExtractorSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
|
||||
if catalog.Extractors == nil {
|
||||
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||
}
|
||||
spec, ok := catalog.Extractors.Spec(module)
|
||||
if !ok {
|
||||
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||
}
|
||||
return spec, nil
|
||||
}
|
||||
|
||||
func cliReferenceMergerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
|
||||
if catalog.Mergers == nil {
|
||||
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||
}
|
||||
spec, ok := catalog.Mergers.Spec(module)
|
||||
if !ok {
|
||||
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||
}
|
||||
return spec, nil
|
||||
}
|
||||
|
||||
func cliReferenceNormalizerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
|
||||
if catalog.Normalizers == nil {
|
||||
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||
}
|
||||
spec, ok := catalog.Normalizers.Spec(module)
|
||||
if !ok {
|
||||
return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
|
||||
}
|
||||
return spec, nil
|
||||
}
|
||||
|
||||
func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} {
|
||||
slotSet := make(map[string]struct{}, len(slots))
|
||||
for _, slot := range slots {
|
||||
slotSet[slot.Name] = struct{}{}
|
||||
}
|
||||
return slotSet
|
||||
}
|
||||
|
||||
func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliReferenceSelector) (selectedReferenceTarget, error) {
|
||||
slotName := strings.TrimSpace(selector.SlotName)
|
||||
if slotName == "" {
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot must not be empty")
|
||||
}
|
||||
if selector.Stage == pipeline.StageChunk {
|
||||
for _, target := range targets {
|
||||
if target.stage != pipeline.StageChunk {
|
||||
continue
|
||||
}
|
||||
if _, ok := target.slots[slotName]; !ok {
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected")
|
||||
}
|
||||
if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageMerge || selector.Stage == pipeline.StageNormalize {
|
||||
if selector.LaneID == "" && selector.Stage == pipeline.StageMerge {
|
||||
return resolveCLIReferenceStageTarget(targets, selector.Stage, slotName)
|
||||
}
|
||||
for _, target := range targets {
|
||||
if target.laneID == selector.LaneID && target.stage == selector.Stage {
|
||||
if _, ok := target.slots[slotName]; !ok {
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
}
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
|
||||
}
|
||||
if strings.TrimSpace(selector.LaneID) != "" {
|
||||
return resolveCLIReferenceLaneTarget(targets, strings.TrimSpace(selector.LaneID), slotName)
|
||||
}
|
||||
return resolveCLIReferenceFlatTarget(targets, slotName)
|
||||
}
|
||||
|
||||
func resolveCLIReferenceStageTarget(targets []selectedReferenceTarget, stage pipeline.ModuleStage, slotName string) (selectedReferenceTarget, error) {
|
||||
matches := make([]selectedReferenceTarget, 0, 2)
|
||||
for _, target := range targets {
|
||||
if target.stage != stage {
|
||||
continue
|
||||
}
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
}
|
||||
}
|
||||
switch len(matches) {
|
||||
case 0:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected %s target", slotName, stage)
|
||||
case 1:
|
||||
return matches[0], nil
|
||||
default:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected %s targets (%s); use a more specific selector such as %s", slotName, stage, targetList(matches), selectorSuggestions(matches, slotName))
|
||||
}
|
||||
}
|
||||
|
||||
func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) {
|
||||
laneSelected := false
|
||||
matches := make([]selectedReferenceTarget, 0, 2)
|
||||
for _, target := range targets {
|
||||
if target.laneID != laneID {
|
||||
continue
|
||||
}
|
||||
laneSelected = true
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
}
|
||||
}
|
||||
if !laneSelected {
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", laneID)
|
||||
}
|
||||
switch len(matches) {
|
||||
case 0:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, laneID)
|
||||
case 1:
|
||||
return matches[0], nil
|
||||
default:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use a more specific selector such as %s", slotName, laneID, targetList(matches), selectorSuggestions(matches, slotName))
|
||||
}
|
||||
}
|
||||
|
||||
func resolveCLIReferenceFlatTarget(targets []selectedReferenceTarget, slotName string) (selectedReferenceTarget, error) {
|
||||
matches := make([]selectedReferenceTarget, 0, 2)
|
||||
for _, target := range targets {
|
||||
if _, ok := target.slots[slotName]; ok {
|
||||
matches = append(matches, target)
|
||||
}
|
||||
}
|
||||
switch len(matches) {
|
||||
case 0:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected reference target", slotName)
|
||||
case 1:
|
||||
return matches[0], nil
|
||||
default:
|
||||
return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets (%s); use a more specific selector such as %s", slotName, targetList(matches), selectorSuggestions(matches, slotName))
|
||||
}
|
||||
}
|
||||
|
||||
func targetList(targets []selectedReferenceTarget) string {
|
||||
labels := make([]string, 0, len(targets))
|
||||
for _, target := range targets {
|
||||
labels = append(labels, targetLabel(target))
|
||||
}
|
||||
sort.Strings(labels)
|
||||
return strings.Join(labels, ", ")
|
||||
}
|
||||
|
||||
func targetLabel(target selectedReferenceTarget) string {
|
||||
if target.stage == pipeline.StageChunk {
|
||||
return "chunk"
|
||||
}
|
||||
return target.laneID + "." + string(target.stage)
|
||||
}
|
||||
|
||||
func selectorSuggestions(targets []selectedReferenceTarget, slotName string) string {
|
||||
suggestions := make([]string, 0, len(targets))
|
||||
for _, target := range targets {
|
||||
if target.stage == pipeline.StageChunk {
|
||||
suggestions = append(suggestions, "chunk."+slotName)
|
||||
continue
|
||||
}
|
||||
suggestions = append(suggestions, target.laneID+"."+string(target.stage)+"."+slotName)
|
||||
}
|
||||
sort.Strings(suggestions)
|
||||
return strings.Join(suggestions, " or ")
|
||||
}
|
||||
|
||||
func sortedPipelineIDs(cfg config.Config) []string {
|
||||
ids := make([]string, 0, len(cfg.Pipelines))
|
||||
for id := range cfg.Pipelines {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
68
internal/cli/scriptorium_profiles.go
Normal file
68
internal/cli/scriptorium_profiles.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing/fstest"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
const profileCheckPromptID = "notarius.profile.check"
|
||||
|
||||
var profileCheckPromptFS = fstest.MapFS{
|
||||
"prompts/profile-check.yaml": &fstest.MapFile{Data: []byte(`id: notarius.profile.check
|
||||
version: "1.0.0"
|
||||
default_profile: mistral-small-3
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: "{{input \"transcript\"}}"
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
repair_attempts: 0
|
||||
`)},
|
||||
}
|
||||
|
||||
func validateExplicitScriptoriumProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
|
||||
if len(profileIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
engine, err := newProfileValidationEngine(cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load Scriptorium profiles: %w", err)
|
||||
}
|
||||
for _, profileID := range profileIDs {
|
||||
if _, err := engine.Prepare(ctx, scriptorium.RunRequest{
|
||||
PromptID: profileCheckPromptID,
|
||||
ProfileID: profileID,
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline("profile check"),
|
||||
},
|
||||
}); err != nil {
|
||||
if errors.Is(err, scriptorium.ErrProfileNotFound) {
|
||||
return fmt.Errorf("Scriptorium profile %q is not configured", profileID)
|
||||
}
|
||||
return fmt.Errorf("validate Scriptorium profile %q: %w", profileID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newProfileValidationEngine(cfg config.Config) (*scriptorium.Engine, error) {
|
||||
opts := []scriptorium.Option{
|
||||
scriptorium.WithPromptFS(profileCheckPromptFS, "prompts"),
|
||||
}
|
||||
if cfg.Scriptorium.ProfileFile != "" {
|
||||
opts = append(opts, scriptorium.WithProfileFile(cfg.Scriptorium.ProfileFile))
|
||||
}
|
||||
return scriptorium.NewEngine(scriptorium.Config{
|
||||
PromptDir: "unused",
|
||||
ProfileDir: cfg.Scriptorium.ProfileDir,
|
||||
}, opts...)
|
||||
}
|
||||
@@ -1,91 +1,95 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
)
|
||||
|
||||
type ArtifactCandidate struct {
|
||||
Index int `json:"index"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
ArtifactType string `json:"artifact_type"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
SourceRefs []source.SourceRef `json:"source_refs,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type Artifact struct {
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
ArtifactType string `json:"artifact_type"`
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
SourceRefs []source.SourceRef `json:"source_refs,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type RejectedArtifact struct {
|
||||
Candidate ArtifactCandidate `json:"candidate"`
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type ArtifactLaneManifest struct {
|
||||
ID string `json:"id"`
|
||||
Extractor string `json:"extractor"`
|
||||
Merger string `json:"merger"`
|
||||
Normalizer string `json:"normalizer"`
|
||||
Validators []string `json:"validators,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type ValidatorChainManifest struct {
|
||||
Stage string `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
Validators []ValidatorManifest `json:"validators"`
|
||||
}
|
||||
|
||||
type ValidatorManifest struct {
|
||||
Key string `json:"key"`
|
||||
ExecutionClass string `json:"execution_class"`
|
||||
}
|
||||
|
||||
type LLMProfileManifest struct {
|
||||
ID string `json:"id"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
}
|
||||
|
||||
type ReferenceProvenance struct {
|
||||
Stage string `json:"stage,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
SlotName string `json:"slot_name"`
|
||||
OriginType string `json:"origin_type"`
|
||||
OriginURI string `json:"origin_uri,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes,omitempty"`
|
||||
BindingSource string `json:"binding_source,omitempty"`
|
||||
}
|
||||
|
||||
type OutputSchemaProvenance struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
}
|
||||
|
||||
type NormalizedOutputManifest struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Schema OutputSchemaProvenance `json:"schema,omitempty"`
|
||||
}
|
||||
|
||||
type RejectedOutputManifest struct {
|
||||
Stage string `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
ValidatorName string `json:"validator_name,omitempty"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
AttemptCount int `json:"attempt_count,omitempty"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
}
|
||||
|
||||
type RunManifest struct {
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
PipelineID string `json:"pipeline_id,omitempty"`
|
||||
PipelineDigest string `json:"pipeline_digest,omitempty"`
|
||||
InputModule string `json:"input_module,omitempty"`
|
||||
Chunker string `json:"chunker,omitempty"`
|
||||
SourceDigests []string `json:"source_digests,omitempty"`
|
||||
Extractors []string `json:"extractors,omitempty"`
|
||||
Merger string `json:"merger,omitempty"`
|
||||
Normalizer string `json:"normalizer,omitempty"`
|
||||
OutputEncoder string `json:"output_encoder,omitempty"`
|
||||
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
||||
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
||||
SchemaVersion string `json:"schema_version,omitempty"`
|
||||
ValidationStatus string `json:"validation_status,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
}
|
||||
|
||||
func ArtifactFromCandidate(candidate ArtifactCandidate) Artifact {
|
||||
return Artifact{
|
||||
ExtractorKey: candidate.ExtractorKey,
|
||||
ArtifactType: candidate.ArtifactType,
|
||||
SchemaVersion: candidate.SchemaVersion,
|
||||
Payload: append(json.RawMessage(nil), candidate.Payload...),
|
||||
SourceRefs: append([]source.SourceRef(nil), candidate.SourceRefs...),
|
||||
Metadata: copyMetadata(candidate.Metadata),
|
||||
}
|
||||
}
|
||||
|
||||
func copyMetadata(metadata map[string]any) map[string]any {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
copied := make(map[string]any, len(metadata))
|
||||
for key, value := range metadata {
|
||||
copied[key] = value
|
||||
}
|
||||
return copied
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
PipelineID string `json:"pipeline_id,omitempty"`
|
||||
PipelineDigest string `json:"pipeline_digest,omitempty"`
|
||||
InputModule string `json:"input_module,omitempty"`
|
||||
Chunker string `json:"chunker,omitempty"`
|
||||
SourceDigests []string `json:"source_digests,omitempty"`
|
||||
Extractors []string `json:"extractors,omitempty"`
|
||||
Merger string `json:"merger,omitempty"`
|
||||
Normalizer string `json:"normalizer,omitempty"`
|
||||
OutputEncoder string `json:"output_encoder,omitempty"`
|
||||
ModuleMetadata map[string]map[string]any `json:"module_metadata,omitempty"`
|
||||
ArtifactLanes []ArtifactLaneManifest `json:"artifact_lanes,omitempty"`
|
||||
ValidatorChains []ValidatorChainManifest `json:"validator_chains,omitempty"`
|
||||
References []ReferenceProvenance `json:"references,omitempty"`
|
||||
NormalizedOutputs []NormalizedOutputManifest `json:"normalized_outputs,omitempty"`
|
||||
RejectedOutputs []RejectedOutputManifest `json:"rejected_outputs,omitempty"`
|
||||
LLMProfiles []LLMProfileManifest `json:"llm_profiles,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
SchemaVersion string `json:"schema_version,omitempty"`
|
||||
ValidationStatus string `json:"validation_status,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
}
|
||||
|
||||
@@ -2,116 +2,9 @@ package artifacts
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
)
|
||||
|
||||
func TestArtifactFromCandidatePreservesCandidateFields(t *testing.T) {
|
||||
candidate := ArtifactCandidate{
|
||||
Index: 7,
|
||||
ExtractorKey: "generic-extractor",
|
||||
ArtifactType: "generic-artifact",
|
||||
SchemaVersion: "v1",
|
||||
Payload: json.RawMessage(`{"name":"example"}`),
|
||||
SourceRefs: []source.SourceRef{
|
||||
{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u2"},
|
||||
},
|
||||
Metadata: map[string]any{
|
||||
"confidence": 0.75,
|
||||
},
|
||||
}
|
||||
|
||||
artifact := ArtifactFromCandidate(candidate)
|
||||
|
||||
if artifact.ExtractorKey != candidate.ExtractorKey {
|
||||
t.Fatalf("ExtractorKey = %q, want %q", artifact.ExtractorKey, candidate.ExtractorKey)
|
||||
}
|
||||
if artifact.ArtifactType != candidate.ArtifactType {
|
||||
t.Fatalf("ArtifactType = %q, want %q", artifact.ArtifactType, candidate.ArtifactType)
|
||||
}
|
||||
if artifact.SchemaVersion != candidate.SchemaVersion {
|
||||
t.Fatalf("SchemaVersion = %q, want %q", artifact.SchemaVersion, candidate.SchemaVersion)
|
||||
}
|
||||
if string(artifact.Payload) != string(candidate.Payload) {
|
||||
t.Fatalf("Payload = %s, want %s", artifact.Payload, candidate.Payload)
|
||||
}
|
||||
if !reflect.DeepEqual(artifact.SourceRefs, candidate.SourceRefs) {
|
||||
t.Fatalf("SourceRefs = %#v, want %#v", artifact.SourceRefs, candidate.SourceRefs)
|
||||
}
|
||||
if !reflect.DeepEqual(artifact.Metadata, candidate.Metadata) {
|
||||
t.Fatalf("Metadata = %#v, want %#v", artifact.Metadata, candidate.Metadata)
|
||||
}
|
||||
|
||||
candidate.Payload[0] = '['
|
||||
candidate.SourceRefs[0].StartUnitID = "changed"
|
||||
candidate.Metadata["confidence"] = 0.5
|
||||
|
||||
if string(artifact.Payload) != `{"name":"example"}` {
|
||||
t.Fatalf("Payload changed after candidate mutation: %s", artifact.Payload)
|
||||
}
|
||||
if artifact.SourceRefs[0].StartUnitID != "u1" {
|
||||
t.Fatalf("SourceRefs changed after candidate mutation: %#v", artifact.SourceRefs)
|
||||
}
|
||||
if artifact.Metadata["confidence"] != 0.75 {
|
||||
t.Fatalf("Metadata changed after candidate mutation: %#v", artifact.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONMarshalUsesExpectedFieldNames(t *testing.T) {
|
||||
candidate := ArtifactCandidate{
|
||||
Index: 1,
|
||||
ExtractorKey: "generic-extractor",
|
||||
ArtifactType: "generic-artifact",
|
||||
SchemaVersion: "v1",
|
||||
Payload: json.RawMessage(`{"value":true}`),
|
||||
SourceRefs: []source.SourceRef{
|
||||
{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u1"},
|
||||
},
|
||||
Metadata: map[string]any{
|
||||
"reviewed": true,
|
||||
},
|
||||
}
|
||||
rejected := RejectedArtifact{
|
||||
Candidate: candidate,
|
||||
ValidatorName: "generic-validator",
|
||||
ReasonCode: "invalid",
|
||||
Message: "candidate was not accepted",
|
||||
}
|
||||
|
||||
gotJSON, err := json.Marshal(rejected)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v", err)
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(gotJSON, &got); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
|
||||
assertHasKeys(t, got, "candidate", "validator_name", "reason_code", "message")
|
||||
|
||||
gotCandidate, ok := got["candidate"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("candidate = %#v, want object", got["candidate"])
|
||||
}
|
||||
assertHasKeys(t, gotCandidate, "index", "extractor_key", "artifact_type", "schema_version", "payload", "source_refs", "metadata")
|
||||
|
||||
gotRefs, ok := gotCandidate["source_refs"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("source_refs = %#v, want array", gotCandidate["source_refs"])
|
||||
}
|
||||
if len(gotRefs) != 1 {
|
||||
t.Fatalf("len(source_refs) = %d, want 1", len(gotRefs))
|
||||
}
|
||||
gotRef, ok := gotRefs[0].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("source_refs[0] = %#v, want object", gotRefs[0])
|
||||
}
|
||||
assertHasKeys(t, gotRef, "source_id", "start_unit_id", "end_unit_id")
|
||||
}
|
||||
|
||||
func TestRunManifestOmitsEmptyOptionalFields(t *testing.T) {
|
||||
gotJSON, err := json.Marshal(RunManifest{})
|
||||
if err != nil {
|
||||
@@ -128,7 +21,7 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
||||
PipelineID: "pipeline-1",
|
||||
PipelineDigest: "sha256:abc123",
|
||||
LLMProfiles: []LLMProfileManifest{
|
||||
{ID: "default", Provider: "openai-compatible", Model: "model-a"},
|
||||
{ID: "default", Provider: "scriptorium", Model: "model-a"},
|
||||
},
|
||||
ArtifactLanes: []ArtifactLaneManifest{
|
||||
{
|
||||
@@ -136,12 +29,21 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
||||
Extractor: "event-extractor",
|
||||
Merger: "appendorder",
|
||||
Normalizer: "noop",
|
||||
Validators: []string{"grounded"},
|
||||
Metadata: map[string]any{
|
||||
"extractor": map[string]any{"prompt_id": "test.prompt"},
|
||||
},
|
||||
},
|
||||
},
|
||||
ValidatorChains: []ValidatorChainManifest{
|
||||
{
|
||||
Stage: "extract",
|
||||
LaneID: "events",
|
||||
ModuleKey: "event-extractor",
|
||||
Validators: []ValidatorManifest{
|
||||
{Key: "grounded", ExecutionClass: "deterministic"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
gotJSON, err := json.Marshal(manifest)
|
||||
@@ -154,7 +56,7 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
|
||||
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "llm_profiles")
|
||||
assertHasKeys(t, got, "pipeline_id", "pipeline_digest", "artifact_lanes", "validator_chains", "llm_profiles")
|
||||
|
||||
profiles, ok := got["llm_profiles"].([]any)
|
||||
if !ok {
|
||||
@@ -180,7 +82,94 @@ func TestRunManifestIncludesPipelineAndArtifactLaneFields(t *testing.T) {
|
||||
if !ok {
|
||||
t.Fatalf("artifact_lanes[0] = %#v, want object", lanes[0])
|
||||
}
|
||||
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "validators", "metadata")
|
||||
assertHasKeys(t, lane, "id", "extractor", "merger", "normalizer", "metadata")
|
||||
|
||||
chains, ok := got["validator_chains"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("validator_chains = %#v, want array", got["validator_chains"])
|
||||
}
|
||||
if len(chains) != 1 {
|
||||
t.Fatalf("len(validator_chains) = %d, want 1", len(chains))
|
||||
}
|
||||
chain, ok := chains[0].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("validator_chains[0] = %#v, want object", chains[0])
|
||||
}
|
||||
assertHasKeys(t, chain, "stage", "lane_id", "module_key", "validators")
|
||||
}
|
||||
|
||||
func TestRunManifestIncludesReferenceProvenance(t *testing.T) {
|
||||
manifest := RunManifest{
|
||||
References: []ReferenceProvenance{
|
||||
{
|
||||
Stage: "extract",
|
||||
LaneID: "events",
|
||||
SlotName: "roster",
|
||||
OriginType: "file",
|
||||
OriginURI: "file:///tmp/roster.txt",
|
||||
Digest: "sha256:reference",
|
||||
MediaType: "text/plain; charset=utf-8",
|
||||
SizeBytes: 12,
|
||||
BindingSource: "config",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
gotJSON, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v", err)
|
||||
}
|
||||
|
||||
var got RunManifest
|
||||
if err := json.Unmarshal(gotJSON, &got); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
if len(got.References) != 1 {
|
||||
t.Fatalf("len(References) = %d, want 1", len(got.References))
|
||||
}
|
||||
reference := got.References[0]
|
||||
if reference.Stage != "extract" || reference.LaneID != "events" || reference.SlotName != "roster" || reference.OriginType != "file" || reference.OriginURI != "file:///tmp/roster.txt" {
|
||||
t.Fatalf("reference provenance = %#v, want lane-scoped origin details", reference)
|
||||
}
|
||||
if reference.Digest != "sha256:reference" || reference.MediaType != "text/plain; charset=utf-8" || reference.SizeBytes != 12 || reference.BindingSource != "config" {
|
||||
t.Fatalf("reference provenance = %#v, want digest/media/size/source details", reference)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunManifestIncludesTopLevelModuleMetadata(t *testing.T) {
|
||||
manifest := RunManifest{
|
||||
ModuleMetadata: map[string]map[string]any{
|
||||
"chunker": {
|
||||
"prompt_id": "dnd.scenes",
|
||||
"prompt_version": "v1",
|
||||
"prompt_sha256": "sha256:abc123",
|
||||
"response_schema_key": "dnd_scenes",
|
||||
"response_schema_name": "dnd_scenes",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
gotJSON, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v", err)
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(gotJSON, &got); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
|
||||
moduleMetadata, ok := got["module_metadata"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("module_metadata = %#v, want object", got["module_metadata"])
|
||||
}
|
||||
assertHasKeys(t, moduleMetadata, "chunker")
|
||||
|
||||
chunkerMetadata, ok := moduleMetadata["chunker"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("module_metadata.chunker = %#v, want object", moduleMetadata["chunker"])
|
||||
}
|
||||
assertHasKeys(t, chunkerMetadata, "prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_name")
|
||||
}
|
||||
|
||||
func assertHasKeys(t *testing.T, values map[string]any, keys ...string) {
|
||||
|
||||
@@ -1,28 +1,26 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const SupportedFileConfigVersion = 1
|
||||
const SupportedFileConfigVersion = 2
|
||||
|
||||
type Config struct {
|
||||
LLMProfiles map[string]LLMProfile `json:"llm_profiles"`
|
||||
Scriptorium ScriptoriumConfig `json:"scriptorium,omitempty"`
|
||||
Pipelines map[string]pipeline.PipelineProfile `json:"pipelines"`
|
||||
Concurrency ConcurrencyConfig `json:"concurrency"`
|
||||
Diagnostics DiagnosticsConfig `json:"diagnostics"`
|
||||
Workspace WorkspaceConfig `json:"workspace"`
|
||||
}
|
||||
|
||||
type LLMProfile struct {
|
||||
Provider string `json:"provider,omitempty"`
|
||||
BaseURL string `json:"base_url,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
APIKey string `json:"api_key,omitempty"`
|
||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
|
||||
MaxRetries int `json:"max_retries,omitempty"`
|
||||
MaxConcurrency int `json:"max_concurrency,omitempty"`
|
||||
type ScriptoriumConfig struct {
|
||||
ProfileDir string `json:"profile_dir,omitempty"`
|
||||
ProfileFile string `json:"profile_file,omitempty"`
|
||||
}
|
||||
|
||||
type ConcurrencyConfig struct {
|
||||
@@ -34,16 +32,30 @@ type DiagnosticsConfig struct {
|
||||
Retention diagnostics.RetentionMode `json:"retention"`
|
||||
}
|
||||
|
||||
type WorkspaceConfig struct {
|
||||
Directory string `json:"directory,omitempty"`
|
||||
Diagnostics WorkspaceDiagnosticsConfig `json:"diagnostics"`
|
||||
Resume WorkspaceResumeConfig `json:"resume"`
|
||||
Debug WorkspaceDebugConfig `json:"debug"`
|
||||
}
|
||||
|
||||
type WorkspaceDiagnosticsConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Retention diagnostics.RetentionMode `json:"retention,omitempty"`
|
||||
enabledSet bool
|
||||
retentionSet bool
|
||||
}
|
||||
|
||||
type WorkspaceResumeConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
type WorkspaceDebugConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
func Default() Config {
|
||||
return Config{
|
||||
LLMProfiles: map[string]LLMProfile{
|
||||
pipeline.DefaultLLMProfile: {
|
||||
Provider: "openai-compatible",
|
||||
TimeoutSeconds: 600,
|
||||
MaxRetries: 3,
|
||||
MaxConcurrency: 1,
|
||||
},
|
||||
},
|
||||
Pipelines: map[string]pipeline.PipelineProfile{},
|
||||
Concurrency: ConcurrencyConfig{
|
||||
TotalLLM: 1,
|
||||
@@ -52,15 +64,43 @@ func Default() Config {
|
||||
WorkDir: "/tmp/notarius",
|
||||
Retention: diagnostics.RetentionAuto,
|
||||
},
|
||||
Workspace: WorkspaceConfig{
|
||||
Diagnostics: WorkspaceDiagnosticsConfig{
|
||||
Enabled: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) RecomputeEffectiveDiagnostics() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
if dir := c.workspaceDirectory(); dir != "" {
|
||||
c.Diagnostics.WorkDir = filepath.Join(dir, "diagnostics")
|
||||
}
|
||||
if c.Workspace.Diagnostics.retentionSet {
|
||||
c.Diagnostics.Retention = c.Workspace.Diagnostics.Retention
|
||||
}
|
||||
}
|
||||
|
||||
func (c Config) DiagnosticsEnabled() bool {
|
||||
if !c.Workspace.Diagnostics.enabledSet {
|
||||
return true
|
||||
}
|
||||
return c.Workspace.Diagnostics.Enabled
|
||||
}
|
||||
|
||||
func (c Config) workspaceDirectory() string {
|
||||
dir := strings.TrimSpace(c.Workspace.Directory)
|
||||
if dir == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Clean(dir)
|
||||
}
|
||||
|
||||
func cloneConfig(in Config) Config {
|
||||
out := in
|
||||
out.LLMProfiles = make(map[string]LLMProfile, len(in.LLMProfiles))
|
||||
for key, profile := range in.LLMProfiles {
|
||||
out.LLMProfiles[key] = profile
|
||||
}
|
||||
out.Pipelines = make(map[string]pipeline.PipelineProfile, len(in.Pipelines))
|
||||
for key, profile := range in.Pipelines {
|
||||
out.Pipelines[key] = clonePipelineProfile(profile)
|
||||
@@ -73,6 +113,7 @@ func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile
|
||||
out.Input = cloneModuleBinding(in.Input)
|
||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||
out.Output = cloneModuleBinding(in.Output)
|
||||
out.References = cloneStringMap(in.References)
|
||||
if len(in.Artifacts) > 0 {
|
||||
out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts))
|
||||
for key, lane := range in.Artifacts {
|
||||
@@ -87,6 +128,7 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
|
||||
out.Extract = cloneModuleBinding(in.Extract)
|
||||
out.Merge = cloneModuleBinding(in.Merge)
|
||||
out.Normalize = cloneModuleBinding(in.Normalize)
|
||||
out.References = cloneStringMap(in.References)
|
||||
if len(in.Validators) > 0 {
|
||||
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
|
||||
for i, binding := range in.Validators {
|
||||
@@ -96,11 +138,35 @@ func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.Artifact
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneStringMap(in map[string]string) map[string]string {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(in))
|
||||
for key, value := range in {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
||||
out := in
|
||||
if len(in.Options) > 0 {
|
||||
out.Options = cloneOptions(in.Options)
|
||||
}
|
||||
out.References = cloneStringMap(in.References)
|
||||
out.Validators = cloneValidatorOverride(in.Validators)
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
|
||||
out := pipeline.ValidatorOverride{Set: in.Set}
|
||||
if len(in.Validators) > 0 {
|
||||
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
|
||||
for i, binding := range in.Validators {
|
||||
out.Validators[i] = cloneModuleBinding(binding)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -4,24 +4,13 @@ import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestDefaultValues(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
defaultProfile, ok := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
|
||||
if !ok {
|
||||
t.Fatalf("expected default LLM profile")
|
||||
}
|
||||
if defaultProfile.Provider != "openai-compatible" {
|
||||
t.Fatalf("unexpected provider: %q", defaultProfile.Provider)
|
||||
}
|
||||
if defaultProfile.BaseURL != "" || defaultProfile.Model != "" {
|
||||
t.Fatalf("default profile should not require base URL/model yet: %+v", defaultProfile)
|
||||
}
|
||||
if defaultProfile.TimeoutSeconds != 600 || defaultProfile.MaxRetries != 3 || defaultProfile.MaxConcurrency != 1 {
|
||||
t.Fatalf("unexpected default LLM operational values: %+v", defaultProfile)
|
||||
if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("unexpected Scriptorium profile source defaults: %+v", cfg.Scriptorium)
|
||||
}
|
||||
if len(cfg.Pipelines) != 0 {
|
||||
t.Fatalf("expected no built-in pipeline profiles, got %v", cfg.Pipelines)
|
||||
@@ -35,14 +24,28 @@ func TestDefaultValues(t *testing.T) {
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionAuto {
|
||||
t.Fatalf("unexpected diagnostics retention: %q", cfg.Diagnostics.Retention)
|
||||
}
|
||||
if cfg.Workspace.Directory != "" {
|
||||
t.Fatalf("unexpected workspace directory: %q", cfg.Workspace.Directory)
|
||||
}
|
||||
if !cfg.Workspace.Diagnostics.Enabled || !cfg.DiagnosticsEnabled() {
|
||||
t.Fatalf("expected workspace diagnostics enabled by default: %+v", cfg.Workspace.Diagnostics)
|
||||
}
|
||||
if cfg.Workspace.Diagnostics.Retention != "" {
|
||||
t.Fatalf("unexpected workspace diagnostics retention: %q", cfg.Workspace.Diagnostics.Retention)
|
||||
}
|
||||
if cfg.Workspace.Resume.Enabled {
|
||||
t.Fatalf("workspace resume should be disabled by default")
|
||||
}
|
||||
if cfg.Workspace.Debug.Enabled {
|
||||
t.Fatalf("workspace debug should be disabled by default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigMergesWithDefaults(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
model: test-model
|
||||
version: 2
|
||||
scriptorium:
|
||||
profile_dir: ./profiles
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -59,12 +62,8 @@ pipelines:
|
||||
t.Fatalf("ApplyFileConfig: %v", err)
|
||||
}
|
||||
|
||||
profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
|
||||
if profile.Model != "test-model" {
|
||||
t.Fatalf("expected file model, got %+v", profile)
|
||||
}
|
||||
if profile.Provider != "openai-compatible" || profile.TimeoutSeconds != 600 || profile.MaxRetries != 3 {
|
||||
t.Fatalf("expected default LLM fields to be preserved, got %+v", profile)
|
||||
if cfg.Scriptorium.ProfileDir != "./profiles" {
|
||||
t.Fatalf("expected Scriptorium profile dir, got %+v", cfg.Scriptorium)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 1 {
|
||||
t.Fatalf("expected default concurrency preserved, got %d", cfg.Concurrency.TotalLLM)
|
||||
|
||||
@@ -3,9 +3,7 @@ package config
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
@@ -14,13 +12,17 @@ type ResolveInput struct {
|
||||
Only []string
|
||||
Catalog pipeline.ModuleCatalog
|
||||
LLMProfileOverride string
|
||||
ReferenceOverrides []pipeline.ReferenceBinding
|
||||
ReferenceUnbinds []pipeline.ReferenceUnbind
|
||||
}
|
||||
|
||||
type EffectiveConfig struct {
|
||||
Config Config
|
||||
PipelineID string
|
||||
Only []string
|
||||
ResolvedPipeline pipeline.ResolvedPipeline
|
||||
Config Config
|
||||
PipelineID string
|
||||
Only []string
|
||||
ReferenceOverrides []pipeline.ReferenceBinding
|
||||
ReferenceUnbinds []pipeline.ReferenceUnbind
|
||||
ResolvedPipeline pipeline.ResolvedPipeline
|
||||
}
|
||||
|
||||
func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
|
||||
@@ -40,36 +42,34 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
|
||||
profile = clonePipelineProfile(profile)
|
||||
profile.ID = pipelineID
|
||||
if override := strings.TrimSpace(input.LLMProfileOverride); override != "" {
|
||||
if !hasLLMProfile(c.LLMProfiles, override) {
|
||||
return EffectiveConfig{}, fmt.Errorf("LLM profile override %q is not configured", override)
|
||||
}
|
||||
applyLLMProfileOverride(&profile, override)
|
||||
}
|
||||
|
||||
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{Only: input.Only}, input.Catalog)
|
||||
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{
|
||||
Only: input.Only,
|
||||
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
|
||||
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
|
||||
}, input.Catalog)
|
||||
if err != nil {
|
||||
return EffectiveConfig{}, fmt.Errorf("resolve pipeline %q: %w", pipelineID, err)
|
||||
}
|
||||
|
||||
return EffectiveConfig{
|
||||
Config: cloneConfig(c),
|
||||
PipelineID: pipelineID,
|
||||
Only: append([]string(nil), input.Only...),
|
||||
ResolvedPipeline: resolved,
|
||||
Config: cloneConfig(c),
|
||||
PipelineID: pipelineID,
|
||||
Only: append([]string(nil), input.Only...),
|
||||
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
|
||||
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),
|
||||
ResolvedPipeline: resolved,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func applyLLMProfileOverride(profile *pipeline.PipelineProfile, profileID string) {
|
||||
profile.Input.LLMProfile = profileID
|
||||
profile.Chunk.LLMProfile = profileID
|
||||
profile.Output.LLMProfile = profileID
|
||||
for laneID, lane := range profile.Artifacts {
|
||||
lane.Extract.LLMProfile = profileID
|
||||
lane.Merge.LLMProfile = profileID
|
||||
lane.Normalize.LLMProfile = profileID
|
||||
for i := range lane.Validators {
|
||||
lane.Validators[i].LLMProfile = profileID
|
||||
}
|
||||
profile.Artifacts[laneID] = lane
|
||||
}
|
||||
}
|
||||
@@ -83,36 +83,3 @@ func lookupPipelineProfile(profiles map[string]pipeline.PipelineProfile, pipelin
|
||||
}
|
||||
return pipeline.PipelineProfile{}, false
|
||||
}
|
||||
|
||||
func (c Config) OpenAICompatibleClientConfig(profileID string) (llm.OpenAICompatibleClientConfig, error) {
|
||||
trimmedID := strings.TrimSpace(profileID)
|
||||
profile, ok := c.LLMProfile(trimmedID)
|
||||
if !ok {
|
||||
return llm.OpenAICompatibleClientConfig{}, fmt.Errorf("LLM profile %q is not configured", trimmedID)
|
||||
}
|
||||
|
||||
provider := strings.TrimSpace(profile.Provider)
|
||||
if provider == "" {
|
||||
provider = providerOpenAICompatible
|
||||
}
|
||||
if provider != providerOpenAICompatible {
|
||||
return llm.OpenAICompatibleClientConfig{}, fmt.Errorf("LLM profile %q provider %q is not supported", trimmedID, provider)
|
||||
}
|
||||
|
||||
baseURL := strings.TrimSpace(profile.BaseURL)
|
||||
if baseURL == "" {
|
||||
return llm.OpenAICompatibleClientConfig{}, fmt.Errorf("LLM profile %q base URL must not be empty", trimmedID)
|
||||
}
|
||||
model := strings.TrimSpace(profile.Model)
|
||||
if model == "" {
|
||||
return llm.OpenAICompatibleClientConfig{}, fmt.Errorf("LLM profile %q model must not be empty", trimmedID)
|
||||
}
|
||||
|
||||
return llm.OpenAICompatibleClientConfig{
|
||||
BaseURL: baseURL,
|
||||
Model: model,
|
||||
APIKey: profile.APIKey,
|
||||
MaxRetries: profile.MaxRetries,
|
||||
RequestTimeout: time.Duration(profile.TimeoutSeconds) * time.Second,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package config
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
@@ -98,6 +97,43 @@ func TestResolveSurfacesMissingCapabilityThroughCatalog(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveCanBindSceneChunkerFromCatalog(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
profile := cfg.Pipelines["example"]
|
||||
profile.Chunk = pipeline.Binding("dnd/scenes")
|
||||
lane := profile.Artifacts["events"]
|
||||
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"events": lane}
|
||||
cfg.Pipelines["example"] = profile
|
||||
|
||||
catalog := fakeCatalog(t,
|
||||
pipeline.ModuleSpec{
|
||||
Key: "fake/input",
|
||||
Stage: pipeline.StageInput,
|
||||
Provides: []string{"source.transcript"},
|
||||
},
|
||||
pipeline.ModuleSpec{
|
||||
Key: "fake/extract",
|
||||
Stage: pipeline.StageExtract,
|
||||
Requires: []string{"chunks", "source.transcript"},
|
||||
Provides: []string{"artifact"},
|
||||
},
|
||||
)
|
||||
mustRegisterChunker(t, catalog.Chunkers, pipeline.ModuleSpec{
|
||||
Key: "dnd/scenes",
|
||||
Stage: pipeline.StageChunk,
|
||||
Requires: []string{"source.transcript"},
|
||||
Provides: []string{"chunks", "chunks.scenes"},
|
||||
})
|
||||
|
||||
effective, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: catalog})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve() error = %v, want nil", err)
|
||||
}
|
||||
if got := effective.ResolvedPipeline.Chunk.Module; got != "dnd/scenes" {
|
||||
t.Fatalf("Chunk.Module = %q, want dnd/scenes", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
first, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
|
||||
@@ -120,7 +156,19 @@ func TestResolveDigestChangesWhenEffectiveConfigChanges(t *testing.T) {
|
||||
|
||||
func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
cfg.LLMProfiles["runtime"] = LLMProfile{Provider: "openai-compatible"}
|
||||
profile := cfg.Pipelines["example"]
|
||||
profile.Input.LLMProfile = "input-profile"
|
||||
profile.Output.LLMProfile = "output-profile"
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Merge.LLMProfile = "merge-profile"
|
||||
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []pipeline.ModuleBinding{
|
||||
{Module: "fake/llm-validator", LLMProfile: "validator-profile"},
|
||||
},
|
||||
}
|
||||
profile.Artifacts["events"] = lane
|
||||
cfg.Pipelines["example"] = profile
|
||||
|
||||
base, err := cfg.Resolve(ResolveInput{PipelineID: "example", Catalog: fakeCatalog(t)})
|
||||
if err != nil {
|
||||
@@ -138,77 +186,43 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
|
||||
if base.ResolvedPipeline.Digest == effective.ResolvedPipeline.Digest {
|
||||
t.Fatalf("expected digest to change after LLM profile override")
|
||||
}
|
||||
for _, binding := range resolvedBindings(effective.ResolvedPipeline) {
|
||||
for _, binding := range llmCapableBindings(effective.ResolvedPipeline) {
|
||||
if binding.LLMProfile != "runtime" {
|
||||
t.Fatalf("binding profile = %q, want runtime", binding.LLMProfile)
|
||||
t.Fatalf("LLM-capable binding profile = %q, want runtime", binding.LLMProfile)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = cfg.Resolve(ResolveInput{
|
||||
PipelineID: "example",
|
||||
Catalog: fakeCatalog(t),
|
||||
LLMProfileOverride: "missing",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "LLM profile override") {
|
||||
t.Fatalf("expected override profile error, got %v", err)
|
||||
if effective.ResolvedPipeline.Input.LLMProfile != "input-profile" {
|
||||
t.Fatalf("input profile = %q, want original input-profile", effective.ResolvedPipeline.Input.LLMProfile)
|
||||
}
|
||||
if effective.ResolvedPipeline.Output.LLMProfile != "output-profile" {
|
||||
t.Fatalf("output profile = %q, want original output-profile", effective.ResolvedPipeline.Output.LLMProfile)
|
||||
}
|
||||
eventLane := effective.ResolvedPipeline.ArtifactLanes[0]
|
||||
if eventLane.Merge.LLMProfile != "runtime" {
|
||||
t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile)
|
||||
}
|
||||
validatorChain := findEffectiveValidatorChain(effective.ResolvedPipeline.ValidatorChains, pipeline.StageExtract, "events", "fake/extract")
|
||||
if validatorChain == nil || len(validatorChain.Validators) != 1 {
|
||||
t.Fatalf("validator chain = %#v, want one extract validator", effective.ResolvedPipeline.ValidatorChains)
|
||||
}
|
||||
if validatorChain.Validators[0].Binding.LLMProfile != "validator-profile" {
|
||||
t.Fatalf("validator profile = %q, want original validator-profile", validatorChain.Validators[0].Binding.LLMProfile)
|
||||
}
|
||||
}
|
||||
|
||||
func resolvedBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
|
||||
bindings := []pipeline.ModuleBinding{resolved.Input, resolved.Chunk, resolved.Output}
|
||||
func findEffectiveValidatorChain(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID string, module string) *pipeline.ResolvedValidatorChain {
|
||||
for i := range chains {
|
||||
if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module {
|
||||
return &chains[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
|
||||
bindings := []pipeline.ModuleBinding{resolved.Chunk}
|
||||
for _, lane := range resolved.ArtifactLanes {
|
||||
bindings = append(bindings, lane.Extract, lane.Merge, lane.Normalize)
|
||||
bindings = append(bindings, lane.Validators...)
|
||||
}
|
||||
return bindings
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientConfigRejectsIncompleteDefaultProfile(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
_, err := cfg.OpenAICompatibleClientConfig("default")
|
||||
if err == nil || !strings.Contains(err.Error(), "base URL") {
|
||||
t.Fatalf("expected incomplete profile error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientConfigSuccess(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.APIKey = "secret"
|
||||
profile.TimeoutSeconds = 45
|
||||
profile.MaxRetries = 4
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
|
||||
llmCfg, err := cfg.OpenAICompatibleClientConfig(" default ")
|
||||
if err != nil {
|
||||
t.Fatalf("OpenAICompatibleClientConfig: %v", err)
|
||||
}
|
||||
|
||||
if llmCfg.BaseURL != "https://example.invalid/v1" || llmCfg.Model != "test-model" || llmCfg.APIKey != "secret" {
|
||||
t.Fatalf("unexpected client config strings: %+v", llmCfg)
|
||||
}
|
||||
if llmCfg.MaxRetries != 4 {
|
||||
t.Fatalf("unexpected max retries: %d", llmCfg.MaxRetries)
|
||||
}
|
||||
if llmCfg.RequestTimeout != 45*time.Second {
|
||||
t.Fatalf("unexpected timeout: %s", llmCfg.RequestTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientConfigRejectsUnknownAndUnsupportedProfiles(t *testing.T) {
|
||||
_, err := validConfig().OpenAICompatibleClientConfig("missing")
|
||||
if err == nil || !strings.Contains(err.Error(), "not configured") {
|
||||
t.Fatalf("expected unknown profile error, got %v", err)
|
||||
}
|
||||
|
||||
cfg := validConfig()
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.Provider = "unsupported"
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
|
||||
_, err = cfg.OpenAICompatibleClientConfig("default")
|
||||
if err == nil || !strings.Contains(err.Error(), "provider") {
|
||||
t.Fatalf("expected unsupported provider error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func LoadFromEnv() (Config, error) {
|
||||
@@ -30,43 +29,6 @@ func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool))
|
||||
if c == nil {
|
||||
return fmt.Errorf("config must not be nil")
|
||||
}
|
||||
if c.LLMProfiles == nil {
|
||||
c.LLMProfiles = map[string]LLMProfile{}
|
||||
}
|
||||
|
||||
defaultProfile := c.LLMProfiles[pipeline.DefaultLLMProfile]
|
||||
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_API_KEY"); ok {
|
||||
defaultProfile.APIKey = raw
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_BASE_URL"); ok {
|
||||
defaultProfile.BaseURL = strings.TrimSpace(raw)
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MODEL"); ok {
|
||||
defaultProfile.Model = strings.TrimSpace(raw)
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS"); ok {
|
||||
value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defaultProfile.TimeoutSeconds = value
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MAX_RETRIES"); ok {
|
||||
value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_MAX_RETRIES", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defaultProfile.MaxRetries = value
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY"); ok {
|
||||
value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defaultProfile.MaxConcurrency = value
|
||||
}
|
||||
c.LLMProfiles[pipeline.DefaultLLMProfile] = defaultProfile
|
||||
|
||||
if raw, ok := lookup("NOTARIUS_TOTAL_LLM_CONCURRENCY"); ok {
|
||||
value, err := parseIntEnv("NOTARIUS_TOTAL_LLM_CONCURRENCY", raw)
|
||||
if err != nil {
|
||||
@@ -80,6 +42,36 @@ func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool))
|
||||
if raw, ok := lookup("NOTARIUS_DIAGNOSTICS_RETENTION"); ok {
|
||||
c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(raw))
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIR"); ok {
|
||||
c.Workspace.Directory = strings.TrimSpace(raw)
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED"); ok {
|
||||
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Workspace.Diagnostics.Enabled = value
|
||||
c.Workspace.Diagnostics.enabledSet = true
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION"); ok {
|
||||
c.Workspace.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(raw))
|
||||
c.Workspace.Diagnostics.retentionSet = true
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_RESUME_ENABLED"); ok {
|
||||
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_RESUME_ENABLED", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Workspace.Resume.Enabled = value
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DEBUG_ENABLED"); ok {
|
||||
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_DEBUG_ENABLED", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Workspace.Debug.Enabled = value
|
||||
}
|
||||
c.RecomputeEffectiveDiagnostics()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -90,3 +82,11 @@ func parseIntEnv(name string, raw string) (int, error) {
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func parseBoolEnv(name string, raw string) (bool, error) {
|
||||
value, err := strconv.ParseBool(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s: must be a boolean", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
@@ -8,39 +8,46 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestApplyEnvOverridesOperationalAndLLMValues(t *testing.T) {
|
||||
func TestApplyEnvOverridesOperationalValues(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Pipelines["example"] = pipeline.PipelineProfile{ID: "example", Input: pipeline.Binding("before")}
|
||||
|
||||
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
|
||||
"NOTARIUS_LLM_DEFAULT_API_KEY": "secret",
|
||||
"NOTARIUS_LLM_DEFAULT_BASE_URL": "https://example.invalid/v1",
|
||||
"NOTARIUS_LLM_DEFAULT_MODEL": "test-model",
|
||||
"NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS": "120",
|
||||
"NOTARIUS_LLM_DEFAULT_MAX_RETRIES": "5",
|
||||
"NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY": "2",
|
||||
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "3",
|
||||
"NOTARIUS_WORK_DIR": "/tmp/notarius-env",
|
||||
"NOTARIUS_DIAGNOSTICS_RETENTION": "never",
|
||||
"NOTARIUS_PIPELINE_INPUT": "after",
|
||||
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "3",
|
||||
"NOTARIUS_WORK_DIR": "/tmp/notarius-env",
|
||||
"NOTARIUS_DIAGNOSTICS_RETENTION": "never",
|
||||
"NOTARIUS_WORKSPACE_DIR": "/var/lib/notarius-env",
|
||||
"NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED": "false",
|
||||
"NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION": "always",
|
||||
"NOTARIUS_WORKSPACE_RESUME_ENABLED": "true",
|
||||
"NOTARIUS_WORKSPACE_DEBUG_ENABLED": "true",
|
||||
"NOTARIUS_PIPELINE_INPUT": "after",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyEnvOverrides: %v", err)
|
||||
}
|
||||
|
||||
profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
|
||||
if profile.APIKey != "secret" || profile.BaseURL != "https://example.invalid/v1" || profile.Model != "test-model" {
|
||||
t.Fatalf("unexpected LLM profile strings: %+v", profile)
|
||||
}
|
||||
if profile.TimeoutSeconds != 120 || profile.MaxRetries != 5 || profile.MaxConcurrency != 2 {
|
||||
t.Fatalf("unexpected LLM profile numeric values: %+v", profile)
|
||||
if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("LLM environment overrides must not change Scriptorium config: %+v", cfg.Scriptorium)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 3 {
|
||||
t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM)
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/tmp/notarius-env" || cfg.Diagnostics.Retention != diagnostics.RetentionNever {
|
||||
if cfg.Workspace.Directory != "/var/lib/notarius-env" {
|
||||
t.Fatalf("unexpected workspace directory: %q", cfg.Workspace.Directory)
|
||||
}
|
||||
if cfg.DiagnosticsEnabled() {
|
||||
t.Fatalf("expected workspace diagnostics disabled")
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/var/lib/notarius-env/diagnostics" || cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
|
||||
t.Fatalf("unexpected diagnostics config: %+v", cfg.Diagnostics)
|
||||
}
|
||||
if !cfg.Workspace.Resume.Enabled {
|
||||
t.Fatalf("expected workspace resume enabled")
|
||||
}
|
||||
if !cfg.Workspace.Debug.Enabled {
|
||||
t.Fatalf("expected workspace debug enabled")
|
||||
}
|
||||
if cfg.Pipelines["example"].Input.Module != "before" {
|
||||
t.Fatalf("environment overrides must not change pipeline wiring: %+v", cfg.Pipelines["example"])
|
||||
}
|
||||
@@ -56,14 +63,52 @@ func TestApplyEnvOverridesRejectsInvalidIntegers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyEnvOverridesRejectsInvalidBooleans(t *testing.T) {
|
||||
for _, name := range []string{
|
||||
"NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED",
|
||||
"NOTARIUS_WORKSPACE_RESUME_ENABLED",
|
||||
"NOTARIUS_WORKSPACE_DEBUG_ENABLED",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{name: "maybe"}))
|
||||
if err == nil || !strings.Contains(err.Error(), name) {
|
||||
t.Fatalf("expected named boolean error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyEnvOverridesLegacyDiagnosticsRemainCompatibleWithoutWorkspace(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
|
||||
"NOTARIUS_WORK_DIR": "/tmp/notarius-env",
|
||||
"NOTARIUS_DIAGNOSTICS_RETENTION": "never",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyEnvOverrides: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Diagnostics.WorkDir != "/tmp/notarius-env" {
|
||||
t.Fatalf("diagnostics work dir = %q, want legacy env", cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionNever {
|
||||
t.Fatalf("diagnostics retention = %q, want legacy env", cfg.Diagnostics.Retention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvUsesDefaultConfig(t *testing.T) {
|
||||
t.Setenv("NOTARIUS_LLM_DEFAULT_MODEL", "env-model")
|
||||
t.Setenv("NOTARIUS_TOTAL_LLM_CONCURRENCY", "2")
|
||||
|
||||
cfg, err := LoadFromEnv()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFromEnv: %v", err)
|
||||
}
|
||||
if cfg.LLMProfiles[pipeline.DefaultLLMProfile].Model != "env-model" {
|
||||
t.Fatalf("expected env model, got %+v", cfg.LLMProfiles[pipeline.DefaultLLMProfile])
|
||||
if cfg.Scriptorium.ProfileDir != "" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("unexpected Scriptorium config from env: %+v", cfg.Scriptorium)
|
||||
}
|
||||
if cfg.Concurrency.TotalLLM != 2 {
|
||||
t.Fatalf("expected env concurrency override, got %+v", cfg.Concurrency)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,41 +4,34 @@ import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
type FileConfig struct {
|
||||
Version int `yaml:"version"`
|
||||
LLMProfiles map[string]FileLLMProfile `yaml:"llm_profiles,omitempty"`
|
||||
Scriptorium *FileScriptoriumConfig `yaml:"scriptorium,omitempty"`
|
||||
Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
|
||||
Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"`
|
||||
Diagnostics *FileDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
||||
Workspace *FileWorkspaceConfig `yaml:"workspace,omitempty"`
|
||||
}
|
||||
|
||||
type FileLLMProfile struct {
|
||||
Provider *string `yaml:"provider,omitempty"`
|
||||
BaseURL *string `yaml:"base_url,omitempty"`
|
||||
Model *string `yaml:"model,omitempty"`
|
||||
APIKeyEnv *string `yaml:"api_key_env,omitempty"`
|
||||
Timeout *fileDurationSeconds `yaml:"timeout,omitempty"`
|
||||
MaxRetries *int `yaml:"max_retries,omitempty"`
|
||||
MaxConcurrency *int `yaml:"max_concurrency,omitempty"`
|
||||
type FileScriptoriumConfig struct {
|
||||
ProfileDir *string `yaml:"profile_dir,omitempty"`
|
||||
ProfileFile *string `yaml:"profile_file,omitempty"`
|
||||
}
|
||||
|
||||
type FilePipelineProfile struct {
|
||||
Input fileModuleBinding `yaml:"input"`
|
||||
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
||||
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
||||
Output *fileModuleBinding `yaml:"output,omitempty"`
|
||||
Input fileModuleBinding `yaml:"input"`
|
||||
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
|
||||
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
|
||||
Output *fileModuleBinding `yaml:"output,omitempty"`
|
||||
References map[string]string `yaml:"references,omitempty"`
|
||||
}
|
||||
|
||||
type FileArtifactLaneProfile struct {
|
||||
@@ -46,6 +39,7 @@ type FileArtifactLaneProfile struct {
|
||||
Merge *fileModuleBinding `yaml:"merge,omitempty"`
|
||||
Normalize *fileModuleBinding `yaml:"normalize,omitempty"`
|
||||
Validators []fileModuleBinding `yaml:"validators,omitempty"`
|
||||
References map[string]string `yaml:"references,omitempty"`
|
||||
}
|
||||
|
||||
type FileConcurrencyConfig struct {
|
||||
@@ -57,46 +51,29 @@ type FileDiagnosticsConfig struct {
|
||||
Retention *string `yaml:"retention,omitempty"`
|
||||
}
|
||||
|
||||
type fileDurationSeconds struct {
|
||||
seconds int
|
||||
type FileWorkspaceConfig struct {
|
||||
Directory *string `yaml:"directory,omitempty"`
|
||||
Diagnostics *FileWorkspaceDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
||||
Resume *FileWorkspaceEnabledConfig `yaml:"resume,omitempty"`
|
||||
Debug *FileWorkspaceEnabledConfig `yaml:"debug,omitempty"`
|
||||
}
|
||||
|
||||
func (d *fileDurationSeconds) UnmarshalYAML(node *yaml.Node) error {
|
||||
if node.Kind != yaml.ScalarNode {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
if node.Tag == "!!int" {
|
||||
var seconds int
|
||||
if err := node.Decode(&seconds); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
d.seconds = seconds
|
||||
return nil
|
||||
}
|
||||
|
||||
var raw string
|
||||
if err := node.Decode(&raw); err != nil {
|
||||
return fmt.Errorf("must be an integer seconds value or duration string")
|
||||
}
|
||||
duration, err := time.ParseDuration(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration %q", raw)
|
||||
}
|
||||
if duration%time.Second != 0 {
|
||||
return fmt.Errorf("duration %q must resolve to whole seconds", raw)
|
||||
}
|
||||
d.seconds = int(duration / time.Second)
|
||||
return nil
|
||||
type FileWorkspaceDiagnosticsConfig struct {
|
||||
Enabled *bool `yaml:"enabled,omitempty"`
|
||||
Retention *string `yaml:"retention,omitempty"`
|
||||
}
|
||||
|
||||
func (d fileDurationSeconds) Seconds() int {
|
||||
return d.seconds
|
||||
type FileWorkspaceEnabledConfig struct {
|
||||
Enabled *bool `yaml:"enabled,omitempty"`
|
||||
}
|
||||
|
||||
type fileModuleBinding struct {
|
||||
Module string
|
||||
LLMProfile string
|
||||
Retries int
|
||||
Options map[string]any
|
||||
References map[string]string
|
||||
Validators pipeline.ValidatorOverride
|
||||
}
|
||||
|
||||
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||
@@ -125,12 +102,34 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||
return err
|
||||
}
|
||||
b.LLMProfile = strings.TrimSpace(llmProfile)
|
||||
case "retries":
|
||||
var retries int
|
||||
if err := valueNode.Decode(&retries); err != nil {
|
||||
return err
|
||||
}
|
||||
b.Retries = retries
|
||||
case "options":
|
||||
var options map[string]any
|
||||
if err := valueNode.Decode(&options); err != nil {
|
||||
return err
|
||||
}
|
||||
b.Options = normalizeOptions(options)
|
||||
case "references":
|
||||
var references map[string]string
|
||||
if err := valueNode.Decode(&references); err != nil {
|
||||
return err
|
||||
}
|
||||
b.References = references
|
||||
case "validators":
|
||||
b.Validators.Set = true
|
||||
var validators []fileModuleBinding
|
||||
if err := valueNode.Decode(&validators); err != nil {
|
||||
return err
|
||||
}
|
||||
b.Validators.Validators = make([]pipeline.ModuleBinding, len(validators))
|
||||
for i, validator := range validators {
|
||||
b.Validators.Validators[i] = validator.toPipelineBinding()
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
|
||||
}
|
||||
@@ -145,7 +144,10 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
||||
return pipeline.ModuleBinding{
|
||||
Module: strings.TrimSpace(b.Module),
|
||||
LLMProfile: strings.TrimSpace(b.LLMProfile),
|
||||
Retries: b.Retries,
|
||||
Options: cloneOptions(b.Options),
|
||||
References: normalizedStringMap(b.References),
|
||||
Validators: b.Validators,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -186,23 +188,17 @@ func (c *Config) ApplyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
}
|
||||
|
||||
func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
|
||||
_ = lookup
|
||||
if c == nil {
|
||||
return fmt.Errorf("config must not be nil")
|
||||
}
|
||||
if fileCfg.Version != SupportedFileConfigVersion {
|
||||
return fmt.Errorf("unsupported config version %d", fileCfg.Version)
|
||||
}
|
||||
if c.LLMProfiles == nil {
|
||||
c.LLMProfiles = map[string]LLMProfile{}
|
||||
}
|
||||
if c.Pipelines == nil {
|
||||
c.Pipelines = map[string]pipeline.PipelineProfile{}
|
||||
}
|
||||
|
||||
profileIDs, rawLLMProfileIDs, err := normalizedMapKeys(fileCfg.LLMProfiles, "llm profile id")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pipelineIDs, rawPipelineIDs, err := normalizedMapKeys(fileCfg.Pipelines, "pipeline id")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -212,38 +208,66 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if filePipeline.Chunk != nil {
|
||||
if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if filePipeline.Output != nil {
|
||||
if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for rawLaneID, fileLane := range filePipeline.Artifacts {
|
||||
laneID := strings.TrimSpace(rawLaneID)
|
||||
if laneID == "" {
|
||||
continue
|
||||
}
|
||||
if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
|
||||
return err
|
||||
}
|
||||
if fileLane.Merge != nil {
|
||||
if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if fileLane.Normalize != nil {
|
||||
if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for i, validator := range fileLane.Validators {
|
||||
if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, profileID := range profileIDs {
|
||||
fileProfile := fileCfg.LLMProfiles[rawLLMProfileIDs[profileID]]
|
||||
profile := c.LLMProfiles[profileID]
|
||||
if fileProfile.Provider != nil {
|
||||
profile.Provider = strings.TrimSpace(*fileProfile.Provider)
|
||||
}
|
||||
if fileProfile.BaseURL != nil {
|
||||
profile.BaseURL = strings.TrimSpace(*fileProfile.BaseURL)
|
||||
}
|
||||
if fileProfile.Model != nil {
|
||||
profile.Model = strings.TrimSpace(*fileProfile.Model)
|
||||
}
|
||||
if fileProfile.APIKeyEnv != nil {
|
||||
apiKey, err := resolveAPIKeyEnv(*fileProfile.APIKeyEnv, lookup)
|
||||
if err != nil {
|
||||
return fmt.Errorf("llm_profiles.%s.api_key_env: %w", profileID, err)
|
||||
if fileCfg.Scriptorium != nil {
|
||||
if fileCfg.Scriptorium.ProfileDir != nil {
|
||||
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileDir)
|
||||
if value == "" {
|
||||
return fmt.Errorf("scriptorium.profile_dir must not be empty when set")
|
||||
}
|
||||
profile.APIKeyEnv = strings.TrimSpace(*fileProfile.APIKeyEnv)
|
||||
profile.APIKey = apiKey
|
||||
c.Scriptorium.ProfileDir = value
|
||||
}
|
||||
if fileProfile.Timeout != nil {
|
||||
profile.TimeoutSeconds = fileProfile.Timeout.Seconds()
|
||||
if fileCfg.Scriptorium.ProfileFile != nil {
|
||||
value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileFile)
|
||||
if value == "" {
|
||||
return fmt.Errorf("scriptorium.profile_file must not be empty when set")
|
||||
}
|
||||
c.Scriptorium.ProfileFile = value
|
||||
}
|
||||
if fileProfile.MaxRetries != nil {
|
||||
profile.MaxRetries = *fileProfile.MaxRetries
|
||||
}
|
||||
if fileProfile.MaxConcurrency != nil {
|
||||
profile.MaxConcurrency = *fileProfile.MaxConcurrency
|
||||
}
|
||||
c.LLMProfiles[profileID] = profile
|
||||
}
|
||||
|
||||
for _, pipelineID := range pipelineIDs {
|
||||
@@ -253,9 +277,10 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
return err
|
||||
}
|
||||
profile := pipeline.PipelineProfile{
|
||||
ID: pipelineID,
|
||||
Input: filePipeline.Input.toPipelineBinding(),
|
||||
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
||||
ID: pipelineID,
|
||||
Input: filePipeline.Input.toPipelineBinding(),
|
||||
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
|
||||
References: normalizedStringMap(filePipeline.References),
|
||||
}
|
||||
if filePipeline.Chunk != nil {
|
||||
profile.Chunk = filePipeline.Chunk.toPipelineBinding()
|
||||
@@ -265,8 +290,11 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
}
|
||||
for _, laneID := range laneIDs {
|
||||
fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
|
||||
extract := fileLane.Extract.toPipelineBinding()
|
||||
extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References)
|
||||
lane := pipeline.ArtifactLaneProfile{
|
||||
Extract: fileLane.Extract.toPipelineBinding(),
|
||||
Extract: extract,
|
||||
References: normalizedStringMap(fileLane.References),
|
||||
}
|
||||
if fileLane.Merge != nil {
|
||||
lane.Merge = fileLane.Merge.toPipelineBinding()
|
||||
@@ -296,6 +324,28 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Diagnostics.Retention))
|
||||
}
|
||||
}
|
||||
if fileCfg.Workspace != nil {
|
||||
if fileCfg.Workspace.Directory != nil {
|
||||
c.Workspace.Directory = strings.TrimSpace(*fileCfg.Workspace.Directory)
|
||||
}
|
||||
if fileCfg.Workspace.Diagnostics != nil {
|
||||
if fileCfg.Workspace.Diagnostics.Enabled != nil {
|
||||
c.Workspace.Diagnostics.Enabled = *fileCfg.Workspace.Diagnostics.Enabled
|
||||
c.Workspace.Diagnostics.enabledSet = true
|
||||
}
|
||||
if fileCfg.Workspace.Diagnostics.Retention != nil {
|
||||
c.Workspace.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Workspace.Diagnostics.Retention))
|
||||
c.Workspace.Diagnostics.retentionSet = true
|
||||
}
|
||||
}
|
||||
if fileCfg.Workspace.Resume != nil && fileCfg.Workspace.Resume.Enabled != nil {
|
||||
c.Workspace.Resume.Enabled = *fileCfg.Workspace.Resume.Enabled
|
||||
}
|
||||
if fileCfg.Workspace.Debug != nil && fileCfg.Workspace.Debug.Enabled != nil {
|
||||
c.Workspace.Debug.Enabled = *fileCfg.Workspace.Debug.Enabled
|
||||
}
|
||||
}
|
||||
c.RecomputeEffectiveDiagnostics()
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -318,19 +368,37 @@ func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, ma
|
||||
return keys, rawByNormalized, nil
|
||||
}
|
||||
|
||||
func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
|
||||
name := strings.TrimSpace(envName)
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("must not be empty")
|
||||
func normalizedStringMap(values map[string]string) map[string]string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !envVarNamePattern.MatchString(name) {
|
||||
return "", fmt.Errorf("must be an environment variable name")
|
||||
out := make(map[string]string, len(values))
|
||||
keys := make([]string, 0, len(values))
|
||||
rawByNormalized := make(map[string]string, len(values))
|
||||
for rawKey := range values {
|
||||
key := strings.TrimSpace(rawKey)
|
||||
rawByNormalized[key] = rawKey
|
||||
keys = append(keys, key)
|
||||
}
|
||||
value, ok := lookup(name)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s is not set", name)
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
|
||||
}
|
||||
return value, nil
|
||||
return out
|
||||
}
|
||||
|
||||
func mergeStringMaps(base map[string]string, override map[string]string) map[string]string {
|
||||
if len(base) == 0 && len(override) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(base)+len(override))
|
||||
for key, value := range base {
|
||||
out[key] = value
|
||||
}
|
||||
for key, value := range override {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeOptions(options map[string]any) map[string]any {
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
|
||||
func TestParseMinimalValidConfig(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
version: 2
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
@@ -24,7 +24,7 @@ version: 1
|
||||
|
||||
func TestLoadFileConfig(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(path, []byte("version: 1\n"), 0o644); err != nil {
|
||||
if err := os.WriteFile(path, []byte("version: 2\n"), 0o644); err != nil {
|
||||
t.Fatalf("write config: %v", err)
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ func TestLoadFileConfig(t *testing.T) {
|
||||
|
||||
func TestParseFileConfigRejectsUnknownYAMLFields(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
version: 2
|
||||
unexpected: true
|
||||
`))
|
||||
if err == nil || !strings.Contains(err.Error(), "field unexpected not found") {
|
||||
@@ -49,7 +49,7 @@ unexpected: true
|
||||
|
||||
func TestParseFileConfigRejectsUnknownModuleBindingFields(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input:
|
||||
@@ -70,8 +70,8 @@ func TestParseFileConfigRejectsMissingAndUnsupportedVersion(t *testing.T) {
|
||||
data string
|
||||
want string
|
||||
}{
|
||||
{name: "missing", data: `llm_profiles: {}`, want: "version is required"},
|
||||
{name: "unsupported", data: `version: 2`, want: "unsupported config version"},
|
||||
{name: "missing", data: `scriptorium: {}`, want: "version is required"},
|
||||
{name: "unsupported", data: `version: 1`, want: "unsupported config version"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
@@ -84,14 +84,50 @@ func TestParseFileConfigRejectsMissingAndUnsupportedVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigRejectsStaleLLMProfiles(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
llm_profiles:
|
||||
default: {}
|
||||
`))
|
||||
if err == nil || !strings.Contains(err.Error(), "llm_profiles") {
|
||||
t.Fatalf("expected stale llm_profiles error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigScriptoriumProfileSources(t *testing.T) {
|
||||
t.Run("profile dir", func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
scriptorium:
|
||||
profile_dir: ./profiles
|
||||
`)
|
||||
if cfg.Scriptorium.ProfileDir != "./profiles" || cfg.Scriptorium.ProfileFile != "" {
|
||||
t.Fatalf("Scriptorium = %+v, want profile_dir", cfg.Scriptorium)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("profile file", func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
scriptorium:
|
||||
profile_file: ./profiles.yml
|
||||
`)
|
||||
if cfg.Scriptorium.ProfileFile != "./profiles.yml" || cfg.Scriptorium.ProfileDir != "" {
|
||||
t.Fatalf("Scriptorium = %+v, want profile_file", cfg.Scriptorium)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseFileConfigModuleBindingForms(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
chunk:
|
||||
module: generic
|
||||
retries: 2
|
||||
options:
|
||||
size: 10
|
||||
flags:
|
||||
@@ -103,9 +139,12 @@ pipelines:
|
||||
extract:
|
||||
module: fake/extract
|
||||
llm_profile: fast
|
||||
retries: 3
|
||||
options:
|
||||
temperature: 0
|
||||
merge: appendorder
|
||||
merge:
|
||||
module: appendorder
|
||||
retries: 1
|
||||
normalize:
|
||||
module: noop
|
||||
output: json
|
||||
@@ -118,6 +157,9 @@ pipelines:
|
||||
if profile.Chunk.Module != "generic" {
|
||||
t.Fatalf("unexpected chunk binding: %+v", profile.Chunk)
|
||||
}
|
||||
if profile.Chunk.Retries != 2 {
|
||||
t.Fatalf("chunk retries = %d, want 2", profile.Chunk.Retries)
|
||||
}
|
||||
if profile.Chunk.Options["size"] != 10 {
|
||||
t.Fatalf("expected chunk options to preserve scalar, got %#v", profile.Chunk.Options)
|
||||
}
|
||||
@@ -133,6 +175,9 @@ pipelines:
|
||||
if lane.Extract.Module != "fake/extract" || lane.Extract.LLMProfile != "fast" {
|
||||
t.Fatalf("unexpected extract binding: %+v", lane.Extract)
|
||||
}
|
||||
if lane.Extract.Retries != 3 || lane.Merge.Retries != 1 {
|
||||
t.Fatalf("unexpected retries: extract=%d merge=%d", lane.Extract.Retries, lane.Merge.Retries)
|
||||
}
|
||||
if lane.Extract.Options["temperature"] != 0 {
|
||||
t.Fatalf("expected object options, got %#v", lane.Extract.Options)
|
||||
}
|
||||
@@ -144,9 +189,88 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigReferenceMaps(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
references:
|
||||
" roster ": " ./shared-roster.yml "
|
||||
artifacts:
|
||||
events:
|
||||
extract: fake/extract
|
||||
references:
|
||||
" lore ": " ./lore.md "
|
||||
`)
|
||||
|
||||
profile := cfg.Pipelines["example"]
|
||||
if !reflect.DeepEqual(profile.References, map[string]string{"roster": "./shared-roster.yml"}) {
|
||||
t.Fatalf("pipeline references = %#v, want trimmed map", profile.References)
|
||||
}
|
||||
gotLaneRefs := profile.Artifacts["events"].References
|
||||
if !reflect.DeepEqual(gotLaneRefs, map[string]string{"lore": "./lore.md"}) {
|
||||
t.Fatalf("lane references = %#v, want trimmed map", gotLaneRefs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigStageLocalReferenceMaps(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
chunk:
|
||||
module: generic
|
||||
references:
|
||||
" scene_guide ": " ./scenes.md "
|
||||
artifacts:
|
||||
events:
|
||||
extract:
|
||||
module: fake/extract
|
||||
references:
|
||||
" glossary ": " ./glossary.md "
|
||||
" roster ": " ./extract-roster.yml "
|
||||
references:
|
||||
roster: ./legacy-roster.yml
|
||||
lore: ./lore.md
|
||||
merge:
|
||||
module: appendorder
|
||||
references:
|
||||
" merge_notes ": " ./merge.md "
|
||||
normalize:
|
||||
module: noop
|
||||
references:
|
||||
" normalization_notes ": " ./normalization.md "
|
||||
`)
|
||||
|
||||
profile := cfg.Pipelines["example"]
|
||||
if !reflect.DeepEqual(profile.Chunk.References, map[string]string{"scene_guide": "./scenes.md"}) {
|
||||
t.Fatalf("chunk references = %#v, want trimmed map", profile.Chunk.References)
|
||||
}
|
||||
lane := profile.Artifacts["events"]
|
||||
if !reflect.DeepEqual(lane.References, map[string]string{"lore": "./lore.md", "roster": "./legacy-roster.yml"}) {
|
||||
t.Fatalf("legacy lane references = %#v, want trimmed map", lane.References)
|
||||
}
|
||||
wantExtract := map[string]string{
|
||||
"glossary": "./glossary.md",
|
||||
"lore": "./lore.md",
|
||||
"roster": "./extract-roster.yml",
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Extract.References, wantExtract) {
|
||||
t.Fatalf("extract references = %#v, want legacy merged with extract override %#v", lane.Extract.References, wantExtract)
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Merge.References, map[string]string{"merge_notes": "./merge.md"}) {
|
||||
t.Fatalf("merge references = %#v, want trimmed map", lane.Merge.References)
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Normalize.References, map[string]string{"normalization_notes": "./normalization.md"}) {
|
||||
t.Fatalf("normalize references = %#v, want trimmed map", lane.Normalize.References)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -176,87 +300,64 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigDurationParsing(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want int
|
||||
}{
|
||||
{name: "integer seconds", raw: "600", want: 600},
|
||||
{name: "duration string", raw: "10m", want: 600},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
timeout: `+tc.raw+`
|
||||
func TestParseFileConfigStageLocalValidatorOverrides(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
chunk:
|
||||
module: generic
|
||||
validators: []
|
||||
artifacts:
|
||||
events:
|
||||
extract:
|
||||
module: fake/extract
|
||||
validators:
|
||||
- fake/validator
|
||||
- module: fake/llm-validator
|
||||
llm_profile: careful
|
||||
options:
|
||||
threshold: 0.7
|
||||
merge:
|
||||
module: appendorder
|
||||
validators: []
|
||||
normalize:
|
||||
module: noop
|
||||
`)
|
||||
if got := cfg.LLMProfiles["default"].TimeoutSeconds; got != tc.want {
|
||||
t.Fatalf("TimeoutSeconds = %d, want %d", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileConfigRejectsSubsecondDuration(t *testing.T) {
|
||||
_, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
timeout: 1500ms
|
||||
`))
|
||||
if err == nil || !strings.Contains(err.Error(), "whole seconds") {
|
||||
t.Fatalf("expected whole-seconds duration error, got %v", err)
|
||||
profile := cfg.Pipelines["example"]
|
||||
if !profile.Chunk.Validators.Set || len(profile.Chunk.Validators.Validators) != 0 {
|
||||
t.Fatalf("chunk validator override = %#v, want explicit empty", profile.Chunk.Validators)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigResolvesAPIKeyEnv(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
api_key_env: NOTARIUS_TEST_API_KEY
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
lane := profile.Artifacts["events"]
|
||||
if !lane.Extract.Validators.Set {
|
||||
t.Fatalf("extract validator override Set = false, want true")
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{"NOTARIUS_TEST_API_KEY": "secret"})); err != nil {
|
||||
t.Fatalf("ApplyFileConfig: %v", err)
|
||||
validators := lane.Extract.Validators.Validators
|
||||
if len(validators) != 2 {
|
||||
t.Fatalf("extract validators = %#v, want two validators", validators)
|
||||
}
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
if profile.APIKeyEnv != "NOTARIUS_TEST_API_KEY" || profile.APIKey != "secret" {
|
||||
t.Fatalf("unexpected resolved API key: %+v", profile)
|
||||
if validators[0].Module != "fake/validator" {
|
||||
t.Fatalf("first validator = %#v, want fake/validator", validators[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsDuplicateTrimmedLLMProfileIDs(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
model: first
|
||||
" default ":
|
||||
model: second
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" {
|
||||
t.Fatalf("second validator = %#v, want LLM validator with profile", validators[1])
|
||||
}
|
||||
|
||||
cfg := Default()
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
|
||||
if err == nil || !strings.Contains(err.Error(), "llm profile id") || !strings.Contains(err.Error(), "duplicated") {
|
||||
t.Fatalf("expected duplicate LLM profile ID error, got %v", err)
|
||||
if validators[1].Options["threshold"] != 0.7 {
|
||||
t.Fatalf("second validator options = %#v, want threshold", validators[1].Options)
|
||||
}
|
||||
if !lane.Merge.Validators.Set || len(lane.Merge.Validators.Validators) != 0 {
|
||||
t.Fatalf("merge validator override = %#v, want explicit empty", lane.Merge.Validators)
|
||||
}
|
||||
if lane.Normalize.Validators.Set {
|
||||
t.Fatalf("normalize validator override Set = true, want omitted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -276,7 +377,7 @@ pipelines:
|
||||
|
||||
func TestApplyFileConfigRejectsDuplicateTrimmedArtifactLaneIDs(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
@@ -297,49 +398,134 @@ pipelines:
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigAllowsRetryOnlyLLMProfile(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 1
|
||||
llm_profiles:
|
||||
retry-only:
|
||||
max_retries: 3
|
||||
`)
|
||||
|
||||
profile := cfg.LLMProfiles["retry-only"]
|
||||
if profile.MaxRetries != 3 {
|
||||
t.Fatalf("unexpected max retries: %d", profile.MaxRetries)
|
||||
}
|
||||
if profile.TimeoutSeconds != 0 {
|
||||
t.Fatalf("expected unset timeout, got %d", profile.TimeoutSeconds)
|
||||
}
|
||||
if profile.MaxConcurrency != 0 {
|
||||
t.Fatalf("expected unset max concurrency, got %d", profile.MaxConcurrency)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsInvalidAPIKeyEnv(t *testing.T) {
|
||||
func TestApplyFileConfigRejectsDuplicateTrimmedReferenceSlots(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
env string
|
||||
raw string
|
||||
want string
|
||||
}{
|
||||
{name: "invalid name", env: "NOTARIUS-KEY", want: "environment variable name"},
|
||||
{name: "not set", env: "NOTARIUS_TEST_API_KEY", want: "is not set"},
|
||||
{
|
||||
name: "pipeline",
|
||||
raw: `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
references:
|
||||
roster: ./first.yml
|
||||
" roster ": ./second.yml
|
||||
`,
|
||||
want: `pipeline "example" reference slot`,
|
||||
},
|
||||
{
|
||||
name: "lane",
|
||||
raw: `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
artifacts:
|
||||
events:
|
||||
extract: fake/extract
|
||||
references:
|
||||
roster: ./first.yml
|
||||
" roster ": ./second.yml
|
||||
`,
|
||||
want: `pipeline "example" lane "events" reference slot`,
|
||||
},
|
||||
{
|
||||
name: "chunk",
|
||||
raw: `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
chunk:
|
||||
module: generic
|
||||
references:
|
||||
roster: ./first.yml
|
||||
" roster ": ./second.yml
|
||||
`,
|
||||
want: `pipeline "example" chunk reference slot`,
|
||||
},
|
||||
{
|
||||
name: "extract",
|
||||
raw: `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
artifacts:
|
||||
events:
|
||||
extract:
|
||||
module: fake/extract
|
||||
references:
|
||||
roster: ./first.yml
|
||||
" roster ": ./second.yml
|
||||
`,
|
||||
want: `pipeline "example" lane "events" extract reference slot`,
|
||||
},
|
||||
{
|
||||
name: "normalize",
|
||||
raw: `
|
||||
version: 2
|
||||
pipelines:
|
||||
example:
|
||||
input: fake/input
|
||||
artifacts:
|
||||
events:
|
||||
extract: fake/extract
|
||||
normalize:
|
||||
module: noop
|
||||
references:
|
||||
roster: ./first.yml
|
||||
" roster ": ./second.yml
|
||||
`,
|
||||
want: `pipeline "example" lane "events" normalize reference slot`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 1
|
||||
llm_profiles:
|
||||
default:
|
||||
api_key_env: ` + tc.env + `
|
||||
`))
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(tc.raw))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
}
|
||||
cfg := Default()
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), "duplicated") {
|
||||
t.Fatalf("expected duplicate reference slot error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigRejectsInvalidScriptoriumSources(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want string
|
||||
}{
|
||||
{name: "empty profile dir", raw: "profile_dir: ' '", want: "profile_dir"},
|
||||
{name: "empty profile file", raw: "profile_file: ' '", want: "profile_file"},
|
||||
{name: "both sources", raw: "profile_dir: ./profiles\n profile_file: ./profiles.yml", want: "mutually exclusive"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||
version: 2
|
||||
scriptorium:
|
||||
` + tc.raw + `
|
||||
`))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFileConfigYAML: %v", err)
|
||||
}
|
||||
err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
|
||||
if err == nil {
|
||||
err = cfg.Validate()
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.want, err)
|
||||
}
|
||||
@@ -349,7 +535,7 @@ llm_profiles:
|
||||
|
||||
func TestApplyFileConfigOperationalSections(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 1
|
||||
version: 2
|
||||
concurrency:
|
||||
total_llm: 4
|
||||
diagnostics:
|
||||
@@ -368,6 +554,95 @@ diagnostics:
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigWorkspaceSection(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
enabled: false
|
||||
retention: never
|
||||
resume:
|
||||
enabled: true
|
||||
debug:
|
||||
enabled: true
|
||||
diagnostics:
|
||||
work_dir: /tmp/legacy
|
||||
retention: always
|
||||
`)
|
||||
|
||||
if cfg.Workspace.Directory != "/var/lib/notarius" {
|
||||
t.Fatalf("workspace directory = %q, want /var/lib/notarius", cfg.Workspace.Directory)
|
||||
}
|
||||
if cfg.DiagnosticsEnabled() {
|
||||
t.Fatalf("expected diagnostics disabled")
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/var/lib/notarius/diagnostics" {
|
||||
t.Fatalf("effective diagnostics work dir = %q, want workspace diagnostics root", cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionNever {
|
||||
t.Fatalf("effective diagnostics retention = %q, want workspace override", cfg.Diagnostics.Retention)
|
||||
}
|
||||
if !cfg.Workspace.Resume.Enabled {
|
||||
t.Fatalf("expected resume enabled")
|
||||
}
|
||||
if !cfg.Workspace.Debug.Enabled {
|
||||
t.Fatalf("expected debug enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigLegacyDiagnosticsRemainCompatible(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
diagnostics:
|
||||
work_dir: /tmp/legacy
|
||||
retention: always
|
||||
`)
|
||||
|
||||
if cfg.Workspace.Directory != "" {
|
||||
t.Fatalf("workspace directory = %q, want unset", cfg.Workspace.Directory)
|
||||
}
|
||||
if !cfg.DiagnosticsEnabled() {
|
||||
t.Fatalf("expected diagnostics enabled")
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/tmp/legacy" {
|
||||
t.Fatalf("effective diagnostics work dir = %q, want legacy", cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
|
||||
t.Fatalf("effective diagnostics retention = %q, want legacy", cfg.Diagnostics.Retention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigWorkspaceRetentionOverridesLegacyRetentionOnlyWhenSet(t *testing.T) {
|
||||
t.Run("legacy retained", func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
retention: never
|
||||
`)
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionNever {
|
||||
t.Fatalf("effective diagnostics retention = %q, want legacy", cfg.Diagnostics.Retention)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("workspace overrides", func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
retention: always
|
||||
diagnostics:
|
||||
retention: never
|
||||
`)
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
|
||||
t.Fatalf("effective diagnostics retention = %q, want workspace", cfg.Diagnostics.Retention)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func parseAndApplyConfig(t *testing.T, raw string) Config {
|
||||
t.Helper()
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
|
||||
@@ -2,17 +2,8 @@ package config
|
||||
|
||||
import "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
|
||||
const redactedSecret = "[REDACTED]"
|
||||
|
||||
func (c Config) Redacted() Config {
|
||||
redacted := cloneConfig(c)
|
||||
for id, profile := range redacted.LLMProfiles {
|
||||
if profile.APIKey != "" {
|
||||
profile.APIKey = redactedSecret
|
||||
}
|
||||
redacted.LLMProfiles[id] = profile
|
||||
}
|
||||
return redacted
|
||||
return cloneConfig(c)
|
||||
}
|
||||
|
||||
func (c Config) RedactedDiagnosticsPayload() any {
|
||||
@@ -21,10 +12,12 @@ func (c Config) RedactedDiagnosticsPayload() any {
|
||||
|
||||
func (e EffectiveConfig) RedactedDiagnosticsPayload() any {
|
||||
return EffectiveConfig{
|
||||
Config: e.Config.Redacted(),
|
||||
PipelineID: e.PipelineID,
|
||||
Only: append([]string(nil), e.Only...),
|
||||
ResolvedPipeline: cloneResolvedPipeline(e.ResolvedPipeline),
|
||||
Config: e.Config.Redacted(),
|
||||
PipelineID: e.PipelineID,
|
||||
Only: append([]string(nil), e.Only...),
|
||||
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), e.ReferenceOverrides...),
|
||||
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), e.ReferenceUnbinds...),
|
||||
ResolvedPipeline: cloneResolvedPipeline(e.ResolvedPipeline),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,7 +25,14 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
|
||||
out := in
|
||||
out.Input = cloneModuleBinding(in.Input)
|
||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
|
||||
out.Output = cloneModuleBinding(in.Output)
|
||||
if len(in.ValidatorChains) > 0 {
|
||||
out.ValidatorChains = make([]pipeline.ResolvedValidatorChain, len(in.ValidatorChains))
|
||||
for i, chain := range in.ValidatorChains {
|
||||
out.ValidatorChains[i] = cloneResolvedValidatorChain(chain)
|
||||
}
|
||||
}
|
||||
if len(in.ArtifactLanes) > 0 {
|
||||
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
|
||||
for i, lane := range in.ArtifactLanes {
|
||||
@@ -42,11 +42,28 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneResolvedValidatorChain(in pipeline.ResolvedValidatorChain) pipeline.ResolvedValidatorChain {
|
||||
out := in
|
||||
if len(in.Validators) > 0 {
|
||||
out.Validators = make([]pipeline.ResolvedValidator, len(in.Validators))
|
||||
for i, validator := range in.Validators {
|
||||
out.Validators[i] = pipeline.ResolvedValidator{
|
||||
Binding: cloneModuleBinding(validator.Binding),
|
||||
ExecutionClass: validator.ExecutionClass,
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.ResolvedArtifactLane {
|
||||
out := in
|
||||
out.Extract = cloneModuleBinding(in.Extract)
|
||||
out.Merge = cloneModuleBinding(in.Merge)
|
||||
out.Normalize = cloneModuleBinding(in.Normalize)
|
||||
out.ExtractReferences = pipeline.CloneReferenceTarget(in.ExtractReferences)
|
||||
out.MergeReferences = pipeline.CloneReferenceTarget(in.MergeReferences)
|
||||
out.NormalizeReferences = pipeline.CloneReferenceTarget(in.NormalizeReferences)
|
||||
if len(in.Validators) > 0 {
|
||||
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
|
||||
for i, binding := range in.Validators {
|
||||
|
||||
@@ -3,74 +3,92 @@ package config
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestRedactedConfigRemovesAPIKeyValues(t *testing.T) {
|
||||
func TestRedactedConfigCopiesScriptoriumConfig(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = LLMProfile{
|
||||
Provider: "openai-compatible",
|
||||
BaseURL: "https://example.invalid/v1",
|
||||
Model: "test-model",
|
||||
APIKey: "secret",
|
||||
APIKeyEnv: "NOTARIUS_TEST_API_KEY",
|
||||
TimeoutSeconds: 600,
|
||||
MaxRetries: 3,
|
||||
MaxConcurrency: 1,
|
||||
}
|
||||
cfg.LLMProfiles["other"] = LLMProfile{APIKey: "other-secret", Model: "other-model"}
|
||||
cfg.Scriptorium.ProfileDir = "./profiles"
|
||||
cfg.Workspace.Directory = "/var/lib/notarius"
|
||||
cfg.Workspace.Resume.Enabled = true
|
||||
|
||||
redacted := cfg.Redacted()
|
||||
|
||||
if redacted.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != redactedSecret {
|
||||
t.Fatalf("expected default API key redacted, got %+v", redacted.LLMProfiles[pipeline.DefaultLLMProfile])
|
||||
if redacted.Scriptorium.ProfileDir != "./profiles" {
|
||||
t.Fatalf("expected Scriptorium profile source preserved, got %+v", redacted.Scriptorium)
|
||||
}
|
||||
if redacted.LLMProfiles["other"].APIKey != redactedSecret {
|
||||
t.Fatalf("expected other API key redacted, got %+v", redacted.LLMProfiles["other"])
|
||||
}
|
||||
if redacted.LLMProfiles[pipeline.DefaultLLMProfile].Model != "test-model" {
|
||||
t.Fatalf("expected non-secret fields preserved, got %+v", redacted.LLMProfiles[pipeline.DefaultLLMProfile])
|
||||
}
|
||||
if cfg.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != "secret" {
|
||||
redacted.Scriptorium.ProfileDir = "./changed"
|
||||
if cfg.Scriptorium.ProfileDir != "./profiles" {
|
||||
t.Fatalf("redaction mutated original config")
|
||||
}
|
||||
if redacted.Workspace.Directory != "/var/lib/notarius" || !redacted.Workspace.Resume.Enabled {
|
||||
t.Fatalf("expected workspace config preserved, got %+v", redacted.Workspace)
|
||||
}
|
||||
redacted.Workspace.Directory = "/changed"
|
||||
if cfg.Workspace.Directory != "/var/lib/notarius" {
|
||||
t.Fatalf("redaction mutated original workspace config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigRedactedDiagnosticsPayloadRedactsAPIKeys(t *testing.T) {
|
||||
func TestConfigRedactedDiagnosticsPayloadCopiesConfig(t *testing.T) {
|
||||
cfg := Default()
|
||||
profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
|
||||
profile.APIKey = "secret"
|
||||
profile.Model = "test-model"
|
||||
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile
|
||||
cfg.Scriptorium.ProfileFile = "./profiles.yml"
|
||||
|
||||
payload, ok := cfg.RedactedDiagnosticsPayload().(Config)
|
||||
if !ok {
|
||||
t.Fatalf("expected Config payload, got %T", cfg.RedactedDiagnosticsPayload())
|
||||
}
|
||||
if payload.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != redactedSecret {
|
||||
t.Fatalf("expected API key redacted, got %+v", payload.LLMProfiles[pipeline.DefaultLLMProfile])
|
||||
}
|
||||
if payload.LLMProfiles[pipeline.DefaultLLMProfile].Model != "test-model" {
|
||||
t.Fatalf("expected non-secret fields preserved, got %+v", payload.LLMProfiles[pipeline.DefaultLLMProfile])
|
||||
}
|
||||
if cfg.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != "secret" {
|
||||
t.Fatalf("redacted diagnostics payload mutated original config")
|
||||
if payload.Scriptorium.ProfileFile != "./profiles.yml" {
|
||||
t.Fatalf("expected Scriptorium profile file preserved, got %+v", payload.Scriptorium)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T) {
|
||||
func TestEffectiveConfigRedactedDiagnosticsPayloadCopies(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
|
||||
profile.APIKey = "secret"
|
||||
cfg.LLMProfiles[pipeline.DefaultLLMProfile] = profile
|
||||
lane := cfg.Pipelines["example"].Artifacts["events"]
|
||||
lane.Extract.Options = map[string]any{"temperature": 0.2}
|
||||
lane.References = map[string]string{"roster": "./roster.yml"}
|
||||
lane.Extract.References = map[string]string{"glossary": "./glossary.md"}
|
||||
lane.Normalize.References = map[string]string{"notes": "./normalize.md"}
|
||||
cfg.Pipelines["example"].Artifacts["events"] = lane
|
||||
pipelineProfile := cfg.Pipelines["example"]
|
||||
pipelineProfile.Chunk.References = map[string]string{"scene_guide": "./scene.md"}
|
||||
cfg.Pipelines["example"] = pipelineProfile
|
||||
|
||||
effective, err := cfg.Resolve(ResolveInput{
|
||||
PipelineID: "example",
|
||||
Only: []string{"events"},
|
||||
Catalog: fakeCatalog(t),
|
||||
Catalog: fakeCatalog(t,
|
||||
pipeline.ModuleSpec{
|
||||
Key: "generic",
|
||||
Stage: pipeline.StageChunk,
|
||||
Requires: []string{"source"},
|
||||
Provides: []string{"chunks"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "scene_guide"},
|
||||
},
|
||||
},
|
||||
pipeline.ModuleSpec{
|
||||
Key: "fake/extract",
|
||||
Stage: pipeline.StageExtract,
|
||||
Requires: []string{"chunks"},
|
||||
Provides: []string{"artifact"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "glossary"},
|
||||
{Name: "roster"},
|
||||
},
|
||||
},
|
||||
pipeline.ModuleSpec{
|
||||
Key: "noop",
|
||||
Stage: pipeline.StageNormalize,
|
||||
Requires: []string{"merged"},
|
||||
Provides: []string{"normalized"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "notes"},
|
||||
},
|
||||
},
|
||||
),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Resolve: %v", err)
|
||||
@@ -80,12 +98,6 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
|
||||
if !ok {
|
||||
t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedDiagnosticsPayload())
|
||||
}
|
||||
if payload.Config.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != redactedSecret {
|
||||
t.Fatalf("expected nested API key redacted, got %+v", payload.Config.LLMProfiles[pipeline.DefaultLLMProfile])
|
||||
}
|
||||
if cfg.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != "secret" {
|
||||
t.Fatalf("redacted diagnostics payload mutated source config")
|
||||
}
|
||||
if payload.PipelineID != effective.PipelineID || payload.ResolvedPipeline.Digest != effective.ResolvedPipeline.Digest {
|
||||
t.Fatalf("expected pipeline metadata preserved, got %+v", payload)
|
||||
}
|
||||
@@ -98,4 +110,52 @@ func TestEffectiveConfigRedactedDiagnosticsPayloadRedactsAndCopies(t *testing.T)
|
||||
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.Options["temperature"] != 0.2 {
|
||||
t.Fatalf("expected resolved pipeline options to be copied")
|
||||
}
|
||||
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings[0].Source = "./changed.yml"
|
||||
if referenceBindingSource(effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.Bindings, "roster") != "./roster.yml" {
|
||||
t.Fatalf("expected resolved pipeline references to be copied")
|
||||
}
|
||||
payload.ResolvedPipeline.Chunk.References["scene_guide"] = "./changed-scene.md"
|
||||
if effective.ResolvedPipeline.Chunk.References["scene_guide"] != "./scene.md" {
|
||||
t.Fatalf("expected chunk references to be copied")
|
||||
}
|
||||
payload.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] = "./changed-glossary.md"
|
||||
if effective.ResolvedPipeline.ArtifactLanes[0].Extract.References["glossary"] != "./glossary.md" {
|
||||
t.Fatalf("expected extract references to be copied")
|
||||
}
|
||||
payload.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] = "./changed-normalize.md"
|
||||
if effective.ResolvedPipeline.ArtifactLanes[0].Normalize.References["notes"] != "./normalize.md" {
|
||||
t.Fatalf("expected normalize references to be copied")
|
||||
}
|
||||
|
||||
effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = contracts.ReferenceSet{
|
||||
Slots: map[string]contracts.ResolvedReferenceSlot{
|
||||
"roster": {
|
||||
Slot: contracts.ReferenceSlot{Name: "roster"},
|
||||
Items: []contracts.ReferenceItem{
|
||||
{
|
||||
SlotName: "roster",
|
||||
Content: []byte("reference content"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
payload, ok = effective.RedactedDiagnosticsPayload().(EffectiveConfig)
|
||||
if !ok {
|
||||
t.Fatalf("expected EffectiveConfig payload, got %T", effective.RedactedDiagnosticsPayload())
|
||||
}
|
||||
payload.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content[0] = 'X'
|
||||
got := effective.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Content
|
||||
if string(got) != "reference content" {
|
||||
t.Fatalf("expected materialized reference content to be copied, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func referenceBindingSource(bindings []pipeline.ReferenceBinding, slotName string) string {
|
||||
for _, binding := range bindings {
|
||||
if binding.SlotName == slotName {
|
||||
return binding.Source
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -8,10 +8,11 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const providerOpenAICompatible = "openai-compatible"
|
||||
|
||||
func (c Config) Validate() error {
|
||||
if err := validateLLMProfiles(c.LLMProfiles); err != nil {
|
||||
if err := validateScriptorium(c.Scriptorium); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateWorkspace(c.Workspace); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateDiagnostics(c.Diagnostics); err != nil {
|
||||
@@ -20,43 +21,22 @@ func (c Config) Validate() error {
|
||||
if c.Concurrency.TotalLLM <= 0 {
|
||||
return fmt.Errorf("total LLM concurrency must be greater than zero")
|
||||
}
|
||||
return validatePipelineProfiles(c.Pipelines, c.LLMProfiles)
|
||||
return validatePipelineProfiles(c.Pipelines)
|
||||
}
|
||||
|
||||
func (c Config) LLMProfile(id string) (LLMProfile, bool) {
|
||||
trimmedID := strings.TrimSpace(id)
|
||||
for rawID, profile := range c.LLMProfiles {
|
||||
if strings.TrimSpace(rawID) == trimmedID {
|
||||
return profile, true
|
||||
}
|
||||
func validateScriptorium(cfg ScriptoriumConfig) error {
|
||||
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
|
||||
return fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive")
|
||||
}
|
||||
return LLMProfile{}, false
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateLLMProfiles(profiles map[string]LLMProfile) error {
|
||||
seen := make(map[string]struct{}, len(profiles))
|
||||
for rawID, profile := range profiles {
|
||||
id := strings.TrimSpace(rawID)
|
||||
if id == "" {
|
||||
return fmt.Errorf("LLM profile id must not be empty")
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
return fmt.Errorf("LLM profile id %q is duplicated after trimming", id)
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
|
||||
provider := strings.TrimSpace(profile.Provider)
|
||||
if provider != "" && provider != providerOpenAICompatible {
|
||||
return fmt.Errorf("LLM profile %q provider %q is not supported", id, provider)
|
||||
}
|
||||
if profile.TimeoutSeconds < 0 {
|
||||
return fmt.Errorf("LLM profile %q timeout seconds must not be negative", id)
|
||||
}
|
||||
if profile.MaxRetries < 0 {
|
||||
return fmt.Errorf("LLM profile %q max retries must not be negative", id)
|
||||
}
|
||||
if profile.MaxConcurrency < 0 {
|
||||
return fmt.Errorf("LLM profile %q max concurrency must not be negative", id)
|
||||
func validateWorkspace(cfg WorkspaceConfig) error {
|
||||
if cfg.Diagnostics.retentionSet {
|
||||
switch cfg.Diagnostics.Retention {
|
||||
case "", diagnostics.RetentionAuto, diagnostics.RetentionAlways, diagnostics.RetentionNever:
|
||||
default:
|
||||
return fmt.Errorf("workspace diagnostics retention %q is not supported", cfg.Diagnostics.Retention)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -74,7 +54,7 @@ func validateDiagnostics(cfg DiagnosticsConfig) error {
|
||||
}
|
||||
}
|
||||
|
||||
func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmProfiles map[string]LLMProfile) error {
|
||||
func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) error {
|
||||
seen := make(map[string]struct{}, len(profiles))
|
||||
for rawID, profile := range profiles {
|
||||
id := strings.TrimSpace(rawID)
|
||||
@@ -89,13 +69,16 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
|
||||
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
|
||||
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
|
||||
}
|
||||
if err := validateBindingLLMProfile(id, "", "input", profile.Input, llmProfiles); err != nil {
|
||||
if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateBindingLLMProfile(id, "", "chunk", profile.Chunk, llmProfiles); err != nil {
|
||||
if err := validateBinding(id, "", "chunk", profile.Chunk, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateBindingLLMProfile(id, "", "output", profile.Output, llmProfiles); err != nil {
|
||||
if err := validateBinding(id, "", "output", profile.Output, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateReferenceMap(id, "", profile.References); err != nil {
|
||||
return err
|
||||
}
|
||||
for rawLaneID, lane := range profile.Artifacts {
|
||||
@@ -103,51 +86,133 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile, llmP
|
||||
if laneID == "" {
|
||||
return fmt.Errorf("pipeline %q artifact lane id must not be empty", id)
|
||||
}
|
||||
if err := validateBindingLLMProfile(id, laneID, "extract", lane.Extract, llmProfiles); err != nil {
|
||||
if err := validateReferenceMap(id, laneID, lane.References); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateBindingLLMProfile(id, laneID, "merge", lane.Merge, llmProfiles); err != nil {
|
||||
if err := validateBinding(id, laneID, "extract", lane.Extract, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateBindingLLMProfile(id, laneID, "normalize", lane.Normalize, llmProfiles); err != nil {
|
||||
if err := validateBinding(id, laneID, "merge", lane.Merge, true); err != nil {
|
||||
return err
|
||||
}
|
||||
for i, validator := range lane.Validators {
|
||||
if err := validateBindingLLMProfile(id, laneID, fmt.Sprintf("validator[%d]", i), validator, llmProfiles); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateBinding(id, laneID, "normalize", lane.Normalize, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(lane.Validators) > 0 {
|
||||
return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", id, laneID)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateBinding(
|
||||
pipelineID string,
|
||||
laneID string,
|
||||
slot string,
|
||||
binding pipeline.ModuleBinding,
|
||||
referencesAllowed bool,
|
||||
) error {
|
||||
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil {
|
||||
return err
|
||||
}
|
||||
if binding.Retries < 0 {
|
||||
if laneID != "" {
|
||||
return fmt.Errorf("pipeline %q lane %q %s retries must be greater than or equal to zero", pipelineID, laneID, slot)
|
||||
}
|
||||
return fmt.Errorf("pipeline %q %s retries must be greater than or equal to zero", pipelineID, slot)
|
||||
}
|
||||
if err := validateValidatorOverride(pipelineID, laneID, slot, binding.Validators); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(binding.References) == 0 {
|
||||
return nil
|
||||
}
|
||||
if !referencesAllowed {
|
||||
if laneID != "" {
|
||||
return fmt.Errorf("pipeline %q lane %q %s references are not supported", pipelineID, laneID, slot)
|
||||
}
|
||||
return fmt.Errorf("pipeline %q %s references are not supported", pipelineID, slot)
|
||||
}
|
||||
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
|
||||
}
|
||||
|
||||
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
|
||||
if !override.Set {
|
||||
return nil
|
||||
}
|
||||
switch slot {
|
||||
case "chunk", "extract", "merge", "normalize":
|
||||
default:
|
||||
return fmt.Errorf("%s validators are not supported", referenceContext(pipelineID, laneID, slot))
|
||||
}
|
||||
for i, validator := range override.Validators {
|
||||
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
|
||||
if strings.TrimSpace(validator.Module) == "" {
|
||||
return fmt.Errorf("%s module must not be empty", context)
|
||||
}
|
||||
if len(validator.References) > 0 {
|
||||
return fmt.Errorf("%s references are not supported", context)
|
||||
}
|
||||
if validator.Validators.Set {
|
||||
return fmt.Errorf("%s nested validators are not supported", context)
|
||||
}
|
||||
if validator.Retries != 0 {
|
||||
return fmt.Errorf("%s retries are not supported", context)
|
||||
}
|
||||
if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" {
|
||||
return fmt.Errorf("%s llm_profile must not be empty when set", context)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
|
||||
return validateReferenceMapForContext(pipelineID, laneID, "", references)
|
||||
}
|
||||
|
||||
func validateReferenceMapForContext(pipelineID string, laneID string, slot string, references map[string]string) error {
|
||||
seen := make(map[string]struct{}, len(references))
|
||||
for rawSlotName, rawSource := range references {
|
||||
slotName := strings.TrimSpace(rawSlotName)
|
||||
if slotName == "" {
|
||||
return fmt.Errorf("%s reference slot name must not be empty", referenceContext(pipelineID, laneID, slot))
|
||||
}
|
||||
if _, ok := seen[slotName]; ok {
|
||||
return fmt.Errorf("%s reference slot %q is duplicated after trimming", referenceContext(pipelineID, laneID, slot), slotName)
|
||||
}
|
||||
seen[slotName] = struct{}{}
|
||||
if strings.TrimSpace(rawSource) == "" {
|
||||
return fmt.Errorf("%s reference slot %q source must not be empty", referenceContext(pipelineID, laneID, slot), slotName)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func referenceContext(pipelineID string, laneID string, slot string) string {
|
||||
if laneID != "" && slot != "" {
|
||||
return fmt.Sprintf("pipeline %q lane %q %s", pipelineID, laneID, slot)
|
||||
}
|
||||
if laneID != "" {
|
||||
return fmt.Sprintf("pipeline %q lane %q", pipelineID, laneID)
|
||||
}
|
||||
if slot != "" {
|
||||
return fmt.Sprintf("pipeline %q %s", pipelineID, slot)
|
||||
}
|
||||
return fmt.Sprintf("pipeline %q", pipelineID)
|
||||
}
|
||||
|
||||
func validateBindingLLMProfile(
|
||||
pipelineID string,
|
||||
laneID string,
|
||||
slot string,
|
||||
binding pipeline.ModuleBinding,
|
||||
profiles map[string]LLMProfile,
|
||||
) error {
|
||||
profileID := strings.TrimSpace(binding.LLMProfile)
|
||||
if profileID == "" {
|
||||
profileID = pipeline.DefaultLLMProfile
|
||||
}
|
||||
if hasLLMProfile(profiles, profileID) {
|
||||
return nil
|
||||
}
|
||||
if laneID != "" {
|
||||
return fmt.Errorf("pipeline %q lane %q %s references unknown LLM profile %q", pipelineID, laneID, slot, profileID)
|
||||
}
|
||||
return fmt.Errorf("pipeline %q %s references unknown LLM profile %q", pipelineID, slot, profileID)
|
||||
}
|
||||
|
||||
func hasLLMProfile(profiles map[string]LLMProfile, profileID string) bool {
|
||||
profileID = strings.TrimSpace(profileID)
|
||||
for rawID := range profiles {
|
||||
if strings.TrimSpace(rawID) == profileID {
|
||||
return true
|
||||
if binding.LLMProfile != "" && strings.TrimSpace(binding.LLMProfile) == "" {
|
||||
if laneID != "" {
|
||||
return fmt.Errorf("pipeline %q lane %q %s llm_profile must not be empty when set", pipelineID, laneID, slot)
|
||||
}
|
||||
return fmt.Errorf("pipeline %q %s llm_profile must not be empty when set", pipelineID, slot)
|
||||
}
|
||||
return false
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -17,27 +17,26 @@ func TestValidateSuccessForValidConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsUnknownLLMProfileReferencedByBinding(t *testing.T) {
|
||||
func TestValidateAllowsExplicitScriptoriumProfileIDOnBinding(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
lane := cfg.Pipelines["example"].Artifacts["events"]
|
||||
lane.Extract.LLMProfile = "missing"
|
||||
lane.Extract.LLMProfile = "scriptorium-profile"
|
||||
cfg.Pipelines["example"].Artifacts["events"] = lane
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown LLM profile") || !strings.Contains(err.Error(), "events") {
|
||||
t.Fatalf("expected unknown LLM profile error with lane context, got %v", err)
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsInvalidProvider(t *testing.T) {
|
||||
func TestValidateRejectsWhitespaceOnlyExplicitLLMProfile(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.Provider = "unsupported"
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
lane := cfg.Pipelines["example"].Artifacts["events"]
|
||||
lane.Extract.LLMProfile = " "
|
||||
cfg.Pipelines["example"].Artifacts["events"] = lane
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil || !strings.Contains(err.Error(), "provider") {
|
||||
t.Fatalf("expected provider error, got %v", err)
|
||||
if err == nil || !strings.Contains(err.Error(), "llm_profile") || !strings.Contains(err.Error(), "events") {
|
||||
t.Fatalf("expected llm_profile error with lane context, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,34 +55,16 @@ func TestValidateRejectsInvalidNumericFields(t *testing.T) {
|
||||
want: "total LLM concurrency",
|
||||
},
|
||||
{
|
||||
name: "timeout",
|
||||
name: "negative retries",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.TimeoutSeconds = -1
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
profile := cfg.Pipelines["example"]
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Merge.Retries = -1
|
||||
profile.Artifacts["events"] = lane
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: "timeout",
|
||||
},
|
||||
{
|
||||
name: "max retries",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.MaxRetries = -1
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: "max retries",
|
||||
},
|
||||
{
|
||||
name: "max concurrency",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.MaxConcurrency = -1
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: "max concurrency",
|
||||
want: "retries",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -97,12 +78,14 @@ func TestValidateRejectsInvalidNumericFields(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAllowsPartialLLMProfileNumericConfig(t *testing.T) {
|
||||
func TestValidateRejectsMutuallyExclusiveScriptoriumProfileSources(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
cfg.LLMProfiles["retry-only"] = LLMProfile{MaxRetries: 3}
|
||||
cfg.Scriptorium.ProfileDir = "./profiles"
|
||||
cfg.Scriptorium.ProfileFile = "./profiles.yml"
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate: %v", err)
|
||||
err := cfg.Validate()
|
||||
if err == nil || !strings.Contains(err.Error(), "mutually exclusive") {
|
||||
t.Fatalf("expected Scriptorium source conflict, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,20 +99,201 @@ func TestValidateRejectsInvalidDiagnosticsRetention(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsInvalidWorkspaceDiagnosticsRetention(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
cfg.Workspace.Diagnostics.Retention = diagnostics.RetentionMode("sometimes")
|
||||
cfg.Workspace.Diagnostics.retentionSet = true
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil || !strings.Contains(err.Error(), "workspace diagnostics retention") {
|
||||
t.Fatalf("expected workspace retention error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(Config) Config
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "empty chunk slot",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.Pipelines["example"]
|
||||
profile.Chunk.References = map[string]string{" ": "./roster.yml"}
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: []string{"example", "chunk", "reference slot", "empty"},
|
||||
},
|
||||
{
|
||||
name: "empty chunk source",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.Pipelines["example"]
|
||||
profile.Chunk.References = map[string]string{"roster": " "}
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: []string{"example", "chunk", "roster", "source", "empty"},
|
||||
},
|
||||
{
|
||||
name: "empty extract slot",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.Pipelines["example"]
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.References = map[string]string{" ": "./roster.yml"}
|
||||
profile.Artifacts["events"] = lane
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: []string{"example", "events", "extract", "reference slot", "empty"},
|
||||
},
|
||||
{
|
||||
name: "empty extract source",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.Pipelines["example"]
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.References = map[string]string{"roster": " "}
|
||||
profile.Artifacts["events"] = lane
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: []string{"example", "events", "extract", "roster", "source", "empty"},
|
||||
},
|
||||
{
|
||||
name: "empty normalize slot",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.Pipelines["example"]
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Normalize.References = map[string]string{" ": "./roster.yml"}
|
||||
profile.Artifacts["events"] = lane
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: []string{"example", "events", "normalize", "reference slot", "empty"},
|
||||
},
|
||||
{
|
||||
name: "empty normalize source",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.Pipelines["example"]
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Normalize.References = map[string]string{"roster": " "}
|
||||
profile.Artifacts["events"] = lane
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: []string{"example", "events", "normalize", "roster", "source", "empty"},
|
||||
},
|
||||
{
|
||||
name: "empty pipeline slot",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.Pipelines["example"]
|
||||
profile.References = map[string]string{" ": "./roster.yml"}
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: []string{"example", "reference slot", "empty"},
|
||||
},
|
||||
{
|
||||
name: "empty pipeline source",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.Pipelines["example"]
|
||||
profile.References = map[string]string{"roster": " "}
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: []string{"example", "roster", "source", "empty"},
|
||||
},
|
||||
{
|
||||
name: "empty lane slot",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.Pipelines["example"]
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.References = map[string]string{" ": "./roster.yml"}
|
||||
profile.Artifacts["events"] = lane
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: []string{"example", "events", "reference slot", "empty"},
|
||||
},
|
||||
{
|
||||
name: "empty lane source",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.Pipelines["example"]
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.References = map[string]string{"roster": " "}
|
||||
profile.Artifacts["events"] = lane
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: []string{"example", "events", "roster", "source", "empty"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := tc.mutate(validConfig()).Validate()
|
||||
if err == nil {
|
||||
t.Fatal("Validate() error = nil, want error")
|
||||
}
|
||||
for _, want := range tc.want {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsReferencesOnUnsupportedBindings(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(Config) Config
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "input",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.Pipelines["example"]
|
||||
profile.Input.References = map[string]string{"roster": "./roster.yml"}
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: []string{"example", "input", "references", "not supported"},
|
||||
},
|
||||
{
|
||||
name: "output",
|
||||
mutate: func(cfg Config) Config {
|
||||
profile := cfg.Pipelines["example"]
|
||||
profile.Output.References = map[string]string{"roster": "./roster.yml"}
|
||||
cfg.Pipelines["example"] = profile
|
||||
return cfg
|
||||
},
|
||||
want: []string{"example", "output", "references", "not supported"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := tc.mutate(validConfig()).Validate()
|
||||
if err == nil {
|
||||
t.Fatal("Validate() error = nil, want error")
|
||||
}
|
||||
for _, want := range tc.want {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsEmptyIDs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(Config) Config
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "LLM profile",
|
||||
mutate: func(cfg Config) Config {
|
||||
cfg.LLMProfiles[" "] = LLMProfile{}
|
||||
return cfg
|
||||
},
|
||||
want: "LLM profile id",
|
||||
},
|
||||
{
|
||||
name: "pipeline",
|
||||
mutate: func(cfg Config) Config {
|
||||
@@ -156,14 +320,6 @@ func TestValidateRejectsIDsDuplicatedAfterTrimming(t *testing.T) {
|
||||
mutate func(Config) Config
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "LLM profile",
|
||||
mutate: func(cfg Config) Config {
|
||||
cfg.LLMProfiles[" default "] = cfg.LLMProfiles["default"]
|
||||
return cfg
|
||||
},
|
||||
want: "duplicated",
|
||||
},
|
||||
{
|
||||
name: "pipeline",
|
||||
mutate: func(cfg Config) Config {
|
||||
@@ -184,31 +340,101 @@ func TestValidateRejectsIDsDuplicatedAfterTrimming(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUsesTrimmedLLMProfileIDs(t *testing.T) {
|
||||
func TestValidateRejectsConfiguredValidators(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
cfg.LLMProfiles[" default "] = cfg.LLMProfiles["default"]
|
||||
delete(cfg.LLMProfiles, "default")
|
||||
profile := cfg.Pipelines["example"]
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Validators = []pipeline.ModuleBinding{pipeline.Binding("fake/validator")}
|
||||
profile.Artifacts["events"] = lane
|
||||
cfg.Pipelines["example"] = profile
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Fatal("Validate() error = nil, want configured validators error")
|
||||
}
|
||||
for _, want := range []string{"example", "events", "validators", "extract.validators", "merge.validators", "normalize.validators"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateAcceptsStageLocalValidatorOverrides(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
profile := cfg.Pipelines["example"]
|
||||
profile.Chunk.Validators = pipeline.ValidatorOverride{Set: true}
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []pipeline.ModuleBinding{
|
||||
pipeline.Binding("fake/validator"),
|
||||
{Module: "fake/llm-validator", LLMProfile: "careful", Options: map[string]any{"threshold": 0.7}},
|
||||
},
|
||||
}
|
||||
lane.Merge.Validators = pipeline.ValidatorOverride{Set: true}
|
||||
profile.Artifacts["events"] = lane
|
||||
cfg.Pipelines["example"] = profile
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatalf("Validate: %v", err)
|
||||
t.Fatalf("Validate() error = %v, want nil", err)
|
||||
}
|
||||
if _, ok := cfg.LLMProfile("default"); !ok {
|
||||
t.Fatalf("expected trimmed LLM profile lookup to succeed")
|
||||
}
|
||||
|
||||
func TestValidateRejectsInvalidValidatorBindings(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
binding pipeline.ModuleBinding
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty module",
|
||||
binding: pipeline.ModuleBinding{},
|
||||
want: "module must not be empty",
|
||||
},
|
||||
{
|
||||
name: "references",
|
||||
binding: pipeline.ModuleBinding{Module: "fake/validator", References: map[string]string{"roster": "./roster.txt"}},
|
||||
want: "references are not supported",
|
||||
},
|
||||
{
|
||||
name: "nested validators",
|
||||
binding: pipeline.ModuleBinding{Module: "fake/validator", Validators: pipeline.ValidatorOverride{Set: true}},
|
||||
want: "nested validators are not supported",
|
||||
},
|
||||
{
|
||||
name: "retries",
|
||||
binding: pipeline.ModuleBinding{Module: "fake/validator", Retries: 1},
|
||||
want: "retries are not supported",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
profile := cfg.Pipelines["example"]
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []pipeline.ModuleBinding{test.binding},
|
||||
}
|
||||
profile.Artifacts["events"] = lane
|
||||
cfg.Pipelines["example"] = profile
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("Validate() error = %v, want %q", err, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func validConfig() Config {
|
||||
cfg := Default()
|
||||
profile := cfg.LLMProfiles["default"]
|
||||
profile.BaseURL = "https://example.invalid/v1"
|
||||
profile.Model = "test-model"
|
||||
cfg.LLMProfiles["default"] = profile
|
||||
cfg.Pipelines["example"] = pipeline.PipelineProfile{
|
||||
Input: pipeline.Binding("fake/input"),
|
||||
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||
"events": {
|
||||
Extract: pipeline.Binding("fake/extract"),
|
||||
Validators: []pipeline.ModuleBinding{pipeline.Binding("fake/validator")},
|
||||
Extract: pipeline.Binding("fake/extract"),
|
||||
},
|
||||
"notes": {
|
||||
Extract: pipeline.Binding("fake/extract"),
|
||||
@@ -256,6 +482,12 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
|
||||
Requires: []string{"normalized"},
|
||||
Provides: []string{"validated"},
|
||||
},
|
||||
"fake/llm-validator": {
|
||||
Key: "fake/llm-validator",
|
||||
Stage: pipeline.StageValidate,
|
||||
Requires: []string{"normalized"},
|
||||
Provides: []string{"validated"},
|
||||
},
|
||||
"json": {
|
||||
Key: "json",
|
||||
Stage: pipeline.StageOutput,
|
||||
@@ -280,16 +512,18 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
|
||||
mustRegisterMerger(t, mergers, specs["appendorder"])
|
||||
mustRegisterNormalizer(t, normalizers, specs["noop"])
|
||||
mustRegisterValidator(t, validators, specs["fake/validator"])
|
||||
mustRegisterValidator(t, validators, specs["fake/llm-validator"])
|
||||
mustRegisterOutput(t, outputs, specs["json"])
|
||||
|
||||
return pipeline.ModuleCatalog{
|
||||
Inputs: inputs,
|
||||
Chunkers: chunkers,
|
||||
Extractors: extractors,
|
||||
Mergers: mergers,
|
||||
Normalizers: normalizers,
|
||||
Validators: validators,
|
||||
Outputs: outputs,
|
||||
Inputs: inputs,
|
||||
Chunkers: chunkers,
|
||||
Extractors: extractors,
|
||||
Mergers: mergers,
|
||||
Normalizers: normalizers,
|
||||
Validators: validators,
|
||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: outputs,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -330,7 +564,12 @@ func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry,
|
||||
|
||||
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterWithSpec(spec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
|
||||
executionClass := contracts.ExecutionClassDeterministic
|
||||
if spec.Key == "fake/llm-validator" {
|
||||
executionClass = contracts.ExecutionClassLLMBacked
|
||||
}
|
||||
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: executionClass}
|
||||
if err := registry.RegisterWithSpec(validatorSpec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
|
||||
t.Fatalf("register validator: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ const (
|
||||
ArtifactInvocationMetadata = "invocation.json"
|
||||
ArtifactEffectiveConfig = "effective-config.json"
|
||||
ArtifactResolvedPipeline = "resolved-pipeline.json"
|
||||
ArtifactResolvedReferences = "resolved-references.json"
|
||||
ArtifactCheckpointEvents = "checkpoint-events.json"
|
||||
ArtifactSourceDocument = "source-document.json"
|
||||
ArtifactRunManifest = "run-manifest.json"
|
||||
ArtifactRunReport = "run-report.json"
|
||||
|
||||
@@ -7,6 +7,7 @@ func TestArtifactNamesUseExtractionOrientedNames(t *testing.T) {
|
||||
ArtifactInvocationMetadata,
|
||||
ArtifactEffectiveConfig,
|
||||
ArtifactResolvedPipeline,
|
||||
ArtifactResolvedReferences,
|
||||
ArtifactSourceDocument,
|
||||
ArtifactRunManifest,
|
||||
ArtifactRunReport,
|
||||
|
||||
@@ -51,6 +51,7 @@ type InvocationMetadata struct {
|
||||
Operation string `json:"operation"`
|
||||
PipelineID string `json:"pipeline_id,omitempty"`
|
||||
PipelineDigest string `json:"pipeline_digest,omitempty"`
|
||||
Resume bool `json:"resume,omitempty"`
|
||||
InputPath string `json:"input_path,omitempty"`
|
||||
ConfigPath string `json:"config_path,omitempty"`
|
||||
ConfigSource string `json:"config_source,omitempty"`
|
||||
@@ -149,6 +150,14 @@ func (r *RunDirectory) WriteResolvedPipeline(payload any) error {
|
||||
return r.WriteJSONArtifact(ArtifactResolvedPipeline, payload)
|
||||
}
|
||||
|
||||
func (r *RunDirectory) WriteResolvedReferences(payload any) error {
|
||||
return r.WriteJSONArtifact(ArtifactResolvedReferences, payload)
|
||||
}
|
||||
|
||||
func (r *RunDirectory) WriteCheckpointEvents(payload any) error {
|
||||
return r.WriteJSONArtifact(ArtifactCheckpointEvents, payload)
|
||||
}
|
||||
|
||||
func (r *RunDirectory) WriteSourceDocument(payload any) error {
|
||||
return r.WriteJSONArtifact(ArtifactSourceDocument, payload)
|
||||
}
|
||||
|
||||
@@ -191,6 +191,9 @@ func TestWriteTypedArtifacts(t *testing.T) {
|
||||
if err := runDir.WriteResolvedPipeline(map[string]any{"pipeline": "test"}); err != nil {
|
||||
t.Fatalf("WriteResolvedPipeline: %v", err)
|
||||
}
|
||||
if err := runDir.WriteResolvedReferences([]artifacts.ReferenceProvenance{{LaneID: "events", SlotName: "roster"}}); err != nil {
|
||||
t.Fatalf("WriteResolvedReferences: %v", err)
|
||||
}
|
||||
if err := runDir.WriteSourceDocument(map[string]any{"source_id": "source-1"}); err != nil {
|
||||
t.Fatalf("WriteSourceDocument: %v", err)
|
||||
}
|
||||
@@ -207,6 +210,7 @@ func TestWriteTypedArtifacts(t *testing.T) {
|
||||
for _, name := range []string{
|
||||
ArtifactEffectiveConfig,
|
||||
ArtifactResolvedPipeline,
|
||||
ArtifactResolvedReferences,
|
||||
ArtifactSourceDocument,
|
||||
ArtifactRunManifest,
|
||||
ArtifactRunReport,
|
||||
|
||||
@@ -10,7 +10,7 @@ type SourceDocument struct {
|
||||
}
|
||||
|
||||
type SourceUnit struct {
|
||||
ID string `json:"id"`
|
||||
ID int `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Text string `json:"text"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
@@ -18,6 +18,6 @@ type SourceUnit struct {
|
||||
|
||||
type SourceRef struct {
|
||||
SourceID string `json:"source_id"`
|
||||
StartUnitID string `json:"start_unit_id"`
|
||||
EndUnitID string `json:"end_unit_id"`
|
||||
StartUnitID int `json:"start_unit_id"`
|
||||
EndUnitID int `json:"end_unit_id"`
|
||||
}
|
||||
|
||||
@@ -96,13 +96,8 @@ func TestValidateDocumentMissingUnitFields(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "id",
|
||||
mutate: func(doc *SourceDocument) { doc.Units[1].ID = "" },
|
||||
wantErr: "source unit[1].id must not be empty",
|
||||
},
|
||||
{
|
||||
name: "id surrounding whitespace",
|
||||
mutate: func(doc *SourceDocument) { doc.Units[1].ID = " u2 " },
|
||||
wantErr: "source unit[1].id \" u2 \" must not contain leading or trailing whitespace",
|
||||
mutate: func(doc *SourceDocument) { doc.Units[1].ID = 0 },
|
||||
wantErr: "source unit[1].id must be positive",
|
||||
},
|
||||
{
|
||||
name: "kind",
|
||||
@@ -135,14 +130,14 @@ func TestValidateDocumentMissingUnitFields(t *testing.T) {
|
||||
|
||||
func TestValidateDocumentDuplicateUnitIDs(t *testing.T) {
|
||||
doc := validDocument()
|
||||
doc.Units[1].ID = "u1"
|
||||
doc.Units[1].ID = 1
|
||||
|
||||
err := ValidateDocument(doc)
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("ValidateDocument() error = nil, want error")
|
||||
}
|
||||
if err.Error() != "source unit id \"u1\" is duplicated" {
|
||||
if err.Error() != "source unit id 1 is duplicated" {
|
||||
t.Fatalf("ValidateDocument() error = %q", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -151,8 +146,8 @@ func TestValidateRefValid(t *testing.T) {
|
||||
doc := validDocument()
|
||||
ref := SourceRef{
|
||||
SourceID: "source-1",
|
||||
StartUnitID: "u1",
|
||||
EndUnitID: "u2",
|
||||
StartUnitID: 1,
|
||||
EndUnitID: 2,
|
||||
}
|
||||
|
||||
if err := ValidateRef(doc, ref); err != nil {
|
||||
@@ -164,8 +159,8 @@ func TestValidateRefSourceIDMismatch(t *testing.T) {
|
||||
doc := validDocument()
|
||||
ref := SourceRef{
|
||||
SourceID: "source-2",
|
||||
StartUnitID: "u1",
|
||||
EndUnitID: "u2",
|
||||
StartUnitID: 1,
|
||||
EndUnitID: 2,
|
||||
}
|
||||
|
||||
err := ValidateRef(doc, ref)
|
||||
@@ -186,43 +181,33 @@ func TestValidateRefMissingUnitIDs(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "missing source id",
|
||||
ref: SourceRef{StartUnitID: "u1", EndUnitID: "u2"},
|
||||
ref: SourceRef{StartUnitID: 1, EndUnitID: 2},
|
||||
wantErr: "source ref source_id must not be empty",
|
||||
},
|
||||
{
|
||||
name: "source id surrounding whitespace",
|
||||
ref: SourceRef{SourceID: " source-1 ", StartUnitID: "u1", EndUnitID: "u2"},
|
||||
ref: SourceRef{SourceID: " source-1 ", StartUnitID: 1, EndUnitID: 2},
|
||||
wantErr: "source ref source_id \" source-1 \" must not contain leading or trailing whitespace",
|
||||
},
|
||||
{
|
||||
name: "missing start id",
|
||||
ref: SourceRef{SourceID: "source-1", EndUnitID: "u2"},
|
||||
wantErr: "source ref start_unit_id must not be empty",
|
||||
},
|
||||
{
|
||||
name: "start id surrounding whitespace",
|
||||
ref: SourceRef{SourceID: "source-1", StartUnitID: " u1 ", EndUnitID: "u2"},
|
||||
wantErr: "source ref start_unit_id \" u1 \" must not contain leading or trailing whitespace",
|
||||
ref: SourceRef{SourceID: "source-1", EndUnitID: 2},
|
||||
wantErr: "source ref start_unit_id must be positive",
|
||||
},
|
||||
{
|
||||
name: "missing end id",
|
||||
ref: SourceRef{SourceID: "source-1", StartUnitID: "u1"},
|
||||
wantErr: "source ref end_unit_id must not be empty",
|
||||
},
|
||||
{
|
||||
name: "end id surrounding whitespace",
|
||||
ref: SourceRef{SourceID: "source-1", StartUnitID: "u1", EndUnitID: " u2 "},
|
||||
wantErr: "source ref end_unit_id \" u2 \" must not contain leading or trailing whitespace",
|
||||
ref: SourceRef{SourceID: "source-1", StartUnitID: 1},
|
||||
wantErr: "source ref end_unit_id must be positive",
|
||||
},
|
||||
{
|
||||
name: "unknown start id",
|
||||
ref: SourceRef{SourceID: "source-1", StartUnitID: "u9", EndUnitID: "u2"},
|
||||
wantErr: "source ref start_unit_id \"u9\" was not found",
|
||||
ref: SourceRef{SourceID: "source-1", StartUnitID: 9, EndUnitID: 2},
|
||||
wantErr: "source ref start_unit_id 9 was not found",
|
||||
},
|
||||
{
|
||||
name: "unknown end id",
|
||||
ref: SourceRef{SourceID: "source-1", StartUnitID: "u1", EndUnitID: "u9"},
|
||||
wantErr: "source ref end_unit_id \"u9\" was not found",
|
||||
ref: SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 9},
|
||||
wantErr: "source ref end_unit_id 9 was not found",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -244,8 +229,8 @@ func TestValidateRefReversedUnitOrder(t *testing.T) {
|
||||
doc := validDocument()
|
||||
ref := SourceRef{
|
||||
SourceID: "source-1",
|
||||
StartUnitID: "u2",
|
||||
EndUnitID: "u1",
|
||||
StartUnitID: 2,
|
||||
EndUnitID: 1,
|
||||
}
|
||||
|
||||
err := ValidateRef(doc, ref)
|
||||
@@ -261,7 +246,7 @@ func TestValidateRefReversedUnitOrder(t *testing.T) {
|
||||
func TestUnitIndex(t *testing.T) {
|
||||
doc := validDocument()
|
||||
|
||||
index, ok := UnitIndex(doc, "u2")
|
||||
index, ok := UnitIndex(doc, 2)
|
||||
if !ok {
|
||||
t.Fatal("UnitIndex() ok = false, want true")
|
||||
}
|
||||
@@ -269,7 +254,7 @@ func TestUnitIndex(t *testing.T) {
|
||||
t.Fatalf("UnitIndex() index = %d, want 1", index)
|
||||
}
|
||||
|
||||
index, ok = UnitIndex(doc, "u9")
|
||||
index, ok = UnitIndex(doc, 9)
|
||||
if ok {
|
||||
t.Fatal("UnitIndex() ok = true, want false")
|
||||
}
|
||||
@@ -286,12 +271,12 @@ func validDocument() *SourceDocument {
|
||||
Digest: "sha256:abc123",
|
||||
Units: []SourceUnit{
|
||||
{
|
||||
ID: "u1",
|
||||
ID: 1,
|
||||
Kind: "paragraph",
|
||||
Text: "First unit.",
|
||||
},
|
||||
{
|
||||
ID: "u2",
|
||||
ID: 2,
|
||||
Kind: "paragraph",
|
||||
Text: "Second unit.",
|
||||
},
|
||||
|
||||
@@ -28,13 +28,10 @@ func ValidateDocument(doc *SourceDocument) error {
|
||||
return fmt.Errorf("source document units must not be empty")
|
||||
}
|
||||
|
||||
seenUnitIDs := make(map[string]struct{}, len(doc.Units))
|
||||
seenUnitIDs := make(map[int]struct{}, len(doc.Units))
|
||||
for i, unit := range doc.Units {
|
||||
if isBlank(unit.ID) {
|
||||
return fmt.Errorf("source unit[%d].id must not be empty", i)
|
||||
}
|
||||
if hasSurroundingWhitespace(unit.ID) {
|
||||
return fmt.Errorf("source unit[%d].id %q must not contain leading or trailing whitespace", i, unit.ID)
|
||||
if unit.ID <= 0 {
|
||||
return fmt.Errorf("source unit[%d].id must be positive", i)
|
||||
}
|
||||
if isBlank(unit.Kind) {
|
||||
return fmt.Errorf("source unit[%d].kind must not be empty", i)
|
||||
@@ -43,7 +40,7 @@ func ValidateDocument(doc *SourceDocument) error {
|
||||
return fmt.Errorf("source unit[%d].text must not be empty", i)
|
||||
}
|
||||
if _, ok := seenUnitIDs[unit.ID]; ok {
|
||||
return fmt.Errorf("source unit id %q is duplicated", unit.ID)
|
||||
return fmt.Errorf("source unit id %d is duplicated", unit.ID)
|
||||
}
|
||||
seenUnitIDs[unit.ID] = struct{}{}
|
||||
}
|
||||
@@ -61,17 +58,11 @@ func ValidateRef(doc *SourceDocument, ref SourceRef) error {
|
||||
if hasSurroundingWhitespace(ref.SourceID) {
|
||||
return fmt.Errorf("source ref source_id %q must not contain leading or trailing whitespace", ref.SourceID)
|
||||
}
|
||||
if isBlank(ref.StartUnitID) {
|
||||
return fmt.Errorf("source ref start_unit_id must not be empty")
|
||||
if ref.StartUnitID <= 0 {
|
||||
return fmt.Errorf("source ref start_unit_id must be positive")
|
||||
}
|
||||
if hasSurroundingWhitespace(ref.StartUnitID) {
|
||||
return fmt.Errorf("source ref start_unit_id %q must not contain leading or trailing whitespace", ref.StartUnitID)
|
||||
}
|
||||
if isBlank(ref.EndUnitID) {
|
||||
return fmt.Errorf("source ref end_unit_id must not be empty")
|
||||
}
|
||||
if hasSurroundingWhitespace(ref.EndUnitID) {
|
||||
return fmt.Errorf("source ref end_unit_id %q must not contain leading or trailing whitespace", ref.EndUnitID)
|
||||
if ref.EndUnitID <= 0 {
|
||||
return fmt.Errorf("source ref end_unit_id must be positive")
|
||||
}
|
||||
if ref.SourceID != doc.ID {
|
||||
return fmt.Errorf("source ref source_id %q does not match document id %q", ref.SourceID, doc.ID)
|
||||
@@ -79,20 +70,20 @@ func ValidateRef(doc *SourceDocument, ref SourceRef) error {
|
||||
|
||||
startIndex, ok := UnitIndex(doc, ref.StartUnitID)
|
||||
if !ok {
|
||||
return fmt.Errorf("source ref start_unit_id %q was not found", ref.StartUnitID)
|
||||
return fmt.Errorf("source ref start_unit_id %d was not found", ref.StartUnitID)
|
||||
}
|
||||
endIndex, ok := UnitIndex(doc, ref.EndUnitID)
|
||||
if !ok {
|
||||
return fmt.Errorf("source ref end_unit_id %q was not found", ref.EndUnitID)
|
||||
return fmt.Errorf("source ref end_unit_id %d was not found", ref.EndUnitID)
|
||||
}
|
||||
if startIndex > endIndex {
|
||||
return fmt.Errorf("source ref start_unit_id %q appears after end_unit_id %q", ref.StartUnitID, ref.EndUnitID)
|
||||
return fmt.Errorf("source ref start_unit_id %d appears after end_unit_id %d", ref.StartUnitID, ref.EndUnitID)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func UnitIndex(doc *SourceDocument, unitID string) (int, bool) {
|
||||
func UnitIndex(doc *SourceDocument, unitID int) (int, bool) {
|
||||
if doc == nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
114
internal/core/workspace/files.go
Normal file
114
internal/core/workspace/files.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func SafePath(root string, name string) (string, error) {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" {
|
||||
return "", fmt.Errorf("workspace root must not be empty")
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("workspace artifact name must not be empty")
|
||||
}
|
||||
if strings.Contains(name, `\`) {
|
||||
return "", fmt.Errorf("workspace artifact name %q must use slash-separated relative paths", name)
|
||||
}
|
||||
if path.IsAbs(name) || filepath.IsAbs(name) {
|
||||
return "", fmt.Errorf("workspace artifact name %q must be relative", name)
|
||||
}
|
||||
if name == "." || strings.Contains(name, "..") {
|
||||
return "", fmt.Errorf("workspace artifact name %q must not contain ..", name)
|
||||
}
|
||||
cleaned := path.Clean(name)
|
||||
if cleaned != name {
|
||||
return "", fmt.Errorf("workspace artifact name %q must be clean", name)
|
||||
}
|
||||
|
||||
absRoot, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve workspace root %q: %w", root, err)
|
||||
}
|
||||
target, err := filepath.Abs(filepath.Join(absRoot, filepath.FromSlash(cleaned)))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve workspace artifact %q: %w", name, err)
|
||||
}
|
||||
rel, err := filepath.Rel(absRoot, target)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve workspace artifact %q: %w", name, err)
|
||||
}
|
||||
if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("workspace artifact name %q resolves outside workspace root", name)
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
|
||||
func WriteJSON(root string, name string, payload any) error {
|
||||
target, err := SafePath(root, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(payload, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal workspace artifact %q: %w", name, err)
|
||||
}
|
||||
data = append(data, '\n')
|
||||
if err := writeFileAtomic(target, data, 0o644); err != nil {
|
||||
return fmt.Errorf("write workspace artifact %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func WriteBytes(root string, name string, data []byte) error {
|
||||
target, err := SafePath(root, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeFileAtomic(target, data, 0o644); err != nil {
|
||||
return fmt.Errorf("write workspace artifact %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeFileAtomic(target string, data []byte, perm os.FileMode) error {
|
||||
dir := filepath.Dir(target)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
temp, err := os.CreateTemp(dir, "."+filepath.Base(target)+".tmp-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tempPath := temp.Name()
|
||||
removeTemp := true
|
||||
defer func() {
|
||||
if removeTemp {
|
||||
_ = os.Remove(tempPath)
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := temp.Write(data); err != nil {
|
||||
_ = temp.Close()
|
||||
return err
|
||||
}
|
||||
if err := temp.Chmod(perm); err != nil {
|
||||
_ = temp.Close()
|
||||
return err
|
||||
}
|
||||
if err := temp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tempPath, target); err != nil {
|
||||
return err
|
||||
}
|
||||
removeTemp = false
|
||||
return nil
|
||||
}
|
||||
148
internal/core/workspace/files_test.go
Normal file
148
internal/core/workspace/files_test.go
Normal file
@@ -0,0 +1,148 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSafePathAcceptsCleanRelativePaths(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
|
||||
got, err := SafePath(root, "source/manifest.json")
|
||||
if err != nil {
|
||||
t.Fatalf("SafePath: %v", err)
|
||||
}
|
||||
|
||||
want := filepath.Join(root, "source", "manifest.json")
|
||||
if got != want {
|
||||
t.Fatalf("SafePath = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafePathRejectsUnsafeNames(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{name: "empty", path: " ", want: "empty"},
|
||||
{name: "absolute", path: filepath.Join(root, "artifact.json"), want: "relative"},
|
||||
{name: "parent segment", path: "../artifact.json", want: ".."},
|
||||
{name: "embedded parent", path: "source/../artifact.json", want: ".."},
|
||||
{name: "backslash", path: `source\artifact.json`, want: "slash-separated"},
|
||||
{name: "unclean", path: "source//artifact.json", want: "clean"},
|
||||
{name: "dot", path: ".", want: ".."},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := SafePath(root, tc.path)
|
||||
if err == nil {
|
||||
t.Fatalf("SafePath returned %q, want error", got)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("SafePath error = %v, want containing %q", err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafePathRejectsEmptyRoot(t *testing.T) {
|
||||
got, err := SafePath(" ", "artifact.json")
|
||||
if err == nil {
|
||||
t.Fatalf("SafePath returned %q, want error", got)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "root") {
|
||||
t.Fatalf("SafePath error = %v, want root error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafePathDoesNotPermitEscapingRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for _, name := range []string{
|
||||
"..",
|
||||
"../outside.json",
|
||||
"nested/../../outside.json",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, err := SafePath(root, name)
|
||||
if err == nil {
|
||||
t.Fatalf("SafePath returned %q, want error", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteJSONWritesIndentedAtomicArtifact(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
|
||||
err := WriteJSON(root, "source/manifest.json", map[string]any{
|
||||
"status": "succeeded",
|
||||
"count": 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WriteJSON: %v", err)
|
||||
}
|
||||
|
||||
got := string(readFile(t, filepath.Join(root, "source", "manifest.json")))
|
||||
if !strings.HasSuffix(got, "\n") {
|
||||
t.Fatalf("expected trailing newline, got %q", got)
|
||||
}
|
||||
if !strings.Contains(got, `"status": "succeeded"`) || !strings.Contains(got, `"count": 2`) {
|
||||
t.Fatalf("unexpected JSON: %s", got)
|
||||
}
|
||||
assertNoTempFiles(t, filepath.Join(root, "source"))
|
||||
}
|
||||
|
||||
func TestWriteBytesWritesNestedArtifact(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
|
||||
if err := WriteBytes(root, "chunk/chunks.json", []byte("payload")); err != nil {
|
||||
t.Fatalf("WriteBytes: %v", err)
|
||||
}
|
||||
|
||||
got := string(readFile(t, filepath.Join(root, "chunk", "chunks.json")))
|
||||
if got != "payload" {
|
||||
t.Fatalf("bytes = %q, want payload", got)
|
||||
}
|
||||
assertNoTempFiles(t, filepath.Join(root, "chunk"))
|
||||
}
|
||||
|
||||
func TestWritersRejectUnsafePaths(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
|
||||
if err := WriteBytes(root, "../outside.json", []byte("payload")); err == nil {
|
||||
t.Fatalf("WriteBytes accepted unsafe path")
|
||||
}
|
||||
if err := WriteJSON(root, `debug\trace.json`, map[string]string{"x": "y"}); err == nil {
|
||||
t.Fatalf("WriteJSON accepted unsafe path")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "..", "outside.json")); !os.IsNotExist(err) {
|
||||
t.Fatalf("outside path stat err = %v, want not exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %q: %v", path, err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func assertNoTempFiles(t *testing.T, dir string) {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read dir %q: %v", dir, err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if strings.Contains(entry.Name(), ".tmp-") {
|
||||
t.Fatalf("temporary file was not cleaned up: %s", entry.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
276
internal/core/workspace/identity.go
Normal file
276
internal/core/workspace/identity.go
Normal file
@@ -0,0 +1,276 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const digestPrefixLength = 16
|
||||
|
||||
type Fingerprint struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type CheckpointIdentityInput struct {
|
||||
Pipeline pipeline.ResolvedPipeline
|
||||
InputKey string
|
||||
RawInputDigest string
|
||||
SourceDigest string
|
||||
SelectedLanes []string
|
||||
RuntimeOverrides []Fingerprint
|
||||
References []artifacts.ReferenceProvenance
|
||||
ProvenanceFingerprints []Fingerprint
|
||||
}
|
||||
|
||||
type CheckpointIdentity struct {
|
||||
Digest string `json:"digest"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
PipelineDigest string `json:"pipeline_digest"`
|
||||
InputKey string `json:"input_key"`
|
||||
RawInputDigest string `json:"raw_input_digest,omitempty"`
|
||||
SourceDigest string `json:"source_digest,omitempty"`
|
||||
SelectedLanes []string `json:"selected_lanes,omitempty"`
|
||||
RuntimeOverrides []Fingerprint `json:"runtime_overrides,omitempty"`
|
||||
ReferenceDigests []Fingerprint `json:"reference_digests,omitempty"`
|
||||
ProvenanceFingerprints []Fingerprint `json:"provenance_fingerprints,omitempty"`
|
||||
}
|
||||
|
||||
func NewCheckpointIdentity(input CheckpointIdentityInput) (CheckpointIdentity, error) {
|
||||
pipelineID := strings.TrimSpace(input.Pipeline.ID)
|
||||
if pipelineID == "" {
|
||||
return CheckpointIdentity{}, fmt.Errorf("checkpoint identity pipeline id must not be empty")
|
||||
}
|
||||
pipelineDigest := strings.TrimSpace(input.Pipeline.Digest)
|
||||
if pipelineDigest == "" {
|
||||
return CheckpointIdentity{}, fmt.Errorf("checkpoint identity pipeline digest must not be empty")
|
||||
}
|
||||
inputKey := strings.TrimSpace(input.InputKey)
|
||||
if inputKey == "" {
|
||||
inputKey = strings.TrimSpace(input.Pipeline.Input.Module)
|
||||
}
|
||||
if inputKey == "" {
|
||||
return CheckpointIdentity{}, fmt.Errorf("checkpoint identity input key must not be empty")
|
||||
}
|
||||
|
||||
rawInputDigest := strings.TrimSpace(input.RawInputDigest)
|
||||
sourceDigest := strings.TrimSpace(input.SourceDigest)
|
||||
if rawInputDigest == "" && sourceDigest == "" {
|
||||
return CheckpointIdentity{}, fmt.Errorf("checkpoint identity raw input digest or source digest must be set")
|
||||
}
|
||||
|
||||
identity := CheckpointIdentity{
|
||||
PipelineID: pipelineID,
|
||||
PipelineDigest: pipelineDigest,
|
||||
InputKey: inputKey,
|
||||
RawInputDigest: rawInputDigest,
|
||||
SourceDigest: sourceDigest,
|
||||
SelectedLanes: normalizedLanes(input.SelectedLanes, input.Pipeline.ArtifactLanes),
|
||||
RuntimeOverrides: normalizeFingerprints(input.RuntimeOverrides),
|
||||
ReferenceDigests: referenceFingerprints(input.References),
|
||||
ProvenanceFingerprints: normalizeFingerprints(input.ProvenanceFingerprints),
|
||||
}
|
||||
digest, err := identityDigest(identity)
|
||||
if err != nil {
|
||||
return CheckpointIdentity{}, err
|
||||
}
|
||||
identity.Digest = digest
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func (s Settings) CheckpointDirectory(identity CheckpointIdentity) (string, error) {
|
||||
if !s.ResumeEnabled || strings.TrimSpace(s.CheckpointsRoot) == "" {
|
||||
return "", nil
|
||||
}
|
||||
relative, err := identity.RelativePath()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return SafePath(s.CheckpointsRoot, relative)
|
||||
}
|
||||
|
||||
func (i CheckpointIdentity) RelativePath() (string, error) {
|
||||
pipelineID, err := safePathComponent(i.PipelineID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("checkpoint identity pipeline id: %w", err)
|
||||
}
|
||||
inputKey, err := safePathComponent(i.InputKey)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("checkpoint identity input key: %w", err)
|
||||
}
|
||||
sourceDigest := digestPrefix(i.SourceDigest)
|
||||
if sourceDigest == "" {
|
||||
sourceDigest = digestPrefix(i.RawInputDigest)
|
||||
}
|
||||
if sourceDigest == "" {
|
||||
return "", fmt.Errorf("checkpoint identity source digest prefix must not be empty")
|
||||
}
|
||||
pipelineDigest := digestPrefix(i.PipelineDigest)
|
||||
if pipelineDigest == "" {
|
||||
return "", fmt.Errorf("checkpoint identity pipeline digest prefix must not be empty")
|
||||
}
|
||||
sourceComponent, err := safePathComponent(sourceDigest)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("checkpoint identity source digest: %w", err)
|
||||
}
|
||||
pipelineComponent, err := safePathComponent(pipelineDigest)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("checkpoint identity pipeline digest: %w", err)
|
||||
}
|
||||
identityDigest := digestPrefix(i.Digest)
|
||||
if identityDigest == "" {
|
||||
return "", fmt.Errorf("checkpoint identity digest prefix must not be empty")
|
||||
}
|
||||
identityComponent, err := safePathComponent(identityDigest)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("checkpoint identity digest: %w", err)
|
||||
}
|
||||
return filepath.ToSlash(filepath.Join(pipelineID, inputKey+"-"+sourceComponent, pipelineComponent, identityComponent)), nil
|
||||
}
|
||||
|
||||
func identityDigest(identity CheckpointIdentity) (string, error) {
|
||||
payload := identity
|
||||
payload.Digest = ""
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal checkpoint identity: %w", err)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return "sha256:" + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func normalizedLanes(selected []string, resolved []pipeline.ResolvedArtifactLane) []string {
|
||||
if len(selected) > 0 {
|
||||
return normalizeStrings(selected)
|
||||
}
|
||||
lanes := make([]string, 0, len(resolved))
|
||||
for _, lane := range resolved {
|
||||
lanes = append(lanes, lane.ID)
|
||||
}
|
||||
return normalizeStrings(lanes)
|
||||
}
|
||||
|
||||
func normalizeFingerprints(values []Fingerprint) []Fingerprint {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
byName := make(map[string]string, len(values))
|
||||
for _, value := range values {
|
||||
name := strings.TrimSpace(value.Name)
|
||||
fingerprint := strings.TrimSpace(value.Value)
|
||||
if name == "" || fingerprint == "" {
|
||||
continue
|
||||
}
|
||||
byName[name] = fingerprint
|
||||
}
|
||||
if len(byName) == 0 {
|
||||
return nil
|
||||
}
|
||||
names := make([]string, 0, len(byName))
|
||||
for name := range byName {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
out := make([]Fingerprint, 0, len(names))
|
||||
for _, name := range names {
|
||||
out = append(out, Fingerprint{Name: name, Value: byName[name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func referenceFingerprints(references []artifacts.ReferenceProvenance) []Fingerprint {
|
||||
if len(references) == 0 {
|
||||
return nil
|
||||
}
|
||||
values := make([]Fingerprint, 0, len(references))
|
||||
for _, reference := range references {
|
||||
digest := strings.TrimSpace(reference.Digest)
|
||||
if digest == "" {
|
||||
continue
|
||||
}
|
||||
parts := []string{
|
||||
strings.TrimSpace(reference.Stage),
|
||||
strings.TrimSpace(reference.LaneID),
|
||||
strings.TrimSpace(reference.SlotName),
|
||||
strings.TrimSpace(reference.OriginURI),
|
||||
}
|
||||
values = append(values, Fingerprint{
|
||||
Name: strings.Join(parts, ":"),
|
||||
Value: digest,
|
||||
})
|
||||
}
|
||||
return normalizeFingerprints(values)
|
||||
}
|
||||
|
||||
func normalizeStrings(values []string) []string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for value := range seen {
|
||||
out = append(out, value)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func digestPrefix(digest string) string {
|
||||
digest = strings.TrimSpace(digest)
|
||||
if digest == "" {
|
||||
return ""
|
||||
}
|
||||
if idx := strings.Index(digest, ":"); idx >= 0 {
|
||||
digest = digest[idx+1:]
|
||||
}
|
||||
digest = strings.TrimSpace(digest)
|
||||
if len(digest) > digestPrefixLength {
|
||||
return digest[:digestPrefixLength]
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
func safePathComponent(value string) (string, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("must not be empty")
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
b.WriteRune(r)
|
||||
case r >= 'A' && r <= 'Z':
|
||||
b.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '-' || r == '_' || r == '.':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteString(fmt.Sprintf("~%x", r))
|
||||
}
|
||||
}
|
||||
encoded := b.String()
|
||||
if encoded == "." || encoded == ".." || strings.Contains(encoded, "..") || strings.ContainsAny(encoded, `/\`) {
|
||||
return "", fmt.Errorf("%q is not filesystem safe", value)
|
||||
}
|
||||
return encoded, nil
|
||||
}
|
||||
266
internal/core/workspace/identity_test.go
Normal file
266
internal/core/workspace/identity_test.go
Normal file
@@ -0,0 +1,266 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestCheckpointIdentityIsDeterministic(t *testing.T) {
|
||||
first := mustIdentity(t, identityInput())
|
||||
second := mustIdentity(t, identityInput())
|
||||
|
||||
if first.Digest != second.Digest {
|
||||
t.Fatalf("digest changed for same input: %q != %q", first.Digest, second.Digest)
|
||||
}
|
||||
if !strings.HasPrefix(first.Digest, "sha256:") {
|
||||
t.Fatalf("digest = %q, want sha256 prefix", first.Digest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckpointIdentityChangesWhenInputsChange(t *testing.T) {
|
||||
base := mustIdentity(t, identityInput())
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(CheckpointIdentityInput) CheckpointIdentityInput
|
||||
}{
|
||||
{
|
||||
name: "pipeline digest",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.Pipeline.Digest = "sha256:pipeline-b"
|
||||
return input
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "raw input digest",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.RawInputDigest = "sha256:raw-b"
|
||||
return input
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "selected lanes",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.SelectedLanes = []string{"items"}
|
||||
return input
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "reference digest",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.References[0].Digest = "sha256:reference-b"
|
||||
return input
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "runtime override",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.RuntimeOverrides = []Fingerprint{{Name: "llm_profile", Value: "careful"}}
|
||||
return input
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "provenance fingerprint",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.ProvenanceFingerprints = []Fingerprint{{Name: "prompt:dnd.spells", Value: "sha256:prompt-b"}}
|
||||
return input
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
changed := mustIdentity(t, tc.mutate(identityInput()))
|
||||
if changed.Digest == base.Digest {
|
||||
t.Fatalf("digest did not change after %s mutation: %q", tc.name, changed.Digest)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckpointIdentityNormalizesOrder(t *testing.T) {
|
||||
input := identityInput()
|
||||
input.SelectedLanes = []string{"spells", "items", "spells"}
|
||||
input.RuntimeOverrides = []Fingerprint{
|
||||
{Name: "z", Value: "2"},
|
||||
{Name: "a", Value: "1"},
|
||||
}
|
||||
input.ProvenanceFingerprints = []Fingerprint{
|
||||
{Name: "schema", Value: "sha256:schema"},
|
||||
{Name: "prompt", Value: "sha256:prompt"},
|
||||
}
|
||||
|
||||
identity := mustIdentity(t, input)
|
||||
|
||||
if got := strings.Join(identity.SelectedLanes, ","); got != "items,spells" {
|
||||
t.Fatalf("selected lanes = %q, want sorted unique values", got)
|
||||
}
|
||||
if identity.RuntimeOverrides[0].Name != "a" || identity.ProvenanceFingerprints[0].Name != "prompt" {
|
||||
t.Fatalf("fingerprints not sorted: runtime=%+v provenance=%+v", identity.RuntimeOverrides, identity.ProvenanceFingerprints)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckpointIdentityPathIsFilesystemSafe(t *testing.T) {
|
||||
input := identityInput()
|
||||
input.Pipeline.ID = "campaign/main"
|
||||
input.InputKey = "seriatim/input"
|
||||
input.SourceDigest = "sha256:abcdef0123456789ffffffff"
|
||||
input.Pipeline.Digest = "sha256:1234567890abcdefeeeeeeee"
|
||||
identity := mustIdentity(t, input)
|
||||
|
||||
relative, err := identity.RelativePath()
|
||||
if err != nil {
|
||||
t.Fatalf("RelativePath: %v", err)
|
||||
}
|
||||
if strings.Contains(relative, `\`) || strings.Contains(relative, "..") {
|
||||
t.Fatalf("relative path is not filesystem safe: %q", relative)
|
||||
}
|
||||
identityDigest := digestPrefix(identity.Digest)
|
||||
wantRelative := "campaign~2fmain/seriatim~2finput-abcdef0123456789/1234567890abcdef/" + identityDigest
|
||||
if relative != wantRelative {
|
||||
t.Fatalf("relative path = %q", relative)
|
||||
}
|
||||
|
||||
root := t.TempDir()
|
||||
settings := Settings{
|
||||
CheckpointsRoot: filepath.Join(root, "checkpoints"),
|
||||
ResumeEnabled: true,
|
||||
}
|
||||
got, err := settings.CheckpointDirectory(identity)
|
||||
if err != nil {
|
||||
t.Fatalf("CheckpointDirectory: %v", err)
|
||||
}
|
||||
want := filepath.Join(root, "checkpoints", "campaign~2fmain", "seriatim~2finput-abcdef0123456789", "1234567890abcdef", identityDigest)
|
||||
if got != want {
|
||||
t.Fatalf("checkpoint directory = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckpointIdentityPathIncludesInvocationIdentity(t *testing.T) {
|
||||
base := mustIdentity(t, identityInput())
|
||||
changedInput := identityInput()
|
||||
changedInput.References[0].Digest = "sha256:reference-b"
|
||||
changed := mustIdentity(t, changedInput)
|
||||
|
||||
if base.Digest == changed.Digest {
|
||||
t.Fatalf("test setup produced same identity digest: %q", base.Digest)
|
||||
}
|
||||
basePath, err := base.RelativePath()
|
||||
if err != nil {
|
||||
t.Fatalf("base RelativePath: %v", err)
|
||||
}
|
||||
changedPath, err := changed.RelativePath()
|
||||
if err != nil {
|
||||
t.Fatalf("changed RelativePath: %v", err)
|
||||
}
|
||||
if basePath == changedPath {
|
||||
t.Fatalf("relative path did not change with invocation identity: %q", basePath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckpointDirectoryDisabledReturnsEmptyPath(t *testing.T) {
|
||||
settings := Settings{CheckpointsRoot: filepath.Join(t.TempDir(), "checkpoints")}
|
||||
got, err := settings.CheckpointDirectory(mustIdentity(t, identityInput()))
|
||||
if err != nil {
|
||||
t.Fatalf("CheckpointDirectory: %v", err)
|
||||
}
|
||||
if got != "" {
|
||||
t.Fatalf("CheckpointDirectory = %q, want empty path", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCheckpointIdentityRequiresCoreInputs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(CheckpointIdentityInput) CheckpointIdentityInput
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "pipeline id",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.Pipeline.ID = ""
|
||||
return input
|
||||
},
|
||||
want: "pipeline id",
|
||||
},
|
||||
{
|
||||
name: "pipeline digest",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.Pipeline.Digest = ""
|
||||
return input
|
||||
},
|
||||
want: "pipeline digest",
|
||||
},
|
||||
{
|
||||
name: "input key",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.InputKey = ""
|
||||
input.Pipeline.Input.Module = ""
|
||||
return input
|
||||
},
|
||||
want: "input key",
|
||||
},
|
||||
{
|
||||
name: "input digest",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.RawInputDigest = ""
|
||||
input.SourceDigest = ""
|
||||
return input
|
||||
},
|
||||
want: "raw input digest or source digest",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := NewCheckpointIdentity(tc.mutate(identityInput()))
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.want, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func identityInput() CheckpointIdentityInput {
|
||||
return CheckpointIdentityInput{
|
||||
Pipeline: pipeline.ResolvedPipeline{
|
||||
ID: "dnd-session",
|
||||
Digest: "sha256:pipeline-a",
|
||||
Input: pipeline.Binding("seriatim"),
|
||||
ArtifactLanes: []pipeline.ResolvedArtifactLane{
|
||||
{ID: "spells"},
|
||||
{ID: "items"},
|
||||
},
|
||||
},
|
||||
InputKey: "seriatim",
|
||||
RawInputDigest: "sha256:raw-a",
|
||||
SelectedLanes: []string{"spells"},
|
||||
RuntimeOverrides: []Fingerprint{
|
||||
{Name: "llm_profile", Value: "fast"},
|
||||
},
|
||||
References: []artifacts.ReferenceProvenance{
|
||||
{
|
||||
Stage: "extract",
|
||||
LaneID: "spells",
|
||||
SlotName: "party",
|
||||
OriginURI: "file:///party.yml",
|
||||
Digest: "sha256:reference-a",
|
||||
},
|
||||
},
|
||||
ProvenanceFingerprints: []Fingerprint{
|
||||
{Name: "prompt:dnd.spells", Value: "sha256:prompt-a"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func mustIdentity(t *testing.T, input CheckpointIdentityInput) CheckpointIdentity {
|
||||
t.Helper()
|
||||
identity, err := NewCheckpointIdentity(input)
|
||||
if err != nil {
|
||||
t.Fatalf("NewCheckpointIdentity: %v", err)
|
||||
}
|
||||
return identity
|
||||
}
|
||||
82
internal/core/workspace/manifest.go
Normal file
82
internal/core/workspace/manifest.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package workspace
|
||||
|
||||
import "time"
|
||||
|
||||
const WorkspaceSchemaVersion = "notarius.workspace.v1"
|
||||
|
||||
type StageName string
|
||||
|
||||
const (
|
||||
StageSource StageName = "source"
|
||||
StageChunk StageName = "chunk"
|
||||
StageExtract StageName = "extract"
|
||||
StageMerge StageName = "merge"
|
||||
StageNormalize StageName = "normalize"
|
||||
)
|
||||
|
||||
type StageStatus string
|
||||
|
||||
const (
|
||||
StatusPending StageStatus = "pending"
|
||||
StatusRunning StageStatus = "running"
|
||||
StatusSucceeded StageStatus = "succeeded"
|
||||
StatusSucceededWithRejections StageStatus = "succeeded_with_rejections"
|
||||
StatusFailed StageStatus = "failed"
|
||||
StatusInvalidated StageStatus = "invalidated"
|
||||
)
|
||||
|
||||
type StageManifest struct {
|
||||
WorkspaceSchemaVersion string `json:"workspace_schema_version"`
|
||||
Stage StageName `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
DependencyFingerprints []Fingerprint `json:"dependency_fingerprints,omitempty"`
|
||||
Status StageStatus `json:"status"`
|
||||
OutputDigests []Fingerprint `json:"output_digests,omitempty"`
|
||||
ValidationStatus string `json:"validation_status,omitempty"`
|
||||
Rejections []RejectionSummary `json:"rejections,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type RejectionSummary struct {
|
||||
ValidatorName string `json:"validator_name,omitempty"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Count int `json:"count,omitempty"`
|
||||
}
|
||||
|
||||
type SourceManifest struct {
|
||||
StageManifest
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
}
|
||||
|
||||
type ChunkManifest struct {
|
||||
StageManifest
|
||||
ChunkCount int `json:"chunk_count,omitempty"`
|
||||
}
|
||||
|
||||
type ExtractLaneManifest struct {
|
||||
StageManifest
|
||||
ChunkCount int `json:"chunk_count,omitempty"`
|
||||
OutputCount int `json:"output_count,omitempty"`
|
||||
}
|
||||
|
||||
type MergeLaneManifest struct {
|
||||
StageManifest
|
||||
InputCount int `json:"input_count,omitempty"`
|
||||
}
|
||||
|
||||
type NormalizeLaneManifest struct {
|
||||
StageManifest
|
||||
InputCount int `json:"input_count,omitempty"`
|
||||
}
|
||||
|
||||
func NewStageManifest(stage StageName, status StageStatus) StageManifest {
|
||||
return StageManifest{
|
||||
WorkspaceSchemaVersion: WorkspaceSchemaVersion,
|
||||
Stage: stage,
|
||||
Status: status,
|
||||
}
|
||||
}
|
||||
143
internal/core/workspace/manifest_test.go
Normal file
143
internal/core/workspace/manifest_test.go
Normal file
@@ -0,0 +1,143 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStageManifestDefaults(t *testing.T) {
|
||||
manifest := NewStageManifest(StageExtract, StatusRunning)
|
||||
|
||||
if manifest.WorkspaceSchemaVersion != WorkspaceSchemaVersion {
|
||||
t.Fatalf("schema version = %q, want %q", manifest.WorkspaceSchemaVersion, WorkspaceSchemaVersion)
|
||||
}
|
||||
if manifest.Stage != StageExtract {
|
||||
t.Fatalf("stage = %q, want extract", manifest.Stage)
|
||||
}
|
||||
if manifest.Status != StatusRunning {
|
||||
t.Fatalf("status = %q, want running", manifest.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestJSONRoundTrips(t *testing.T) {
|
||||
started := time.Unix(100, 0).UTC()
|
||||
completed := time.Unix(200, 0).UTC()
|
||||
|
||||
t.Run("source", func(t *testing.T) {
|
||||
manifest := SourceManifest{
|
||||
StageManifest: populatedManifest(StageSource, "", "seriatim", started, completed),
|
||||
SourceID: "source-1",
|
||||
}
|
||||
var got SourceManifest
|
||||
roundTripManifest(t, manifest, &got)
|
||||
if got.SourceID != manifest.SourceID || got.Stage != StageSource {
|
||||
t.Fatalf("round trip source manifest = %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("chunk", func(t *testing.T) {
|
||||
manifest := ChunkManifest{
|
||||
StageManifest: populatedManifest(StageChunk, "", "generic", started, completed),
|
||||
ChunkCount: 3,
|
||||
}
|
||||
var got ChunkManifest
|
||||
roundTripManifest(t, manifest, &got)
|
||||
if got.ChunkCount != manifest.ChunkCount || got.Stage != StageChunk {
|
||||
t.Fatalf("round trip chunk manifest = %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("extract", func(t *testing.T) {
|
||||
manifest := ExtractLaneManifest{
|
||||
StageManifest: populatedManifest(StageExtract, "spells", "dnd/spells", started, completed),
|
||||
ChunkCount: 3,
|
||||
OutputCount: 2,
|
||||
}
|
||||
var got ExtractLaneManifest
|
||||
roundTripManifest(t, manifest, &got)
|
||||
if got.LaneID != "spells" || got.OutputCount != manifest.OutputCount || got.Stage != StageExtract {
|
||||
t.Fatalf("round trip extract manifest = %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("merge", func(t *testing.T) {
|
||||
manifest := MergeLaneManifest{
|
||||
StageManifest: populatedManifest(StageMerge, "spells", "appendorder", started, completed),
|
||||
InputCount: 2,
|
||||
}
|
||||
var got MergeLaneManifest
|
||||
roundTripManifest(t, manifest, &got)
|
||||
if got.InputCount != manifest.InputCount || got.Stage != StageMerge {
|
||||
t.Fatalf("round trip merge manifest = %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("normalize", func(t *testing.T) {
|
||||
manifest := NormalizeLaneManifest{
|
||||
StageManifest: populatedManifest(StageNormalize, "spells", "noop", started, completed),
|
||||
InputCount: 1,
|
||||
}
|
||||
var got NormalizeLaneManifest
|
||||
roundTripManifest(t, manifest, &got)
|
||||
if got.InputCount != manifest.InputCount || got.Stage != StageNormalize {
|
||||
t.Fatalf("round trip normalize manifest = %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestStatusValues(t *testing.T) {
|
||||
values := []StageStatus{
|
||||
StatusPending,
|
||||
StatusRunning,
|
||||
StatusSucceeded,
|
||||
StatusSucceededWithRejections,
|
||||
StatusFailed,
|
||||
StatusInvalidated,
|
||||
}
|
||||
want := []string{
|
||||
"pending",
|
||||
"running",
|
||||
"succeeded",
|
||||
"succeeded_with_rejections",
|
||||
"failed",
|
||||
"invalidated",
|
||||
}
|
||||
for i, value := range values {
|
||||
if string(value) != want[i] {
|
||||
t.Fatalf("status[%d] = %q, want %q", i, value, want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func populatedManifest(stage StageName, laneID string, moduleKey string, started time.Time, completed time.Time) StageManifest {
|
||||
manifest := NewStageManifest(stage, StatusSucceededWithRejections)
|
||||
manifest.LaneID = laneID
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = []Fingerprint{{Name: "source", Value: "sha256:source"}}
|
||||
manifest.OutputDigests = []Fingerprint{{Name: "output", Value: "sha256:output"}}
|
||||
manifest.ValidationStatus = "approved_with_warnings"
|
||||
manifest.Rejections = []RejectionSummary{
|
||||
{
|
||||
ValidatorName: "shape",
|
||||
ReasonCode: "invalid_shape",
|
||||
Message: "invalid output shape",
|
||||
Count: 1,
|
||||
},
|
||||
}
|
||||
manifest.StartedAt = &started
|
||||
manifest.CompletedAt = &completed
|
||||
manifest.Metadata = map[string]string{"attempt": "1"}
|
||||
return manifest
|
||||
}
|
||||
|
||||
func roundTripManifest(t *testing.T, in any, out any) {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal manifest: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
t.Fatalf("unmarshal manifest: %v", err)
|
||||
}
|
||||
}
|
||||
76
internal/core/workspace/settings.go
Normal file
76
internal/core/workspace/settings.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
)
|
||||
|
||||
type Settings struct {
|
||||
RootDir string
|
||||
DiagnosticsRoot string
|
||||
CheckpointsRoot string
|
||||
DebugRoot string
|
||||
DiagnosticsEnabled bool
|
||||
ResumeEnabled bool
|
||||
DebugEnabled bool
|
||||
}
|
||||
|
||||
func FromConfig(cfg config.Config) Settings {
|
||||
root := cleanPath(cfg.Workspace.Directory)
|
||||
settings := Settings{
|
||||
RootDir: root,
|
||||
DiagnosticsEnabled: cfg.DiagnosticsEnabled(),
|
||||
}
|
||||
if settings.DiagnosticsEnabled {
|
||||
settings.DiagnosticsRoot = cleanPath(cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if root == "" {
|
||||
return settings
|
||||
}
|
||||
|
||||
settings.CheckpointsRoot = filepath.Join(root, "checkpoints")
|
||||
settings.DebugRoot = filepath.Join(root, "debug")
|
||||
settings.ResumeEnabled = cfg.Workspace.Resume.Enabled
|
||||
settings.DebugEnabled = cfg.Workspace.Debug.Enabled
|
||||
return settings
|
||||
}
|
||||
|
||||
func (s Settings) DiagnosticsRunDirectory(runID string) (string, error) {
|
||||
if !s.DiagnosticsEnabled || strings.TrimSpace(s.DiagnosticsRoot) == "" {
|
||||
return "", nil
|
||||
}
|
||||
return safeSingleDirectory(s.DiagnosticsRoot, runID, "diagnostics run ID")
|
||||
}
|
||||
|
||||
func (s Settings) CheckpointIdentityDirectory(identity string) (string, error) {
|
||||
if !s.ResumeEnabled || strings.TrimSpace(s.CheckpointsRoot) == "" {
|
||||
return "", nil
|
||||
}
|
||||
return SafePath(s.CheckpointsRoot, identity)
|
||||
}
|
||||
|
||||
func (s Settings) DebugRunDirectory(runID string) (string, error) {
|
||||
if !s.DebugEnabled || strings.TrimSpace(s.DebugRoot) == "" {
|
||||
return "", nil
|
||||
}
|
||||
return safeSingleDirectory(s.DebugRoot, runID, "debug run ID")
|
||||
}
|
||||
|
||||
func cleanPath(path string) string {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Clean(path)
|
||||
}
|
||||
|
||||
func safeSingleDirectory(root string, name string, label string) (string, error) {
|
||||
name = strings.TrimSpace(name)
|
||||
if strings.Contains(name, "/") || strings.Contains(name, `\`) {
|
||||
return "", fmt.Errorf("%s %q must be a single directory name", label, name)
|
||||
}
|
||||
return SafePath(root, name)
|
||||
}
|
||||
127
internal/core/workspace/settings_test.go
Normal file
127
internal/core/workspace/settings_test.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
)
|
||||
|
||||
func TestFromConfigBuildsWorkspaceRoots(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.Workspace.Directory = "/var/lib/notarius"
|
||||
cfg.Workspace.Resume.Enabled = true
|
||||
cfg.Workspace.Debug.Enabled = true
|
||||
cfg.RecomputeEffectiveDiagnostics()
|
||||
|
||||
settings := FromConfig(cfg)
|
||||
|
||||
if settings.RootDir != "/var/lib/notarius" {
|
||||
t.Fatalf("RootDir = %q, want /var/lib/notarius", settings.RootDir)
|
||||
}
|
||||
if settings.DiagnosticsRoot != "/var/lib/notarius/diagnostics" || !settings.DiagnosticsEnabled {
|
||||
t.Fatalf("diagnostics settings = %+v, want workspace diagnostics root enabled", settings)
|
||||
}
|
||||
if settings.CheckpointsRoot != "/var/lib/notarius/checkpoints" || !settings.ResumeEnabled {
|
||||
t.Fatalf("checkpoint settings = %+v, want workspace checkpoints root enabled", settings)
|
||||
}
|
||||
if settings.DebugRoot != "/var/lib/notarius/debug" || !settings.DebugEnabled {
|
||||
t.Fatalf("debug settings = %+v, want workspace debug root enabled", settings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromConfigKeepsLegacyDiagnosticsRootWithoutWorkspaceRoot(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.Diagnostics.WorkDir = "/tmp/notarius-legacy"
|
||||
cfg.Workspace.Resume.Enabled = true
|
||||
cfg.Workspace.Debug.Enabled = true
|
||||
|
||||
settings := FromConfig(cfg)
|
||||
|
||||
if settings.RootDir != "" {
|
||||
t.Fatalf("RootDir = %q, want empty", settings.RootDir)
|
||||
}
|
||||
if settings.DiagnosticsRoot != "/tmp/notarius-legacy" || !settings.DiagnosticsEnabled {
|
||||
t.Fatalf("diagnostics settings = %+v, want legacy diagnostics root enabled", settings)
|
||||
}
|
||||
if settings.CheckpointsRoot != "" || settings.ResumeEnabled {
|
||||
t.Fatalf("checkpoint settings = %+v, want disabled empty root", settings)
|
||||
}
|
||||
if settings.DebugRoot != "" || settings.DebugEnabled {
|
||||
t.Fatalf("debug settings = %+v, want disabled empty root", settings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathConstructors(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
settings := Settings{
|
||||
RootDir: root,
|
||||
DiagnosticsRoot: filepath.Join(root, "diagnostics"),
|
||||
CheckpointsRoot: filepath.Join(root, "checkpoints"),
|
||||
DebugRoot: filepath.Join(root, "debug"),
|
||||
DiagnosticsEnabled: true,
|
||||
ResumeEnabled: true,
|
||||
DebugEnabled: true,
|
||||
}
|
||||
|
||||
diagnosticsDir, err := settings.DiagnosticsRunDirectory("run-123")
|
||||
if err != nil {
|
||||
t.Fatalf("DiagnosticsRunDirectory: %v", err)
|
||||
}
|
||||
if diagnosticsDir != filepath.Join(root, "diagnostics", "run-123") {
|
||||
t.Fatalf("diagnostics dir = %q", diagnosticsDir)
|
||||
}
|
||||
|
||||
checkpointDir, err := settings.CheckpointIdentityDirectory("pipeline/input-digest/pipeline-digest")
|
||||
if err != nil {
|
||||
t.Fatalf("CheckpointIdentityDirectory: %v", err)
|
||||
}
|
||||
if checkpointDir != filepath.Join(root, "checkpoints", "pipeline", "input-digest", "pipeline-digest") {
|
||||
t.Fatalf("checkpoint dir = %q", checkpointDir)
|
||||
}
|
||||
|
||||
debugDir, err := settings.DebugRunDirectory("run-456")
|
||||
if err != nil {
|
||||
t.Fatalf("DebugRunDirectory: %v", err)
|
||||
}
|
||||
if debugDir != filepath.Join(root, "debug", "run-456") {
|
||||
t.Fatalf("debug dir = %q", debugDir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisabledPathConstructorsReturnEmptyPaths(t *testing.T) {
|
||||
settings := Settings{}
|
||||
|
||||
for name, call := range map[string]func() (string, error){
|
||||
"diagnostics": func() (string, error) { return settings.DiagnosticsRunDirectory("run-1") },
|
||||
"checkpoint": func() (string, error) { return settings.CheckpointIdentityDirectory("identity") },
|
||||
"debug": func() (string, error) { return settings.DebugRunDirectory("run-1") },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, err := call()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got != "" {
|
||||
t.Fatalf("path = %q, want empty", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDirectoryConstructorsRejectNestedNames(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
settings := Settings{
|
||||
DiagnosticsRoot: filepath.Join(root, "diagnostics"),
|
||||
DebugRoot: filepath.Join(root, "debug"),
|
||||
DiagnosticsEnabled: true,
|
||||
DebugEnabled: true,
|
||||
}
|
||||
|
||||
if got, err := settings.DiagnosticsRunDirectory("run-1/nested"); err == nil {
|
||||
t.Fatalf("DiagnosticsRunDirectory returned %q, want error", got)
|
||||
}
|
||||
if got, err := settings.DebugRunDirectory("run-1/nested"); err == nil {
|
||||
t.Fatalf("DebugRunDirectory returned %q, want error", got)
|
||||
}
|
||||
}
|
||||
345
internal/framework/checkpoint/loader.go
Normal file
345
internal/framework/checkpoint/loader.go
Normal file
@@ -0,0 +1,345 @@
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
type WorkspaceLoader struct {
|
||||
root string
|
||||
identityDigest string
|
||||
}
|
||||
|
||||
func NewWorkspaceLoader(settings coreworkspace.Settings, identity coreworkspace.CheckpointIdentity) (pipeline.CheckpointLoader, error) {
|
||||
root, err := settings.CheckpointDirectory(identity)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return pipeline.NoopCheckpointLoader(), nil
|
||||
}
|
||||
return &WorkspaceLoader{root: root, identityDigest: identity.Digest}, nil
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) Enabled() bool {
|
||||
return l != nil && strings.TrimSpace(l.root) != ""
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) Source(moduleKey string) (pipeline.SourceCheckpoint, pipeline.CheckpointDecision) {
|
||||
var manifest coreworkspace.SourceManifest
|
||||
if decision := l.readJSON("source/manifest.json", &manifest); !decision.Reused {
|
||||
return pipeline.SourceCheckpoint{}, decision
|
||||
}
|
||||
if decision := l.validateManifest(manifest.StageManifest, coreworkspace.StageSource, "", moduleKey, coreworkspace.StatusSucceeded, nil); !decision.Reused {
|
||||
return pipeline.SourceCheckpoint{}, decision
|
||||
}
|
||||
var payload sourceDocumentEnvelope
|
||||
if decision := l.readJSON("source/source-document.json", &payload); !decision.Reused {
|
||||
return pipeline.SourceCheckpoint{}, decision
|
||||
}
|
||||
doc := cloneSourceDocument(payload.Document)
|
||||
if err := source.ValidateDocument(&doc); err != nil {
|
||||
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint document is invalid: %v", err)
|
||||
}
|
||||
if strings.TrimSpace(manifest.SourceID) != "" && manifest.SourceID != doc.ID {
|
||||
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint source id does not match payload")
|
||||
}
|
||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), digestFingerprints("source_document", doc.Digest)) {
|
||||
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint output digest does not match payload")
|
||||
}
|
||||
return pipeline.SourceCheckpoint{Document: &doc}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) Chunk(moduleKey string, sourceDigest string) (pipeline.ChunkCheckpoint, pipeline.CheckpointDecision) {
|
||||
expectedDependencies := digestFingerprints("source_document", sourceDigest)
|
||||
var manifest coreworkspace.ChunkManifest
|
||||
if decision := l.readJSON("chunk/manifest.json", &manifest); !decision.Reused {
|
||||
return pipeline.ChunkCheckpoint{}, decision
|
||||
}
|
||||
if decision := l.validateManifest(manifest.StageManifest, coreworkspace.StageChunk, "", moduleKey, coreworkspace.StatusSucceeded, expectedDependencies); !decision.Reused {
|
||||
return pipeline.ChunkCheckpoint{}, decision
|
||||
}
|
||||
var payload chunksEnvelope
|
||||
if decision := l.readJSON("chunk/chunks.json", &payload); !decision.Reused {
|
||||
return pipeline.ChunkCheckpoint{}, decision
|
||||
}
|
||||
chunks, err := sourceChunksFromEnvelope(payload.Chunks)
|
||||
if err != nil {
|
||||
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint payload is invalid: %v", err)
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint payload has no chunks")
|
||||
}
|
||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), chunkOutputDigests(chunks)) {
|
||||
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint output digests do not match payload")
|
||||
}
|
||||
return pipeline.ChunkCheckpoint{Chunks: chunks, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) Extract(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
|
||||
var manifest coreworkspace.ExtractLaneManifest
|
||||
if decision := l.readJSON(laneManifestPath("extract", laneID), &manifest); !decision.Reused {
|
||||
return pipeline.ExtractCheckpoint{}, decision
|
||||
}
|
||||
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageExtract, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded, coreworkspace.StatusSucceededWithRejections); !decision.Reused {
|
||||
return pipeline.ExtractCheckpoint{}, decision
|
||||
}
|
||||
var payload extractOutputsEnvelope
|
||||
if decision := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !decision.Reused {
|
||||
return pipeline.ExtractCheckpoint{}, decision
|
||||
}
|
||||
outputs, err := extractOutputsFromEnvelope(payload.Outputs)
|
||||
if err != nil {
|
||||
return pipeline.ExtractCheckpoint{}, invalidDecision("extract checkpoint payload is invalid: %v", err)
|
||||
}
|
||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests(extractPayloads(outputs))) {
|
||||
return pipeline.ExtractCheckpoint{}, invalidDecision("extract checkpoint output digests do not match payload")
|
||||
}
|
||||
return pipeline.ExtractCheckpoint{
|
||||
Outputs: outputs,
|
||||
Rejected: cloneRejectedOutputs(payload.Rejected),
|
||||
Warnings: cloneWarnings(payload.Warnings),
|
||||
}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) Merge(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
|
||||
var manifest coreworkspace.MergeLaneManifest
|
||||
if decision := l.readJSON(laneManifestPath("merge", laneID), &manifest); !decision.Reused {
|
||||
return pipeline.MergeCheckpoint{}, decision
|
||||
}
|
||||
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageMerge, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !decision.Reused {
|
||||
return pipeline.MergeCheckpoint{}, decision
|
||||
}
|
||||
var payload mergeOutputEnvelope
|
||||
if decision := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !decision.Reused {
|
||||
return pipeline.MergeCheckpoint{}, decision
|
||||
}
|
||||
output, err := mergeOutputFromEnvelope(payload.Output)
|
||||
if err != nil {
|
||||
return pipeline.MergeCheckpoint{}, invalidDecision("merge checkpoint payload is invalid: %v", err)
|
||||
}
|
||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests([]contracts.RawPayload{output.Payload})) {
|
||||
return pipeline.MergeCheckpoint{}, invalidDecision("merge checkpoint output digest does not match payload")
|
||||
}
|
||||
return pipeline.MergeCheckpoint{Output: output, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) Normalize(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
|
||||
var manifest coreworkspace.NormalizeLaneManifest
|
||||
if decision := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !decision.Reused {
|
||||
return pipeline.NormalizeCheckpoint{}, decision
|
||||
}
|
||||
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageNormalize, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !decision.Reused {
|
||||
return pipeline.NormalizeCheckpoint{}, decision
|
||||
}
|
||||
var payload normalizeOutputEnvelope
|
||||
if decision := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !decision.Reused {
|
||||
return pipeline.NormalizeCheckpoint{}, decision
|
||||
}
|
||||
output, err := normalizeOutputFromEnvelope(payload.Output)
|
||||
if err != nil {
|
||||
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize checkpoint payload is invalid: %v", err)
|
||||
}
|
||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests([]contracts.RawPayload{output.Payload})) {
|
||||
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize checkpoint output digest does not match payload")
|
||||
}
|
||||
return pipeline.NormalizeCheckpoint{Output: output, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) readJSON(name string, out any) pipeline.CheckpointDecision {
|
||||
if !l.Enabled() {
|
||||
return pipeline.CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
target, err := coreworkspace.SafePath(l.root, name)
|
||||
if err != nil {
|
||||
return invalidDecision("checkpoint path is invalid: %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return pipeline.CheckpointDecision{Reason: "checkpoint artifact is missing"}
|
||||
}
|
||||
return invalidDecision("read checkpoint artifact: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
return invalidDecision("decode checkpoint artifact: %v", err)
|
||||
}
|
||||
return reusedDecision()
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) validateManifest(manifest coreworkspace.StageManifest, stage coreworkspace.StageName, laneID string, moduleKey string, status coreworkspace.StageStatus, dependencies []pipeline.CheckpointFingerprint) pipeline.CheckpointDecision {
|
||||
return l.validateLaneManifest(manifest, stage, laneID, moduleKey, dependencies, status)
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) validateLaneManifest(manifest coreworkspace.StageManifest, stage coreworkspace.StageName, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, statuses ...coreworkspace.StageStatus) pipeline.CheckpointDecision {
|
||||
if manifest.WorkspaceSchemaVersion != coreworkspace.WorkspaceSchemaVersion {
|
||||
return invalidDecision("checkpoint workspace schema version %q is not supported", manifest.WorkspaceSchemaVersion)
|
||||
}
|
||||
if strings.TrimSpace(l.identityDigest) != "" && manifest.Metadata["checkpoint_identity_digest"] != l.identityDigest {
|
||||
return invalidDecision("checkpoint identity digest does not match current invocation")
|
||||
}
|
||||
if manifest.Stage != stage {
|
||||
return invalidDecision("checkpoint stage %q does not match %q", manifest.Stage, stage)
|
||||
}
|
||||
if strings.TrimSpace(laneID) != "" && manifest.LaneID != laneID {
|
||||
return invalidDecision("checkpoint lane %q does not match %q", manifest.LaneID, laneID)
|
||||
}
|
||||
if strings.TrimSpace(moduleKey) != "" && manifest.ModuleKey != moduleKey {
|
||||
return invalidDecision("checkpoint module %q does not match %q", manifest.ModuleKey, moduleKey)
|
||||
}
|
||||
statusOK := false
|
||||
for _, status := range statuses {
|
||||
if manifest.Status == status {
|
||||
statusOK = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !statusOK {
|
||||
return invalidDecision("checkpoint status %q cannot be reused", manifest.Status)
|
||||
}
|
||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.DependencyFingerprints), dependencies) {
|
||||
return invalidDecision("checkpoint dependency fingerprints do not match")
|
||||
}
|
||||
return reusedDecision()
|
||||
}
|
||||
|
||||
func sourceChunksFromEnvelope(values []chunkEnvelope) ([]contracts.SourceChunk, error) {
|
||||
if len(values) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
out := make([]contracts.SourceChunk, 0, len(values))
|
||||
for _, value := range values {
|
||||
content, err := contentFromEnvelope(value.Content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, contracts.SourceChunk{
|
||||
ID: value.ID,
|
||||
SourceID: value.SourceID,
|
||||
Index: value.Index,
|
||||
StartUnitID: value.StartUnitID,
|
||||
EndUnitID: value.EndUnitID,
|
||||
Content: content,
|
||||
MediaType: value.Content.MediaType,
|
||||
Units: cloneSourceUnits(value.Units),
|
||||
Metadata: cloneMetadata(value.Metadata),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func extractOutputsFromEnvelope(values []extractOutputEnvelope) ([]contracts.ExtractOutput, error) {
|
||||
if len(values) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
out := make([]contracts.ExtractOutput, 0, len(values))
|
||||
for _, value := range values {
|
||||
payload, err := rawPayloadFromEnvelope(value.Payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, contracts.ExtractOutput{
|
||||
LaneID: value.LaneID,
|
||||
ExtractorKey: value.ExtractorKey,
|
||||
SourceID: value.SourceID,
|
||||
ChunkID: value.ChunkID,
|
||||
ChunkIndex: value.ChunkIndex,
|
||||
Schema: value.Schema,
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func mergeOutputFromEnvelope(value mergeOutputPayload) (contracts.MergeOutput, error) {
|
||||
payload, err := rawPayloadFromEnvelope(value.Payload)
|
||||
if err != nil {
|
||||
return contracts.MergeOutput{}, err
|
||||
}
|
||||
return contracts.MergeOutput{
|
||||
LaneID: value.LaneID,
|
||||
MergerKey: value.MergerKey,
|
||||
SourceID: value.SourceID,
|
||||
Schema: value.Schema,
|
||||
Payload: payload,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func normalizeOutputFromEnvelope(value normalizeOutputPayload) (contracts.NormalizeOutput, error) {
|
||||
payload, err := rawPayloadFromEnvelope(value.Payload)
|
||||
if err != nil {
|
||||
return contracts.NormalizeOutput{}, err
|
||||
}
|
||||
return contracts.NormalizeOutput{
|
||||
LaneID: value.LaneID,
|
||||
NormalizerKey: value.NormalizerKey,
|
||||
SourceID: value.SourceID,
|
||||
Schema: value.Schema,
|
||||
Payload: payload,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func rawPayloadFromEnvelope(value binaryEnvelope) (contracts.RawPayload, error) {
|
||||
content, err := contentFromEnvelope(value)
|
||||
if err != nil {
|
||||
return contracts.RawPayload{}, err
|
||||
}
|
||||
return contracts.RawPayload{
|
||||
Content: content,
|
||||
MediaType: value.MediaType,
|
||||
Metadata: cloneMetadata(value.Metadata),
|
||||
Warnings: cloneWarnings(value.Warnings),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func contentFromEnvelope(value binaryEnvelope) ([]byte, error) {
|
||||
content, err := base64.StdEncoding.DecodeString(value.ContentBase64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode content_base64: %w", err)
|
||||
}
|
||||
if digest := strings.TrimSpace(value.ContentDigest); digest != "" && digest != contentDigest(content) {
|
||||
return nil, fmt.Errorf("content digest mismatch")
|
||||
}
|
||||
return content, nil
|
||||
}
|
||||
|
||||
func coreworkspaceToPipelineFingerprints(values []coreworkspace.Fingerprint) []pipeline.CheckpointFingerprint {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]pipeline.CheckpointFingerprint, 0, len(values))
|
||||
for _, value := range values {
|
||||
out = append(out, pipeline.CheckpointFingerprint{Name: value.Name, Value: value.Value})
|
||||
}
|
||||
return normalizeFingerprints(out)
|
||||
}
|
||||
|
||||
func fingerprintsEqual(a []pipeline.CheckpointFingerprint, b []pipeline.CheckpointFingerprint) bool {
|
||||
a = normalizeFingerprints(a)
|
||||
b = normalizeFingerprints(b)
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func reusedDecision() pipeline.CheckpointDecision {
|
||||
return pipeline.CheckpointDecision{Reused: true, Reason: "checkpoint is valid"}
|
||||
}
|
||||
|
||||
func invalidDecision(format string, args ...any) pipeline.CheckpointDecision {
|
||||
return pipeline.CheckpointDecision{Reason: fmt.Sprintf(format, args...)}
|
||||
}
|
||||
634
internal/framework/checkpoint/recorder.go
Normal file
634
internal/framework/checkpoint/recorder.go
Normal file
@@ -0,0 +1,634 @@
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
type WorkspaceRecorder struct {
|
||||
root string
|
||||
identityDigest string
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewWorkspaceRecorder(settings coreworkspace.Settings, identity coreworkspace.CheckpointIdentity) (pipeline.CheckpointRecorder, error) {
|
||||
root, err := settings.CheckpointDirectory(identity)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return pipeline.NoopCheckpointRecorder(), nil
|
||||
}
|
||||
return &WorkspaceRecorder{root: root, identityDigest: identity.Digest, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) SourceRunning(moduleKey string) error {
|
||||
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusRunning)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) SourceSucceeded(moduleKey string, doc *source.SourceDocument) error {
|
||||
if doc == nil {
|
||||
return fmt.Errorf("checkpoint source document must not be nil")
|
||||
}
|
||||
if err := r.writePayload("source/source-document.json", sourceDocumentEnvelope{Document: cloneSourceDocument(*doc)}); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusSucceeded)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.OutputDigests = workspaceFingerprints(digestFingerprints("source_document", doc.Digest))
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{
|
||||
StageManifest: manifest,
|
||||
SourceID: doc.ID,
|
||||
})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) SourceFailed(moduleKey string, err error) error {
|
||||
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusFailed)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ChunkRunning(moduleKey string, sourceDigest string) error {
|
||||
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusRunning)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ChunkSucceeded(moduleKey string, sourceDigest string, chunks []contracts.SourceChunk, warnings []contracts.Warning) error {
|
||||
payload := chunksEnvelope{Chunks: chunkEnvelopes(chunks), Warnings: cloneWarnings(warnings)}
|
||||
if err := r.writePayload("chunk/chunks.json", payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceeded)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
||||
manifest.OutputDigests = workspaceFingerprints(chunkOutputDigests(chunks))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{
|
||||
StageManifest: manifest,
|
||||
ChunkCount: len(chunks),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error {
|
||||
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceededWithRejections)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
||||
manifest.ValidationStatus = "rejected"
|
||||
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ChunkFailed(moduleKey string, sourceDigest string, err error) error {
|
||||
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusFailed)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ExtractRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ExtractSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []contracts.ExtractOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
|
||||
payload := extractOutputsEnvelope{
|
||||
Outputs: extractOutputEnvelopes(outputs),
|
||||
Rejected: cloneRejectedOutputs(rejected),
|
||||
Warnings: cloneWarnings(warnings),
|
||||
}
|
||||
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.laneManifest(coreworkspace.StageExtract, statusForRejected(rejected), laneID, moduleKey, dependencies)
|
||||
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests(extractPayloads(outputs)))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, rejected)
|
||||
manifest.Rejections = rejectionSummaries(rejected)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{
|
||||
StageManifest: manifest,
|
||||
ChunkCount: len(outputs) + len(rejected),
|
||||
OutputCount: len(outputs),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ExtractFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) MergeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) MergeSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output contracts.MergeOutput, warnings []contracts.Warning) error {
|
||||
payload := mergeOutputEnvelope{Output: mergeOutputEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}
|
||||
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
|
||||
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{
|
||||
StageManifest: manifest,
|
||||
InputCount: len(dependencies),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) MergeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
|
||||
manifest.ValidationStatus = "rejected"
|
||||
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) MergeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) NormalizeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) NormalizeSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output contracts.NormalizeOutput, warnings []contracts.Warning) error {
|
||||
payload := normalizeOutputEnvelope{Output: normalizeOutputEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}
|
||||
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
|
||||
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) NormalizeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
|
||||
manifest.ValidationStatus = "rejected"
|
||||
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) NormalizeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) writeManifest(name string, payload any) error {
|
||||
return r.writeJSON(name, payload)
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) writePayload(name string, payload any) error {
|
||||
return r.writeJSON(name, payload)
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) writeJSON(name string, payload any) error {
|
||||
if r == nil || strings.TrimSpace(r.root) == "" {
|
||||
return nil
|
||||
}
|
||||
return coreworkspace.WriteJSON(r.root, name, payload)
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) timestamp() time.Time {
|
||||
if r == nil || r.now == nil {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
return r.now().UTC()
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) newStageManifest(stage coreworkspace.StageName, status coreworkspace.StageStatus) coreworkspace.StageManifest {
|
||||
manifest := coreworkspace.NewStageManifest(stage, status)
|
||||
if strings.TrimSpace(r.identityDigest) != "" {
|
||||
manifest.Metadata = map[string]string{"checkpoint_identity_digest": r.identityDigest}
|
||||
}
|
||||
return manifest
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) laneManifest(stage coreworkspace.StageName, status coreworkspace.StageStatus, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) coreworkspace.StageManifest {
|
||||
manifest := r.newStageManifest(stage, status)
|
||||
manifest.LaneID = laneID
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(dependencies)
|
||||
return manifest
|
||||
}
|
||||
|
||||
type sourceDocumentEnvelope struct {
|
||||
Document source.SourceDocument `json:"document"`
|
||||
}
|
||||
|
||||
type chunksEnvelope struct {
|
||||
Chunks []chunkEnvelope `json:"chunks"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type chunkEnvelope struct {
|
||||
ID string `json:"id"`
|
||||
SourceID string `json:"source_id"`
|
||||
Index int `json:"index"`
|
||||
StartUnitID int `json:"start_unit_id"`
|
||||
EndUnitID int `json:"end_unit_id"`
|
||||
Content binaryEnvelope `json:"content"`
|
||||
Units []source.SourceUnit `json:"units,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type extractOutputsEnvelope struct {
|
||||
Outputs []extractOutputEnvelope `json:"outputs"`
|
||||
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type extractOutputEnvelope struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
SourceID string `json:"source_id"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ChunkIndex int `json:"chunk_index"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload binaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type mergeOutputEnvelope struct {
|
||||
Output mergeOutputPayload `json:"output"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type mergeOutputPayload struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
MergerKey string `json:"merger_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload binaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type normalizeOutputEnvelope struct {
|
||||
Output normalizeOutputPayload `json:"output"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type normalizeOutputPayload struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
NormalizerKey string `json:"normalizer_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload binaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type binaryEnvelope struct {
|
||||
ContentBase64 string `json:"content_base64,omitempty"`
|
||||
ContentDigest string `json:"content_digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
func chunkEnvelopes(chunks []contracts.SourceChunk) []chunkEnvelope {
|
||||
if len(chunks) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]chunkEnvelope, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
out = append(out, chunkEnvelope{
|
||||
ID: chunk.ID,
|
||||
SourceID: chunk.SourceID,
|
||||
Index: chunk.Index,
|
||||
StartUnitID: chunk.StartUnitID,
|
||||
EndUnitID: chunk.EndUnitID,
|
||||
Content: binaryEnvelopeFromContent(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
|
||||
Units: cloneSourceUnits(chunk.Units),
|
||||
Metadata: cloneMetadata(chunk.Metadata),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func extractOutputEnvelopes(outputs []contracts.ExtractOutput) []extractOutputEnvelope {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]extractOutputEnvelope, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
out = append(out, extractOutputEnvelope{
|
||||
LaneID: output.LaneID,
|
||||
ExtractorKey: output.ExtractorKey,
|
||||
SourceID: output.SourceID,
|
||||
ChunkID: output.ChunkID,
|
||||
ChunkIndex: output.ChunkIndex,
|
||||
Schema: schemaEnvelope(output.Schema),
|
||||
Payload: binaryEnvelopeFromPayload(output.Payload),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mergeOutputEnvelopeFromOutput(output contracts.MergeOutput) mergeOutputPayload {
|
||||
return mergeOutputPayload{
|
||||
LaneID: output.LaneID,
|
||||
MergerKey: output.MergerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: schemaEnvelope(output.Schema),
|
||||
Payload: binaryEnvelopeFromPayload(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeOutputEnvelopeFromOutput(output contracts.NormalizeOutput) normalizeOutputPayload {
|
||||
return normalizeOutputPayload{
|
||||
LaneID: output.LaneID,
|
||||
NormalizerKey: output.NormalizerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: schemaEnvelope(output.Schema),
|
||||
Payload: binaryEnvelopeFromPayload(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func schemaEnvelope(schema contracts.ResponseSchema) contracts.ResponseSchema {
|
||||
schema.JSONSchema = nil
|
||||
return schema
|
||||
}
|
||||
|
||||
func binaryEnvelopeFromPayload(payload contracts.RawPayload) binaryEnvelope {
|
||||
return binaryEnvelopeFromContent(payload.Content, payload.MediaType, payload.Metadata, payload.Warnings)
|
||||
}
|
||||
|
||||
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope {
|
||||
return binaryEnvelope{
|
||||
ContentBase64: base64.StdEncoding.EncodeToString(content),
|
||||
ContentDigest: contentDigest(content),
|
||||
MediaType: mediaType,
|
||||
Metadata: cloneMetadata(metadata),
|
||||
Warnings: cloneWarnings(warnings),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneSourceDocument(doc source.SourceDocument) source.SourceDocument {
|
||||
doc.Units = cloneSourceUnits(doc.Units)
|
||||
doc.Metadata = cloneMetadata(doc.Metadata)
|
||||
return doc
|
||||
}
|
||||
|
||||
func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
|
||||
if len(units) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]source.SourceUnit, 0, len(units))
|
||||
for _, unit := range units {
|
||||
out = append(out, source.SourceUnit{
|
||||
ID: unit.ID,
|
||||
Kind: unit.Kind,
|
||||
Text: unit.Text,
|
||||
Metadata: cloneMetadata(unit.Metadata),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
|
||||
if len(warnings) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]contracts.Warning(nil), warnings...)
|
||||
}
|
||||
|
||||
func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
|
||||
if len(rejected) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]contracts.RejectedOutput(nil), rejected...)
|
||||
}
|
||||
|
||||
func cloneMetadata(metadata map[string]any) map[string]any {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(metadata))
|
||||
for key, value := range metadata {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rawOutputDigests(payloads []contracts.RawPayload) []pipeline.CheckpointFingerprint {
|
||||
values := make([]pipeline.CheckpointFingerprint, 0, len(payloads))
|
||||
for i, payload := range payloads {
|
||||
values = append(values, pipeline.CheckpointFingerprint{
|
||||
Name: fmt.Sprintf("payload[%d]", i),
|
||||
Value: contentDigest(payload.Content),
|
||||
})
|
||||
}
|
||||
return normalizeFingerprints(values)
|
||||
}
|
||||
|
||||
func extractPayloads(outputs []contracts.ExtractOutput) []contracts.RawPayload {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
payloads := make([]contracts.RawPayload, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
payloads = append(payloads, output.Payload)
|
||||
}
|
||||
return payloads
|
||||
}
|
||||
|
||||
func chunkOutputDigests(chunks []contracts.SourceChunk) []pipeline.CheckpointFingerprint {
|
||||
values := make([]pipeline.CheckpointFingerprint, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
values = append(values, pipeline.CheckpointFingerprint{
|
||||
Name: chunk.ID,
|
||||
Value: contentDigest(chunk.Content),
|
||||
})
|
||||
}
|
||||
return normalizeFingerprints(values)
|
||||
}
|
||||
|
||||
func digestFingerprints(name string, digest string) []pipeline.CheckpointFingerprint {
|
||||
digest = strings.TrimSpace(digest)
|
||||
if digest == "" {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{{Name: name, Value: digest}}
|
||||
}
|
||||
|
||||
func workspaceFingerprints(values []pipeline.CheckpointFingerprint) []coreworkspace.Fingerprint {
|
||||
normalized := normalizeFingerprints(values)
|
||||
if len(normalized) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]coreworkspace.Fingerprint, 0, len(normalized))
|
||||
for _, value := range normalized {
|
||||
out = append(out, coreworkspace.Fingerprint{Name: value.Name, Value: value.Value})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeFingerprints(values []pipeline.CheckpointFingerprint) []pipeline.CheckpointFingerprint {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
byName := make(map[string]string, len(values))
|
||||
for _, value := range values {
|
||||
name := strings.TrimSpace(value.Name)
|
||||
fingerprint := strings.TrimSpace(value.Value)
|
||||
if name == "" || fingerprint == "" {
|
||||
continue
|
||||
}
|
||||
byName[name] = fingerprint
|
||||
}
|
||||
if len(byName) == 0 {
|
||||
return nil
|
||||
}
|
||||
names := make([]string, 0, len(byName))
|
||||
for name := range byName {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
out := make([]pipeline.CheckpointFingerprint, 0, len(names))
|
||||
for _, name := range names {
|
||||
out = append(out, pipeline.CheckpointFingerprint{Name: name, Value: byName[name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rejectionSummaries(rejected []contracts.RejectedOutput) []coreworkspace.RejectionSummary {
|
||||
if len(rejected) == 0 {
|
||||
return nil
|
||||
}
|
||||
type key struct {
|
||||
validatorName string
|
||||
reasonCode string
|
||||
message string
|
||||
}
|
||||
counts := make(map[key]int, len(rejected))
|
||||
for _, item := range rejected {
|
||||
k := key{validatorName: item.ValidatorName, reasonCode: item.ReasonCode, message: item.Message}
|
||||
counts[k]++
|
||||
}
|
||||
keys := make([]key, 0, len(counts))
|
||||
for k := range counts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].validatorName != keys[j].validatorName {
|
||||
return keys[i].validatorName < keys[j].validatorName
|
||||
}
|
||||
if keys[i].reasonCode != keys[j].reasonCode {
|
||||
return keys[i].reasonCode < keys[j].reasonCode
|
||||
}
|
||||
return keys[i].message < keys[j].message
|
||||
})
|
||||
out := make([]coreworkspace.RejectionSummary, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
out = append(out, coreworkspace.RejectionSummary{
|
||||
ValidatorName: k.validatorName,
|
||||
ReasonCode: k.reasonCode,
|
||||
Message: k.message,
|
||||
Count: counts[k],
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func statusForRejected(rejected []contracts.RejectedOutput) coreworkspace.StageStatus {
|
||||
if len(rejected) > 0 {
|
||||
return coreworkspace.StatusSucceededWithRejections
|
||||
}
|
||||
return coreworkspace.StatusSucceeded
|
||||
}
|
||||
|
||||
func validationStatusString(warnings []contracts.Warning, rejected []contracts.RejectedOutput) string {
|
||||
if len(rejected) > 0 {
|
||||
return "rejected"
|
||||
}
|
||||
if len(warnings) > 0 {
|
||||
return "approved_with_warnings"
|
||||
}
|
||||
return "approved"
|
||||
}
|
||||
|
||||
func errorMetadata(err error) map[string]string {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{"error": err.Error()}
|
||||
}
|
||||
|
||||
func laneManifestPath(stage string, laneID string) string {
|
||||
return lanePayloadPath(stage, laneID, "manifest.json")
|
||||
}
|
||||
|
||||
func lanePayloadPath(stage string, laneID string, file string) string {
|
||||
return path.Join(stage, checkpointPathComponent(laneID), file)
|
||||
}
|
||||
|
||||
func checkpointPathComponent(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return "_"
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
b.WriteRune(r)
|
||||
case r >= 'A' && r <= 'Z':
|
||||
b.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '-' || r == '_' || r == '.':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteString(fmt.Sprintf("~%x", r))
|
||||
}
|
||||
}
|
||||
out := b.String()
|
||||
if out == "." || out == ".." || strings.Contains(out, "..") {
|
||||
return "_"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func contentDigest(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func timePtr(t time.Time) *time.Time {
|
||||
return &t
|
||||
}
|
||||
344
internal/framework/checkpoint/recorder_test.go
Normal file
344
internal/framework/checkpoint/recorder_test.go
Normal file
@@ -0,0 +1,344 @@
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
doc := &source.SourceDocument{
|
||||
ID: "source-1",
|
||||
Kind: "document",
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:source",
|
||||
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello"}},
|
||||
}
|
||||
chunks := []contracts.SourceChunk{
|
||||
{
|
||||
ID: "chunk-1",
|
||||
SourceID: "source-1",
|
||||
Index: 0,
|
||||
StartUnitID: 1,
|
||||
EndUnitID: 1,
|
||||
Content: []byte("chunk content"),
|
||||
MediaType: "text/plain",
|
||||
Units: doc.Units,
|
||||
},
|
||||
}
|
||||
|
||||
if err := recorder.SourceRunning("seriatim"); err != nil {
|
||||
t.Fatalf("SourceRunning: %v", err)
|
||||
}
|
||||
assertManifestStatus(t, filepath.Join(root, "source", "manifest.json"), coreworkspace.StatusRunning)
|
||||
if err := recorder.SourceSucceeded("seriatim", doc); err != nil {
|
||||
t.Fatalf("SourceSucceeded: %v", err)
|
||||
}
|
||||
assertManifestStatus(t, filepath.Join(root, "source", "manifest.json"), coreworkspace.StatusSucceeded)
|
||||
if _, err := os.Stat(filepath.Join(root, "source", "source-document.json")); err != nil {
|
||||
t.Fatalf("expected source checkpoint payload: %v", err)
|
||||
}
|
||||
|
||||
if err := recorder.ChunkRunning("generic", doc.Digest); err != nil {
|
||||
t.Fatalf("ChunkRunning: %v", err)
|
||||
}
|
||||
if err := recorder.ChunkSucceeded("generic", doc.Digest, chunks, nil); err != nil {
|
||||
t.Fatalf("ChunkSucceeded: %v", err)
|
||||
}
|
||||
assertManifestStatus(t, filepath.Join(root, "chunk", "manifest.json"), coreworkspace.StatusSucceeded)
|
||||
var chunkPayload struct {
|
||||
Chunks []struct {
|
||||
Content struct {
|
||||
ContentBase64 string `json:"content_base64"`
|
||||
ContentDigest string `json:"content_digest"`
|
||||
} `json:"content"`
|
||||
} `json:"chunks"`
|
||||
}
|
||||
readJSON(t, filepath.Join(root, "chunk", "chunks.json"), &chunkPayload)
|
||||
if len(chunkPayload.Chunks) != 1 {
|
||||
t.Fatalf("checkpoint chunks = %#v, want one", chunkPayload.Chunks)
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(chunkPayload.Chunks[0].Content.ContentBase64)
|
||||
if err != nil {
|
||||
t.Fatalf("decode chunk content: %v", err)
|
||||
}
|
||||
if string(decoded) != "chunk content" {
|
||||
t.Fatalf("chunk content = %q, want original content", decoded)
|
||||
}
|
||||
if got, want := chunkPayload.Chunks[0].Content.ContentDigest, contentDigest([]byte("chunk content")); got != want {
|
||||
t.Fatalf("content digest = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceLoaderReusesSuccessfulCheckpointFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
loader := &WorkspaceLoader{root: root}
|
||||
doc := &source.SourceDocument{
|
||||
ID: "source-1",
|
||||
Kind: "document",
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:source",
|
||||
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello"}},
|
||||
}
|
||||
chunks := []contracts.SourceChunk{
|
||||
{
|
||||
ID: "chunk-1",
|
||||
SourceID: "source-1",
|
||||
Index: 0,
|
||||
StartUnitID: 1,
|
||||
EndUnitID: 1,
|
||||
Content: []byte("chunk content"),
|
||||
MediaType: "text/plain",
|
||||
Units: doc.Units,
|
||||
},
|
||||
}
|
||||
extractOutput := contracts.ExtractOutput{
|
||||
LaneID: "spells",
|
||||
ExtractorKey: "dnd/spells",
|
||||
SourceID: doc.ID,
|
||||
ChunkID: "chunk-1",
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"spell":"cure wounds"}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
mergeOutput := contracts.MergeOutput{
|
||||
LaneID: "spells",
|
||||
MergerKey: "appendorder",
|
||||
SourceID: doc.ID,
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"merged":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
normalizeOutput := contracts.NormalizeOutput{
|
||||
LaneID: "spells",
|
||||
NormalizerKey: "noop",
|
||||
SourceID: doc.ID,
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"normalized":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
|
||||
if err := recorder.SourceSucceeded("seriatim", doc); err != nil {
|
||||
t.Fatalf("SourceSucceeded: %v", err)
|
||||
}
|
||||
if err := recorder.ChunkSucceeded("generic", doc.Digest, chunks, nil); err != nil {
|
||||
t.Fatalf("ChunkSucceeded: %v", err)
|
||||
}
|
||||
extractDeps := []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}
|
||||
if err := recorder.ExtractSucceeded("spells", "dnd/spells", extractDeps, []contracts.ExtractOutput{extractOutput}, nil, nil); err != nil {
|
||||
t.Fatalf("ExtractSucceeded: %v", err)
|
||||
}
|
||||
mergeDeps := rawOutputDigests([]contracts.RawPayload{extractOutput.Payload})
|
||||
if err := recorder.MergeSucceeded("spells", "appendorder", mergeDeps, mergeOutput, nil); err != nil {
|
||||
t.Fatalf("MergeSucceeded: %v", err)
|
||||
}
|
||||
normalizeDeps := rawOutputDigests([]contracts.RawPayload{mergeOutput.Payload})
|
||||
if err := recorder.NormalizeSucceeded("spells", "noop", normalizeDeps, normalizeOutput, nil); err != nil {
|
||||
t.Fatalf("NormalizeSucceeded: %v", err)
|
||||
}
|
||||
|
||||
sourceCheckpoint, decision := loader.Source("seriatim")
|
||||
if !decision.Reused || sourceCheckpoint.Document.ID != "source-1" {
|
||||
t.Fatalf("source decision = %#v checkpoint=%#v, want reused", decision, sourceCheckpoint)
|
||||
}
|
||||
chunkCheckpoint, decision := loader.Chunk("generic", doc.Digest)
|
||||
if !decision.Reused || len(chunkCheckpoint.Chunks) != 1 || string(chunkCheckpoint.Chunks[0].Content) != "chunk content" {
|
||||
t.Fatalf("chunk decision = %#v checkpoint=%#v, want reused", decision, chunkCheckpoint)
|
||||
}
|
||||
extractCheckpoint, decision := loader.Extract("spells", "dnd/spells", extractDeps)
|
||||
if !decision.Reused || len(extractCheckpoint.Outputs) != 1 || string(extractCheckpoint.Outputs[0].Payload.Content) != `{"spell":"cure wounds"}` {
|
||||
t.Fatalf("extract decision = %#v checkpoint=%#v, want reused", decision, extractCheckpoint)
|
||||
}
|
||||
mergeCheckpoint, decision := loader.Merge("spells", "appendorder", mergeDeps)
|
||||
if !decision.Reused || string(mergeCheckpoint.Output.Payload.Content) != `{"merged":true}` {
|
||||
t.Fatalf("merge decision = %#v checkpoint=%#v, want reused", decision, mergeCheckpoint)
|
||||
}
|
||||
normalizeCheckpoint, decision := loader.Normalize("spells", "noop", normalizeDeps)
|
||||
if !decision.Reused || string(normalizeCheckpoint.Output.Payload.Content) != `{"normalized":true}` {
|
||||
t.Fatalf("normalize decision = %#v checkpoint=%#v, want reused", decision, normalizeCheckpoint)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceLoaderInvalidatesMissingCorruptAndMismatchedCheckpoints(t *testing.T) {
|
||||
t.Run("missing", func(t *testing.T) {
|
||||
loader := &WorkspaceLoader{root: t.TempDir()}
|
||||
if _, decision := loader.Source("seriatim"); decision.Reused || !strings.Contains(decision.Reason, "missing") {
|
||||
t.Fatalf("decision = %#v, want missing invalidation", decision)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("dependency mismatch", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
chunks := []contracts.SourceChunk{{
|
||||
ID: "chunk-1",
|
||||
SourceID: "source-1",
|
||||
Content: []byte("chunk content"),
|
||||
MediaType: "text/plain",
|
||||
}}
|
||||
if err := recorder.ChunkSucceeded("generic", "sha256:source-a", chunks, nil); err != nil {
|
||||
t.Fatalf("ChunkSucceeded: %v", err)
|
||||
}
|
||||
loader := &WorkspaceLoader{root: root}
|
||||
if _, decision := loader.Chunk("generic", "sha256:source-b"); decision.Reused || !strings.Contains(decision.Reason, "dependency") {
|
||||
t.Fatalf("decision = %#v, want dependency invalidation", decision)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("corrupt payload", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
chunks := []contracts.SourceChunk{{
|
||||
ID: "chunk-1",
|
||||
SourceID: "source-1",
|
||||
Content: []byte("chunk content"),
|
||||
MediaType: "text/plain",
|
||||
}}
|
||||
if err := recorder.ChunkSucceeded("generic", "sha256:source", chunks, nil); err != nil {
|
||||
t.Fatalf("ChunkSucceeded: %v", err)
|
||||
}
|
||||
payloadPath := filepath.Join(root, "chunk", "chunks.json")
|
||||
data := strings.ReplaceAll(string(readFile(t, payloadPath)), contentDigest([]byte("chunk content")), "sha256:bad")
|
||||
if err := os.WriteFile(payloadPath, []byte(data), 0o644); err != nil {
|
||||
t.Fatalf("corrupt chunk payload: %v", err)
|
||||
}
|
||||
loader := &WorkspaceLoader{root: root}
|
||||
if _, decision := loader.Chunk("generic", "sha256:source"); decision.Reused || !strings.Contains(decision.Reason, "invalid") {
|
||||
t.Fatalf("decision = %#v, want corrupt payload invalidation", decision)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestWorkspaceRecorderRecordsRejectedExtractOutputs(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
rejected := []contracts.RejectedOutput{
|
||||
{
|
||||
Stage: string(pipeline.StageExtract),
|
||||
LaneID: "spells",
|
||||
ModuleKey: "dnd/spells",
|
||||
ChunkID: "chunk-1",
|
||||
ValidatorName: "shape",
|
||||
ReasonCode: "invalid_shape",
|
||||
Message: "bad shape",
|
||||
},
|
||||
}
|
||||
|
||||
if err := recorder.ExtractRunning("spells", "dnd/spells", []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}); err != nil {
|
||||
t.Fatalf("ExtractRunning: %v", err)
|
||||
}
|
||||
if err := recorder.ExtractSucceeded("spells", "dnd/spells", nil, nil, rejected, nil); err != nil {
|
||||
t.Fatalf("ExtractSucceeded: %v", err)
|
||||
}
|
||||
|
||||
var manifest coreworkspace.ExtractLaneManifest
|
||||
readJSON(t, filepath.Join(root, "extract", "spells", "manifest.json"), &manifest)
|
||||
if manifest.Status != coreworkspace.StatusSucceededWithRejections || manifest.ValidationStatus != "rejected" {
|
||||
t.Fatalf("extract manifest status = %q validation=%q", manifest.Status, manifest.ValidationStatus)
|
||||
}
|
||||
if len(manifest.Rejections) != 1 || manifest.Rejections[0].Count != 1 || manifest.Rejections[0].ReasonCode != "invalid_shape" {
|
||||
t.Fatalf("rejections = %#v", manifest.Rejections)
|
||||
}
|
||||
var payload struct {
|
||||
Rejected []contracts.RejectedOutput `json:"rejected"`
|
||||
}
|
||||
readJSON(t, filepath.Join(root, "extract", "spells", "outputs.json"), &payload)
|
||||
if len(payload.Rejected) != 1 || payload.Rejected[0].ChunkID != "chunk-1" {
|
||||
t.Fatalf("checkpoint rejected payload = %#v", payload.Rejected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceRecorderRecordsFailedStages(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
|
||||
if err := recorder.MergeRunning("spells", "appendorder", nil); err != nil {
|
||||
t.Fatalf("MergeRunning: %v", err)
|
||||
}
|
||||
if err := recorder.MergeFailed("spells", "appendorder", nil, assertErr("merge failed")); err != nil {
|
||||
t.Fatalf("MergeFailed: %v", err)
|
||||
}
|
||||
|
||||
var manifest coreworkspace.MergeLaneManifest
|
||||
readJSON(t, filepath.Join(root, "merge", "spells", "manifest.json"), &manifest)
|
||||
if manifest.Status != coreworkspace.StatusFailed {
|
||||
t.Fatalf("status = %q, want failed", manifest.Status)
|
||||
}
|
||||
if !strings.Contains(manifest.Metadata["error"], "merge failed") {
|
||||
t.Fatalf("metadata = %#v, want error", manifest.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceRecorderRecordsWarningOnlyValidation(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
output := contracts.NormalizeOutput{
|
||||
LaneID: "spells",
|
||||
NormalizerKey: "noop",
|
||||
SourceID: "source-1",
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"ok":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
warnings := []contracts.Warning{{ReasonCode: "note", Message: "warning"}}
|
||||
|
||||
if err := recorder.NormalizeSucceeded("spells", "noop", nil, output, warnings); err != nil {
|
||||
t.Fatalf("NormalizeSucceeded: %v", err)
|
||||
}
|
||||
|
||||
var manifest coreworkspace.NormalizeLaneManifest
|
||||
readJSON(t, filepath.Join(root, "normalize", "spells", "manifest.json"), &manifest)
|
||||
if manifest.Status != coreworkspace.StatusSucceeded || manifest.ValidationStatus != "approved_with_warnings" {
|
||||
t.Fatalf("normalize manifest status = %q validation=%q", manifest.Status, manifest.ValidationStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestRecorder(t *testing.T, root string) *WorkspaceRecorder {
|
||||
t.Helper()
|
||||
return &WorkspaceRecorder{root: root}
|
||||
}
|
||||
|
||||
func assertManifestStatus(t *testing.T, path string, want coreworkspace.StageStatus) {
|
||||
t.Helper()
|
||||
var manifest coreworkspace.StageManifest
|
||||
readJSON(t, path, &manifest)
|
||||
if manifest.Status != want {
|
||||
t.Fatalf("%s status = %q, want %q", path, manifest.Status, want)
|
||||
}
|
||||
}
|
||||
|
||||
func readJSON(t *testing.T, path string, out any) {
|
||||
t.Helper()
|
||||
data := readFile(t, path)
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
t.Fatalf("decode %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %q: %v", path, err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
type assertErr string
|
||||
|
||||
func (e assertErr) Error() string { return string(e) }
|
||||
@@ -17,6 +17,7 @@ var _ contracts.Extractor = compositionExtractor{}
|
||||
var _ contracts.Merger = compositionMerger{}
|
||||
var _ contracts.Normalizer = compositionNormalizer{}
|
||||
var _ contracts.Validator = compositionValidator{}
|
||||
var _ contracts.StructuredLLMClient = compositionLLMClient{}
|
||||
var _ contracts.OutputEncoder = compositionOutputEncoder{}
|
||||
|
||||
func TestContractsComposeAcrossPackages(t *testing.T) {
|
||||
@@ -26,7 +27,6 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
|
||||
extractor := compositionExtractor{}
|
||||
merger := compositionMerger{}
|
||||
normalizer := compositionNormalizer{}
|
||||
validator := compositionValidator{}
|
||||
encoder := compositionOutputEncoder{}
|
||||
|
||||
doc, err := adapter.Parse(ctx, contracts.ParseRequest{SourceID: "source-1"})
|
||||
@@ -38,8 +38,9 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
|
||||
}
|
||||
|
||||
chunking, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
||||
Source: doc,
|
||||
Metadata: map[string]any{"max_units": 2},
|
||||
Source: doc,
|
||||
LLMClient: compositionLLMClient{},
|
||||
Metadata: map[string]any{"max_units": 2},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Chunk() error = %v, want nil", err)
|
||||
@@ -56,70 +57,37 @@ func TestContractsComposeAcrossPackages(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
if len(extraction.Candidates) != 1 {
|
||||
t.Fatalf("len(Candidates) = %d, want 1", len(extraction.Candidates))
|
||||
}
|
||||
|
||||
candidate := extraction.Candidates[0]
|
||||
for _, ref := range candidate.SourceRefs {
|
||||
if err := source.ValidateRef(doc, ref); err != nil {
|
||||
t.Fatalf("ValidateRef() error = %v, want nil", err)
|
||||
}
|
||||
if extraction.Output.Payload.MediaType != "application/json" {
|
||||
t.Fatalf("extract media type = %q, want application/json", extraction.Output.Payload.MediaType)
|
||||
}
|
||||
|
||||
merge, err := merger.Merge(ctx, contracts.MergeRequest{
|
||||
Source: doc,
|
||||
LaneID: candidate.ArtifactType,
|
||||
ChunkArtifacts: []contracts.ChunkArtifacts{
|
||||
{
|
||||
Chunk: chunking.Chunks[0],
|
||||
Candidates: extraction.Candidates,
|
||||
},
|
||||
},
|
||||
Source: doc,
|
||||
LaneID: "generic-lane",
|
||||
ExtractOutputs: []contracts.ExtractOutput{extraction.Output},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Merge() error = %v, want nil", err)
|
||||
}
|
||||
if len(merge.Candidates) != 1 {
|
||||
t.Fatalf("len(merge.Candidates) = %d, want 1", len(merge.Candidates))
|
||||
if string(merge.Output.Payload.Content) != `{"value":"example"}` {
|
||||
t.Fatalf("merge output = %s, want extract payload", merge.Output.Payload.Content)
|
||||
}
|
||||
|
||||
normalize, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
|
||||
Source: doc,
|
||||
LaneID: candidate.ArtifactType,
|
||||
Candidates: merge.Candidates,
|
||||
Source: doc,
|
||||
LaneID: "generic-lane",
|
||||
MergeOutput: merge.Output,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||
}
|
||||
if len(normalize.Candidates) != 1 {
|
||||
t.Fatalf("len(normalize.Candidates) = %d, want 1", len(normalize.Candidates))
|
||||
}
|
||||
|
||||
validation, err := validator.Validate(ctx, contracts.ValidationRequest{
|
||||
Source: doc,
|
||||
Candidates: normalize.Candidates,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Validate() error = %v, want nil", err)
|
||||
}
|
||||
if len(validation.Decisions) != 1 {
|
||||
t.Fatalf("len(Decisions) = %d, want 1", len(validation.Decisions))
|
||||
}
|
||||
|
||||
decision := validation.Decisions[0]
|
||||
if !decision.Approved {
|
||||
t.Fatal("Approved = false, want true")
|
||||
}
|
||||
if decision.CandidateIndex != candidate.Index {
|
||||
t.Fatalf("CandidateIndex = %d, want %d", decision.CandidateIndex, candidate.Index)
|
||||
if string(normalize.Output.Payload.Content) != `{"value":"example"}` {
|
||||
t.Fatalf("normalize output = %s, want merge payload", normalize.Output.Payload.Content)
|
||||
}
|
||||
|
||||
output, err := encoder.Encode(ctx, contracts.OutputRequest{
|
||||
Manifest: artifacts.RunManifest{RunID: "run-1"},
|
||||
Approved: []artifacts.Artifact{
|
||||
artifacts.ArtifactFromCandidate(normalize.Candidates[0]),
|
||||
},
|
||||
Manifest: artifacts.RunManifest{RunID: "run-1"},
|
||||
NormalizeOutputs: []contracts.NormalizeOutput{normalize.Output},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v, want nil", err)
|
||||
@@ -148,8 +116,8 @@ func (adapter compositionAdapter) Parse(ctx context.Context, req contracts.Parse
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:abc123",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: "u1", Kind: "unit", Text: "First source unit."},
|
||||
{ID: "u2", Kind: "unit", Text: "Second source unit."},
|
||||
{ID: 1, Kind: "unit", Text: "First source unit."},
|
||||
{ID: 2, Kind: "unit", Text: "Second source unit."},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -160,69 +128,65 @@ func (chunker compositionChunker) Key() string {
|
||||
return "generic-chunker"
|
||||
}
|
||||
|
||||
func (chunker compositionChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (chunker compositionChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||
if req.Source == nil {
|
||||
return contracts.ChunkResult{}, errors.New("source document is required")
|
||||
}
|
||||
if req.LLMClient == nil {
|
||||
return contracts.ChunkResult{}, errors.New("structured llm client is required")
|
||||
}
|
||||
|
||||
return contracts.ChunkResult{
|
||||
Chunks: []contracts.SourceChunk{
|
||||
{
|
||||
ID: req.Source.ID + ":chunk:0",
|
||||
SourceID: req.Source.ID,
|
||||
Index: 0,
|
||||
Units: append([]source.SourceUnit(nil), req.Source.Units...),
|
||||
Metadata: map[string]any{"strategy": "whole-document"},
|
||||
ID: req.Source.ID + ":chunk:0",
|
||||
SourceID: req.Source.ID,
|
||||
Index: 0,
|
||||
StartUnitID: req.Source.Units[0].ID,
|
||||
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
|
||||
Content: []byte(`{"units":[{"id":1,"kind":"unit","text":"First source unit."},{"id":2,"kind":"unit","text":"Second source unit."}]}`),
|
||||
MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), req.Source.Units...),
|
||||
Metadata: map[string]any{"strategy": "whole-document"},
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type compositionLLMClient struct{}
|
||||
|
||||
func (client compositionLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
type compositionExtractor struct{}
|
||||
|
||||
func (extractor compositionExtractor) Key() string {
|
||||
return "generic-extractor"
|
||||
}
|
||||
|
||||
func (extractor compositionExtractor) ArtifactType() string {
|
||||
return "generic-artifact"
|
||||
}
|
||||
|
||||
func (extractor compositionExtractor) SchemaVersion() string {
|
||||
return "v1"
|
||||
}
|
||||
|
||||
func (extractor compositionExtractor) Validators() []contracts.Validator {
|
||||
return []contracts.Validator{compositionValidator{}}
|
||||
func (extractor compositionExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (extractor compositionExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
if req.Source == nil {
|
||||
return contracts.ExtractionResult{}, errors.New("source document is required")
|
||||
}
|
||||
units := req.Source.Units
|
||||
if req.Chunk != nil {
|
||||
units = req.Chunk.Units
|
||||
}
|
||||
if req.AmbientContext["synopsis"] == "" {
|
||||
return contracts.ExtractionResult{}, errors.New("ambient synopsis is required")
|
||||
}
|
||||
|
||||
return contracts.ExtractionResult{
|
||||
Candidates: []artifacts.ArtifactCandidate{
|
||||
{
|
||||
Index: 0,
|
||||
ExtractorKey: extractor.Key(),
|
||||
ArtifactType: extractor.ArtifactType(),
|
||||
SchemaVersion: extractor.SchemaVersion(),
|
||||
Payload: json.RawMessage(`{"value":"example"}`),
|
||||
SourceRefs: []source.SourceRef{
|
||||
{
|
||||
SourceID: req.Source.ID,
|
||||
StartUnitID: units[0].ID,
|
||||
EndUnitID: units[len(units)-1].ID,
|
||||
},
|
||||
},
|
||||
Output: contracts.ExtractOutput{
|
||||
Schema: contracts.ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"value":"example"}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
@@ -235,12 +199,14 @@ func (merger compositionMerger) Key() string {
|
||||
}
|
||||
|
||||
func (merger compositionMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
|
||||
var candidates []artifacts.ArtifactCandidate
|
||||
for _, chunkArtifacts := range req.ChunkArtifacts {
|
||||
candidates = append(candidates, chunkArtifacts.Candidates...)
|
||||
}
|
||||
|
||||
return contracts.MergeResult{Candidates: candidates}, nil
|
||||
output := req.ExtractOutputs[0]
|
||||
return contracts.MergeResult{Output: contracts.MergeOutput{
|
||||
LaneID: req.LaneID,
|
||||
MergerKey: merger.Key(),
|
||||
SourceID: output.SourceID,
|
||||
Schema: output.Schema,
|
||||
Payload: cloneCompositionPayload(output.Payload),
|
||||
}}, nil
|
||||
}
|
||||
|
||||
type compositionNormalizer struct{}
|
||||
@@ -249,8 +215,38 @@ func (normalizer compositionNormalizer) Key() string {
|
||||
return "generic-normalizer"
|
||||
}
|
||||
|
||||
func (normalizer compositionNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (normalizer compositionNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
|
||||
return contracts.NormalizeResult{Output: contracts.NormalizeOutput{
|
||||
LaneID: req.LaneID,
|
||||
NormalizerKey: normalizer.Key(),
|
||||
SourceID: req.MergeOutput.SourceID,
|
||||
Schema: req.MergeOutput.Schema,
|
||||
Payload: cloneCompositionPayload(req.MergeOutput.Payload),
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func cloneCompositionPayload(payload contracts.RawPayload) contracts.RawPayload {
|
||||
return contracts.RawPayload{
|
||||
Content: append([]byte(nil), payload.Content...),
|
||||
MediaType: payload.MediaType,
|
||||
Metadata: cloneCompositionMetadata(payload.Metadata),
|
||||
Warnings: append([]contracts.Warning(nil), payload.Warnings...),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneCompositionMetadata(metadata map[string]any) map[string]any {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(metadata))
|
||||
for key, value := range metadata {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type compositionValidator struct{}
|
||||
@@ -259,20 +255,15 @@ func (validator compositionValidator) Name() string {
|
||||
return "generic-validator"
|
||||
}
|
||||
|
||||
func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
|
||||
for _, candidate := range req.Candidates {
|
||||
decisions = append(decisions, contracts.ValidationDecision{
|
||||
CandidateIndex: candidate.Index,
|
||||
Approved: true,
|
||||
ReasonCode: "accepted",
|
||||
Message: "candidate accepted",
|
||||
})
|
||||
}
|
||||
func (validator compositionValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
|
||||
func (validator compositionValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{
|
||||
ValidatorName: validator.Name(),
|
||||
Decisions: decisions,
|
||||
Approved: true,
|
||||
ReasonCode: "accepted",
|
||||
Message: "output accepted",
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -284,11 +275,11 @@ func (encoder compositionOutputEncoder) Key() string {
|
||||
|
||||
func (encoder compositionOutputEncoder) Encode(ctx context.Context, req contracts.OutputRequest) (contracts.OutputResult, error) {
|
||||
payload := struct {
|
||||
RunID string `json:"run_id"`
|
||||
ApprovedCount int `json:"approved_count"`
|
||||
RunID string `json:"run_id"`
|
||||
OutputCount int `json:"output_count"`
|
||||
}{
|
||||
RunID: req.Manifest.RunID,
|
||||
ApprovedCount: len(req.Approved),
|
||||
RunID: req.Manifest.RunID,
|
||||
OutputCount: len(req.NormalizeOutputs),
|
||||
}
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
|
||||
@@ -8,32 +8,122 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
)
|
||||
|
||||
type LLMMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type StructuredCompletionRequest struct {
|
||||
StageName string `json:"stage_name"`
|
||||
Messages []LLMMessage `json:"messages"`
|
||||
Model string `json:"model,omitempty"`
|
||||
ResponseSchemaName string `json:"response_schema_name,omitempty"`
|
||||
ResponseSchema json.RawMessage `json:"response_schema,omitempty"`
|
||||
StageName string `json:"stage_name"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Inputs LLMInputSet `json:"inputs,omitempty"`
|
||||
Vars map[string]any `json:"vars,omitempty"`
|
||||
}
|
||||
|
||||
type StructuredCompletionResponse struct {
|
||||
Content json.RawMessage `json:"content"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
Content json.RawMessage `json:"content"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
Debug *LLMDebugMaterial `json:"debug,omitempty"`
|
||||
}
|
||||
|
||||
type StructuredLLMClient interface {
|
||||
CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error)
|
||||
}
|
||||
|
||||
type LLMDebugMaterial struct {
|
||||
Prompt *LLMDebugPrompt `json:"prompt,omitempty"`
|
||||
Response *LLMDebugResponse `json:"response,omitempty"`
|
||||
}
|
||||
|
||||
type LLMDebugPrompt struct {
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"`
|
||||
Messages []LLMDebugMessage `json:"messages,omitempty"`
|
||||
EffectiveModelParams map[string]any `json:"effective_model_params,omitempty"`
|
||||
OutputContract map[string]any `json:"output_contract,omitempty"`
|
||||
StructuredOutput map[string]any `json:"structured_output,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
}
|
||||
|
||||
type LLMDebugMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
CacheControl map[string]any `json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
type LLMDebugResponse struct {
|
||||
Content string `json:"content,omitempty"`
|
||||
RunID string `json:"run_id,omitempty"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id,omitempty"`
|
||||
ModelName string `json:"model_name,omitempty"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
EffectiveModelParams map[string]any `json:"effective_model_params,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
Validation map[string]any `json:"validation,omitempty"`
|
||||
Usage LLMDebugUsage `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
type LLMDebugUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
CachedTokens int `json:"cached_tokens,omitempty"`
|
||||
CacheWriteTokens int `json:"cache_write_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type LLMProfileManifestProvider interface {
|
||||
LLMProfileManifests() []artifacts.LLMProfileManifest
|
||||
}
|
||||
|
||||
type LLMInputMaterial struct {
|
||||
Name string `json:"name"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Content []byte `json:"-"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
OriginURI string `json:"origin_uri,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes,omitempty"`
|
||||
}
|
||||
|
||||
func NewLLMInputMaterial(name string, mediaType string, content []byte, digest string, originURI string) LLMInputMaterial {
|
||||
return LLMInputMaterial{
|
||||
Name: name,
|
||||
MediaType: mediaType,
|
||||
Content: append([]byte(nil), content...),
|
||||
Digest: digest,
|
||||
OriginURI: originURI,
|
||||
SizeBytes: int64(len(content)),
|
||||
}
|
||||
}
|
||||
|
||||
func (material LLMInputMaterial) Clone() LLMInputMaterial {
|
||||
material.Content = append([]byte(nil), material.Content...)
|
||||
return material
|
||||
}
|
||||
|
||||
type LLMInputSet map[string]LLMInputMaterial
|
||||
|
||||
func (set LLMInputSet) Clone() LLMInputSet {
|
||||
if len(set) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(LLMInputSet, len(set))
|
||||
for key, material := range set {
|
||||
out[key] = material.Clone()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type ParseRequest struct {
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
@@ -49,18 +139,26 @@ type InputAdapter interface {
|
||||
}
|
||||
|
||||
type SourceChunk struct {
|
||||
ID string `json:"id"`
|
||||
SourceID string `json:"source_id"`
|
||||
Index int `json:"index"`
|
||||
Units []source.SourceUnit `json:"units"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
ID string `json:"id"`
|
||||
SourceID string `json:"source_id"`
|
||||
Index int `json:"index"`
|
||||
StartUnitID int `json:"start_unit_id"`
|
||||
EndUnitID int `json:"end_unit_id"`
|
||||
Content []byte `json:"-"`
|
||||
MediaType string `json:"media_type"`
|
||||
Units []source.SourceUnit `json:"units"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type ChunkRequest struct {
|
||||
Source *source.SourceDocument `json:"-"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Source *source.SourceDocument `json:"-"`
|
||||
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
References ReferenceSet `json:"references,omitempty"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type ChunkResult struct {
|
||||
@@ -70,13 +168,67 @@ type ChunkResult struct {
|
||||
|
||||
type Chunker interface {
|
||||
Key() string
|
||||
ReferenceSlots() []ReferenceSlot
|
||||
Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
|
||||
}
|
||||
|
||||
const (
|
||||
ReferenceBindingSourceConfig = "config"
|
||||
ReferenceBindingSourceCLI = "cli"
|
||||
)
|
||||
|
||||
type ReferenceSlot struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Required bool `json:"required,omitempty"`
|
||||
AcceptedMediaTypes []string `json:"accepted_media_types,omitempty"`
|
||||
Multiple bool `json:"multiple,omitempty"`
|
||||
MaxBytes int64 `json:"max_bytes,omitempty"`
|
||||
}
|
||||
|
||||
func CloneReferenceSlots(slots []ReferenceSlot) []ReferenceSlot {
|
||||
if len(slots) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]ReferenceSlot, len(slots))
|
||||
for i, slot := range slots {
|
||||
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
|
||||
out[i] = slot
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type ReferenceOrigin struct {
|
||||
Type string `json:"type"`
|
||||
URI string `json:"uri,omitempty"`
|
||||
}
|
||||
|
||||
type ReferenceItem struct {
|
||||
SlotName string `json:"slot_name"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Content []byte `json:"-"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Origin ReferenceOrigin `json:"origin"`
|
||||
SizeBytes int64 `json:"size_bytes,omitempty"`
|
||||
BindingSource string `json:"binding_source,omitempty"`
|
||||
}
|
||||
|
||||
type ResolvedReferenceSlot struct {
|
||||
Slot ReferenceSlot `json:"slot"`
|
||||
Items []ReferenceItem `json:"items,omitempty"`
|
||||
}
|
||||
|
||||
type ReferenceSet struct {
|
||||
Slots map[string]ResolvedReferenceSlot `json:"slots,omitempty"`
|
||||
}
|
||||
|
||||
type ExtractionRequest struct {
|
||||
Source *source.SourceDocument `json:"-"`
|
||||
Chunk *SourceChunk `json:"chunk,omitempty"`
|
||||
AmbientContext map[string]any `json:"ambient_context,omitempty"`
|
||||
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
References ReferenceSet `json:"references,omitempty"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
@@ -84,35 +236,108 @@ type ExtractionRequest struct {
|
||||
}
|
||||
|
||||
type ExtractionResult struct {
|
||||
Candidates []artifacts.ArtifactCandidate `json:"candidates,omitempty"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
Output ExtractOutput `json:"output"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type Extractor interface {
|
||||
Key() string
|
||||
ArtifactType() string
|
||||
SchemaVersion() string
|
||||
Validators() []Validator
|
||||
ReferenceSlots() []ReferenceSlot
|
||||
Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error)
|
||||
}
|
||||
|
||||
type ChunkArtifacts struct {
|
||||
Chunk SourceChunk `json:"chunk"`
|
||||
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
|
||||
type RawPayload struct {
|
||||
Content []byte `json:"-"`
|
||||
MediaType string `json:"media_type"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type ExecutionClass string
|
||||
|
||||
const (
|
||||
ExecutionClassDeterministic ExecutionClass = "deterministic"
|
||||
ExecutionClassLLMBacked ExecutionClass = "llm_backed"
|
||||
)
|
||||
|
||||
type ValidationRequest struct {
|
||||
Stage string `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
Source *source.SourceDocument `json:"-"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
References ReferenceSet `json:"references,omitempty"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Schema ResponseSchema `json:"schema,omitempty"`
|
||||
Payload RawPayload `json:"payload"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
Chunk *SourceChunk `json:"chunk,omitempty"`
|
||||
Chunks []SourceChunk `json:"chunks,omitempty"`
|
||||
ExtractOutputs []ExtractOutput `json:"extract_outputs,omitempty"`
|
||||
MergeOutput MergeOutput `json:"merge_output,omitempty"`
|
||||
}
|
||||
|
||||
type ValidationResult struct {
|
||||
Approved bool `json:"approved"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type Validator interface {
|
||||
Name() string
|
||||
ExecutionClass() ExecutionClass
|
||||
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
|
||||
}
|
||||
|
||||
type ResponseSchema struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
JSONSchema []byte `json:"-"`
|
||||
}
|
||||
|
||||
type ExtractOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
SourceID string `json:"source_id"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ChunkIndex int `json:"chunk_index"`
|
||||
Schema ResponseSchema `json:"schema,omitempty"`
|
||||
Payload RawPayload `json:"payload"`
|
||||
}
|
||||
|
||||
type MergeRequest struct {
|
||||
Source *source.SourceDocument `json:"-"`
|
||||
LaneID string `json:"lane_id"`
|
||||
ChunkArtifacts []ChunkArtifacts `json:"chunk_artifacts"`
|
||||
ExtractOutputs []ExtractOutput `json:"extract_outputs"`
|
||||
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
References ReferenceSet `json:"references,omitempty"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type MergeResult struct {
|
||||
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
Output MergeOutput `json:"output"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type MergeOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
MergerKey string `json:"merger_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema ResponseSchema `json:"schema,omitempty"`
|
||||
Payload RawPayload `json:"payload"`
|
||||
}
|
||||
|
||||
type Merger interface {
|
||||
@@ -121,51 +346,37 @@ type Merger interface {
|
||||
}
|
||||
|
||||
type NormalizeRequest struct {
|
||||
Source *source.SourceDocument `json:"-"`
|
||||
LaneID string `json:"lane_id"`
|
||||
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Source *source.SourceDocument `json:"-"`
|
||||
LaneID string `json:"lane_id"`
|
||||
MergeOutput MergeOutput `json:"merge_output"`
|
||||
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
References ReferenceSet `json:"references,omitempty"`
|
||||
LLMClient StructuredLLMClient `json:"-"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type NormalizeResult struct {
|
||||
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
Output NormalizeOutput `json:"output"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type NormalizeOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
NormalizerKey string `json:"normalizer_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema ResponseSchema `json:"schema,omitempty"`
|
||||
Payload RawPayload `json:"payload"`
|
||||
}
|
||||
|
||||
type Normalizer interface {
|
||||
Key() string
|
||||
ReferenceSlots() []ReferenceSlot
|
||||
Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error)
|
||||
}
|
||||
|
||||
type ValidationRequest struct {
|
||||
Source *source.SourceDocument `json:"-"`
|
||||
Candidates []artifacts.ArtifactCandidate `json:"candidates"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type ValidationDecision struct {
|
||||
CandidateIndex int `json:"candidate_index"`
|
||||
Approved bool `json:"approved"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
}
|
||||
|
||||
type ValidationResult struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Decisions []ValidationDecision `json:"decisions"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type Validator interface {
|
||||
Name() string
|
||||
Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error)
|
||||
}
|
||||
|
||||
type Warning struct {
|
||||
Scope string `json:"scope,omitempty"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
@@ -173,13 +384,13 @@ type Warning struct {
|
||||
}
|
||||
|
||||
type OutputRequest struct {
|
||||
Manifest artifacts.RunManifest `json:"manifest"`
|
||||
Approved []artifacts.Artifact `json:"approved,omitempty"`
|
||||
Rejected []artifacts.RejectedArtifact `json:"rejected,omitempty"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Manifest artifacts.RunManifest `json:"manifest"`
|
||||
NormalizeOutputs []NormalizeOutput `json:"normalize_outputs,omitempty"`
|
||||
Rejected []RejectedOutput `json:"rejected,omitempty"`
|
||||
Warnings []Warning `json:"warnings,omitempty"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type OutputFile struct {
|
||||
@@ -198,6 +409,19 @@ type OutputEncoder interface {
|
||||
Encode(ctx context.Context, req OutputRequest) (OutputResult, error)
|
||||
}
|
||||
|
||||
type RejectedOutput struct {
|
||||
Stage string `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
ValidatorName string `json:"validator_name,omitempty"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
Message string `json:"message"`
|
||||
AttemptCount int `json:"attempt_count,omitempty"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
}
|
||||
|
||||
type ManifestMetadataProvider interface {
|
||||
ManifestMetadata() map[string]any
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package contracts
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
@@ -18,13 +19,9 @@ var _ Validator = fakeValidator{}
|
||||
var _ StructuredLLMClient = fakeLLMClient{}
|
||||
var _ OutputEncoder = fakeOutputEncoder{}
|
||||
|
||||
func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
|
||||
validator := fakeValidator{name: "generic-validator"}
|
||||
func TestFakeExtractorReturnsRawOutput(t *testing.T) {
|
||||
extractor := fakeExtractor{
|
||||
key: "generic-extractor",
|
||||
artifactType: "generic-artifact",
|
||||
schemaVersion: "v1",
|
||||
validators: []Validator{validator},
|
||||
key: "generic-extractor",
|
||||
}
|
||||
doc := &source.SourceDocument{
|
||||
ID: "source-1",
|
||||
@@ -32,7 +29,7 @@ func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:abc123",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: "u1", Kind: "section", Text: "Source text."},
|
||||
{ID: 1, Kind: "section", Text: "Source text."},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -44,37 +41,14 @@ func TestFakeExtractorReturnsCandidateAndValidator(t *testing.T) {
|
||||
if extractor.Key() != "generic-extractor" {
|
||||
t.Fatalf("Key() = %q, want generic-extractor", extractor.Key())
|
||||
}
|
||||
if extractor.ArtifactType() != "generic-artifact" {
|
||||
t.Fatalf("ArtifactType() = %q, want generic-artifact", extractor.ArtifactType())
|
||||
if result.Output.ExtractorKey != "" {
|
||||
t.Fatalf("ExtractorKey = %q, want runner-owned empty value", result.Output.ExtractorKey)
|
||||
}
|
||||
if extractor.SchemaVersion() != "v1" {
|
||||
t.Fatalf("SchemaVersion() = %q, want v1", extractor.SchemaVersion())
|
||||
if result.Output.Schema.Version != "v1" {
|
||||
t.Fatalf("Schema.Version = %q, want v1", result.Output.Schema.Version)
|
||||
}
|
||||
if len(extractor.Validators()) != 1 {
|
||||
t.Fatalf("len(Validators()) = %d, want 1", len(extractor.Validators()))
|
||||
}
|
||||
if extractor.Validators()[0].Name() != "generic-validator" {
|
||||
t.Fatalf("Validators()[0].Name() = %q, want generic-validator", extractor.Validators()[0].Name())
|
||||
}
|
||||
if len(result.Candidates) != 1 {
|
||||
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
|
||||
}
|
||||
|
||||
candidate := result.Candidates[0]
|
||||
if candidate.Index != 0 {
|
||||
t.Fatalf("ArtifactCandidate.Index = %d, want 0", candidate.Index)
|
||||
}
|
||||
if candidate.ExtractorKey != extractor.Key() {
|
||||
t.Fatalf("ArtifactCandidate.ExtractorKey = %q, want %q", candidate.ExtractorKey, extractor.Key())
|
||||
}
|
||||
if candidate.ArtifactType != extractor.ArtifactType() {
|
||||
t.Fatalf("ArtifactCandidate.ArtifactType = %q, want %q", candidate.ArtifactType, extractor.ArtifactType())
|
||||
}
|
||||
if candidate.SchemaVersion != extractor.SchemaVersion() {
|
||||
t.Fatalf("ArtifactCandidate.SchemaVersion = %q, want %q", candidate.SchemaVersion, extractor.SchemaVersion())
|
||||
}
|
||||
if string(candidate.Payload) != `{"value":"example"}` {
|
||||
t.Fatalf("ArtifactCandidate.Payload = %s, want example payload", candidate.Payload)
|
||||
if result.Output.Payload.MediaType != "application/json" || string(result.Output.Payload.Content) != `{"value":"example"}` {
|
||||
t.Fatalf("payload = %q %s, want JSON raw output", result.Output.Payload.MediaType, result.Output.Payload.Content)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,7 +59,7 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:abc123",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: "u1", Kind: "section", Text: "Source text."},
|
||||
{ID: 1, Kind: "section", Text: "Source text."},
|
||||
},
|
||||
}
|
||||
chunker := fakeChunker{key: "generic-chunker"}
|
||||
@@ -112,32 +86,59 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
|
||||
if chunk.Index != 0 {
|
||||
t.Fatalf("SourceChunk.Index = %d, want 0", chunk.Index)
|
||||
}
|
||||
if chunk.StartUnitID != 1 || chunk.EndUnitID != 1 {
|
||||
t.Fatalf("SourceChunk boundaries = %d-%d, want 1-1", chunk.StartUnitID, chunk.EndUnitID)
|
||||
}
|
||||
if chunk.MediaType != "application/json" || string(chunk.Content) != `{"units":[{"id":1,"kind":"section","text":"Source text."}]}` {
|
||||
t.Fatalf("SourceChunk payload = %q %s, want JSON units", chunk.MediaType, chunk.Content)
|
||||
}
|
||||
if len(chunk.Units) != 1 {
|
||||
t.Fatalf("len(SourceChunk.Units) = %d, want 1", len(chunk.Units))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
|
||||
extractor := fakeExtractor{
|
||||
key: "generic-extractor",
|
||||
artifactType: "generic-artifact",
|
||||
schemaVersion: "v1",
|
||||
}
|
||||
func TestFakeChunkerReceivesLLMClient(t *testing.T) {
|
||||
doc := &source.SourceDocument{
|
||||
ID: "source-1",
|
||||
Kind: "document",
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:abc123",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: "u1", Kind: "section", Text: "First source text."},
|
||||
{ID: "u2", Kind: "section", Text: "Second source text."},
|
||||
{ID: 1, Kind: "section", Text: "Source text."},
|
||||
},
|
||||
}
|
||||
client := fakeLLMClient{}
|
||||
chunker := &recordingChunker{key: "llm-chunker"}
|
||||
|
||||
if _, err := chunker.Chunk(context.Background(), ChunkRequest{Source: doc, LLMClient: client}); err != nil {
|
||||
t.Fatalf("Chunk() error = %v, want nil", err)
|
||||
}
|
||||
if chunker.request.LLMClient == nil {
|
||||
t.Fatal("ChunkRequest.LLMClient = nil, want structured LLM client")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
|
||||
extractor := fakeExtractor{key: "generic-extractor"}
|
||||
doc := &source.SourceDocument{
|
||||
ID: "source-1",
|
||||
Kind: "document",
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:abc123",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: 1, Kind: "section", Text: "First source text."},
|
||||
{ID: 2, Kind: "section", Text: "Second source text."},
|
||||
},
|
||||
}
|
||||
chunk := SourceChunk{
|
||||
ID: "source-1:chunk:1",
|
||||
SourceID: doc.ID,
|
||||
Index: 1,
|
||||
Units: []source.SourceUnit{doc.Units[1]},
|
||||
ID: "source-1:chunk:1",
|
||||
SourceID: doc.ID,
|
||||
Index: 1,
|
||||
StartUnitID: 2,
|
||||
EndUnitID: 2,
|
||||
Content: []byte(`{"units":[{"id":2,"kind":"section","text":"Second source text."}]}`),
|
||||
MediaType: "application/json",
|
||||
Units: []source.SourceUnit{doc.Units[1]},
|
||||
}
|
||||
|
||||
result, err := extractor.Extract(context.Background(), ExtractionRequest{
|
||||
@@ -148,37 +149,217 @@ func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Extract() error = %v, want nil", err)
|
||||
}
|
||||
if len(result.Candidates) != 1 {
|
||||
t.Fatalf("len(Candidates) = %d, want 1", len(result.Candidates))
|
||||
if result.Output.ChunkID != "" || result.Output.ChunkIndex != 0 {
|
||||
t.Fatalf("chunk provenance = %q/%d, want runner-owned zero values", result.Output.ChunkID, result.Output.ChunkIndex)
|
||||
}
|
||||
if string(result.Output.Payload.Content) != `{"value":"chunked"}` {
|
||||
t.Fatalf("Payload.Content = %s, want chunked payload", result.Output.Payload.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceSetDataTypes(t *testing.T) {
|
||||
references := ReferenceSet{
|
||||
Slots: map[string]ResolvedReferenceSlot{
|
||||
"roster": {
|
||||
Slot: ReferenceSlot{
|
||||
Name: "roster",
|
||||
Description: "Known characters",
|
||||
Required: true,
|
||||
AcceptedMediaTypes: []string{"text/plain"},
|
||||
Multiple: true,
|
||||
MaxBytes: 4096,
|
||||
},
|
||||
Items: []ReferenceItem{
|
||||
{
|
||||
SlotName: "roster",
|
||||
MediaType: "text/plain",
|
||||
Content: []byte("Aria\nBryn\n"),
|
||||
Digest: "sha256:reference",
|
||||
Origin: ReferenceOrigin{
|
||||
Type: "file",
|
||||
URI: "file:///tmp/roster.txt",
|
||||
},
|
||||
SizeBytes: 10,
|
||||
BindingSource: ReferenceBindingSourceConfig,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
candidate := result.Candidates[0]
|
||||
if string(candidate.Payload) != `{"value":"chunked"}` {
|
||||
t.Fatalf("ArtifactCandidate.Payload = %s, want chunked payload", candidate.Payload)
|
||||
item := references.Slots["roster"].Items[0]
|
||||
if item.SlotName != "roster" || item.MediaType != "text/plain" || string(item.Content) != "Aria\nBryn\n" {
|
||||
t.Fatalf("reference item = %#v, want constructed item fields", item)
|
||||
}
|
||||
if len(candidate.SourceRefs) != 1 {
|
||||
t.Fatalf("len(SourceRefs) = %d, want 1", len(candidate.SourceRefs))
|
||||
if item.BindingSource != ReferenceBindingSourceConfig {
|
||||
t.Fatalf("BindingSource = %q, want %q", item.BindingSource, ReferenceBindingSourceConfig)
|
||||
}
|
||||
ref := candidate.SourceRefs[0]
|
||||
if ref.StartUnitID != "u2" || ref.EndUnitID != "u2" {
|
||||
t.Fatalf("SourceRef = %+v, want u2 range", ref)
|
||||
}
|
||||
|
||||
func TestCloneReferenceSlotsEmptyInputReturnsNil(t *testing.T) {
|
||||
if got := CloneReferenceSlots(nil); got != nil {
|
||||
t.Fatalf("CloneReferenceSlots(nil) = %#v, want nil", got)
|
||||
}
|
||||
if got := CloneReferenceSlots([]ReferenceSlot{}); got != nil {
|
||||
t.Fatalf("CloneReferenceSlots(empty) = %#v, want nil", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneReferenceSlotsPreservesFields(t *testing.T) {
|
||||
slots := []ReferenceSlot{
|
||||
{
|
||||
Name: "roster",
|
||||
Description: "Known characters",
|
||||
Required: true,
|
||||
AcceptedMediaTypes: []string{"text/plain", "text/markdown"},
|
||||
Multiple: true,
|
||||
MaxBytes: 4096,
|
||||
},
|
||||
{
|
||||
Name: "glossary",
|
||||
Description: "Campaign terms",
|
||||
MaxBytes: 2048,
|
||||
},
|
||||
}
|
||||
|
||||
got := CloneReferenceSlots(slots)
|
||||
|
||||
if !reflect.DeepEqual(got, slots) {
|
||||
t.Fatalf("CloneReferenceSlots() = %#v, want %#v", got, slots)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneReferenceSlotsCopiesAcceptedMediaTypes(t *testing.T) {
|
||||
slots := []ReferenceSlot{
|
||||
{
|
||||
Name: "party",
|
||||
AcceptedMediaTypes: []string{"application/json", "text/plain"},
|
||||
},
|
||||
}
|
||||
|
||||
got := CloneReferenceSlots(slots)
|
||||
got[0].Name = "changed"
|
||||
got[0].AcceptedMediaTypes[0] = "text/markdown"
|
||||
|
||||
if slots[0].Name != "party" {
|
||||
t.Fatalf("source slot name = %q, want unchanged", slots[0].Name)
|
||||
}
|
||||
if slots[0].AcceptedMediaTypes[0] != "application/json" {
|
||||
t.Fatalf("source AcceptedMediaTypes aliased clone: %#v", slots[0].AcceptedMediaTypes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReferenceItemJSONOmitsContent(t *testing.T) {
|
||||
item := ReferenceItem{
|
||||
SlotName: "roster",
|
||||
MediaType: "text/plain",
|
||||
Content: []byte("reference content"),
|
||||
Digest: "sha256:reference",
|
||||
Origin: ReferenceOrigin{Type: "file", URI: "file:///tmp/roster.txt"},
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(encoded, &got); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
|
||||
}
|
||||
if _, ok := got["content"]; ok {
|
||||
t.Fatalf("encoded reference item leaked content: %s", encoded)
|
||||
}
|
||||
if _, ok := got["Content"]; ok {
|
||||
t.Fatalf("encoded reference item leaked Content: %s", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMInputMaterialCopiesContentAndOmitsContentFromJSON(t *testing.T) {
|
||||
content := []byte("raw source bytes")
|
||||
material := NewLLMInputMaterial("transcript", "application/json", content, "sha256:source", "file:///tmp/source.json")
|
||||
content[0] = 'R'
|
||||
if got := string(material.Content); got != "raw source bytes" {
|
||||
t.Fatalf("material content = %q, want defensive copy", got)
|
||||
}
|
||||
if material.SizeBytes != int64(len("raw source bytes")) {
|
||||
t.Fatalf("SizeBytes = %d, want content length", material.SizeBytes)
|
||||
}
|
||||
|
||||
clone := material.Clone()
|
||||
clone.Content[0] = 'X'
|
||||
if got := string(material.Content); got != "raw source bytes" {
|
||||
t.Fatalf("cloned material content aliased original: %q", got)
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(material)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v, want nil", err)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(encoded, &got); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
|
||||
}
|
||||
if _, ok := got["content"]; ok {
|
||||
t.Fatalf("encoded material leaked content: %s", encoded)
|
||||
}
|
||||
if _, ok := got["Content"]; ok {
|
||||
t.Fatalf("encoded material leaked Content: %s", encoded)
|
||||
}
|
||||
if got["digest"] != "sha256:source" || got["origin_uri"] != "file:///tmp/source.json" {
|
||||
t.Fatalf("encoded material = %#v, want non-secret provenance", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMInputSetCloneCopiesContent(t *testing.T) {
|
||||
set := LLMInputSet{
|
||||
"transcript": NewLLMInputMaterial("transcript", "application/json", []byte("source"), "sha256:source", "file:///tmp/source.json"),
|
||||
}
|
||||
clone := set.Clone()
|
||||
clone["transcript"].Content[0] = 'S'
|
||||
if got := string(set["transcript"].Content); got != "source" {
|
||||
t.Fatalf("input set clone aliased content: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResponseSchemaJSONOmitRawSchemaContent(t *testing.T) {
|
||||
schema := ResponseSchema{
|
||||
ID: "schema-id",
|
||||
Name: "schema-name",
|
||||
Version: "v1",
|
||||
JSONSchema: []byte(`{"type":"object"}`),
|
||||
}
|
||||
encoded, err := json.Marshal(schema)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal() error = %v, want nil", err)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(encoded, &got); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v, want nil", err)
|
||||
}
|
||||
if got["id"] != "schema-id" || got["name"] != "schema-name" || got["version"] != "v1" {
|
||||
t.Fatalf("encoded schema = %#v, want schema provenance", got)
|
||||
}
|
||||
if _, ok := got["json_schema"]; ok {
|
||||
t.Fatalf("encoded schema leaked raw schema content: %s", encoded)
|
||||
}
|
||||
if _, ok := got["JSONSchema"]; ok {
|
||||
t.Fatalf("encoded schema leaked raw schema content: %s", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
|
||||
candidate := artifacts.ArtifactCandidate{
|
||||
Index: 0,
|
||||
ExtractorKey: "generic-extractor",
|
||||
ArtifactType: "generic-artifact",
|
||||
SchemaVersion: "v1",
|
||||
Payload: json.RawMessage(`{"value":"example"}`),
|
||||
}
|
||||
chunk := SourceChunk{
|
||||
ID: "source-1:chunk:0",
|
||||
SourceID: "source-1",
|
||||
Index: 0,
|
||||
Units: []source.SourceUnit{
|
||||
{ID: "u1", Kind: "section", Text: "Source text."},
|
||||
extractOutput := ExtractOutput{
|
||||
LaneID: "generic-lane",
|
||||
ExtractorKey: "generic-extractor",
|
||||
SourceID: "source-1",
|
||||
ChunkID: "source-1:chunk:0",
|
||||
ChunkIndex: 0,
|
||||
Schema: ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
|
||||
Payload: RawPayload{
|
||||
Content: []byte(`{"value":"example"}`),
|
||||
MediaType: "application/json",
|
||||
Metadata: map[string]any{"confidence": 0.75},
|
||||
},
|
||||
}
|
||||
merger := fakeMerger{key: "generic-merger"}
|
||||
@@ -186,13 +367,8 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
|
||||
encoder := fakeOutputEncoder{key: "generic-output"}
|
||||
|
||||
merged, err := merger.Merge(context.Background(), MergeRequest{
|
||||
LaneID: "generic-artifact",
|
||||
ChunkArtifacts: []ChunkArtifacts{
|
||||
{
|
||||
Chunk: chunk,
|
||||
Candidates: []artifacts.ArtifactCandidate{candidate},
|
||||
},
|
||||
},
|
||||
LaneID: "generic-lane",
|
||||
ExtractOutputs: []ExtractOutput{extractOutput},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Merge() error = %v, want nil", err)
|
||||
@@ -200,13 +376,13 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
|
||||
if merger.Key() != "generic-merger" {
|
||||
t.Fatalf("Merger.Key() = %q, want generic-merger", merger.Key())
|
||||
}
|
||||
if len(merged.Candidates) != 1 {
|
||||
t.Fatalf("len(merged.Candidates) = %d, want 1", len(merged.Candidates))
|
||||
if string(merged.Output.Payload.Content) != `{"value":"example"}` {
|
||||
t.Fatalf("merged content = %s, want raw extract content", merged.Output.Payload.Content)
|
||||
}
|
||||
|
||||
normalized, err := normalizer.Normalize(context.Background(), NormalizeRequest{
|
||||
LaneID: "generic-artifact",
|
||||
Candidates: merged.Candidates,
|
||||
LaneID: "generic-lane",
|
||||
MergeOutput: merged.Output,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize() error = %v, want nil", err)
|
||||
@@ -214,15 +390,13 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
|
||||
if normalizer.Key() != "generic-normalizer" {
|
||||
t.Fatalf("Normalizer.Key() = %q, want generic-normalizer", normalizer.Key())
|
||||
}
|
||||
if len(normalized.Candidates) != 1 {
|
||||
t.Fatalf("len(normalized.Candidates) = %d, want 1", len(normalized.Candidates))
|
||||
if string(normalized.Output.Payload.Content) != `{"value":"example"}` {
|
||||
t.Fatalf("normalized content = %s, want raw merge content", normalized.Output.Payload.Content)
|
||||
}
|
||||
|
||||
encoded, err := encoder.Encode(context.Background(), OutputRequest{
|
||||
Manifest: artifacts.RunManifest{RunID: "run-1"},
|
||||
Approved: []artifacts.Artifact{
|
||||
artifacts.ArtifactFromCandidate(normalized.Candidates[0]),
|
||||
},
|
||||
Manifest: artifacts.RunManifest{RunID: "run-1"},
|
||||
NormalizeOutputs: []NormalizeOutput{normalized.Output},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v, want nil", err)
|
||||
@@ -236,7 +410,7 @@ func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
|
||||
if encoded.Files[0].ContentType != "application/json" {
|
||||
t.Fatalf("ContentType = %q, want application/json", encoded.Files[0].ContentType)
|
||||
}
|
||||
if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","approved_count":1}` {
|
||||
if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","output_count":1}` {
|
||||
t.Fatalf("Bytes = %s, want encoded output", encoded.Files[0].Bytes)
|
||||
}
|
||||
}
|
||||
@@ -292,67 +466,69 @@ func (chunker fakeChunker) Key() string {
|
||||
return chunker.key
|
||||
}
|
||||
|
||||
func (chunker fakeChunker) ReferenceSlots() []ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
|
||||
return ChunkResult{
|
||||
Chunks: []SourceChunk{
|
||||
{
|
||||
ID: req.Source.ID + ":chunk:0",
|
||||
SourceID: req.Source.ID,
|
||||
Index: 0,
|
||||
Units: append([]source.SourceUnit(nil), req.Source.Units...),
|
||||
ID: req.Source.ID + ":chunk:0",
|
||||
SourceID: req.Source.ID,
|
||||
Index: 0,
|
||||
StartUnitID: req.Source.Units[0].ID,
|
||||
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
|
||||
Content: []byte(`{"units":[{"id":1,"kind":"section","text":"Source text."}]}`),
|
||||
MediaType: "application/json",
|
||||
Units: append([]source.SourceUnit(nil), req.Source.Units...),
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type recordingChunker struct {
|
||||
key string
|
||||
request ChunkRequest
|
||||
}
|
||||
|
||||
func (chunker *recordingChunker) Key() string {
|
||||
return chunker.key
|
||||
}
|
||||
|
||||
func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
|
||||
chunker.request = req
|
||||
return fakeChunker{key: chunker.key}.Chunk(ctx, req)
|
||||
}
|
||||
|
||||
type fakeExtractor struct {
|
||||
key string
|
||||
artifactType string
|
||||
schemaVersion string
|
||||
validators []Validator
|
||||
key string
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) Key() string {
|
||||
return extractor.key
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) ArtifactType() string {
|
||||
return extractor.artifactType
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) SchemaVersion() string {
|
||||
return extractor.schemaVersion
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) Validators() []Validator {
|
||||
return extractor.validators
|
||||
func (extractor fakeExtractor) ReferenceSlots() []ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (extractor fakeExtractor) Extract(ctx context.Context, req ExtractionRequest) (ExtractionResult, error) {
|
||||
units := req.Source.Units
|
||||
if req.Chunk != nil {
|
||||
units = req.Chunk.Units
|
||||
}
|
||||
payload := json.RawMessage(`{"value":"example"}`)
|
||||
if req.AmbientContext["mode"] == "chunked" {
|
||||
payload = json.RawMessage(`{"value":"chunked"}`)
|
||||
}
|
||||
|
||||
return ExtractionResult{
|
||||
Candidates: []artifacts.ArtifactCandidate{
|
||||
{
|
||||
Index: 0,
|
||||
ExtractorKey: extractor.key,
|
||||
ArtifactType: extractor.artifactType,
|
||||
SchemaVersion: extractor.schemaVersion,
|
||||
Payload: payload,
|
||||
SourceRefs: []source.SourceRef{
|
||||
{
|
||||
SourceID: req.Source.ID,
|
||||
StartUnitID: units[0].ID,
|
||||
EndUnitID: units[len(units)-1].ID,
|
||||
},
|
||||
},
|
||||
Output: ExtractOutput{
|
||||
Schema: ResponseSchema{ID: "schema-id", Name: "schema-name", Version: "v1"},
|
||||
Payload: RawPayload{
|
||||
Content: append([]byte(nil), payload...),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
@@ -367,12 +543,14 @@ func (merger fakeMerger) Key() string {
|
||||
}
|
||||
|
||||
func (merger fakeMerger) Merge(ctx context.Context, req MergeRequest) (MergeResult, error) {
|
||||
var candidates []artifacts.ArtifactCandidate
|
||||
for _, chunkArtifacts := range req.ChunkArtifacts {
|
||||
candidates = append(candidates, chunkArtifacts.Candidates...)
|
||||
}
|
||||
|
||||
return MergeResult{Candidates: candidates}, nil
|
||||
output := req.ExtractOutputs[0]
|
||||
return MergeResult{Output: MergeOutput{
|
||||
LaneID: req.LaneID,
|
||||
MergerKey: merger.key,
|
||||
SourceID: output.SourceID,
|
||||
Schema: output.Schema,
|
||||
Payload: cloneTestRawPayload(output.Payload),
|
||||
}}, nil
|
||||
}
|
||||
|
||||
type fakeNormalizer struct {
|
||||
@@ -383,8 +561,38 @@ func (normalizer fakeNormalizer) Key() string {
|
||||
return normalizer.key
|
||||
}
|
||||
|
||||
func (normalizer fakeNormalizer) ReferenceSlots() []ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (normalizer fakeNormalizer) Normalize(ctx context.Context, req NormalizeRequest) (NormalizeResult, error) {
|
||||
return NormalizeResult{Candidates: req.Candidates}, nil
|
||||
return NormalizeResult{Output: NormalizeOutput{
|
||||
LaneID: req.LaneID,
|
||||
NormalizerKey: normalizer.key,
|
||||
SourceID: req.MergeOutput.SourceID,
|
||||
Schema: req.MergeOutput.Schema,
|
||||
Payload: cloneTestRawPayload(req.MergeOutput.Payload),
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func cloneTestRawPayload(payload RawPayload) RawPayload {
|
||||
return RawPayload{
|
||||
Content: append([]byte(nil), payload.Content...),
|
||||
MediaType: payload.MediaType,
|
||||
Metadata: cloneTestMetadata(payload.Metadata),
|
||||
Warnings: append([]Warning(nil), payload.Warnings...),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneTestMetadata(metadata map[string]any) map[string]any {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(metadata))
|
||||
for key, value := range metadata {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type fakeValidator struct {
|
||||
@@ -395,20 +603,15 @@ func (validator fakeValidator) Name() string {
|
||||
return validator.name
|
||||
}
|
||||
|
||||
func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) {
|
||||
decisions := make([]ValidationDecision, 0, len(req.Candidates))
|
||||
for _, candidate := range req.Candidates {
|
||||
decisions = append(decisions, ValidationDecision{
|
||||
CandidateIndex: candidate.Index,
|
||||
Approved: true,
|
||||
ReasonCode: "accepted",
|
||||
Message: "candidate accepted",
|
||||
})
|
||||
}
|
||||
func (validator fakeValidator) ExecutionClass() ExecutionClass {
|
||||
return ExecutionClassDeterministic
|
||||
}
|
||||
|
||||
func (validator fakeValidator) Validate(ctx context.Context, req ValidationRequest) (ValidationResult, error) {
|
||||
return ValidationResult{
|
||||
ValidatorName: validator.name,
|
||||
Decisions: decisions,
|
||||
Approved: true,
|
||||
ReasonCode: "accepted",
|
||||
Message: "output accepted",
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -434,7 +637,7 @@ func (encoder fakeOutputEncoder) Encode(ctx context.Context, req OutputRequest)
|
||||
{
|
||||
Name: "artifacts/generic.json",
|
||||
ContentType: "application/json",
|
||||
Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","approved_count":1}`),
|
||||
Bytes: []byte(`{"run_id":"` + req.Manifest.RunID + `","output_count":1}`),
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
|
||||
41
internal/framework/debug/recorder.go
Normal file
41
internal/framework/debug/recorder.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package debug
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
type WorkspaceRecorder struct {
|
||||
root string
|
||||
}
|
||||
|
||||
func NewWorkspaceRecorder(settings coreworkspace.Settings, runID string) (pipeline.DebugRecorder, error) {
|
||||
root, err := settings.DebugRunDirectory(runID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return pipeline.NoopDebugRecorder(), nil
|
||||
}
|
||||
return &WorkspaceRecorder{root: root}, nil
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) Enabled() bool {
|
||||
return r != nil && strings.TrimSpace(r.root) != ""
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) WriteJSON(name string, payload any) error {
|
||||
if !r.Enabled() {
|
||||
return nil
|
||||
}
|
||||
return coreworkspace.WriteJSON(r.root, name, payload)
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) WriteBytes(name string, data []byte) error {
|
||||
if !r.Enabled() {
|
||||
return nil
|
||||
}
|
||||
return coreworkspace.WriteBytes(r.root, name, data)
|
||||
}
|
||||
333
internal/framework/llm/asset_registry.go
Normal file
333
internal/framework/llm/asset_registry.go
Normal file
@@ -0,0 +1,333 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
type AssetSource struct {
|
||||
FS fs.FS
|
||||
Root string
|
||||
}
|
||||
|
||||
type AssetRegistry struct {
|
||||
prompts []AssetSource
|
||||
schemas []AssetSource
|
||||
}
|
||||
|
||||
type AssetHashPart struct {
|
||||
FS fs.FS
|
||||
Path string
|
||||
}
|
||||
|
||||
func NewAssetRegistry() *AssetRegistry {
|
||||
return &AssetRegistry{}
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) RegisterPromptFS(fsys fs.FS, root string) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("asset registry must not be nil")
|
||||
}
|
||||
source, err := newAssetSource(fsys, root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("register prompt assets: %w", err)
|
||||
}
|
||||
r.prompts = append(r.prompts, source)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) RegisterSchemaFS(fsys fs.FS, root string) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("asset registry must not be nil")
|
||||
}
|
||||
source, err := newAssetSource(fsys, root)
|
||||
if err != nil {
|
||||
return fmt.Errorf("register schema assets: %w", err)
|
||||
}
|
||||
r.schemas = append(r.schemas, source)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) PromptFS() (fs.FS, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("asset registry must not be nil")
|
||||
}
|
||||
return flattenAssetSources(r.prompts)
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) SchemaFS() (fs.FS, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("asset registry must not be nil")
|
||||
}
|
||||
return flattenAssetSources(r.schemas)
|
||||
}
|
||||
|
||||
func (r *AssetRegistry) ScriptoriumOptions() ([]scriptorium.Option, error) {
|
||||
promptFS, err := r.PromptFS()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare prompt assets: %w", err)
|
||||
}
|
||||
schemaFS, err := r.SchemaFS()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("prepare schema assets: %w", err)
|
||||
}
|
||||
return []scriptorium.Option{
|
||||
scriptorium.WithPromptFS(promptFS, "."),
|
||||
scriptorium.WithSchemaFS(schemaFS, "."),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func HashAssets(parts []AssetHashPart) (string, error) {
|
||||
if len(parts) == 0 {
|
||||
return "", fmt.Errorf("asset hash requires at least one part")
|
||||
}
|
||||
hash := sha256.New()
|
||||
for _, part := range parts {
|
||||
cleanPath, err := cleanAssetPath(part.Path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("hash asset %q: %w", part.Path, err)
|
||||
}
|
||||
data, err := fs.ReadFile(part.FS, cleanPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read hash asset %s: %w", cleanPath, err)
|
||||
}
|
||||
if _, err := io.WriteString(hash, cleanPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := hash.Write([]byte{0}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := hash.Write(data); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := hash.Write([]byte{0}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return "sha256:" + hex.EncodeToString(hash.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func newAssetSource(fsys fs.FS, root string) (AssetSource, error) {
|
||||
if fsys == nil {
|
||||
return AssetSource{}, fmt.Errorf("filesystem must not be nil")
|
||||
}
|
||||
cleanRoot, err := cleanAssetRoot(root)
|
||||
if err != nil {
|
||||
return AssetSource{}, err
|
||||
}
|
||||
return AssetSource{FS: fsys, Root: cleanRoot}, nil
|
||||
}
|
||||
|
||||
func flattenAssetSources(sources []AssetSource) (fs.FS, error) {
|
||||
out := assetMapFS{}
|
||||
for _, source := range sources {
|
||||
if err := fs.WalkDir(source.FS, source.Root, func(name string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
rel := name
|
||||
if source.Root != "." {
|
||||
rel = strings.TrimPrefix(name, source.Root+"/")
|
||||
}
|
||||
rel, err := cleanAssetPath(rel)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, exists := out[rel]; exists {
|
||||
return fmt.Errorf("duplicate asset path %q", rel)
|
||||
}
|
||||
data, err := fs.ReadFile(source.FS, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out[rel] = append([]byte(nil), data...)
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("walk asset root %s: %w", source.Root, err)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func cleanAssetRoot(root string) (string, error) {
|
||||
trimmed := strings.TrimSpace(root)
|
||||
if trimmed == "" || trimmed == "." {
|
||||
return ".", nil
|
||||
}
|
||||
return cleanAssetPath(trimmed)
|
||||
}
|
||||
|
||||
func cleanAssetPath(name string) (string, error) {
|
||||
trimmed := strings.TrimSpace(name)
|
||||
if trimmed == "" {
|
||||
return "", fmt.Errorf("path must not be empty")
|
||||
}
|
||||
cleaned := path.Clean(strings.TrimPrefix(trimmed, "/"))
|
||||
if cleaned == "." || !fs.ValidPath(cleaned) {
|
||||
return "", fmt.Errorf("invalid path %q", name)
|
||||
}
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
type assetMapFS map[string][]byte
|
||||
|
||||
func (m assetMapFS) Open(name string) (fs.File, error) {
|
||||
cleaned, err := cleanOpenPath(name)
|
||||
if err != nil {
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: err}
|
||||
}
|
||||
if data, ok := m[cleaned]; ok {
|
||||
return &assetFile{
|
||||
reader: bytes.NewReader(data),
|
||||
info: assetFileInfo{name: path.Base(cleaned), size: int64(len(data))},
|
||||
}, nil
|
||||
}
|
||||
entries := m.dirEntries(cleaned)
|
||||
if entries != nil {
|
||||
return &assetDir{name: path.Base(cleaned), entries: entries}, nil
|
||||
}
|
||||
return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist}
|
||||
}
|
||||
|
||||
func (m assetMapFS) ReadFile(name string) ([]byte, error) {
|
||||
cleaned, err := cleanOpenPath(name)
|
||||
if err != nil {
|
||||
return nil, &fs.PathError{Op: "readfile", Path: name, Err: err}
|
||||
}
|
||||
data, ok := m[cleaned]
|
||||
if !ok {
|
||||
return nil, &fs.PathError{Op: "readfile", Path: name, Err: fs.ErrNotExist}
|
||||
}
|
||||
return append([]byte(nil), data...), nil
|
||||
}
|
||||
|
||||
func (m assetMapFS) ReadDir(name string) ([]fs.DirEntry, error) {
|
||||
cleaned, err := cleanOpenPath(name)
|
||||
if err != nil {
|
||||
return nil, &fs.PathError{Op: "readdir", Path: name, Err: err}
|
||||
}
|
||||
entries := m.dirEntries(cleaned)
|
||||
if entries == nil {
|
||||
return nil, &fs.PathError{Op: "readdir", Path: name, Err: fs.ErrNotExist}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func (m assetMapFS) dirEntries(dir string) []fs.DirEntry {
|
||||
children := map[string]assetDirEntry{}
|
||||
prefix := ""
|
||||
if dir != "." {
|
||||
prefix = dir + "/"
|
||||
}
|
||||
for name, data := range m {
|
||||
if !strings.HasPrefix(name, prefix) {
|
||||
continue
|
||||
}
|
||||
rest := strings.TrimPrefix(name, prefix)
|
||||
if rest == "" {
|
||||
continue
|
||||
}
|
||||
childName, _, hasSlash := strings.Cut(rest, "/")
|
||||
entry := assetDirEntry{name: childName, dir: hasSlash}
|
||||
if !hasSlash {
|
||||
entry.size = int64(len(data))
|
||||
}
|
||||
children[childName] = entry
|
||||
}
|
||||
if len(children) == 0 {
|
||||
return nil
|
||||
}
|
||||
names := make([]string, 0, len(children))
|
||||
for name := range children {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
entries := make([]fs.DirEntry, 0, len(names))
|
||||
for _, name := range names {
|
||||
entries = append(entries, children[name])
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func cleanOpenPath(name string) (string, error) {
|
||||
if name == "." {
|
||||
return ".", nil
|
||||
}
|
||||
return cleanAssetPath(name)
|
||||
}
|
||||
|
||||
type assetFile struct {
|
||||
reader *bytes.Reader
|
||||
info assetFileInfo
|
||||
}
|
||||
|
||||
func (f *assetFile) Stat() (fs.FileInfo, error) { return f.info, nil }
|
||||
func (f *assetFile) Read(p []byte) (int, error) { return f.reader.Read(p) }
|
||||
func (f *assetFile) Close() error { return nil }
|
||||
|
||||
type assetDir struct {
|
||||
name string
|
||||
offset int
|
||||
entries []fs.DirEntry
|
||||
}
|
||||
|
||||
func (d *assetDir) Stat() (fs.FileInfo, error) { return assetFileInfo{name: d.name, dir: true}, nil }
|
||||
func (d *assetDir) Read([]byte) (int, error) { return 0, fmt.Errorf("cannot read directory") }
|
||||
func (d *assetDir) Close() error { return nil }
|
||||
func (d *assetDir) ReadDir(n int) ([]fs.DirEntry, error) {
|
||||
if d.offset >= len(d.entries) {
|
||||
return nil, io.EOF
|
||||
}
|
||||
end := len(d.entries)
|
||||
if n > 0 && d.offset+n < end {
|
||||
end = d.offset + n
|
||||
}
|
||||
out := append([]fs.DirEntry(nil), d.entries[d.offset:end]...)
|
||||
d.offset = end
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type assetDirEntry struct {
|
||||
name string
|
||||
dir bool
|
||||
size int64
|
||||
}
|
||||
|
||||
func (e assetDirEntry) Name() string { return e.name }
|
||||
func (e assetDirEntry) IsDir() bool { return e.dir }
|
||||
func (e assetDirEntry) Type() fs.FileMode { return e.InfoMode().Type() }
|
||||
func (e assetDirEntry) Info() (fs.FileInfo, error) {
|
||||
return assetFileInfo{name: e.name, dir: e.dir, size: e.size}, nil
|
||||
}
|
||||
func (e assetDirEntry) InfoMode() fs.FileMode {
|
||||
if e.dir {
|
||||
return fs.ModeDir | 0o555
|
||||
}
|
||||
return 0o444
|
||||
}
|
||||
|
||||
type assetFileInfo struct {
|
||||
name string
|
||||
dir bool
|
||||
size int64
|
||||
}
|
||||
|
||||
func (i assetFileInfo) Name() string { return i.name }
|
||||
func (i assetFileInfo) Size() int64 { return i.size }
|
||||
func (i assetFileInfo) Mode() fs.FileMode { return assetDirEntry{dir: i.dir}.InfoMode() }
|
||||
func (i assetFileInfo) ModTime() time.Time { return time.Time{} }
|
||||
func (i assetFileInfo) IsDir() bool { return i.dir }
|
||||
func (i assetFileInfo) Sys() any { return nil }
|
||||
198
internal/framework/llm/asset_registry_test.go
Normal file
198
internal/framework/llm/asset_registry_test.go
Normal file
@@ -0,0 +1,198 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestAssetRegistryCombinesPromptAndSchemaSources(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||
"prompts/test.yaml": {Data: []byte(validPromptYAML("schemas/out.json"))},
|
||||
"prompts/messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
|
||||
"prompts/messages/task.tmpl": {Data: []byte("Return JSON.")},
|
||||
"schemas/ignored/schema.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, "prompts")
|
||||
mustRegisterSchemaFS(t, registry, fstest.MapFS{
|
||||
"root/schemas/out.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, "root")
|
||||
|
||||
engine := newAssetTestEngine(t, registry)
|
||||
prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline(`{"ok":true}`),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Prepare() error = %v, want nil", err)
|
||||
}
|
||||
if got := len(prepared.Messages); got != 2 {
|
||||
t.Fatalf("message count = %d, want 2", got)
|
||||
}
|
||||
if prepared.OutputContract.SchemaPath != "schemas/out.json" {
|
||||
t.Fatalf("schema path = %q, want schemas/out.json", prepared.OutputContract.SchemaPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryPrepareFailsForMissingPromptAsset(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||
"test.yaml": {Data: []byte(validPromptYAML("out.json"))},
|
||||
}, ".")
|
||||
mustRegisterSchemaFS(t, registry, fstest.MapFS{
|
||||
"out.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, ".")
|
||||
|
||||
engine := newAssetTestEngine(t, registry)
|
||||
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline(`{"ok":true}`),
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "content_file") {
|
||||
t.Fatalf("Prepare() error = %v, want missing content_file error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryPrepareFailsForMissingSchemaAsset(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||
"test.yaml": {Data: []byte(validPromptYAML("missing.json"))},
|
||||
"messages/user.tmpl": {Data: []byte(`Input: {{ input "transcript" }}`)},
|
||||
"messages/task.tmpl": {Data: []byte("Return JSON.")},
|
||||
}, ".")
|
||||
mustRegisterSchemaFS(t, registry, fstest.MapFS{
|
||||
"present.json": {Data: []byte(`{"type":"object"}`)},
|
||||
}, ".")
|
||||
|
||||
engine := newAssetTestEngine(t, registry)
|
||||
_, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
|
||||
PromptID: "asset.test",
|
||||
ProfileID: "asset-test-profile",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.Inline(`{"ok":true}`),
|
||||
},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "missing.json") {
|
||||
t.Fatalf("Prepare() error = %v, want missing schema error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryRejectsDuplicateAssetPaths(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"one/prompt.yaml": {Data: []byte("id: one")}}, "one")
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{"two/prompt.yaml": {Data: []byte("id: two")}}, "two")
|
||||
|
||||
_, err := registry.PromptFS()
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate asset path") {
|
||||
t.Fatalf("PromptFS() error = %v, want duplicate path error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetRegistryCombinesNamespacedPromptSources(t *testing.T) {
|
||||
registry := NewAssetRegistry()
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||
"dnd.spells/dnd.spells.yaml": {Data: []byte(validPromptYAML("schema.json"))},
|
||||
"dnd.spells/task.md": {Data: []byte("spell task")},
|
||||
"dnd.spells/instructions.md": {Data: []byte("spell instructions")},
|
||||
}, ".")
|
||||
mustRegisterPromptFS(t, registry, fstest.MapFS{
|
||||
"dnd.scenes/dnd.scenes.yaml": {Data: []byte(validPromptYAML("schema.json"))},
|
||||
"dnd.scenes/task.md": {Data: []byte("scene task")},
|
||||
"dnd.scenes/instructions.md": {Data: []byte("scene instructions")},
|
||||
}, ".")
|
||||
|
||||
fsys, err := registry.PromptFS()
|
||||
if err != nil {
|
||||
t.Fatalf("PromptFS() error = %v, want nil", err)
|
||||
}
|
||||
for _, name := range []string{
|
||||
"dnd.spells/dnd.spells.yaml",
|
||||
"dnd.spells/task.md",
|
||||
"dnd.spells/instructions.md",
|
||||
"dnd.scenes/dnd.scenes.yaml",
|
||||
"dnd.scenes/task.md",
|
||||
"dnd.scenes/instructions.md",
|
||||
} {
|
||||
if _, err := fsys.Open(name); err != nil {
|
||||
t.Fatalf("PromptFS().Open(%q) error = %v, want nil", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashAssetsOmitsRawAssetContent(t *testing.T) {
|
||||
hash, err := HashAssets([]AssetHashPart{{
|
||||
FS: fstest.MapFS{"prompt.md": {Data: []byte("secret prompt text")}},
|
||||
Path: "prompt.md",
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("HashAssets() error = %v, want nil", err)
|
||||
}
|
||||
if !strings.HasPrefix(hash, "sha256:") {
|
||||
t.Fatalf("hash = %q, want sha256-prefixed value", hash)
|
||||
}
|
||||
if strings.Contains(hash, "secret prompt text") {
|
||||
t.Fatalf("hash leaked asset content")
|
||||
}
|
||||
}
|
||||
|
||||
func newAssetTestEngine(t *testing.T, registry *AssetRegistry) *scriptorium.Engine {
|
||||
t.Helper()
|
||||
options, err := registry.ScriptoriumOptions()
|
||||
if err != nil {
|
||||
t.Fatalf("ScriptoriumOptions() error = %v, want nil", err)
|
||||
}
|
||||
options = append(options, scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "asset-test-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "asset-test-model",
|
||||
})))
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{Timeout: time.Second}, options...)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func mustRegisterPromptFS(t *testing.T, registry *AssetRegistry, fsys fstest.MapFS, root string) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterPromptFS(fsys, root); err != nil {
|
||||
t.Fatalf("RegisterPromptFS() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRegisterSchemaFS(t *testing.T, registry *AssetRegistry, fsys fstest.MapFS, root string) {
|
||||
t.Helper()
|
||||
if err := registry.RegisterSchemaFS(fsys, root); err != nil {
|
||||
t.Fatalf("RegisterSchemaFS() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
func validPromptYAML(schemaPath string) string {
|
||||
return `id: asset.test
|
||||
version: "v1"
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: user
|
||||
content_file: ./messages/user.tmpl
|
||||
- role: user
|
||||
content_file: ./messages/task.tmpl
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: ` + schemaPath + `
|
||||
repair_attempts: 0
|
||||
`
|
||||
}
|
||||
@@ -1,364 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
const openAICompatibleProviderName = "openai-compatible"
|
||||
|
||||
// OpenAICompatibleClientConfig configures the direct HTTP structured-output adapter.
|
||||
type OpenAICompatibleClientConfig struct {
|
||||
BaseURL string
|
||||
Model string
|
||||
APIKey string
|
||||
MaxRetries int
|
||||
HTTPClient *http.Client
|
||||
RequestTimeout time.Duration
|
||||
}
|
||||
|
||||
// OpenAICompatibleClient sends OpenAI-compatible chat-completion requests with
|
||||
// response_format.type=json_schema.
|
||||
type OpenAICompatibleClient struct {
|
||||
baseURL string
|
||||
model string
|
||||
apiKey string
|
||||
maxRetries int
|
||||
httpClient *http.Client
|
||||
requestTimeout time.Duration
|
||||
}
|
||||
|
||||
var _ contracts.StructuredLLMClient = (*OpenAICompatibleClient)(nil)
|
||||
|
||||
func NewOpenAICompatibleClient(cfg OpenAICompatibleClientConfig) (*OpenAICompatibleClient, error) {
|
||||
normalized, err := normalizeOpenAICompatibleConfig(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := normalized.HTTPClient
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
|
||||
return &OpenAICompatibleClient{
|
||||
baseURL: normalized.BaseURL,
|
||||
model: normalized.Model,
|
||||
apiKey: normalized.APIKey,
|
||||
maxRetries: normalized.MaxRetries,
|
||||
httpClient: client,
|
||||
requestTimeout: normalized.RequestTimeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) CompleteStructured(
|
||||
ctx context.Context,
|
||||
req contracts.StructuredCompletionRequest,
|
||||
out any,
|
||||
) (contracts.StructuredCompletionResponse, error) {
|
||||
if c == nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("openai-compatible client must not be nil")
|
||||
}
|
||||
if err := validateOutputTarget(out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
|
||||
model := strings.TrimSpace(req.Model)
|
||||
if model == "" {
|
||||
model = c.model
|
||||
}
|
||||
if model == "" {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion model must not be empty")
|
||||
}
|
||||
|
||||
schemaName := strings.TrimSpace(req.ResponseSchemaName)
|
||||
if schemaName == "" {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema name must not be empty")
|
||||
}
|
||||
if len(bytes.TrimSpace(req.ResponseSchema)) == 0 || !json.Valid(req.ResponseSchema) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema JSON must be valid")
|
||||
}
|
||||
|
||||
messages, err := toOpenAICompatibleMessages(req.Messages)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
|
||||
endpoint := buildChatCompletionsURL(c.baseURL)
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= c.maxRetries; attempt++ {
|
||||
content, metadata, callErr := c.completeStructuredOnce(ctx, endpoint, model, messages, schemaName, req.ResponseSchema)
|
||||
if callErr == nil {
|
||||
if decodeErr := json.Unmarshal(content, out); decodeErr != nil {
|
||||
callErr = retryableError{err: fmt.Errorf("decode structured output: %w", decodeErr)}
|
||||
} else {
|
||||
return contracts.StructuredCompletionResponse{
|
||||
Content: content,
|
||||
Provider: openAICompatibleProviderName,
|
||||
Model: firstNonEmpty(metadata.Model, model),
|
||||
PromptTokens: metadata.PromptTokens,
|
||||
CompletionTokens: metadata.CompletionTokens,
|
||||
TotalTokens: metadata.TotalTokens,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
}
|
||||
lastErr = c.redactError(callErr)
|
||||
if !canRetry(ctx, attempt, c.maxRetries, callErr) {
|
||||
return contracts.StructuredCompletionResponse{}, lastErr
|
||||
}
|
||||
}
|
||||
|
||||
if lastErr == nil {
|
||||
lastErr = fmt.Errorf("structured completion failed")
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, lastErr
|
||||
}
|
||||
|
||||
type openAICompatibleMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
type openAICompatibleRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []openAICompatibleMessage `json:"messages"`
|
||||
ResponseFormat openAICompatibleStructuredOutputShape `json:"response_format"`
|
||||
}
|
||||
|
||||
type openAICompatibleStructuredOutputShape struct {
|
||||
Type string `json:"type"`
|
||||
JSONSchema openAICompatibleSchemaEnvelope `json:"json_schema"`
|
||||
}
|
||||
|
||||
type openAICompatibleSchemaEnvelope struct {
|
||||
Name string `json:"name"`
|
||||
Strict bool `json:"strict"`
|
||||
Schema json.RawMessage `json:"schema"`
|
||||
}
|
||||
|
||||
type openAICompatibleChatCompletionsResponse struct {
|
||||
Model string `json:"model"`
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content json.RawMessage `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
Usage *openAICompatibleUsage `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
type openAICompatibleUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
}
|
||||
|
||||
type openAICompatibleResponseMetadata struct {
|
||||
Model string
|
||||
PromptTokens int
|
||||
CompletionTokens int
|
||||
TotalTokens int
|
||||
}
|
||||
|
||||
func normalizeOpenAICompatibleConfig(cfg OpenAICompatibleClientConfig) (OpenAICompatibleClientConfig, error) {
|
||||
cfg.BaseURL = strings.TrimSpace(cfg.BaseURL)
|
||||
cfg.Model = strings.TrimSpace(cfg.Model)
|
||||
cfg.APIKey = strings.TrimSpace(cfg.APIKey)
|
||||
if cfg.MaxRetries < 0 {
|
||||
return OpenAICompatibleClientConfig{}, fmt.Errorf("max retries must be zero or greater")
|
||||
}
|
||||
if cfg.BaseURL == "" {
|
||||
return OpenAICompatibleClientConfig{}, fmt.Errorf("base URL must not be empty")
|
||||
}
|
||||
if _, err := url.ParseRequestURI(cfg.BaseURL); err != nil {
|
||||
return OpenAICompatibleClientConfig{}, fmt.Errorf("base URL must be valid: %w", err)
|
||||
}
|
||||
if cfg.Model == "" {
|
||||
return OpenAICompatibleClientConfig{}, fmt.Errorf("model must not be empty")
|
||||
}
|
||||
cfg.BaseURL = strings.TrimRight(cfg.BaseURL, "/")
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) completeStructuredOnce(
|
||||
ctx context.Context,
|
||||
endpoint string,
|
||||
model string,
|
||||
messages []openAICompatibleMessage,
|
||||
responseSchemaName string,
|
||||
responseSchemaJSON json.RawMessage,
|
||||
) (json.RawMessage, openAICompatibleResponseMetadata, error) {
|
||||
requestCtx := ctx
|
||||
var cancel context.CancelFunc
|
||||
if c.requestTimeout > 0 {
|
||||
requestCtx, cancel = context.WithTimeout(ctx, c.requestTimeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
requestBody := openAICompatibleRequest{
|
||||
Model: model,
|
||||
Messages: messages,
|
||||
ResponseFormat: openAICompatibleStructuredOutputShape{
|
||||
Type: "json_schema",
|
||||
JSONSchema: openAICompatibleSchemaEnvelope{
|
||||
Name: responseSchemaName,
|
||||
Strict: true,
|
||||
Schema: responseSchemaJSON,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("marshal provider request: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(requestCtx, http.MethodPost, endpoint, bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("build provider request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if c.apiKey != "" {
|
||||
httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)
|
||||
}
|
||||
|
||||
httpResp, err := c.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("provider request failed: %w", err)}
|
||||
}
|
||||
defer func() {
|
||||
_ = httpResp.Body.Close()
|
||||
}()
|
||||
|
||||
rawResp, err := io.ReadAll(httpResp.Body)
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("read provider response: %w", err)}
|
||||
}
|
||||
|
||||
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
||||
statusErr := parseProviderErrorBody(httpResp.StatusCode, rawResp)
|
||||
if httpResp.StatusCode == http.StatusTooManyRequests || httpResp.StatusCode >= 500 {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: statusErr}
|
||||
}
|
||||
return nil, openAICompatibleResponseMetadata{}, statusErr
|
||||
}
|
||||
|
||||
return decodeChatCompletionsResponse(rawResp)
|
||||
}
|
||||
|
||||
func toOpenAICompatibleMessages(messages []contracts.LLMMessage) ([]openAICompatibleMessage, error) {
|
||||
if len(messages) == 0 {
|
||||
return nil, fmt.Errorf("structured completion messages must not be empty")
|
||||
}
|
||||
|
||||
result := make([]openAICompatibleMessage, len(messages))
|
||||
for i, message := range messages {
|
||||
role := strings.TrimSpace(message.Role)
|
||||
content := strings.TrimSpace(message.Content)
|
||||
if role == "" {
|
||||
return nil, fmt.Errorf("message[%d] role must not be empty", i)
|
||||
}
|
||||
if content == "" {
|
||||
return nil, fmt.Errorf("message[%d] content must not be empty", i)
|
||||
}
|
||||
result[i] = openAICompatibleMessage{
|
||||
Role: role,
|
||||
Content: content,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func buildChatCompletionsURL(baseURL string) string {
|
||||
return strings.TrimRight(baseURL, "/") + "/chat/completions"
|
||||
}
|
||||
|
||||
func decodeChatCompletionsResponse(raw []byte) (json.RawMessage, openAICompatibleResponseMetadata, error) {
|
||||
var parsed openAICompatibleChatCompletionsResponse
|
||||
if err := json.Unmarshal(raw, &parsed); err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("decode provider response envelope: %w", err)}
|
||||
}
|
||||
if len(parsed.Choices) == 0 {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("provider response missing choices")}
|
||||
}
|
||||
|
||||
content, err := extractAssistantContentJSON(parsed.Choices[0].Message.Content)
|
||||
if err != nil {
|
||||
return nil, openAICompatibleResponseMetadata{}, retryableError{err: err}
|
||||
}
|
||||
|
||||
metadata := openAICompatibleResponseMetadata{
|
||||
Model: parsed.Model,
|
||||
}
|
||||
if parsed.Usage != nil {
|
||||
metadata.PromptTokens = parsed.Usage.PromptTokens
|
||||
metadata.CompletionTokens = parsed.Usage.CompletionTokens
|
||||
metadata.TotalTokens = parsed.Usage.TotalTokens
|
||||
}
|
||||
return content, metadata, nil
|
||||
}
|
||||
|
||||
func extractAssistantContentJSON(raw json.RawMessage) (json.RawMessage, error) {
|
||||
trimmedRaw := bytes.TrimSpace(raw)
|
||||
if len(trimmedRaw) == 0 || bytes.Equal(trimmedRaw, []byte("null")) {
|
||||
return nil, fmt.Errorf("provider response missing assistant message content")
|
||||
}
|
||||
|
||||
var textContent string
|
||||
if err := json.Unmarshal(trimmedRaw, &textContent); err == nil {
|
||||
textContent = strings.TrimSpace(textContent)
|
||||
if textContent == "" {
|
||||
return nil, fmt.Errorf("provider response assistant message content is empty")
|
||||
}
|
||||
if !json.Valid([]byte(textContent)) {
|
||||
return nil, fmt.Errorf("provider response assistant message content is not valid JSON")
|
||||
}
|
||||
return json.RawMessage(textContent), nil
|
||||
}
|
||||
|
||||
if json.Valid(trimmedRaw) {
|
||||
return append(json.RawMessage(nil), trimmedRaw...), nil
|
||||
}
|
||||
return nil, fmt.Errorf("provider response assistant message content is not valid JSON")
|
||||
}
|
||||
|
||||
func parseProviderErrorBody(status int, body []byte) error {
|
||||
trimmed := strings.TrimSpace(string(body))
|
||||
if trimmed == "" {
|
||||
return fmt.Errorf("provider returned status %d", status)
|
||||
}
|
||||
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(body, &payload); err == nil {
|
||||
if nested, ok := payload["error"].(map[string]any); ok {
|
||||
if msg, ok := nested["message"].(string); ok && strings.TrimSpace(msg) != "" {
|
||||
return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg))
|
||||
}
|
||||
}
|
||||
if msg, ok := payload["message"].(string); ok && strings.TrimSpace(msg) != "" {
|
||||
return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg))
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("provider returned status %d: %s", status, trimmed)
|
||||
}
|
||||
|
||||
func (c *OpenAICompatibleClient) redactError(err error) error {
|
||||
secrets := []string{c.apiKey}
|
||||
if c.apiKey != "" {
|
||||
secrets = append(secrets, "Bearer "+c.apiKey)
|
||||
}
|
||||
return ErrorWithSecretsRedacted(err, secrets)
|
||||
}
|
||||
@@ -1,494 +0,0 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type testArtifact struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
func TestNewOpenAICompatibleClientValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg OpenAICompatibleClientConfig
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty base URL",
|
||||
cfg: OpenAICompatibleClientConfig{
|
||||
BaseURL: " ",
|
||||
Model: "model",
|
||||
},
|
||||
want: "base URL",
|
||||
},
|
||||
{
|
||||
name: "invalid base URL",
|
||||
cfg: OpenAICompatibleClientConfig{
|
||||
BaseURL: "://bad",
|
||||
Model: "model",
|
||||
},
|
||||
want: "base URL",
|
||||
},
|
||||
{
|
||||
name: "empty model",
|
||||
cfg: OpenAICompatibleClientConfig{
|
||||
BaseURL: "https://example.test/v1",
|
||||
Model: " ",
|
||||
},
|
||||
want: "model",
|
||||
},
|
||||
{
|
||||
name: "negative retries",
|
||||
cfg: OpenAICompatibleClientConfig{
|
||||
BaseURL: "https://example.test/v1",
|
||||
Model: "model",
|
||||
MaxRetries: -1,
|
||||
},
|
||||
want: "max retries",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := NewOpenAICompatibleClient(tc.cfg)
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.want, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientSuccessfulStructuredCompletion(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{
|
||||
"model":"provider-model",
|
||||
"choices":[{"message":{"content":"{\"value\":\"ok\"}"}}],
|
||||
"usage":{"prompt_tokens":11,"completion_tokens":7,"total_tokens":18}
|
||||
}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := newTestClient(t, server.URL, "default-model", 0)
|
||||
var out testArtifact
|
||||
resp, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
|
||||
if out.Value != "ok" {
|
||||
t.Fatalf("unexpected decoded output: %+v", out)
|
||||
}
|
||||
if string(resp.Content) != `{"value":"ok"}` {
|
||||
t.Fatalf("unexpected raw content: %s", resp.Content)
|
||||
}
|
||||
if resp.Provider != openAICompatibleProviderName {
|
||||
t.Fatalf("unexpected provider: %q", resp.Provider)
|
||||
}
|
||||
if resp.Model != "provider-model" {
|
||||
t.Fatalf("unexpected model: %q", resp.Model)
|
||||
}
|
||||
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
|
||||
t.Fatalf("unexpected token metadata: %+v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRequestBodyIncludesStructuredOutputShape(t *testing.T) {
|
||||
var seenPath string
|
||||
var seenAuthorization string
|
||||
var seenReq map[string]any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
seenPath = r.URL.Path
|
||||
seenAuthorization = r.Header.Get("Authorization")
|
||||
if err := json.NewDecoder(r.Body).Decode(&seenReq); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"value\":\"ok\"}"}}]}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL + "/v1",
|
||||
Model: "default-model",
|
||||
APIKey: "secret-key",
|
||||
MaxRetries: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out testArtifact
|
||||
_, err = client.CompleteStructured(context.Background(), validStructuredRequest("request-model"), &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
|
||||
if seenPath != "/v1/chat/completions" {
|
||||
t.Fatalf("unexpected request path: %q", seenPath)
|
||||
}
|
||||
if seenAuthorization != "Bearer secret-key" {
|
||||
t.Fatalf("unexpected authorization header: %q", seenAuthorization)
|
||||
}
|
||||
if seenReq["model"] != "request-model" {
|
||||
t.Fatalf("unexpected model: %v", seenReq["model"])
|
||||
}
|
||||
|
||||
messages, ok := seenReq["messages"].([]any)
|
||||
if !ok || len(messages) != 1 {
|
||||
t.Fatalf("unexpected messages: %#v", seenReq["messages"])
|
||||
}
|
||||
message, ok := messages[0].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("unexpected message shape: %#v", messages[0])
|
||||
}
|
||||
if message["role"] != "user" || message["content"] != "extract this" {
|
||||
t.Fatalf("unexpected message: %#v", message)
|
||||
}
|
||||
|
||||
responseFormat, ok := seenReq["response_format"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected response_format object, got %T", seenReq["response_format"])
|
||||
}
|
||||
if responseFormat["type"] != "json_schema" {
|
||||
t.Fatalf("unexpected response_format.type: %v", responseFormat["type"])
|
||||
}
|
||||
jsonSchema, ok := responseFormat["json_schema"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected response_format.json_schema object, got %T", responseFormat["json_schema"])
|
||||
}
|
||||
if jsonSchema["name"] != "test_artifact" {
|
||||
t.Fatalf("unexpected schema name: %v", jsonSchema["name"])
|
||||
}
|
||||
if jsonSchema["strict"] != true {
|
||||
t.Fatalf("expected strict=true, got %v", jsonSchema["strict"])
|
||||
}
|
||||
schema, ok := jsonSchema["schema"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected schema object, got %T", jsonSchema["schema"])
|
||||
}
|
||||
if schema["type"] != "object" {
|
||||
t.Fatalf("unexpected schema: %#v", schema)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientDefaultModelFallbackAndOverride(t *testing.T) {
|
||||
var seenModels []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var req map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
seenModels = append(seenModels, fmt.Sprint(req["model"]))
|
||||
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"value\":\"ok\"}"}}]}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := newTestClient(t, server.URL, "default-model", 0)
|
||||
var first testArtifact
|
||||
if _, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), &first); err != nil {
|
||||
t.Fatalf("first CompleteStructured: %v", err)
|
||||
}
|
||||
var second testArtifact
|
||||
if _, err := client.CompleteStructured(context.Background(), validStructuredRequest("override-model"), &second); err != nil {
|
||||
t.Fatalf("second CompleteStructured: %v", err)
|
||||
}
|
||||
|
||||
if len(seenModels) != 2 || seenModels[0] != "default-model" || seenModels[1] != "override-model" {
|
||||
t.Fatalf("unexpected models: %v", seenModels)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientInvalidOutputTarget(t *testing.T) {
|
||||
client := newTestClient(t, "https://example.test/v1", "default-model", 0)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
out any
|
||||
}{
|
||||
{name: "nil", out: nil},
|
||||
{name: "non-pointer", out: testArtifact{}},
|
||||
{name: "nil pointer", out: (*testArtifact)(nil)},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), tc.out)
|
||||
if err == nil || !strings.Contains(err.Error(), "output target") {
|
||||
t.Fatalf("expected output target error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientMissingAndInvalidSchema(t *testing.T) {
|
||||
client := newTestClient(t, "https://example.test/v1", "default-model", 0)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*contracts.StructuredCompletionRequest)
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "missing schema name",
|
||||
mutate: func(req *contracts.StructuredCompletionRequest) {
|
||||
req.ResponseSchemaName = " "
|
||||
},
|
||||
want: "schema name",
|
||||
},
|
||||
{
|
||||
name: "missing schema JSON",
|
||||
mutate: func(req *contracts.StructuredCompletionRequest) {
|
||||
req.ResponseSchema = nil
|
||||
},
|
||||
want: "schema JSON",
|
||||
},
|
||||
{
|
||||
name: "invalid schema JSON",
|
||||
mutate: func(req *contracts.StructuredCompletionRequest) {
|
||||
req.ResponseSchema = json.RawMessage(`{"type":`)
|
||||
},
|
||||
want: "schema JSON",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := validStructuredRequest("")
|
||||
tc.mutate(&req)
|
||||
var out testArtifact
|
||||
_, err := client.CompleteStructured(context.Background(), req, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.want, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRejectsEmptyMessages(t *testing.T) {
|
||||
client := newTestClient(t, "https://example.test/v1", "default-model", 0)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*contracts.StructuredCompletionRequest)
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "no messages",
|
||||
mutate: func(req *contracts.StructuredCompletionRequest) {
|
||||
req.Messages = nil
|
||||
},
|
||||
want: "messages",
|
||||
},
|
||||
{
|
||||
name: "empty role",
|
||||
mutate: func(req *contracts.StructuredCompletionRequest) {
|
||||
req.Messages[0].Role = " "
|
||||
},
|
||||
want: "role",
|
||||
},
|
||||
{
|
||||
name: "empty content",
|
||||
mutate: func(req *contracts.StructuredCompletionRequest) {
|
||||
req.Messages[0].Content = " "
|
||||
},
|
||||
want: "content",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := validStructuredRequest("")
|
||||
tc.mutate(&req)
|
||||
var out testArtifact
|
||||
_, err := client.CompleteStructured(context.Background(), req, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.want, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientProviderNon2xxBehavior(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = io.WriteString(w, `{"error":{"message":"bad request"}}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := newTestClient(t, server.URL, "default-model", 0)
|
||||
var out testArtifact
|
||||
_, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "status 400: bad request") {
|
||||
t.Fatalf("expected provider status error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRetries429And5xx(t *testing.T) {
|
||||
var attempts atomic.Int32
|
||||
statuses := []int{http.StatusTooManyRequests, http.StatusInternalServerError, http.StatusOK}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempt := int(attempts.Add(1)) - 1
|
||||
if statuses[attempt] != http.StatusOK {
|
||||
w.WriteHeader(statuses[attempt])
|
||||
_, _ = io.WriteString(w, `{"error":{"message":"try again"}}`)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"value\":\"ok\"}"}}]}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := newTestClient(t, server.URL, "default-model", 2)
|
||||
var out testArtifact
|
||||
if _, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), &out); err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if attempts.Load() != 3 {
|
||||
t.Fatalf("expected 3 attempts, got %d", attempts.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRetriesMalformedResponses(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
firstBody string
|
||||
}{
|
||||
{
|
||||
name: "malformed provider envelope",
|
||||
firstBody: `{"choices":[]}`,
|
||||
},
|
||||
{
|
||||
name: "malformed assistant JSON",
|
||||
firstBody: `{"choices":[{"message":{"content":"{"}}]}`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var attempts atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if attempts.Add(1) == 1 {
|
||||
_, _ = io.WriteString(w, tc.firstBody)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"{\"value\":\"ok\"}"}}]}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := newTestClient(t, server.URL, "default-model", 1)
|
||||
var out testArtifact
|
||||
if _, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), &out); err != nil {
|
||||
t.Fatalf("CompleteStructured: %v", err)
|
||||
}
|
||||
if attempts.Load() != 2 {
|
||||
t.Fatalf("expected 2 attempts, got %d", attempts.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientNoRetryForNonRetryable4xx(t *testing.T) {
|
||||
var attempts atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
attempts.Add(1)
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = io.WriteString(w, `{"error":{"message":"forbidden"}}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client := newTestClient(t, server.URL, "default-model", 3)
|
||||
var out testArtifact
|
||||
_, err := client.CompleteStructured(context.Background(), validStructuredRequest(""), &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "status 403") {
|
||||
t.Fatalf("expected forbidden error, got %v", err)
|
||||
}
|
||||
if attempts.Load() != 1 {
|
||||
t.Fatalf("expected 1 attempt, got %d", attempts.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientProviderErrorRedactsAPIKey(t *testing.T) {
|
||||
const apiKey = "secret-api-key"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = io.WriteString(w, `{"error":{"message":"Bearer secret-api-key failed for secret-api-key"}}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: server.URL,
|
||||
Model: "default-model",
|
||||
APIKey: apiKey,
|
||||
MaxRetries: 0,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
|
||||
var out testArtifact
|
||||
_, err = client.CompleteStructured(context.Background(), validStructuredRequest(""), &out)
|
||||
if err == nil {
|
||||
t.Fatalf("expected provider error")
|
||||
}
|
||||
if strings.Contains(err.Error(), apiKey) || strings.Contains(err.Error(), "Bearer "+apiKey) {
|
||||
t.Fatalf("expected API key to be redacted, got %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleClientRespectsContextCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
client := newTestClient(t, "https://example.test/v1", "default-model", 1)
|
||||
var out testArtifact
|
||||
_, err := client.CompleteStructured(ctx, validStructuredRequest(""), &out)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected context canceled, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestClient(t *testing.T, baseURL string, model string, maxRetries int) *OpenAICompatibleClient {
|
||||
t.Helper()
|
||||
client, err := NewOpenAICompatibleClient(OpenAICompatibleClientConfig{
|
||||
BaseURL: baseURL,
|
||||
Model: model,
|
||||
MaxRetries: maxRetries,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewOpenAICompatibleClient: %v", err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
func validStructuredRequest(model string) contracts.StructuredCompletionRequest {
|
||||
return contracts.StructuredCompletionRequest{
|
||||
Messages: []contracts.LLMMessage{
|
||||
{Role: " user ", Content: " extract this "},
|
||||
},
|
||||
Model: model,
|
||||
ResponseSchemaName: " test_artifact ",
|
||||
ResponseSchema: testResponseSchema(),
|
||||
}
|
||||
}
|
||||
|
||||
func testResponseSchema() json.RawMessage {
|
||||
return json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"value": {"type": "string"}
|
||||
},
|
||||
"required": ["value"],
|
||||
"additionalProperties": false
|
||||
}`)
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
@@ -41,3 +42,14 @@ func (c *scheduledClient) CompleteStructured(ctx context.Context, req contracts.
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (c *scheduledClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
|
||||
if c == nil || c.client == nil {
|
||||
return nil
|
||||
}
|
||||
provider, ok := c.client.(contracts.LLMProfileManifestProvider)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return provider.LLMProfileManifests()
|
||||
}
|
||||
|
||||
175
internal/framework/llm/scriptorium_api_test.go
Normal file
175
internal/framework/llm/scriptorium_api_test.go
Normal file
@@ -0,0 +1,175 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestScriptoriumPublicAPIGrounding(t *testing.T) {
|
||||
// Keep this compile-time grounding close to the future Notarius adapter so
|
||||
// dependency upgrades reveal API drift before the runtime cutover.
|
||||
engine, err := scriptorium.NewEngine(
|
||||
scriptorium.Config{
|
||||
PromptDir: "unused-when-prompt-option-is-set",
|
||||
ProfileDir: "",
|
||||
SchemaDir: "",
|
||||
Timeout: time.Second,
|
||||
},
|
||||
scriptorium.WithPromptFS(fstest.MapFS{}, "."),
|
||||
scriptorium.WithProfileFS(fstest.MapFS{}, "."),
|
||||
scriptorium.WithSchemaFS(fstest.MapFS{}, "."),
|
||||
scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "test-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "test-model",
|
||||
APIKeyRequired: true,
|
||||
ExtraParams: map[string]any{"mode": "test"},
|
||||
})),
|
||||
scriptorium.WithLLMClient(scriptoriumGroundingLLMClient{}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine() error = %v, want nil", err)
|
||||
}
|
||||
if engine == nil {
|
||||
t.Fatalf("NewEngine() = nil, want engine")
|
||||
}
|
||||
|
||||
var (
|
||||
_ func(string) scriptorium.Option = scriptorium.WithPromptFile
|
||||
_ func(string) scriptorium.Option = scriptorium.WithProfileFile
|
||||
_ func(string) scriptorium.Option = scriptorium.WithSchemaFile
|
||||
)
|
||||
|
||||
req := scriptorium.RunRequest{
|
||||
PromptID: "dnd.spells",
|
||||
PromptVersion: "v1",
|
||||
ProfileID: "test-profile",
|
||||
APIKey: "request-scoped-secret",
|
||||
Inputs: map[string]scriptorium.ArtifactRef{
|
||||
"transcript": scriptorium.InlineWithURI("file:///tmp/transcript.json", `{"segments":[]}`),
|
||||
"glossary": scriptorium.Inline(""),
|
||||
"roster": scriptorium.File("/tmp/roster.txt"),
|
||||
},
|
||||
Vars: map[string]string{
|
||||
"session_id": "session-1",
|
||||
},
|
||||
Execution: &scriptorium.ExecutionTargetOverride{
|
||||
Model: "override-model",
|
||||
Temperature: ptr(0.2),
|
||||
MaxTokens: ptr(100),
|
||||
TopP: ptr(0.9),
|
||||
TimeoutSeconds: ptr(30),
|
||||
ServiceTier: "standard",
|
||||
ReasoningEffort: "low",
|
||||
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
||||
ExtraParams: map[string]any{"provider_option": "value"},
|
||||
},
|
||||
Validation: &scriptorium.OutputContract{
|
||||
Format: scriptorium.FormatJSON,
|
||||
ValidationMode: scriptorium.ValidationJSONSchema,
|
||||
SchemaPath: "schemas/dnd_spells.v1.json",
|
||||
RepairAttempts: 1,
|
||||
},
|
||||
Metadata: map[string]string{
|
||||
"artifact_kind": "dnd_spell",
|
||||
},
|
||||
}
|
||||
if req.Inputs["transcript"].Type != scriptorium.ArtifactRefInline {
|
||||
t.Fatalf("inline input type = %q, want %q", req.Inputs["transcript"].Type, scriptorium.ArtifactRefInline)
|
||||
}
|
||||
if req.Inputs["roster"].Type != scriptorium.ArtifactRefFile {
|
||||
t.Fatalf("file input type = %q, want %q", req.Inputs["roster"].Type, scriptorium.ArtifactRefFile)
|
||||
}
|
||||
|
||||
result := scriptorium.RunResult{
|
||||
RunID: "run-1",
|
||||
Artifact: scriptorium.Artifact{
|
||||
Name: "output",
|
||||
ContentType: "application/json",
|
||||
Body: []byte(`{"ok":true}`),
|
||||
URI: "inline://output",
|
||||
Size: int64(len(`{"ok":true}`)),
|
||||
Hash: "sha256:abc",
|
||||
},
|
||||
RawOutput: `{"ok":true}`,
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
PromptHash: "prompt-hash",
|
||||
RenderedPromptHash: "rendered-prompt-hash",
|
||||
SelectedProfileID: req.ProfileID,
|
||||
ModelName: "test-model",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
EffectiveModelParams: scriptorium.ExecutionTarget{
|
||||
Model: "test-model",
|
||||
APIKeyEnv: "SCRIPTORIUM_API_KEY",
|
||||
ExtraParams: map[string]any{"provider_option": "value"},
|
||||
ReasoningEffort: "low",
|
||||
},
|
||||
InputHashes: map[string]string{
|
||||
"transcript": "sha256:def",
|
||||
},
|
||||
Validation: scriptorium.ValidationResult{
|
||||
Status: scriptorium.ValidationPassed,
|
||||
Mode: scriptorium.ValidationJSONSchema,
|
||||
SchemaPath: req.Validation.SchemaPath,
|
||||
RepairAttempts: 1,
|
||||
IsValid: true,
|
||||
},
|
||||
Usage: scriptorium.TokenUsage{
|
||||
PromptTokens: 10,
|
||||
CompletionTokens: 5,
|
||||
TotalTokens: 15,
|
||||
CachedTokens: 3,
|
||||
CacheWriteTokens: 2,
|
||||
},
|
||||
StartTime: time.Unix(1, 0),
|
||||
EndTime: time.Unix(2, 0),
|
||||
Duration: time.Second,
|
||||
}
|
||||
if result.Validation.Status != scriptorium.ValidationPassed {
|
||||
t.Fatalf("validation status = %q, want %q", result.Validation.Status, scriptorium.ValidationPassed)
|
||||
}
|
||||
if result.Usage.TotalTokens != 15 {
|
||||
t.Fatalf("total tokens = %d, want 15", result.Usage.TotalTokens)
|
||||
}
|
||||
|
||||
publicErrors := []error{
|
||||
scriptorium.ErrInvalidConfig,
|
||||
scriptorium.ErrInvalidRequest,
|
||||
scriptorium.ErrPromptNotFound,
|
||||
scriptorium.ErrProfileNotFound,
|
||||
scriptorium.ErrPromptLoad,
|
||||
scriptorium.ErrProfileLoad,
|
||||
scriptorium.ErrArtifactLoad,
|
||||
scriptorium.ErrPromptRender,
|
||||
scriptorium.ErrLLMGenerate,
|
||||
scriptorium.ErrValidation,
|
||||
}
|
||||
for _, publicErr := range publicErrors {
|
||||
if !errors.Is(publicErr, publicErr) {
|
||||
t.Fatalf("sentinel error does not match itself: %v", publicErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type scriptoriumGroundingLLMClient struct{}
|
||||
|
||||
func (scriptoriumGroundingLLMClient) Generate(context.Context, scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
|
||||
return &scriptorium.GenerateResponse{
|
||||
Content: `{"ok":true}`,
|
||||
Usage: scriptorium.TokenUsage{
|
||||
PromptTokens: 1,
|
||||
CompletionTokens: 1,
|
||||
TotalTokens: 2,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ptr[T any](v T) *T {
|
||||
return &v
|
||||
}
|
||||
379
internal/framework/llm/scriptorium_client.go
Normal file
379
internal/framework/llm/scriptorium_client.go
Normal file
@@ -0,0 +1,379 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
const scriptoriumProviderName = "scriptorium"
|
||||
|
||||
type ScriptoriumClientConfig struct {
|
||||
ProfileDir string
|
||||
ProfileFile string
|
||||
Assets *AssetRegistry
|
||||
Timeout time.Duration
|
||||
HTTPClient *http.Client
|
||||
EngineOptions []scriptorium.Option
|
||||
Recorder *LLMProfileRecorder
|
||||
}
|
||||
|
||||
type ScriptoriumClient struct {
|
||||
engine *scriptorium.Engine
|
||||
recorder *LLMProfileRecorder
|
||||
}
|
||||
|
||||
type LLMProfileRecorder struct {
|
||||
mu sync.Mutex
|
||||
profiles map[string]artifacts.LLMProfileManifest
|
||||
}
|
||||
|
||||
var _ contracts.StructuredLLMClient = (*ScriptoriumClient)(nil)
|
||||
var _ contracts.LLMProfileManifestProvider = (*ScriptoriumClient)(nil)
|
||||
|
||||
func NewScriptoriumClient(cfg ScriptoriumClientConfig) (*ScriptoriumClient, error) {
|
||||
if cfg.Assets == nil {
|
||||
return nil, fmt.Errorf("scriptorium client assets must not be nil")
|
||||
}
|
||||
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
|
||||
return nil, fmt.Errorf("scriptorium profile_dir and profile_file are mutually exclusive")
|
||||
}
|
||||
options, err := cfg.Assets.ScriptoriumOptions()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if profileFile := strings.TrimSpace(cfg.ProfileFile); profileFile != "" {
|
||||
options = append(options, scriptorium.WithProfileFile(profileFile))
|
||||
}
|
||||
options = append(options, cfg.EngineOptions...)
|
||||
|
||||
engine, err := scriptorium.NewEngine(scriptorium.Config{
|
||||
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
|
||||
Timeout: cfg.Timeout,
|
||||
HTTPClient: cfg.HTTPClient,
|
||||
}, options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create Scriptorium engine: %w", err)
|
||||
}
|
||||
recorder := cfg.Recorder
|
||||
if recorder == nil {
|
||||
recorder = NewLLMProfileRecorder()
|
||||
}
|
||||
return &ScriptoriumClient{
|
||||
engine: engine,
|
||||
recorder: recorder,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *ScriptoriumClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
if c == nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client must not be nil")
|
||||
}
|
||||
if c.engine == nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("scriptorium client engine must not be nil")
|
||||
}
|
||||
if err := validateOutputTarget(out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
promptID := strings.TrimSpace(req.PromptID)
|
||||
if promptID == "" {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion prompt_id must not be empty")
|
||||
}
|
||||
|
||||
runReq := scriptorium.RunRequest{
|
||||
PromptID: promptID,
|
||||
PromptVersion: strings.TrimSpace(req.PromptVersion),
|
||||
ProfileID: strings.TrimSpace(req.ProfileID),
|
||||
Inputs: scriptoriumInputs(req.Inputs),
|
||||
Vars: scriptoriumVars(req),
|
||||
Metadata: scriptoriumMetadata(req),
|
||||
}
|
||||
prepared, err := c.engine.Prepare(ctx, runReq)
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return contracts.StructuredCompletionResponse{}, ctxErr
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("prepare Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
|
||||
}
|
||||
result, err := c.engine.Run(ctx, runReq)
|
||||
if err != nil {
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return contracts.StructuredCompletionResponse{}, ctxErr
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: %w", promptID, redactScriptoriumError(err))
|
||||
}
|
||||
if result == nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("run Scriptorium prompt %q: empty result", promptID)
|
||||
}
|
||||
response := c.responseFromResult(result, prepared)
|
||||
if result.Validation.Status == scriptorium.ValidationFailed || !result.Validation.IsValid {
|
||||
return response, fmt.Errorf("run Scriptorium prompt %q: validation failed: %s", promptID, strings.Join(result.Validation.Errors, "; "))
|
||||
}
|
||||
if len(strings.TrimSpace(string(response.Content))) == 0 {
|
||||
return response, fmt.Errorf("run Scriptorium prompt %q: empty structured output", promptID)
|
||||
}
|
||||
if err := json.Unmarshal(response.Content, out); err != nil {
|
||||
return response, fmt.Errorf("decode Scriptorium structured output for prompt %q: %w", promptID, err)
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (c *ScriptoriumClient) responseFromResult(result *scriptorium.RunResult, prepared *scriptorium.PreparedRun) contracts.StructuredCompletionResponse {
|
||||
content := result.Artifact.Body
|
||||
if len(content) == 0 {
|
||||
content = []byte(result.RawOutput)
|
||||
}
|
||||
profile := artifacts.LLMProfileManifest{
|
||||
ID: strings.TrimSpace(result.SelectedProfileID),
|
||||
Provider: scriptoriumProviderName,
|
||||
Model: firstNonEmpty(result.ModelName, result.EffectiveModelParams.Model),
|
||||
}
|
||||
if c.recorder != nil {
|
||||
c.recorder.Record(profile)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{
|
||||
Content: append(json.RawMessage(nil), content...),
|
||||
Provider: profile.Provider,
|
||||
Model: profile.Model,
|
||||
ProfileID: profile.ID,
|
||||
PromptTokens: result.Usage.PromptTokens,
|
||||
CompletionTokens: result.Usage.CompletionTokens,
|
||||
TotalTokens: result.Usage.TotalTokens,
|
||||
Debug: scriptoriumDebugMaterial(prepared, result),
|
||||
}
|
||||
}
|
||||
|
||||
func scriptoriumDebugMaterial(prepared *scriptorium.PreparedRun, result *scriptorium.RunResult) *contracts.LLMDebugMaterial {
|
||||
material := &contracts.LLMDebugMaterial{}
|
||||
if prepared != nil {
|
||||
material.Prompt = scriptoriumDebugPrompt(prepared)
|
||||
}
|
||||
if result != nil {
|
||||
material.Response = scriptoriumDebugResponse(result)
|
||||
}
|
||||
if material.Prompt == nil && material.Response == nil {
|
||||
return nil
|
||||
}
|
||||
return material
|
||||
}
|
||||
|
||||
func scriptoriumDebugPrompt(prepared *scriptorium.PreparedRun) *contracts.LLMDebugPrompt {
|
||||
if prepared == nil {
|
||||
return nil
|
||||
}
|
||||
messages := make([]contracts.LLMDebugMessage, 0, len(prepared.Messages))
|
||||
for _, message := range prepared.Messages {
|
||||
messages = append(messages, contracts.LLMDebugMessage{
|
||||
Role: message.Role,
|
||||
Content: message.Content,
|
||||
CacheControl: jsonObject(message.CacheControl),
|
||||
})
|
||||
}
|
||||
return &contracts.LLMDebugPrompt{
|
||||
PromptID: prepared.PromptID,
|
||||
PromptVersion: prepared.PromptVersion,
|
||||
PromptHash: prepared.PromptHash,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
SessionID: prepared.SessionID,
|
||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||
Messages: messages,
|
||||
EffectiveModelParams: jsonObject(prepared.EffectiveModelParams),
|
||||
OutputContract: jsonObject(prepared.OutputContract),
|
||||
StructuredOutput: jsonObject(prepared.StructuredOutput),
|
||||
InputHashes: cloneStringMap(prepared.InputHashes),
|
||||
}
|
||||
}
|
||||
|
||||
func scriptoriumDebugResponse(result *scriptorium.RunResult) *contracts.LLMDebugResponse {
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
content := result.RawOutput
|
||||
if content == "" {
|
||||
content = string(result.Artifact.Body)
|
||||
}
|
||||
return &contracts.LLMDebugResponse{
|
||||
Content: content,
|
||||
RunID: result.RunID,
|
||||
PromptID: result.PromptID,
|
||||
PromptVersion: result.PromptVersion,
|
||||
PromptHash: result.PromptHash,
|
||||
RenderedPromptHash: result.RenderedPromptHash,
|
||||
SelectedProfileID: result.SelectedProfileID,
|
||||
ModelName: result.ModelName,
|
||||
Endpoint: result.Endpoint,
|
||||
EffectiveModelParams: jsonObject(result.EffectiveModelParams),
|
||||
InputHashes: cloneStringMap(result.InputHashes),
|
||||
Validation: jsonObject(result.Validation),
|
||||
Usage: contracts.LLMDebugUsage{
|
||||
PromptTokens: result.Usage.PromptTokens,
|
||||
CompletionTokens: result.Usage.CompletionTokens,
|
||||
TotalTokens: result.Usage.TotalTokens,
|
||||
CachedTokens: result.Usage.CachedTokens,
|
||||
CacheWriteTokens: result.Usage.CacheWriteTokens,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func jsonObject(value any) map[string]any {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil || string(data) == "null" {
|
||||
return nil
|
||||
}
|
||||
var out map[string]any
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneStringMap(values map[string]string) map[string]string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(values))
|
||||
for key, value := range values {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *ScriptoriumClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
|
||||
if c == nil || c.recorder == nil {
|
||||
return nil
|
||||
}
|
||||
return c.recorder.Manifests()
|
||||
}
|
||||
|
||||
func NewLLMProfileRecorder() *LLMProfileRecorder {
|
||||
return &LLMProfileRecorder{profiles: map[string]artifacts.LLMProfileManifest{}}
|
||||
}
|
||||
|
||||
func (r *LLMProfileRecorder) Record(profile artifacts.LLMProfileManifest) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
profile.ID = strings.TrimSpace(profile.ID)
|
||||
profile.Provider = strings.TrimSpace(profile.Provider)
|
||||
profile.Model = strings.TrimSpace(profile.Model)
|
||||
key := profile.ID + "\x00" + profile.Provider + "\x00" + profile.Model
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.profiles == nil {
|
||||
r.profiles = map[string]artifacts.LLMProfileManifest{}
|
||||
}
|
||||
r.profiles[key] = profile
|
||||
}
|
||||
|
||||
func (r *LLMProfileRecorder) Manifests() []artifacts.LLMProfileManifest {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if len(r.profiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(r.profiles))
|
||||
for key := range r.profiles {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := make([]artifacts.LLMProfileManifest, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
out = append(out, r.profiles[key])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scriptoriumInputs(inputs contracts.LLMInputSet) map[string]scriptorium.ArtifactRef {
|
||||
if len(inputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]scriptorium.ArtifactRef, len(inputs))
|
||||
for key, material := range inputs {
|
||||
name := strings.TrimSpace(key)
|
||||
if name == "" {
|
||||
name = strings.TrimSpace(material.Name)
|
||||
}
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
body := string(material.Content)
|
||||
if body == "" {
|
||||
body = " "
|
||||
}
|
||||
if origin := strings.TrimSpace(material.OriginURI); origin != "" {
|
||||
out[name] = scriptorium.InlineWithURI(origin, body)
|
||||
} else {
|
||||
out[name] = scriptorium.Inline(body)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scriptoriumVars(req contracts.StructuredCompletionRequest) map[string]string {
|
||||
vars := make(map[string]string, len(req.Vars)+1)
|
||||
for key, value := range req.Vars {
|
||||
name := strings.TrimSpace(key)
|
||||
if name == "" || value == nil {
|
||||
continue
|
||||
}
|
||||
vars[name] = fmt.Sprint(value)
|
||||
}
|
||||
if sessionID := strings.TrimSpace(req.SessionID); sessionID != "" {
|
||||
vars["session_id"] = sessionID
|
||||
}
|
||||
if len(vars) == 0 {
|
||||
return nil
|
||||
}
|
||||
return vars
|
||||
}
|
||||
|
||||
func scriptoriumMetadata(req contracts.StructuredCompletionRequest) map[string]string {
|
||||
metadata := map[string]string{}
|
||||
if stageName := strings.TrimSpace(req.StageName); stageName != "" {
|
||||
metadata["stage_name"] = stageName
|
||||
}
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
var bearerTokenPattern = regexp.MustCompile(`(?i)Bearer\s+[A-Za-z0-9._~+/=-]+`)
|
||||
|
||||
func redactScriptoriumError(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return redactedProviderError{err: err}
|
||||
}
|
||||
|
||||
type redactedProviderError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
func (e redactedProviderError) Error() string {
|
||||
return bearerTokenPattern.ReplaceAllString(e.err.Error(), "Bearer "+secretReplacement)
|
||||
}
|
||||
|
||||
func (e redactedProviderError) Unwrap() error {
|
||||
return e.err
|
||||
}
|
||||
352
internal/framework/llm/scriptorium_client_test.go
Normal file
352
internal/framework/llm/scriptorium_client_test.go
Normal file
@@ -0,0 +1,352 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/scriptorium"
|
||||
)
|
||||
|
||||
func TestScriptoriumClientMapsPromptRequestAndUnmarshalsOutput(t *testing.T) {
|
||||
fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
|
||||
client := newTestScriptoriumClient(t, fake)
|
||||
|
||||
var out struct {
|
||||
OK bool `json:"ok"`
|
||||
}
|
||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
StageName: "test-stage",
|
||||
PromptID: "adapter.test",
|
||||
PromptVersion: "v1",
|
||||
ProfileID: "explicit-profile",
|
||||
SessionID: "session-123",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "sha256:source", "file:///source.json"),
|
||||
},
|
||||
Vars: map[string]any{"custom": "value"},
|
||||
}, &out)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteStructured() error = %v, want nil", err)
|
||||
}
|
||||
if !out.OK {
|
||||
t.Fatalf("decoded output OK = false, want true")
|
||||
}
|
||||
if resp.Provider != scriptoriumProviderName || resp.Model != "explicit-model" || resp.ProfileID != "explicit-profile" {
|
||||
t.Fatalf("response metadata = %#v", resp)
|
||||
}
|
||||
if resp.PromptTokens != 11 || resp.CompletionTokens != 7 || resp.TotalTokens != 18 {
|
||||
t.Fatalf("usage = %#v, want mapped token counts", resp)
|
||||
}
|
||||
if resp.Debug == nil || resp.Debug.Prompt == nil {
|
||||
t.Fatalf("debug prompt = nil, want prepared prompt material")
|
||||
}
|
||||
if resp.Debug.Prompt.PromptID != "adapter.test" || resp.Debug.Prompt.SelectedProfileID != "explicit-profile" {
|
||||
t.Fatalf("debug prompt metadata = %#v, want prompt/profile", resp.Debug.Prompt)
|
||||
}
|
||||
if len(resp.Debug.Prompt.Messages) != 1 || !strings.Contains(resp.Debug.Prompt.Messages[0].Content, `{"source":true}`) {
|
||||
t.Fatalf("debug prompt messages = %#v, want rendered input content", resp.Debug.Prompt.Messages)
|
||||
}
|
||||
if resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
|
||||
t.Fatalf("debug response = %#v, want raw response content", resp.Debug.Response)
|
||||
}
|
||||
debugJSON, err := json.Marshal(resp.Debug)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal debug material: %v", err)
|
||||
}
|
||||
if strings.Contains(string(debugJSON), "secret-token") || strings.Contains(string(debugJSON), "sk-") {
|
||||
t.Fatalf("debug material contains secret material: %s", debugJSON)
|
||||
}
|
||||
gotReq := fake.lastRequest()
|
||||
if gotReq.Prompt.SessionID != "session-123" {
|
||||
t.Fatalf("session id = %q, want session-123", gotReq.Prompt.SessionID)
|
||||
}
|
||||
if gotReq.Target.Model != "explicit-model" {
|
||||
t.Fatalf("model = %q, want explicit-model", gotReq.Target.Model)
|
||||
}
|
||||
if len(gotReq.Prompt.Messages) != 1 || !strings.Contains(gotReq.Prompt.Messages[0].Content, `{"source":true}`) {
|
||||
t.Fatalf("rendered messages = %#v, want transcript input content", gotReq.Prompt.Messages)
|
||||
}
|
||||
if gotReq.StructuredOutput == nil {
|
||||
t.Fatalf("structured output = nil, want JSON schema")
|
||||
}
|
||||
manifests := client.LLMProfileManifests()
|
||||
if len(manifests) != 1 || manifests[0].ID != "explicit-profile" || manifests[0].Model != "explicit-model" {
|
||||
t.Fatalf("profile manifests = %#v", manifests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumClientUsesPromptDefaultProfileWhenRequestProfileEmpty(t *testing.T) {
|
||||
fake := &fakeScriptoriumLLM{content: `{"ok":true}`}
|
||||
client := newTestScriptoriumClient(t, fake)
|
||||
|
||||
var out map[string]any
|
||||
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "session-123",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out); err != nil {
|
||||
t.Fatalf("CompleteStructured() error = %v, want nil", err)
|
||||
}
|
||||
if got := fake.lastRequest().Target.Model; got != "default-model" {
|
||||
t.Fatalf("model = %q, want prompt default profile model", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumClientValidationFailureReturnsError(t *testing.T) {
|
||||
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"bad":true}`})
|
||||
|
||||
var out map[string]any
|
||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "session-123",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "validation failed") {
|
||||
t.Fatalf("CompleteStructured() error = %v, want validation failure", err)
|
||||
}
|
||||
if got := string(resp.Content); got != `{"bad":true}` {
|
||||
t.Fatalf("response content = %q, want raw failed output", got)
|
||||
}
|
||||
if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"bad":true}` {
|
||||
t.Fatalf("debug response = %#v, want raw failed output", resp.Debug)
|
||||
}
|
||||
if resp.Debug.Prompt == nil || len(resp.Debug.Prompt.Messages) == 0 {
|
||||
t.Fatalf("debug prompt = %#v, want prepared prompt material", resp.Debug.Prompt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumClientDecodeFailureReturnsRawResponse(t *testing.T) {
|
||||
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
|
||||
|
||||
var out []any
|
||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "session-123",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if err == nil || !strings.Contains(err.Error(), "decode Scriptorium structured output") {
|
||||
t.Fatalf("CompleteStructured() error = %v, want decode failure", err)
|
||||
}
|
||||
if got := string(resp.Content); got != `{"ok":true}` {
|
||||
t.Fatalf("response content = %q, want raw decode-failed output", got)
|
||||
}
|
||||
if resp.Debug == nil || resp.Debug.Response == nil || resp.Debug.Response.Content != `{"ok":true}` {
|
||||
t.Fatalf("debug response = %#v, want raw decode-failed output", resp.Debug)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumClientProviderFailureIncludesContextAndRedactsBearerToken(t *testing.T) {
|
||||
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{err: errors.New("provider failed with Bearer secret-token")})
|
||||
|
||||
var out map[string]any
|
||||
resp, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "session-123",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if err == nil {
|
||||
t.Fatalf("CompleteStructured() error = nil, want provider error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), `run Scriptorium prompt "adapter.test"`) {
|
||||
t.Fatalf("error = %q, want operation context", err.Error())
|
||||
}
|
||||
if strings.Contains(err.Error(), "secret-token") || !strings.Contains(err.Error(), "Bearer [REDACTED]") {
|
||||
t.Fatalf("error = %q, want redacted bearer token", err.Error())
|
||||
}
|
||||
if resp.Debug != nil {
|
||||
t.Fatalf("debug material = %#v, want none for provider failure without result", resp.Debug)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScriptoriumClientContextCancellationIsRespected(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
|
||||
|
||||
var out map[string]any
|
||||
_, err := client.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("CompleteStructured() error = %v, want context canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScheduledScriptoriumClientBoundsConcurrentCalls(t *testing.T) {
|
||||
fake := &fakeScriptoriumLLM{
|
||||
content: `{"ok":true}`,
|
||||
block: make(chan struct{}),
|
||||
}
|
||||
client := newTestScriptoriumClient(t, fake)
|
||||
scheduler, err := NewScheduler(1)
|
||||
if err != nil {
|
||||
t.Fatalf("NewScheduler() error = %v, want nil", err)
|
||||
}
|
||||
scheduled := NewScheduledClient(client, scheduler)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 3; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
var out map[string]any
|
||||
_, callErr := scheduled.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{
|
||||
PromptID: "adapter.test",
|
||||
SessionID: "session-123",
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": contracts.NewLLMInputMaterial("transcript", "application/json", []byte(`{"source":true}`), "", ""),
|
||||
},
|
||||
}, &out)
|
||||
if callErr != nil {
|
||||
t.Errorf("CompleteStructured() error = %v, want nil", callErr)
|
||||
}
|
||||
}()
|
||||
}
|
||||
waitForAtomicAtLeast(t, &fake.calls, 1)
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if got := atomic.LoadInt32(&fake.maxInFlight); got > 1 {
|
||||
t.Fatalf("max in-flight calls = %d, want <= 1", got)
|
||||
}
|
||||
close(fake.block)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestScriptoriumClientValidatesRequest(t *testing.T) {
|
||||
client := newTestScriptoriumClient(t, &fakeScriptoriumLLM{content: `{"ok":true}`})
|
||||
var out map[string]any
|
||||
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{}, &out); err == nil || !strings.Contains(err.Error(), "prompt_id") {
|
||||
t.Fatalf("missing prompt id error = %v, want prompt_id validation", err)
|
||||
}
|
||||
if _, err := client.CompleteStructured(context.Background(), contracts.StructuredCompletionRequest{PromptID: "adapter.test"}, nil); err == nil || !strings.Contains(err.Error(), "non-nil pointer") {
|
||||
t.Fatalf("nil output error = %v, want output validation", err)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestScriptoriumClient(t *testing.T, fake *fakeScriptoriumLLM) *ScriptoriumClient {
|
||||
t.Helper()
|
||||
registry := NewAssetRegistry()
|
||||
if err := registry.RegisterPromptFS(fstest.MapFS{
|
||||
"adapter.test.yaml": {Data: []byte(`id: adapter.test
|
||||
version: "v1"
|
||||
default_profile: default-profile
|
||||
session_id: "{{ .session_id }}"
|
||||
inputs:
|
||||
- name: transcript
|
||||
required: true
|
||||
content_type: application/json
|
||||
messages:
|
||||
- role: user
|
||||
content: "Transcript: {{ input \"transcript\" }}"
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: adapter.schema.json
|
||||
repair_attempts: 0
|
||||
`)},
|
||||
}, "."); err != nil {
|
||||
t.Fatalf("RegisterPromptFS() error = %v", err)
|
||||
}
|
||||
if err := registry.RegisterSchemaFS(fstest.MapFS{
|
||||
"adapter.schema.json": {Data: []byte(`{"type":"object","required":["ok"],"properties":{"ok":{"type":"boolean"}}}`)},
|
||||
}, "."); err != nil {
|
||||
t.Fatalf("RegisterSchemaFS() error = %v", err)
|
||||
}
|
||||
client, err := NewScriptoriumClient(ScriptoriumClientConfig{
|
||||
Assets: registry,
|
||||
EngineOptions: []scriptorium.Option{
|
||||
scriptorium.WithProfiles(
|
||||
scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "default-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "default-model",
|
||||
}),
|
||||
scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
|
||||
ID: "explicit-profile",
|
||||
Endpoint: "http://127.0.0.1:1/v1",
|
||||
Model: "explicit-model",
|
||||
}),
|
||||
),
|
||||
scriptorium.WithLLMClient(fake),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewScriptoriumClient() error = %v, want nil", err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
type fakeScriptoriumLLM struct {
|
||||
content string
|
||||
err error
|
||||
block chan struct{}
|
||||
mu sync.Mutex
|
||||
last scriptorium.GenerateRequest
|
||||
calls int32
|
||||
inFlight int32
|
||||
maxInFlight int32
|
||||
}
|
||||
|
||||
func (f *fakeScriptoriumLLM) Generate(ctx context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
|
||||
f.mu.Lock()
|
||||
f.last = req
|
||||
f.mu.Unlock()
|
||||
atomic.AddInt32(&f.calls, 1)
|
||||
current := atomic.AddInt32(&f.inFlight, 1)
|
||||
for {
|
||||
seen := atomic.LoadInt32(&f.maxInFlight)
|
||||
if current <= seen || atomic.CompareAndSwapInt32(&f.maxInFlight, seen, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
defer atomic.AddInt32(&f.inFlight, -1)
|
||||
if f.block != nil {
|
||||
select {
|
||||
case <-f.block:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
content := f.content
|
||||
if content == "" {
|
||||
content = `{"ok":true}`
|
||||
}
|
||||
if !json.Valid([]byte(content)) {
|
||||
return nil, errors.New("test fake must return JSON content")
|
||||
}
|
||||
return &scriptorium.GenerateResponse{
|
||||
Content: content,
|
||||
Usage: scriptorium.TokenUsage{
|
||||
PromptTokens: 11,
|
||||
CompletionTokens: 7,
|
||||
TotalTokens: 18,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *fakeScriptoriumLLM) lastRequest() scriptorium.GenerateRequest {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.last
|
||||
}
|
||||
227
internal/framework/pipeline/checkpoint.go
Normal file
227
internal/framework/pipeline/checkpoint.go
Normal file
@@ -0,0 +1,227 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type CheckpointFingerprint struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type CheckpointRecorder interface {
|
||||
SourceRunning(moduleKey string) error
|
||||
SourceSucceeded(moduleKey string, doc *source.SourceDocument) error
|
||||
SourceFailed(moduleKey string, err error) error
|
||||
ChunkRunning(moduleKey string, sourceDigest string) error
|
||||
ChunkSucceeded(moduleKey string, sourceDigest string, chunks []contracts.SourceChunk, warnings []contracts.Warning) error
|
||||
ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error
|
||||
ChunkFailed(moduleKey string, sourceDigest string, err error) error
|
||||
ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []contracts.ExtractOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
|
||||
ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
MergeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output contracts.MergeOutput, warnings []contracts.Warning) error
|
||||
MergeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
|
||||
MergeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
NormalizeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output contracts.NormalizeOutput, warnings []contracts.Warning) error
|
||||
NormalizeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
|
||||
NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
}
|
||||
|
||||
type CheckpointDecision struct {
|
||||
Reused bool `json:"reused"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
type CheckpointEvent struct {
|
||||
Stage string `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
Action string `json:"action"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
type SourceCheckpoint struct {
|
||||
Document *source.SourceDocument
|
||||
}
|
||||
|
||||
type ChunkCheckpoint struct {
|
||||
Chunks []contracts.SourceChunk
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type ExtractCheckpoint struct {
|
||||
Outputs []contracts.ExtractOutput
|
||||
Rejected []contracts.RejectedOutput
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type MergeCheckpoint struct {
|
||||
Output contracts.MergeOutput
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type NormalizeCheckpoint struct {
|
||||
Output contracts.NormalizeOutput
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type CheckpointLoader interface {
|
||||
Enabled() bool
|
||||
Source(moduleKey string) (SourceCheckpoint, CheckpointDecision)
|
||||
Chunk(moduleKey string, sourceDigest string) (ChunkCheckpoint, CheckpointDecision)
|
||||
Extract(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision)
|
||||
Merge(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision)
|
||||
Normalize(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision)
|
||||
}
|
||||
|
||||
type noopCheckpointRecorder struct{}
|
||||
type noopCheckpointLoader struct{}
|
||||
|
||||
func NoopCheckpointRecorder() CheckpointRecorder { return noopCheckpointRecorder{} }
|
||||
func NoopCheckpointLoader() CheckpointLoader { return noopCheckpointLoader{} }
|
||||
|
||||
func (noopCheckpointRecorder) SourceRunning(string) error { return nil }
|
||||
func (noopCheckpointRecorder) SourceSucceeded(string, *source.SourceDocument) error { return nil }
|
||||
func (noopCheckpointRecorder) SourceFailed(string, error) error { return nil }
|
||||
func (noopCheckpointRecorder) ChunkRunning(string, string) error { return nil }
|
||||
func (noopCheckpointRecorder) ChunkSucceeded(string, string, []contracts.SourceChunk, []contracts.Warning) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) ChunkRejected(string, string, contracts.RejectedOutput) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) ChunkFailed(string, string, error) error { return nil }
|
||||
func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []contracts.ExtractOutput, []contracts.RejectedOutput, []contracts.Warning) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) ExtractFailed(string, string, []CheckpointFingerprint, error) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) MergeRunning(string, string, []CheckpointFingerprint) error { return nil }
|
||||
func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, contracts.MergeOutput, []contracts.Warning) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) MergeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) MergeFailed(string, string, []CheckpointFingerprint, error) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) NormalizeRunning(string, string, []CheckpointFingerprint) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, contracts.NormalizeOutput, []contracts.Warning) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) NormalizeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) NormalizeFailed(string, string, []CheckpointFingerprint, error) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (noopCheckpointLoader) Enabled() bool { return false }
|
||||
func (noopCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) {
|
||||
return SourceCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
func (noopCheckpointLoader) Chunk(string, string) (ChunkCheckpoint, CheckpointDecision) {
|
||||
return ChunkCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
func (noopCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||
return ExtractCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
func (noopCheckpointLoader) Merge(string, string, []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
|
||||
return MergeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
|
||||
return NormalizeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
|
||||
func rawOutputDigests(payloads []contracts.RawPayload) []CheckpointFingerprint {
|
||||
values := make([]CheckpointFingerprint, 0, len(payloads))
|
||||
for i, payload := range payloads {
|
||||
values = append(values, CheckpointFingerprint{
|
||||
Name: fmt.Sprintf("payload[%d]", i),
|
||||
Value: checkpointContentDigest(payload.Content),
|
||||
})
|
||||
}
|
||||
return normalizeCheckpointFingerprints(values)
|
||||
}
|
||||
|
||||
func extractPayloads(outputs []contracts.ExtractOutput) []contracts.RawPayload {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
payloads := make([]contracts.RawPayload, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
payloads = append(payloads, output.Payload)
|
||||
}
|
||||
return payloads
|
||||
}
|
||||
|
||||
func digestFingerprints(name string, digest string) []CheckpointFingerprint {
|
||||
digest = strings.TrimSpace(digest)
|
||||
if digest == "" {
|
||||
return nil
|
||||
}
|
||||
return []CheckpointFingerprint{{Name: name, Value: digest}}
|
||||
}
|
||||
|
||||
func joinedChunkDigest(chunks []contracts.SourceChunk) string {
|
||||
if len(chunks) == 0 {
|
||||
return ""
|
||||
}
|
||||
values := make([]string, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
values = append(values, chunk.ID+"="+checkpointContentDigest(chunk.Content))
|
||||
}
|
||||
sort.Strings(values)
|
||||
sum := sha256.Sum256([]byte(strings.Join(values, "\n")))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func normalizeCheckpointFingerprints(values []CheckpointFingerprint) []CheckpointFingerprint {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
byName := make(map[string]string, len(values))
|
||||
for _, value := range values {
|
||||
name := strings.TrimSpace(value.Name)
|
||||
fingerprint := strings.TrimSpace(value.Value)
|
||||
if name == "" || fingerprint == "" {
|
||||
continue
|
||||
}
|
||||
byName[name] = fingerprint
|
||||
}
|
||||
if len(byName) == 0 {
|
||||
return nil
|
||||
}
|
||||
names := make([]string, 0, len(byName))
|
||||
for name := range byName {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
out := make([]CheckpointFingerprint, 0, len(names))
|
||||
for _, name := range names {
|
||||
out = append(out, CheckpointFingerprint{Name: name, Value: byName[name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func checkpointContentDigest(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
107
internal/framework/pipeline/chunk_validation.go
Normal file
107
internal/framework/pipeline/chunk_validation.go
Normal file
@@ -0,0 +1,107 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) {
|
||||
if len(chunks) == 0 {
|
||||
return nil, fmt.Errorf("chunks must not be empty")
|
||||
}
|
||||
|
||||
sourceUnitIndexes := make(map[int]int, len(doc.Units))
|
||||
sourceUnits := make(map[int]source.SourceUnit, len(doc.Units))
|
||||
for index, unit := range doc.Units {
|
||||
sourceUnitIndexes[unit.ID] = index
|
||||
sourceUnits[unit.ID] = unit
|
||||
}
|
||||
|
||||
canonicalChunks := make([]contracts.SourceChunk, 0, len(chunks))
|
||||
seenChunkIDs := make(map[string]struct{}, len(chunks))
|
||||
for chunkIndex, chunk := range chunks {
|
||||
if strings.TrimSpace(chunk.ID) == "" {
|
||||
return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
|
||||
}
|
||||
if _, ok := seenChunkIDs[chunk.ID]; ok {
|
||||
return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
|
||||
}
|
||||
seenChunkIDs[chunk.ID] = struct{}{}
|
||||
|
||||
if chunk.SourceID != doc.ID {
|
||||
return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
|
||||
}
|
||||
if chunk.Index != chunkIndex {
|
||||
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
|
||||
}
|
||||
startIndex, ok := sourceUnitIndexes[chunk.StartUnitID]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("chunk %q start_unit_id %d was not found in source document %q", chunk.ID, chunk.StartUnitID, doc.ID)
|
||||
}
|
||||
endIndex, ok := sourceUnitIndexes[chunk.EndUnitID]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("chunk %q end_unit_id %d was not found in source document %q", chunk.ID, chunk.EndUnitID, doc.ID)
|
||||
}
|
||||
if startIndex > endIndex {
|
||||
return nil, fmt.Errorf("chunk %q start_unit_id %d appears after end_unit_id %d", chunk.ID, chunk.StartUnitID, chunk.EndUnitID)
|
||||
}
|
||||
if len(chunk.Units) == 0 {
|
||||
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
|
||||
}
|
||||
if len(chunk.Content) == 0 {
|
||||
return nil, fmt.Errorf("chunk %q content must not be empty", chunk.ID)
|
||||
}
|
||||
if strings.TrimSpace(chunk.MediaType) == "" {
|
||||
return nil, fmt.Errorf("chunk %q media_type must not be empty", chunk.ID)
|
||||
}
|
||||
|
||||
seenUnitIDs := make(map[int]struct{}, len(chunk.Units))
|
||||
previousSourceIndex := -1
|
||||
canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
|
||||
for unitIndex, unit := range chunk.Units {
|
||||
if unit.ID <= 0 {
|
||||
return nil, fmt.Errorf("chunk %q unit[%d].id must be positive", chunk.ID, unitIndex)
|
||||
}
|
||||
if _, ok := seenUnitIDs[unit.ID]; ok {
|
||||
return nil, fmt.Errorf("chunk %q repeats source unit %d", chunk.ID, unit.ID)
|
||||
}
|
||||
seenUnitIDs[unit.ID] = struct{}{}
|
||||
|
||||
sourceIndex, ok := sourceUnitIndexes[unit.ID]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("chunk %q source unit %d was not found in source document %q", chunk.ID, unit.ID, doc.ID)
|
||||
}
|
||||
if sourceIndex <= previousSourceIndex {
|
||||
return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
|
||||
}
|
||||
previousSourceIndex = sourceIndex
|
||||
canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
|
||||
}
|
||||
|
||||
canonicalChunks = append(canonicalChunks, contracts.SourceChunk{
|
||||
ID: chunk.ID,
|
||||
SourceID: chunk.SourceID,
|
||||
Index: chunk.Index,
|
||||
StartUnitID: chunk.StartUnitID,
|
||||
EndUnitID: chunk.EndUnitID,
|
||||
Content: append([]byte(nil), chunk.Content...),
|
||||
MediaType: chunk.MediaType,
|
||||
Units: canonicalUnits,
|
||||
Metadata: cloneMetadata(chunk.Metadata),
|
||||
})
|
||||
}
|
||||
|
||||
return canonicalChunks, nil
|
||||
}
|
||||
|
||||
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
|
||||
return source.SourceUnit{
|
||||
ID: unit.ID,
|
||||
Kind: unit.Kind,
|
||||
Text: unit.Text,
|
||||
Metadata: cloneMetadata(unit.Metadata),
|
||||
}
|
||||
}
|
||||
@@ -330,6 +330,10 @@ func (chunker registryChunker) Key() string {
|
||||
return chunker.key
|
||||
}
|
||||
|
||||
func (chunker registryChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||
return contracts.ChunkResult{}, nil
|
||||
}
|
||||
@@ -354,6 +358,10 @@ func (normalizer registryNormalizer) Key() string {
|
||||
return normalizer.key
|
||||
}
|
||||
|
||||
func (normalizer registryNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (normalizer registryNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||
return contracts.NormalizeResult{}, nil
|
||||
}
|
||||
@@ -378,6 +386,10 @@ func (validator registryValidator) Name() string {
|
||||
return validator.name
|
||||
}
|
||||
|
||||
func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{}, nil
|
||||
func (validator registryValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassDeterministic
|
||||
}
|
||||
|
||||
func (validator registryValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
806
internal/framework/pipeline/debug.go
Normal file
806
internal/framework/pipeline/debug.go
Normal file
@@ -0,0 +1,806 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type DebugRecorder interface {
|
||||
Enabled() bool
|
||||
WriteJSON(name string, payload any) error
|
||||
WriteBytes(name string, data []byte) error
|
||||
}
|
||||
|
||||
type noopDebugRecorder struct{}
|
||||
|
||||
func NoopDebugRecorder() DebugRecorder { return noopDebugRecorder{} }
|
||||
|
||||
func (noopDebugRecorder) Enabled() bool { return false }
|
||||
func (noopDebugRecorder) WriteJSON(string, any) error { return nil }
|
||||
func (noopDebugRecorder) WriteBytes(string, []byte) error { return nil }
|
||||
func debugPathComponent(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return "_"
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
b.WriteRune(r)
|
||||
case r >= 'A' && r <= 'Z':
|
||||
b.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '-' || r == '_' || r == '.':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteString(fmt.Sprintf("~%x", r))
|
||||
}
|
||||
}
|
||||
out := b.String()
|
||||
if out == "." || out == ".." || strings.Contains(out, "..") {
|
||||
return "_"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type debugTimedEnvelope struct {
|
||||
Stage string `json:"stage,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
Attempt int `json:"attempt,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Payload any `json:"payload,omitempty"`
|
||||
LLMCalls []debugLLMCallReference `json:"llm_calls,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type debugBinaryEnvelope struct {
|
||||
ContentBase64 string `json:"content_base64,omitempty"`
|
||||
ContentDigest string `json:"content_digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type debugRawPayload struct {
|
||||
Content debugBinaryEnvelope `json:"content"`
|
||||
}
|
||||
|
||||
type debugSourceInput struct {
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Raw debugBinaryEnvelope `json:"raw,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type debugSourceDocument struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Format string `json:"format,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Units []source.SourceUnit `json:"units,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type debugSourceChunk struct {
|
||||
ID string `json:"id"`
|
||||
SourceID string `json:"source_id"`
|
||||
Index int `json:"index"`
|
||||
StartUnitID int `json:"start_unit_id"`
|
||||
EndUnitID int `json:"end_unit_id"`
|
||||
Content debugBinaryEnvelope `json:"content"`
|
||||
Units []source.SourceUnit `json:"units,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type debugExtractOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
SourceID string `json:"source_id"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ChunkIndex int `json:"chunk_index"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload debugBinaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type debugMergeOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
MergerKey string `json:"merger_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload debugBinaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type debugNormalizeOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
NormalizerKey string `json:"normalizer_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload debugBinaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type debugLLMInputMaterial struct {
|
||||
Name string `json:"name"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Content string `json:"content_base64,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
OriginURI string `json:"origin_uri,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes,omitempty"`
|
||||
}
|
||||
|
||||
type debugStructuredCompletionRequest struct {
|
||||
StageName string `json:"stage_name"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Inputs map[string]debugLLMInputMaterial `json:"inputs,omitempty"`
|
||||
Vars map[string]any `json:"vars,omitempty"`
|
||||
}
|
||||
|
||||
type debugStructuredCompletionResponse struct {
|
||||
Content string `json:"content,omitempty"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type debugLLMPromptArtifact struct {
|
||||
CallID string `json:"call_id"`
|
||||
Prompt *contracts.LLMDebugPrompt `json:"prompt,omitempty"`
|
||||
}
|
||||
|
||||
type debugLLMResponseArtifact struct {
|
||||
CallID string `json:"call_id"`
|
||||
Response *contracts.LLMDebugResponse `json:"response,omitempty"`
|
||||
Fallback *debugStructuredCompletionResponse `json:"fallback,omitempty"`
|
||||
ContentPath string `json:"content_path,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type debugLLMCallReference struct {
|
||||
CallID string `json:"call_id"`
|
||||
PromptPath string `json:"prompt_path,omitempty"`
|
||||
ResponsePath string `json:"response_path"`
|
||||
ResponseContentPath string `json:"response_content_path,omitempty"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Error bool `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type debugValidationRequest struct {
|
||||
Stage string `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload *debugBinaryEnvelope `json:"payload,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
Chunk *debugSourceChunk `json:"chunk,omitempty"`
|
||||
Chunks []debugSourceChunk `json:"chunks,omitempty"`
|
||||
ExtractOutputs []debugExtractOutput `json:"extract_outputs,omitempty"`
|
||||
MergeOutput *debugMergeOutput `json:"merge_output,omitempty"`
|
||||
}
|
||||
|
||||
type debugValidationCall struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Request debugValidationRequest `json:"request"`
|
||||
Result contracts.ValidationResult `json:"result,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type debugLLMClient struct {
|
||||
inner contracts.StructuredLLMClient
|
||||
recorder DebugRecorder
|
||||
mu sync.Mutex
|
||||
counter int
|
||||
}
|
||||
|
||||
type debugLLMScope struct {
|
||||
prefix string
|
||||
parent *debugLLMScope
|
||||
mu sync.Mutex
|
||||
calls []debugLLMCallReference
|
||||
}
|
||||
|
||||
type debugLLMScopeContextKey struct{}
|
||||
|
||||
func wrapDebugLLMClient(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient {
|
||||
if client == nil || recorder == nil || !recorder.Enabled() {
|
||||
return client
|
||||
}
|
||||
return &debugLLMClient{inner: client, recorder: recorder}
|
||||
}
|
||||
|
||||
func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.mu.Lock()
|
||||
client.counter++
|
||||
callID := fmt.Sprintf("%04d", client.counter)
|
||||
client.mu.Unlock()
|
||||
|
||||
started := time.Now().UTC()
|
||||
response, err := client.inner.CompleteStructured(ctx, req, out)
|
||||
completed := time.Now().UTC()
|
||||
errorText := ""
|
||||
if err != nil {
|
||||
errorText = err.Error()
|
||||
}
|
||||
|
||||
scopePrefix := cleanDebugPath(req.StageName)
|
||||
if scopePrefix == "_" {
|
||||
scopePrefix = "llm"
|
||||
}
|
||||
if scope := debugLLMScopeFromContext(ctx); scope != nil {
|
||||
scopePrefix = scope.prefix
|
||||
}
|
||||
promptPath := ""
|
||||
var writeErr error
|
||||
if response.Debug != nil && response.Debug.Prompt != nil {
|
||||
promptPath = path.Join(scopePrefix, "prompt-"+callID+".json")
|
||||
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, promptPath, debugTimedEnvelope{
|
||||
Stage: req.StageName,
|
||||
ModuleKey: req.StageName,
|
||||
StartedAt: started,
|
||||
CompletedAt: completed,
|
||||
DurationMS: completed.Sub(started).Milliseconds(),
|
||||
Payload: debugLLMPromptArtifact{
|
||||
CallID: callID,
|
||||
Prompt: response.Debug.Prompt,
|
||||
},
|
||||
}))
|
||||
}
|
||||
responsePath := path.Join(scopePrefix, "response-"+callID+".json")
|
||||
responseMaterial := debugResponseMaterial(response)
|
||||
fallbackMaterial := debugCompletionFallback(response)
|
||||
responseContentPath, responseForArtifact, fallbackForArtifact, contentErr := writeDebugResponseContent(client.recorder, scopePrefix, callID, responseMaterial, fallbackMaterial)
|
||||
writeErr = errors.Join(writeErr, contentErr)
|
||||
writeErr = errors.Join(writeErr, writeDebugTimed(client.recorder, responsePath, debugTimedEnvelope{
|
||||
Stage: req.StageName,
|
||||
ModuleKey: req.StageName,
|
||||
StartedAt: started,
|
||||
CompletedAt: completed,
|
||||
DurationMS: completed.Sub(started).Milliseconds(),
|
||||
Payload: debugLLMResponseArtifact{
|
||||
CallID: callID,
|
||||
Response: responseForArtifact,
|
||||
Fallback: fallbackForArtifact,
|
||||
ContentPath: responseContentPath,
|
||||
Error: errorText,
|
||||
},
|
||||
Error: errorText,
|
||||
}))
|
||||
callRef := debugLLMCallReference{
|
||||
CallID: callID,
|
||||
PromptPath: promptPath,
|
||||
ResponsePath: responsePath,
|
||||
ResponseContentPath: responseContentPath,
|
||||
PromptID: req.PromptID,
|
||||
ProfileID: debugFirstNonEmptyString(response.ProfileID, req.ProfileID),
|
||||
Model: debugFirstNonEmptyString(response.Model, debugResponseModel(response)),
|
||||
Error: err != nil,
|
||||
}
|
||||
if scope := debugLLMScopeFromContext(ctx); scope != nil {
|
||||
scope.record(callRef)
|
||||
}
|
||||
if err != nil {
|
||||
return response, err
|
||||
}
|
||||
if writeErr != nil {
|
||||
return response, fmt.Errorf("write LLM debug artifact: %w", writeErr)
|
||||
}
|
||||
return response, err
|
||||
}
|
||||
|
||||
func withDebugLLMScope(ctx context.Context, prefix string) (context.Context, *debugLLMScope) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
prefix = cleanDebugPath(prefix)
|
||||
scope := &debugLLMScope{
|
||||
prefix: prefix,
|
||||
parent: debugLLMScopeFromContext(ctx),
|
||||
}
|
||||
return context.WithValue(ctx, debugLLMScopeContextKey{}, scope), scope
|
||||
}
|
||||
|
||||
func debugLLMScopeFromContext(ctx context.Context) *debugLLMScope {
|
||||
if ctx == nil {
|
||||
return nil
|
||||
}
|
||||
scope, _ := ctx.Value(debugLLMScopeContextKey{}).(*debugLLMScope)
|
||||
return scope
|
||||
}
|
||||
|
||||
func (scope *debugLLMScope) record(ref debugLLMCallReference) {
|
||||
if scope == nil {
|
||||
return
|
||||
}
|
||||
scope.mu.Lock()
|
||||
scope.calls = append(scope.calls, ref)
|
||||
scope.mu.Unlock()
|
||||
if scope.parent != nil {
|
||||
scope.parent.record(ref)
|
||||
}
|
||||
}
|
||||
|
||||
func (scope *debugLLMScope) references() []debugLLMCallReference {
|
||||
if scope == nil {
|
||||
return nil
|
||||
}
|
||||
scope.mu.Lock()
|
||||
defer scope.mu.Unlock()
|
||||
if len(scope.calls) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]debugLLMCallReference, len(scope.calls))
|
||||
copy(out, scope.calls)
|
||||
return out
|
||||
}
|
||||
|
||||
func cleanDebugPath(value string) string {
|
||||
parts := strings.Split(path.Clean(strings.TrimSpace(value)), "/")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
out = append(out, debugPathComponent(part))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return "_"
|
||||
}
|
||||
return path.Join(out...)
|
||||
}
|
||||
|
||||
func debugFirstNonEmptyString(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (client *debugLLMClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
|
||||
provider, ok := client.inner.(contracts.LLMProfileManifestProvider)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return provider.LLMProfileManifests()
|
||||
}
|
||||
|
||||
func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnvelope) error {
|
||||
if recorder == nil || !recorder.Enabled() {
|
||||
return nil
|
||||
}
|
||||
if envelope.CompletedAt.IsZero() {
|
||||
envelope.CompletedAt = time.Now().UTC()
|
||||
}
|
||||
if envelope.StartedAt.IsZero() {
|
||||
envelope.StartedAt = envelope.CompletedAt
|
||||
}
|
||||
if envelope.DurationMS == 0 {
|
||||
envelope.DurationMS = envelope.CompletedAt.Sub(envelope.StartedAt).Milliseconds()
|
||||
}
|
||||
return recorder.WriteJSON(name, envelope)
|
||||
}
|
||||
|
||||
func debugEnvelopeWithLLMCalls(envelope debugTimedEnvelope, scope *debugLLMScope) debugTimedEnvelope {
|
||||
if scope != nil {
|
||||
envelope.LLMCalls = scope.references()
|
||||
}
|
||||
return envelope
|
||||
}
|
||||
|
||||
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) debugBinaryEnvelope {
|
||||
content = redactSecretBytes(content)
|
||||
return debugBinaryEnvelope{
|
||||
ContentBase64: base64.StdEncoding.EncodeToString(content),
|
||||
ContentDigest: debugContentDigest(content),
|
||||
MediaType: mediaType,
|
||||
Metadata: redactSensitiveMap(metadata),
|
||||
Warnings: cloneWarnings(warnings),
|
||||
}
|
||||
}
|
||||
|
||||
func debugPayloadEnvelope(payload contracts.RawPayload) debugBinaryEnvelope {
|
||||
return debugContentEnvelope(payload.Content, payload.MediaType, payload.Metadata, payload.Warnings)
|
||||
}
|
||||
|
||||
func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocument {
|
||||
if doc == nil {
|
||||
return nil
|
||||
}
|
||||
return &debugSourceDocument{
|
||||
ID: doc.ID,
|
||||
Kind: doc.Kind,
|
||||
Format: doc.Format,
|
||||
Digest: doc.Digest,
|
||||
Units: cloneSourceUnits(doc.Units),
|
||||
Metadata: redactSensitiveMap(doc.Metadata),
|
||||
}
|
||||
}
|
||||
|
||||
func debugSourceChunkEnvelope(chunk contracts.SourceChunk) debugSourceChunk {
|
||||
return debugSourceChunk{
|
||||
ID: chunk.ID,
|
||||
SourceID: chunk.SourceID,
|
||||
Index: chunk.Index,
|
||||
StartUnitID: chunk.StartUnitID,
|
||||
EndUnitID: chunk.EndUnitID,
|
||||
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
|
||||
Units: cloneSourceUnits(chunk.Units),
|
||||
Metadata: redactSensitiveMap(chunk.Metadata),
|
||||
}
|
||||
}
|
||||
|
||||
func debugSourceChunkEnvelopes(chunks []contracts.SourceChunk) []debugSourceChunk {
|
||||
if len(chunks) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]debugSourceChunk, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
out = append(out, debugSourceChunkEnvelope(chunk))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugExtractOutputEnvelope(output contracts.ExtractOutput) debugExtractOutput {
|
||||
output.Schema.JSONSchema = nil
|
||||
return debugExtractOutput{
|
||||
LaneID: output.LaneID,
|
||||
ExtractorKey: output.ExtractorKey,
|
||||
SourceID: output.SourceID,
|
||||
ChunkID: output.ChunkID,
|
||||
ChunkIndex: output.ChunkIndex,
|
||||
Schema: output.Schema,
|
||||
Payload: debugPayloadEnvelope(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func debugExtractOutputEnvelopes(outputs []contracts.ExtractOutput) []debugExtractOutput {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]debugExtractOutput, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
out = append(out, debugExtractOutputEnvelope(output))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugMergeOutputEnvelope(output contracts.MergeOutput) debugMergeOutput {
|
||||
output.Schema.JSONSchema = nil
|
||||
return debugMergeOutput{
|
||||
LaneID: output.LaneID,
|
||||
MergerKey: output.MergerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: output.Schema,
|
||||
Payload: debugPayloadEnvelope(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func debugNormalizeOutputEnvelope(output contracts.NormalizeOutput) debugNormalizeOutput {
|
||||
output.Schema.JSONSchema = nil
|
||||
return debugNormalizeOutput{
|
||||
LaneID: output.LaneID,
|
||||
NormalizerKey: output.NormalizerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: output.Schema,
|
||||
Payload: debugPayloadEnvelope(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func debugNormalizeOutputEnvelopes(outputs []contracts.NormalizeOutput) []debugNormalizeOutput {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]debugNormalizeOutput, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
out = append(out, debugNormalizeOutputEnvelope(output))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type debugOutputFile struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
Content debugBinaryEnvelope `json:"content"`
|
||||
}
|
||||
|
||||
func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]debugOutputFile, 0, len(files))
|
||||
for _, file := range files {
|
||||
out = append(out, debugOutputFile{
|
||||
Name: file.Name,
|
||||
ContentType: file.ContentType,
|
||||
Content: debugContentEnvelope(file.Bytes, file.ContentType, nil, nil),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest {
|
||||
inputs := make(map[string]debugLLMInputMaterial, len(req.Inputs))
|
||||
for key, material := range req.Inputs {
|
||||
inputs[key] = debugLLMInputMaterial{
|
||||
Name: material.Name,
|
||||
MediaType: material.MediaType,
|
||||
Content: base64.StdEncoding.EncodeToString(redactSecretBytes(material.Content)),
|
||||
Digest: material.Digest,
|
||||
OriginURI: material.OriginURI,
|
||||
SizeBytes: material.SizeBytes,
|
||||
}
|
||||
}
|
||||
if len(inputs) == 0 {
|
||||
inputs = nil
|
||||
}
|
||||
return debugStructuredCompletionRequest{
|
||||
StageName: req.StageName,
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: inputs,
|
||||
Vars: redactSensitiveMap(req.Vars),
|
||||
}
|
||||
}
|
||||
|
||||
func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse {
|
||||
return debugStructuredCompletionResponse{
|
||||
Content: string(response.Content),
|
||||
Provider: response.Provider,
|
||||
Model: response.Model,
|
||||
ProfileID: response.ProfileID,
|
||||
PromptTokens: response.PromptTokens,
|
||||
CompletionTokens: response.CompletionTokens,
|
||||
TotalTokens: response.TotalTokens,
|
||||
}
|
||||
}
|
||||
|
||||
func debugCompletionFallback(response contracts.StructuredCompletionResponse) *debugStructuredCompletionResponse {
|
||||
if response.Debug != nil && response.Debug.Response != nil {
|
||||
return nil
|
||||
}
|
||||
fallback := debugCompletionResponse(response)
|
||||
if fallback.Content == "" &&
|
||||
fallback.Provider == "" &&
|
||||
fallback.Model == "" &&
|
||||
fallback.ProfileID == "" &&
|
||||
fallback.PromptTokens == 0 &&
|
||||
fallback.CompletionTokens == 0 &&
|
||||
fallback.TotalTokens == 0 {
|
||||
return nil
|
||||
}
|
||||
return &fallback
|
||||
}
|
||||
|
||||
func writeDebugResponseContent(recorder DebugRecorder, scopePrefix string, callID string, response *contracts.LLMDebugResponse, fallback *debugStructuredCompletionResponse) (string, *contracts.LLMDebugResponse, *debugStructuredCompletionResponse, error) {
|
||||
responseCopy := cloneDebugResponseWithoutContent(response)
|
||||
fallbackCopy := cloneDebugFallbackWithoutContent(fallback)
|
||||
content := ""
|
||||
if response != nil {
|
||||
content = response.Content
|
||||
}
|
||||
if content == "" && fallback != nil {
|
||||
content = fallback.Content
|
||||
}
|
||||
if content == "" {
|
||||
return "", responseCopy, fallbackCopy, nil
|
||||
}
|
||||
|
||||
contentPath, data := debugResponseContentFile(scopePrefix, callID, content)
|
||||
if recorder == nil || !recorder.Enabled() {
|
||||
return contentPath, responseCopy, fallbackCopy, nil
|
||||
}
|
||||
if err := recorder.WriteBytes(contentPath, data); err != nil {
|
||||
return contentPath, responseCopy, fallbackCopy, err
|
||||
}
|
||||
return contentPath, responseCopy, fallbackCopy, nil
|
||||
}
|
||||
|
||||
func cloneDebugResponseWithoutContent(response *contracts.LLMDebugResponse) *contracts.LLMDebugResponse {
|
||||
if response == nil {
|
||||
return nil
|
||||
}
|
||||
clone := *response
|
||||
clone.Content = ""
|
||||
return &clone
|
||||
}
|
||||
|
||||
func cloneDebugFallbackWithoutContent(fallback *debugStructuredCompletionResponse) *debugStructuredCompletionResponse {
|
||||
if fallback == nil {
|
||||
return nil
|
||||
}
|
||||
clone := *fallback
|
||||
clone.Content = ""
|
||||
return &clone
|
||||
}
|
||||
|
||||
func debugResponseContentFile(scopePrefix string, callID string, content string) (string, []byte) {
|
||||
raw := []byte(content)
|
||||
if json.Valid(raw) {
|
||||
var formatted bytes.Buffer
|
||||
if err := json.Indent(&formatted, raw, "", " "); err == nil {
|
||||
formatted.WriteByte('\n')
|
||||
return path.Join(scopePrefix, "response-content-"+callID+".json"), formatted.Bytes()
|
||||
}
|
||||
}
|
||||
return path.Join(scopePrefix, "response-content-"+callID+".txt"), raw
|
||||
}
|
||||
|
||||
func debugResponseMaterial(response contracts.StructuredCompletionResponse) *contracts.LLMDebugResponse {
|
||||
if response.Debug == nil {
|
||||
return nil
|
||||
}
|
||||
return response.Debug.Response
|
||||
}
|
||||
|
||||
func debugResponseModel(response contracts.StructuredCompletionResponse) string {
|
||||
if response.Debug == nil || response.Debug.Response == nil {
|
||||
return ""
|
||||
}
|
||||
return response.Debug.Response.ModelName
|
||||
}
|
||||
|
||||
func debugValidationRequestEnvelope(req contracts.ValidationRequest) debugValidationRequest {
|
||||
req.Schema.JSONSchema = nil
|
||||
out := debugValidationRequest{
|
||||
Stage: req.Stage,
|
||||
LaneID: req.LaneID,
|
||||
ModuleKey: req.ModuleKey,
|
||||
SourceID: req.SourceID,
|
||||
SessionID: req.SessionID,
|
||||
LLMProfile: req.LLMProfile,
|
||||
Options: redactSensitiveMap(req.Options),
|
||||
Metadata: redactSensitiveMap(req.Metadata),
|
||||
Schema: req.Schema,
|
||||
ChunkID: req.ChunkID,
|
||||
ChunkIndex: req.ChunkIndex,
|
||||
}
|
||||
payload := debugPayloadEnvelope(req.Payload)
|
||||
out.Payload = &payload
|
||||
if req.Chunk != nil {
|
||||
chunk := debugSourceChunkEnvelope(*req.Chunk)
|
||||
out.Chunk = &chunk
|
||||
}
|
||||
out.Chunks = debugSourceChunkEnvelopes(req.Chunks)
|
||||
out.ExtractOutputs = debugExtractOutputEnvelopes(req.ExtractOutputs)
|
||||
if len(req.MergeOutput.Payload.Content) > 0 || req.MergeOutput.LaneID != "" {
|
||||
merge := debugMergeOutputEnvelope(req.MergeOutput)
|
||||
out.MergeOutput = &merge
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
|
||||
result.Message = string(redactSecretBytes([]byte(result.Message)))
|
||||
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))
|
||||
for i := range result.Warnings {
|
||||
result.Warnings[i].Message = string(redactSecretBytes([]byte(result.Warnings[i].Message)))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func debugRejectedOutputEnvelope(rejected contracts.RejectedOutput) contracts.RejectedOutput {
|
||||
rejected.Message = string(redactSecretBytes([]byte(rejected.Message)))
|
||||
rejected.DiagnosticArtifactPath = string(redactSecretBytes([]byte(rejected.DiagnosticArtifactPath)))
|
||||
return rejected
|
||||
}
|
||||
|
||||
func debugRejectedOutputPtr(rejected *contracts.RejectedOutput) any {
|
||||
if rejected == nil {
|
||||
return nil
|
||||
}
|
||||
out := debugRejectedOutputEnvelope(*rejected)
|
||||
return out
|
||||
}
|
||||
|
||||
func debugRejectedOutputEnvelopes(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
|
||||
if len(rejected) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]contracts.RejectedOutput, 0, len(rejected))
|
||||
for _, item := range rejected {
|
||||
out = append(out, debugRejectedOutputEnvelope(item))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugContentDigest(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
var secretPatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)bearer\s+[a-z0-9._~+/=-]{8,}`),
|
||||
regexp.MustCompile(`(?i)sk-[a-z0-9_-]{8,}`),
|
||||
}
|
||||
|
||||
func redactSecretBytes(content []byte) []byte {
|
||||
if len(content) == 0 || !utf8.Valid(content) {
|
||||
return append([]byte(nil), content...)
|
||||
}
|
||||
text := string(content)
|
||||
for _, pattern := range secretPatterns {
|
||||
text = pattern.ReplaceAllString(text, "[REDACTED]")
|
||||
}
|
||||
return []byte(text)
|
||||
}
|
||||
|
||||
func redactSensitiveMap(values map[string]any) map[string]any {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(values))
|
||||
for key, value := range values {
|
||||
if sensitiveKey(key) {
|
||||
out[key] = "[REDACTED]"
|
||||
continue
|
||||
}
|
||||
out[key] = redactSensitiveValue(value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func redactSensitiveValue(value any) any {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return string(redactSecretBytes([]byte(typed)))
|
||||
case map[string]any:
|
||||
return redactSensitiveMap(typed)
|
||||
case map[string]string:
|
||||
out := make(map[string]string, len(typed))
|
||||
for key, value := range typed {
|
||||
if sensitiveKey(key) {
|
||||
out[key] = "[REDACTED]"
|
||||
} else {
|
||||
out[key] = string(redactSecretBytes([]byte(value)))
|
||||
}
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func sensitiveKey(key string) bool {
|
||||
key = strings.ToLower(key)
|
||||
return strings.Contains(key, "api_key") ||
|
||||
strings.Contains(key, "apikey") ||
|
||||
strings.Contains(key, "authorization") ||
|
||||
strings.Contains(key, "bearer") ||
|
||||
strings.Contains(key, "password") ||
|
||||
strings.Contains(key, "secret") ||
|
||||
strings.Contains(key, "token")
|
||||
}
|
||||
@@ -92,12 +92,13 @@ func defaultModuleCatalog(t *testing.T) pipeline.ModuleCatalog {
|
||||
}
|
||||
|
||||
return pipeline.ModuleCatalog{
|
||||
Inputs: inputs,
|
||||
Chunkers: chunkers,
|
||||
Extractors: extractors,
|
||||
Mergers: mergers,
|
||||
Normalizers: normalizers,
|
||||
Outputs: outputs,
|
||||
Inputs: inputs,
|
||||
Chunkers: chunkers,
|
||||
Extractors: extractors,
|
||||
Mergers: mergers,
|
||||
Normalizers: normalizers,
|
||||
ValidatorChains: pipeline.NewValidatorChainRegistry(),
|
||||
Outputs: outputs,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -113,11 +114,7 @@ type defaultExtractor struct{}
|
||||
|
||||
func (defaultExtractor) Key() string { return "extract" }
|
||||
|
||||
func (defaultExtractor) ArtifactType() string { return "record" }
|
||||
|
||||
func (defaultExtractor) SchemaVersion() string { return "v1" }
|
||||
|
||||
func (defaultExtractor) Validators() []contracts.Validator { return nil }
|
||||
func (defaultExtractor) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (defaultExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
return contracts.ExtractionResult{}, nil
|
||||
|
||||
@@ -49,6 +49,20 @@ func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
|
||||
Stage: StageExtract,
|
||||
Provides: []string{" generic-artifact ", "source-citations", "generic-artifact", ""},
|
||||
Requires: []string{" source-document ", "source-document", ""},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{
|
||||
Name: " glossary ",
|
||||
Description: " Supporting terms ",
|
||||
AcceptedMediaTypes: []string{" text/plain ", "text/markdown", "text/plain", ""},
|
||||
MaxBytes: 1024,
|
||||
},
|
||||
{
|
||||
Name: " roster ",
|
||||
Description: " Characters ",
|
||||
Required: true,
|
||||
Multiple: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if err := registry.RegisterWithSpec(spec, fakeExtractorConstructor("generic-extractor")); err != nil {
|
||||
@@ -64,12 +78,28 @@ func TestExtractorRegistryRegisterWithSpecStoresMetadata(t *testing.T) {
|
||||
Stage: StageExtract,
|
||||
Provides: []string{"generic-artifact", "source-citations"},
|
||||
Requires: []string{"source-document"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{
|
||||
Name: "glossary",
|
||||
Description: "Supporting terms",
|
||||
AcceptedMediaTypes: []string{"text/markdown", "text/plain"},
|
||||
MaxBytes: 1024,
|
||||
},
|
||||
{
|
||||
Name: "roster",
|
||||
Description: "Characters",
|
||||
Required: true,
|
||||
Multiple: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("Spec() = %#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
got.Provides[0] = "changed"
|
||||
got.ReferenceSlots[0].Name = "changed"
|
||||
got.ReferenceSlots[0].AcceptedMediaTypes[0] = "changed"
|
||||
again, ok := registry.Spec("generic-extractor")
|
||||
if !ok {
|
||||
t.Fatal("Spec() after caller mutation ok = false, want true")
|
||||
@@ -109,6 +139,50 @@ func TestExtractorRegistryRegisterWithSpecRejectsWrongStage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistryRejectsInvalidReferenceSlots(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
slots []contracts.ReferenceSlot
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty name",
|
||||
slots: []contracts.ReferenceSlot{{Name: " "}},
|
||||
want: "name",
|
||||
},
|
||||
{
|
||||
name: "duplicate name after trim",
|
||||
slots: []contracts.ReferenceSlot{
|
||||
{Name: "roster"},
|
||||
{Name: " roster "},
|
||||
},
|
||||
want: "duplicated",
|
||||
},
|
||||
{
|
||||
name: "negative max bytes",
|
||||
slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
|
||||
want: "max_bytes",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
err := registry.RegisterWithSpec(ModuleSpec{
|
||||
Key: "generic-extractor",
|
||||
Stage: StageExtract,
|
||||
ReferenceSlots: test.slots,
|
||||
}, fakeExtractorConstructor("generic-extractor"))
|
||||
if err == nil {
|
||||
t.Fatal("RegisterWithSpec() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), test.want) {
|
||||
t.Fatalf("RegisterWithSpec() error = %q, want %q", err.Error(), test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractorRegistrySpecRejectsUnknownKey(t *testing.T) {
|
||||
registry := NewExtractorRegistry()
|
||||
|
||||
@@ -293,15 +367,7 @@ func (extractor registryFakeExtractor) Key() string {
|
||||
return extractor.key
|
||||
}
|
||||
|
||||
func (extractor registryFakeExtractor) ArtifactType() string {
|
||||
return "generic-artifact"
|
||||
}
|
||||
|
||||
func (extractor registryFakeExtractor) SchemaVersion() string {
|
||||
return "v1"
|
||||
}
|
||||
|
||||
func (extractor registryFakeExtractor) Validators() []contracts.Validator {
|
||||
func (extractor registryFakeExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -301,7 +301,7 @@ func (adapter fakeAdapter) Parse(ctx context.Context, req contracts.ParseRequest
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:abc123",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: "u1", Kind: "unit", Text: "Source unit."},
|
||||
{ID: 1, Kind: "unit", Text: "Source unit."},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type ModuleStage string
|
||||
@@ -19,10 +21,11 @@ const (
|
||||
)
|
||||
|
||||
type ModuleSpec struct {
|
||||
Key string
|
||||
Stage ModuleStage
|
||||
Provides []string
|
||||
Requires []string
|
||||
Key string
|
||||
Stage ModuleStage
|
||||
Provides []string
|
||||
Requires []string
|
||||
ReferenceSlots []contracts.ReferenceSlot
|
||||
}
|
||||
|
||||
func defaultModuleSpec(key string, stage ModuleStage) ModuleSpec {
|
||||
@@ -34,10 +37,11 @@ func defaultModuleSpec(key string, stage ModuleStage) ModuleSpec {
|
||||
|
||||
func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
|
||||
return ModuleSpec{
|
||||
Key: strings.TrimSpace(spec.Key),
|
||||
Stage: spec.Stage,
|
||||
Provides: normalizeCapabilities(spec.Provides),
|
||||
Requires: normalizeCapabilities(spec.Requires),
|
||||
Key: strings.TrimSpace(spec.Key),
|
||||
Stage: spec.Stage,
|
||||
Provides: normalizeCapabilities(spec.Provides),
|
||||
Requires: normalizeCapabilities(spec.Requires),
|
||||
ReferenceSlots: normalizeReferenceSlots(spec.ReferenceSlots),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,10 +72,11 @@ func normalizeCapabilities(values []string) []string {
|
||||
|
||||
func cloneModuleSpec(spec ModuleSpec) ModuleSpec {
|
||||
return ModuleSpec{
|
||||
Key: spec.Key,
|
||||
Stage: spec.Stage,
|
||||
Provides: append([]string(nil), spec.Provides...),
|
||||
Requires: append([]string(nil), spec.Requires...),
|
||||
Key: spec.Key,
|
||||
Stage: spec.Stage,
|
||||
Provides: append([]string(nil), spec.Provides...),
|
||||
Requires: append([]string(nil), spec.Requires...),
|
||||
ReferenceSlots: contracts.CloneReferenceSlots(spec.ReferenceSlots),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,9 +87,19 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
|
||||
if spec.Stage != expectedStage {
|
||||
return fmt.Errorf("%s %q must use %q stage, got %q", kind, spec.Key, expectedStage, spec.Stage)
|
||||
}
|
||||
if !referenceSlotStage(spec.Stage) && len(spec.ReferenceSlots) > 0 {
|
||||
return fmt.Errorf("%s %q must not declare reference slots", kind, spec.Key)
|
||||
}
|
||||
if err := validateReferenceSlots(spec.ReferenceSlots); err != nil {
|
||||
return fmt.Errorf("%s %q reference slots: %w", kind, spec.Key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func referenceSlotStage(stage ModuleStage) bool {
|
||||
return stage == StageChunk || stage == StageExtract || stage == StageMerge || stage == StageNormalize
|
||||
}
|
||||
|
||||
func sortedRegistryKeys[C any](constructors map[string]C) []string {
|
||||
if len(constructors) == 0 {
|
||||
return nil
|
||||
@@ -97,3 +112,62 @@ func sortedRegistryKeys[C any](constructors map[string]C) []string {
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func normalizeReferenceSlots(slots []contracts.ReferenceSlot) []contracts.ReferenceSlot {
|
||||
if len(slots) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
normalized := make([]contracts.ReferenceSlot, 0, len(slots))
|
||||
for _, slot := range slots {
|
||||
slot.Name = strings.TrimSpace(slot.Name)
|
||||
slot.Description = strings.TrimSpace(slot.Description)
|
||||
slot.AcceptedMediaTypes = normalizeStringSet(slot.AcceptedMediaTypes)
|
||||
normalized = append(normalized, slot)
|
||||
}
|
||||
sort.SliceStable(normalized, func(i, j int) bool {
|
||||
return normalized[i].Name < normalized[j].Name
|
||||
})
|
||||
return normalized
|
||||
}
|
||||
|
||||
func normalizeStringSet(values []string) []string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
normalized := strings.TrimSpace(value)
|
||||
if normalized == "" {
|
||||
continue
|
||||
}
|
||||
seen[normalized] = struct{}{}
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
out := make([]string, 0, len(seen))
|
||||
for value := range seen {
|
||||
out = append(out, value)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func validateReferenceSlots(slots []contracts.ReferenceSlot) error {
|
||||
seen := make(map[string]struct{}, len(slots))
|
||||
for i, slot := range slots {
|
||||
if slot.Name == "" {
|
||||
return fmt.Errorf("slot[%d].name must not be empty", i)
|
||||
}
|
||||
if _, ok := seen[slot.Name]; ok {
|
||||
return fmt.Errorf("slot name %q is duplicated", slot.Name)
|
||||
}
|
||||
seen[slot.Name] = struct{}{}
|
||||
if slot.MaxBytes < 0 {
|
||||
return fmt.Errorf("slot %q max_bytes must not be negative", slot.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
124
internal/framework/pipeline/module_test.go
Normal file
124
internal/framework/pipeline/module_test.go
Normal file
@@ -0,0 +1,124 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestValidateModuleSpecAllowsReferenceSlotsForEligibleStages(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
kind string
|
||||
stage ModuleStage
|
||||
}{
|
||||
{name: "chunker", kind: "chunker", stage: StageChunk},
|
||||
{name: "extractor", kind: "extractor", stage: StageExtract},
|
||||
{name: "merger", kind: "merger", stage: StageMerge},
|
||||
{name: "normalizer", kind: "normalizer", stage: StageNormalize},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
spec := normalizeModuleSpec(ModuleSpec{
|
||||
Key: "module",
|
||||
Stage: test.stage,
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "roster", Description: "Character roster", MaxBytes: 1024},
|
||||
},
|
||||
})
|
||||
err := validateModuleSpec(test.kind, test.stage, spec)
|
||||
if err != nil {
|
||||
t.Fatalf("validateModuleSpec() error = %v, want nil", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateModuleSpecRejectsReferenceSlotsForIneligibleStages(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
kind string
|
||||
stage ModuleStage
|
||||
}{
|
||||
{name: "input", kind: "input adapter", stage: StageInput},
|
||||
{name: "validate", kind: "validator", stage: StageValidate},
|
||||
{name: "output", kind: "output encoder", stage: StageOutput},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
spec := normalizeModuleSpec(ModuleSpec{
|
||||
Key: "module",
|
||||
Stage: test.stage,
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "roster"},
|
||||
},
|
||||
})
|
||||
err := validateModuleSpec(test.kind, test.stage, spec)
|
||||
if err == nil {
|
||||
t.Fatal("validateModuleSpec() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "reference slots") {
|
||||
t.Fatalf("validateModuleSpec() error = %q, want reference slots context", err.Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateModuleSpecRejectsInvalidReferenceSlotsForEligibleStages(t *testing.T) {
|
||||
invalidSlots := []struct {
|
||||
name string
|
||||
slots []contracts.ReferenceSlot
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty name",
|
||||
slots: []contracts.ReferenceSlot{{Name: " "}},
|
||||
want: "name",
|
||||
},
|
||||
{
|
||||
name: "duplicate name after trim",
|
||||
slots: []contracts.ReferenceSlot{
|
||||
{Name: "roster"},
|
||||
{Name: " roster "},
|
||||
},
|
||||
want: "duplicated",
|
||||
},
|
||||
{
|
||||
name: "negative max bytes",
|
||||
slots: []contracts.ReferenceSlot{{Name: "roster", MaxBytes: -1}},
|
||||
want: "max_bytes",
|
||||
},
|
||||
}
|
||||
eligibleStages := []struct {
|
||||
name string
|
||||
kind string
|
||||
stage ModuleStage
|
||||
}{
|
||||
{name: "chunk", kind: "chunker", stage: StageChunk},
|
||||
{name: "extract", kind: "extractor", stage: StageExtract},
|
||||
{name: "merge", kind: "merger", stage: StageMerge},
|
||||
{name: "normalize", kind: "normalizer", stage: StageNormalize},
|
||||
}
|
||||
|
||||
for _, stage := range eligibleStages {
|
||||
for _, invalid := range invalidSlots {
|
||||
t.Run(stage.name+"/"+invalid.name, func(t *testing.T) {
|
||||
spec := normalizeModuleSpec(ModuleSpec{
|
||||
Key: "module",
|
||||
Stage: stage.stage,
|
||||
ReferenceSlots: invalid.slots,
|
||||
})
|
||||
err := validateModuleSpec(stage.kind, stage.stage, spec)
|
||||
if err == nil {
|
||||
t.Fatal("validateModuleSpec() error = nil, want error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), invalid.want) {
|
||||
t.Fatalf("validateModuleSpec() error = %q, want %q", err.Error(), invalid.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -18,55 +20,130 @@ const (
|
||||
)
|
||||
|
||||
type ModuleBinding struct {
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Retries int `json:"retries,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
References map[string]string `json:"references,omitempty"`
|
||||
Validators ValidatorOverride `json:"validators,omitempty"`
|
||||
}
|
||||
|
||||
type ArtifactLaneProfile struct {
|
||||
Extract ModuleBinding `json:"extract"`
|
||||
Merge ModuleBinding `json:"merge,omitempty"`
|
||||
Normalize ModuleBinding `json:"normalize,omitempty"`
|
||||
type ValidatorOverride struct {
|
||||
Set bool `json:"set,omitempty"`
|
||||
Validators []ModuleBinding `json:"validators,omitempty"`
|
||||
}
|
||||
|
||||
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
||||
type moduleBindingJSON struct {
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Retries int `json:"retries,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
References map[string]string `json:"references,omitempty"`
|
||||
Validators *[]ModuleBinding `json:"validators,omitempty"`
|
||||
}
|
||||
out := moduleBindingJSON{
|
||||
Module: binding.Module,
|
||||
LLMProfile: binding.LLMProfile,
|
||||
Retries: binding.Retries,
|
||||
Options: binding.Options,
|
||||
References: binding.References,
|
||||
}
|
||||
if binding.Validators.Set {
|
||||
validators := cloneModuleBindings(binding.Validators.Validators)
|
||||
out.Validators = &validators
|
||||
}
|
||||
return json.Marshal(out)
|
||||
}
|
||||
|
||||
type ArtifactLaneProfile struct {
|
||||
Extract ModuleBinding `json:"extract"`
|
||||
Merge ModuleBinding `json:"merge,omitempty"`
|
||||
Normalize ModuleBinding `json:"normalize,omitempty"`
|
||||
Validators []ModuleBinding `json:"validators,omitempty"`
|
||||
References map[string]string `json:"references,omitempty"`
|
||||
}
|
||||
|
||||
type PipelineProfile struct {
|
||||
ID string `json:"id"`
|
||||
Input ModuleBinding `json:"input"`
|
||||
Chunk ModuleBinding `json:"chunk,omitempty"`
|
||||
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
|
||||
Output ModuleBinding `json:"output,omitempty"`
|
||||
ID string `json:"id"`
|
||||
Input ModuleBinding `json:"input"`
|
||||
Chunk ModuleBinding `json:"chunk,omitempty"`
|
||||
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
|
||||
Output ModuleBinding `json:"output,omitempty"`
|
||||
References map[string]string `json:"references,omitempty"`
|
||||
}
|
||||
|
||||
type ResolveOptions struct {
|
||||
Only []string
|
||||
Only []string
|
||||
ReferenceOverrides []ReferenceBinding
|
||||
ReferenceUnbinds []ReferenceUnbind
|
||||
}
|
||||
|
||||
type ReferenceBinding struct {
|
||||
Stage ModuleStage `json:"stage,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
SlotName string `json:"slot_name"`
|
||||
Source string `json:"source"`
|
||||
BindingSource string `json:"binding_source,omitempty"`
|
||||
}
|
||||
|
||||
type ReferenceUnbind struct {
|
||||
Stage ModuleStage `json:"stage,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
SlotName string `json:"slot_name"`
|
||||
}
|
||||
|
||||
type ResolvedReferenceTarget struct {
|
||||
Stage ModuleStage `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
Module string `json:"module"`
|
||||
Bindings []ReferenceBinding `json:"bindings,omitempty"`
|
||||
ReferenceSet contracts.ReferenceSet `json:"-"`
|
||||
}
|
||||
|
||||
type ResolvedArtifactLane struct {
|
||||
ID string
|
||||
Extract ModuleBinding
|
||||
Merge ModuleBinding
|
||||
Normalize ModuleBinding
|
||||
Validators []ModuleBinding
|
||||
ID string
|
||||
Extract ModuleBinding
|
||||
Merge ModuleBinding
|
||||
Normalize ModuleBinding
|
||||
Validators []ModuleBinding
|
||||
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
|
||||
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
|
||||
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
|
||||
}
|
||||
|
||||
type ResolvedValidatorChain struct {
|
||||
Stage ModuleStage `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
Validators []ResolvedValidator `json:"validators"`
|
||||
}
|
||||
|
||||
type ResolvedValidator struct {
|
||||
Binding ModuleBinding `json:"binding"`
|
||||
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
||||
}
|
||||
|
||||
type ResolvedPipeline struct {
|
||||
ID string
|
||||
Digest string
|
||||
Input ModuleBinding
|
||||
Chunk ModuleBinding
|
||||
ArtifactLanes []ResolvedArtifactLane
|
||||
Output ModuleBinding
|
||||
ID string
|
||||
Digest string
|
||||
Input ModuleBinding
|
||||
Chunk ModuleBinding
|
||||
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
||||
ArtifactLanes []ResolvedArtifactLane
|
||||
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
|
||||
Output ModuleBinding
|
||||
}
|
||||
|
||||
type ModuleCatalog struct {
|
||||
Inputs *InputAdapterRegistry
|
||||
Chunkers *ChunkerRegistry
|
||||
Extractors *ExtractorRegistry
|
||||
Mergers *MergerRegistry
|
||||
Normalizers *NormalizerRegistry
|
||||
Validators *ValidatorRegistry
|
||||
Outputs *OutputEncoderRegistry
|
||||
Inputs *InputAdapterRegistry
|
||||
Chunkers *ChunkerRegistry
|
||||
Extractors *ExtractorRegistry
|
||||
Mergers *MergerRegistry
|
||||
Normalizers *NormalizerRegistry
|
||||
Validators *ValidatorRegistry
|
||||
ValidatorChains *ValidatorChainRegistry
|
||||
Outputs *OutputEncoderRegistry
|
||||
}
|
||||
|
||||
func Binding(module string) ModuleBinding {
|
||||
@@ -115,22 +192,44 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
||||
if len(selectedLaneIDs) == 0 {
|
||||
return ResolvedPipeline{}, fmt.Errorf("pipeline %q must select at least one artifact lane", pipelineID)
|
||||
}
|
||||
|
||||
resolved := ResolvedPipeline{
|
||||
ID: pipelineID,
|
||||
Input: input,
|
||||
Chunk: chunk,
|
||||
Output: resolveBinding(profile.Output, DefaultOutputModule),
|
||||
if err := validatePipelineReferenceDefaults(pipelineID, profile.References, chunkSpec, lanesByID, catalog); err != nil {
|
||||
return ResolvedPipeline{}, err
|
||||
}
|
||||
|
||||
chunkReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||
PipelineID: pipelineID,
|
||||
Stage: StageChunk,
|
||||
Module: chunk.Module,
|
||||
Slots: chunkSpec.ReferenceSlots,
|
||||
PipelineReferences: profile.References,
|
||||
LocalReferences: chunk.References,
|
||||
Options: options,
|
||||
})
|
||||
if err != nil {
|
||||
return ResolvedPipeline{}, err
|
||||
}
|
||||
resolved := ResolvedPipeline{
|
||||
ID: pipelineID,
|
||||
Input: input,
|
||||
Chunk: chunk,
|
||||
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
||||
Output: resolveBinding(profile.Output, DefaultOutputModule),
|
||||
}
|
||||
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, catalog)
|
||||
if err != nil {
|
||||
return ResolvedPipeline{}, err
|
||||
}
|
||||
resolved.ValidatorChains = append(resolved.ValidatorChains, chunkValidatorChain)
|
||||
outputCapabilities := capabilities.clone()
|
||||
|
||||
for _, laneID := range selectedLaneIDs {
|
||||
laneProfile := lanesByID[laneID]
|
||||
lane, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, capabilities, catalog)
|
||||
lane, validatorChains, laneCapabilities, err := resolveArtifactLane(pipelineID, laneID, laneProfile, profile.References, options, capabilities, catalog)
|
||||
if err != nil {
|
||||
return ResolvedPipeline{}, err
|
||||
}
|
||||
resolved.ArtifactLanes = append(resolved.ArtifactLanes, lane)
|
||||
resolved.ValidatorChains = append(resolved.ValidatorChains, validatorChains...)
|
||||
outputCapabilities.addSet(laneCapabilities)
|
||||
}
|
||||
|
||||
@@ -150,7 +249,15 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile, inherited capabilitySet, catalog ModuleCatalog) (ResolvedArtifactLane, capabilitySet, error) {
|
||||
func resolveArtifactLane(
|
||||
pipelineID string,
|
||||
laneID string,
|
||||
profile ArtifactLaneProfile,
|
||||
pipelineReferences map[string]string,
|
||||
options ResolveOptions,
|
||||
inherited capabilitySet,
|
||||
catalog ModuleCatalog,
|
||||
) (ResolvedArtifactLane, []ResolvedValidatorChain, capabilitySet, error) {
|
||||
lane := ResolvedArtifactLane{
|
||||
ID: laneID,
|
||||
Extract: resolveBinding(profile.Extract, ""),
|
||||
@@ -159,50 +266,488 @@ func resolveArtifactLane(pipelineID, laneID string, profile ArtifactLaneProfile,
|
||||
Validators: resolveBindings(profile.Validators, ""),
|
||||
}
|
||||
if lane.Extract.Module == "" {
|
||||
return ResolvedArtifactLane{}, nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
|
||||
return ResolvedArtifactLane{}, nil, nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
|
||||
}
|
||||
|
||||
capabilities := inherited.clone()
|
||||
|
||||
extractSpec, err := extractorSpec(catalog, lane.Extract.Module)
|
||||
if err != nil {
|
||||
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageExtract, lane.Extract.Module, err)
|
||||
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageExtract, lane.Extract.Module, err)
|
||||
}
|
||||
if missing, ok := capabilities.missing(extractSpec.Requires); ok {
|
||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
|
||||
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageExtract, lane.Extract.Module, missing)
|
||||
}
|
||||
extractReferences := mergeReferenceMaps(profile.References, lane.Extract.References)
|
||||
references, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||
PipelineID: pipelineID,
|
||||
LaneID: laneID,
|
||||
Stage: StageExtract,
|
||||
Module: lane.Extract.Module,
|
||||
Slots: extractSpec.ReferenceSlots,
|
||||
PipelineReferences: pipelineReferences,
|
||||
LocalReferences: extractReferences,
|
||||
Options: options,
|
||||
})
|
||||
if err != nil {
|
||||
return ResolvedArtifactLane{}, nil, nil, err
|
||||
}
|
||||
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
|
||||
capabilities.add(extractSpec.Provides...)
|
||||
|
||||
mergeSpec, err := mergerSpec(catalog, lane.Merge.Module)
|
||||
if err != nil {
|
||||
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageMerge, lane.Merge.Module, err)
|
||||
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageMerge, lane.Merge.Module, err)
|
||||
}
|
||||
if missing, ok := capabilities.missing(mergeSpec.Requires); ok {
|
||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing)
|
||||
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageMerge, lane.Merge.Module, missing)
|
||||
}
|
||||
mergeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||
PipelineID: pipelineID,
|
||||
LaneID: laneID,
|
||||
Stage: StageMerge,
|
||||
Module: lane.Merge.Module,
|
||||
Slots: mergeSpec.ReferenceSlots,
|
||||
PipelineReferences: pipelineReferences,
|
||||
LocalReferences: lane.Merge.References,
|
||||
Options: options,
|
||||
})
|
||||
if err != nil {
|
||||
return ResolvedArtifactLane{}, nil, nil, err
|
||||
}
|
||||
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
|
||||
capabilities.add(mergeSpec.Provides...)
|
||||
|
||||
normalizeSpec, err := normalizerSpec(catalog, lane.Normalize.Module)
|
||||
if err != nil {
|
||||
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, err)
|
||||
return ResolvedArtifactLane{}, nil, nil, moduleLookupError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, err)
|
||||
}
|
||||
if missing, ok := capabilities.missing(normalizeSpec.Requires); ok {
|
||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
|
||||
return ResolvedArtifactLane{}, nil, nil, capabilityError(pipelineID, laneID, StageNormalize, lane.Normalize.Module, missing)
|
||||
}
|
||||
normalizeReferences, err := resolveReferenceTargetBindings(referenceResolutionTarget{
|
||||
PipelineID: pipelineID,
|
||||
LaneID: laneID,
|
||||
Stage: StageNormalize,
|
||||
Module: lane.Normalize.Module,
|
||||
Slots: normalizeSpec.ReferenceSlots,
|
||||
PipelineReferences: pipelineReferences,
|
||||
LocalReferences: lane.Normalize.References,
|
||||
Options: options,
|
||||
})
|
||||
if err != nil {
|
||||
return ResolvedArtifactLane{}, nil, nil, err
|
||||
}
|
||||
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
|
||||
capabilities.add(normalizeSpec.Provides...)
|
||||
|
||||
for _, validator := range lane.Validators {
|
||||
validatorSpec, err := validatorSpec(catalog, validator.Module)
|
||||
if err != nil {
|
||||
return ResolvedArtifactLane{}, nil, moduleLookupError(pipelineID, laneID, StageValidate, validator.Module, err)
|
||||
}
|
||||
if missing, ok := capabilities.missing(validatorSpec.Requires); ok {
|
||||
return ResolvedArtifactLane{}, nil, capabilityError(pipelineID, laneID, StageValidate, validator.Module, missing)
|
||||
}
|
||||
capabilities.add(validatorSpec.Provides...)
|
||||
if len(lane.Validators) > 0 {
|
||||
return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID)
|
||||
}
|
||||
|
||||
return lane, capabilities, nil
|
||||
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, lane.Extract.Validators, catalog)
|
||||
if err != nil {
|
||||
return ResolvedArtifactLane{}, nil, nil, err
|
||||
}
|
||||
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, lane.Merge.Validators, catalog)
|
||||
if err != nil {
|
||||
return ResolvedArtifactLane{}, nil, nil, err
|
||||
}
|
||||
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, lane.Normalize.Validators, catalog)
|
||||
if err != nil {
|
||||
return ResolvedArtifactLane{}, nil, nil, err
|
||||
}
|
||||
validatorChains := []ResolvedValidatorChain{extractValidatorChain, mergeValidatorChain, normalizeValidatorChain}
|
||||
|
||||
return lane, validatorChains, capabilities, nil
|
||||
}
|
||||
|
||||
func configuredValidatorsError(pipelineID string, laneID string) error {
|
||||
return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", pipelineID, laneID)
|
||||
}
|
||||
|
||||
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, override ValidatorOverride, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
|
||||
chain := ResolvedValidatorChain{
|
||||
Stage: stage,
|
||||
LaneID: strings.TrimSpace(laneID),
|
||||
ModuleKey: strings.TrimSpace(module),
|
||||
}
|
||||
if chain.ModuleKey == "" {
|
||||
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain %q module key must not be empty", pipelineID, stage)
|
||||
}
|
||||
switch stage {
|
||||
case StageChunk, StageExtract, StageMerge, StageNormalize:
|
||||
default:
|
||||
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage)
|
||||
}
|
||||
|
||||
var bindings []ModuleBinding
|
||||
if override.Set {
|
||||
bindings = cloneModuleBindings(override.Validators)
|
||||
} else if catalog.ValidatorChains != nil {
|
||||
bindings = catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
|
||||
}
|
||||
if len(bindings) == 0 {
|
||||
return chain, nil
|
||||
}
|
||||
if catalog.Validators == nil {
|
||||
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator registry must not be nil for %s validator chain on module %q", pipelineID, stage, chain.ModuleKey)
|
||||
}
|
||||
chain.Validators = make([]ResolvedValidator, 0, len(bindings))
|
||||
for _, validator := range bindings {
|
||||
spec, ok := catalog.Validators.Spec(validator.Module)
|
||||
if !ok {
|
||||
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
|
||||
}
|
||||
if strings.TrimSpace(validator.LLMProfile) != "" && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q assigns llm_profile to deterministic validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
|
||||
}
|
||||
chain.Validators = append(chain.Validators, ResolvedValidator{
|
||||
Binding: cloneModuleBinding(validator),
|
||||
ExecutionClass: spec.ExecutionClass,
|
||||
})
|
||||
}
|
||||
return chain, nil
|
||||
}
|
||||
|
||||
func cloneResolvedValidatorChains(chains []ResolvedValidatorChain) []ResolvedValidatorChain {
|
||||
if len(chains) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]ResolvedValidatorChain, len(chains))
|
||||
for i, chain := range chains {
|
||||
out[i] = ResolvedValidatorChain{
|
||||
Stage: chain.Stage,
|
||||
LaneID: strings.TrimSpace(chain.LaneID),
|
||||
ModuleKey: strings.TrimSpace(chain.ModuleKey),
|
||||
Validators: cloneResolvedValidators(chain.Validators),
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneResolvedValidators(validators []ResolvedValidator) []ResolvedValidator {
|
||||
if len(validators) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]ResolvedValidator, len(validators))
|
||||
for i, validator := range validators {
|
||||
out[i] = ResolvedValidator{
|
||||
Binding: cloneModuleBinding(validator.Binding),
|
||||
ExecutionClass: validator.ExecutionClass,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func referenceTarget(stage ModuleStage, laneID string, module string, bindings []ReferenceBinding) ResolvedReferenceTarget {
|
||||
return ResolvedReferenceTarget{
|
||||
Stage: stage,
|
||||
LaneID: strings.TrimSpace(laneID),
|
||||
Module: strings.TrimSpace(module),
|
||||
Bindings: append([]ReferenceBinding(nil), bindings...),
|
||||
}
|
||||
}
|
||||
|
||||
func mergeReferenceMaps(base map[string]string, override map[string]string) map[string]string {
|
||||
if len(base) == 0 && len(override) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(base)+len(override))
|
||||
for key, value := range base {
|
||||
out[key] = value
|
||||
}
|
||||
for key, value := range override {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func validatePipelineReferenceDefaults(
|
||||
pipelineID string,
|
||||
pipelineReferences map[string]string,
|
||||
chunkSpec ModuleSpec,
|
||||
lanesByID map[string]ArtifactLaneProfile,
|
||||
catalog ModuleCatalog,
|
||||
) error {
|
||||
normalizedPipelineReferences, err := normalizedReferenceMap(pipelineReferences, fmt.Sprintf("pipeline %q reference slot", pipelineID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(normalizedPipelineReferences) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
declaredByAnyTarget := make(map[string]struct{}, len(normalizedPipelineReferences))
|
||||
for _, slot := range chunkSpec.ReferenceSlots {
|
||||
declaredByAnyTarget[slot.Name] = struct{}{}
|
||||
}
|
||||
for _, laneID := range sortedArtifactLaneProfileKeys(lanesByID) {
|
||||
laneProfile := lanesByID[laneID]
|
||||
extract := resolveBinding(laneProfile.Extract, "")
|
||||
if extract.Module == "" {
|
||||
return fmt.Errorf("pipeline %q lane %q extract module must not be empty", pipelineID, laneID)
|
||||
}
|
||||
extractSpec, err := extractorSpec(catalog, extract.Module)
|
||||
if err != nil {
|
||||
return moduleLookupError(pipelineID, laneID, StageExtract, extract.Module, err)
|
||||
}
|
||||
for _, slot := range extractSpec.ReferenceSlots {
|
||||
declaredByAnyTarget[slot.Name] = struct{}{}
|
||||
}
|
||||
|
||||
merge := resolveBinding(laneProfile.Merge, DefaultMergeModule)
|
||||
mergeSpec, err := mergerSpec(catalog, merge.Module)
|
||||
if err != nil {
|
||||
return moduleLookupError(pipelineID, laneID, StageMerge, merge.Module, err)
|
||||
}
|
||||
for _, slot := range mergeSpec.ReferenceSlots {
|
||||
declaredByAnyTarget[slot.Name] = struct{}{}
|
||||
}
|
||||
|
||||
normalize := resolveBinding(laneProfile.Normalize, DefaultNormalizeModule)
|
||||
normalizeSpec, err := normalizerSpec(catalog, normalize.Module)
|
||||
if err != nil {
|
||||
return moduleLookupError(pipelineID, laneID, StageNormalize, normalize.Module, err)
|
||||
}
|
||||
for _, slot := range normalizeSpec.ReferenceSlots {
|
||||
declaredByAnyTarget[slot.Name] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
|
||||
if _, ok := declaredByAnyTarget[slotName]; !ok {
|
||||
return fmt.Errorf("pipeline %q reference slot %q is not declared by any eligible reference target", pipelineID, slotName)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type referenceResolutionTarget struct {
|
||||
PipelineID string
|
||||
LaneID string
|
||||
Stage ModuleStage
|
||||
Module string
|
||||
Slots []contracts.ReferenceSlot
|
||||
PipelineReferences map[string]string
|
||||
LocalReferences map[string]string
|
||||
Options ResolveOptions
|
||||
}
|
||||
|
||||
func resolveReferenceTargetBindings(target referenceResolutionTarget) ([]ReferenceBinding, error) {
|
||||
slotByName := make(map[string]contracts.ReferenceSlot, len(target.Slots))
|
||||
for _, slot := range target.Slots {
|
||||
slotByName[slot.Name] = slot
|
||||
}
|
||||
|
||||
bindings := make(map[string]ReferenceBinding)
|
||||
addBinding := func(slotName, source, bindingSource string) error {
|
||||
slotName = strings.TrimSpace(slotName)
|
||||
source = strings.TrimSpace(source)
|
||||
if slotName == "" {
|
||||
return fmt.Errorf("%s reference slot name must not be empty", referenceTargetErrorContext(target))
|
||||
}
|
||||
if source == "" {
|
||||
return fmt.Errorf("%s reference slot %q source must not be empty", referenceTargetErrorContext(target), slotName)
|
||||
}
|
||||
if _, ok := slotByName[slotName]; !ok {
|
||||
return fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetErrorContext(target), slotName, target.Stage, target.Module)
|
||||
}
|
||||
bindings[slotName] = ReferenceBinding{
|
||||
LaneID: target.LaneID,
|
||||
SlotName: slotName,
|
||||
Source: source,
|
||||
BindingSource: bindingSource,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
normalizedPipelineReferences, err := normalizedReferenceMap(target.PipelineReferences, fmt.Sprintf("pipeline %q reference slot", target.PipelineID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, slotName := range sortedStringMapKeys(normalizedPipelineReferences) {
|
||||
if _, ok := slotByName[slotName]; !ok {
|
||||
continue
|
||||
}
|
||||
if err := addBinding(slotName, normalizedPipelineReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
normalizedLocalReferences, err := normalizedReferenceMap(target.LocalReferences, referenceTargetSlotLabel(target))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, slotName := range sortedStringMapKeys(normalizedLocalReferences) {
|
||||
if err := addBinding(slotName, normalizedLocalReferences[slotName], contracts.ReferenceBindingSourceConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
for _, override := range target.Options.ReferenceOverrides {
|
||||
match, err := referenceOverrideMatchesTarget(target, override)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !match {
|
||||
continue
|
||||
}
|
||||
source := override.BindingSource
|
||||
if strings.TrimSpace(source) == "" {
|
||||
source = contracts.ReferenceBindingSourceCLI
|
||||
}
|
||||
if err := addBinding(override.SlotName, override.Source, strings.TrimSpace(source)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
for _, unbind := range target.Options.ReferenceUnbinds {
|
||||
match, err := referenceUnbindMatchesTarget(target, unbind)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !match {
|
||||
continue
|
||||
}
|
||||
slotName := strings.TrimSpace(unbind.SlotName)
|
||||
if slotName == "" {
|
||||
return nil, fmt.Errorf("%s reference unbind slot name must not be empty", referenceTargetErrorContext(target))
|
||||
}
|
||||
if _, ok := slotByName[slotName]; !ok {
|
||||
return nil, fmt.Errorf("%s reference slot %q is not declared", referenceTargetErrorContext(target), slotName)
|
||||
}
|
||||
delete(bindings, slotName)
|
||||
}
|
||||
|
||||
for _, slot := range target.Slots {
|
||||
if slot.Required {
|
||||
if _, ok := bindings[slot.Name]; !ok {
|
||||
return nil, fmt.Errorf("%s required reference slot %q is not bound", referenceTargetErrorContext(target), slot.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return sortedReferenceBindings(bindings), nil
|
||||
}
|
||||
|
||||
func referenceOverrideMatchesTarget(target referenceResolutionTarget, override ReferenceBinding) (bool, error) {
|
||||
stage, laneID, err := normalizeReferenceOptionTarget(target.PipelineID, "override", override.Stage, override.LaneID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return stage == target.Stage && laneID == target.LaneID, nil
|
||||
}
|
||||
|
||||
func referenceUnbindMatchesTarget(target referenceResolutionTarget, unbind ReferenceUnbind) (bool, error) {
|
||||
stage, laneID, err := normalizeReferenceOptionTarget(target.PipelineID, "unbind", unbind.Stage, unbind.LaneID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return stage == target.Stage && laneID == target.LaneID, nil
|
||||
}
|
||||
|
||||
func normalizeReferenceOptionTarget(pipelineID string, operation string, stage ModuleStage, laneID string) (ModuleStage, string, error) {
|
||||
stage = ModuleStage(strings.TrimSpace(string(stage)))
|
||||
if stage == "" {
|
||||
stage = StageExtract
|
||||
}
|
||||
laneID = strings.TrimSpace(laneID)
|
||||
|
||||
switch stage {
|
||||
case StageChunk:
|
||||
if laneID != "" {
|
||||
return "", "", fmt.Errorf("pipeline %q reference %s for chunk must not include a lane id", pipelineID, operation)
|
||||
}
|
||||
case StageExtract, StageMerge, StageNormalize:
|
||||
if laneID == "" {
|
||||
return "", "", fmt.Errorf("pipeline %q reference %s lane id must not be empty", pipelineID, operation)
|
||||
}
|
||||
default:
|
||||
return "", "", fmt.Errorf("pipeline %q reference %s stage %q is not supported", pipelineID, operation, stage)
|
||||
}
|
||||
return stage, laneID, nil
|
||||
}
|
||||
|
||||
func sortedReferenceBindings(bindings map[string]ReferenceBinding) []ReferenceBinding {
|
||||
keys := sortedReferenceBindingKeys(bindings)
|
||||
resolved := make([]ReferenceBinding, 0, len(keys))
|
||||
for _, slotName := range keys {
|
||||
resolved = append(resolved, bindings[slotName])
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
func referenceTargetErrorContext(target referenceResolutionTarget) string {
|
||||
if target.LaneID != "" {
|
||||
return fmt.Sprintf("pipeline %q lane %q %s module %q", target.PipelineID, target.LaneID, target.Stage, target.Module)
|
||||
}
|
||||
return fmt.Sprintf("pipeline %q %s module %q", target.PipelineID, target.Stage, target.Module)
|
||||
}
|
||||
|
||||
func referenceTargetSlotLabel(target referenceResolutionTarget) string {
|
||||
if target.LaneID != "" {
|
||||
return fmt.Sprintf("pipeline %q lane %q %s reference slot", target.PipelineID, target.LaneID, target.Stage)
|
||||
}
|
||||
return fmt.Sprintf("pipeline %q %s reference slot", target.PipelineID, target.Stage)
|
||||
}
|
||||
|
||||
func normalizedReferenceMap(values map[string]string, keyName string) (map[string]string, error) {
|
||||
if len(values) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
out := make(map[string]string, len(values))
|
||||
for rawSlotName, rawSource := range values {
|
||||
slotName := strings.TrimSpace(rawSlotName)
|
||||
if slotName == "" {
|
||||
return nil, fmt.Errorf("%s must not be empty", keyName)
|
||||
}
|
||||
if _, ok := out[slotName]; ok {
|
||||
return nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, slotName)
|
||||
}
|
||||
source := strings.TrimSpace(rawSource)
|
||||
if source == "" {
|
||||
return nil, fmt.Errorf("%s %q source must not be empty", keyName, slotName)
|
||||
}
|
||||
out[slotName] = source
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func sortedArtifactLaneProfileKeys(values map[string]ArtifactLaneProfile) []string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(values))
|
||||
for key := range values {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func sortedStringMapKeys(values map[string]string) []string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(values))
|
||||
for key := range values {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func sortedReferenceBindingKeys(values map[string]ReferenceBinding) []string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(values))
|
||||
for key := range values {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
|
||||
@@ -211,13 +756,13 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
|
||||
module = defaultModule
|
||||
}
|
||||
llmProfile := strings.TrimSpace(binding.LLMProfile)
|
||||
if llmProfile == "" {
|
||||
llmProfile = DefaultLLMProfile
|
||||
}
|
||||
return ModuleBinding{
|
||||
Module: module,
|
||||
LLMProfile: llmProfile,
|
||||
Retries: binding.Retries,
|
||||
Options: cloneOptions(binding.Options),
|
||||
References: normalizeReferenceMap(binding.References),
|
||||
Validators: cloneValidatorOverride(binding.Validators),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -246,6 +791,25 @@ func cloneOptions(options map[string]any) map[string]any {
|
||||
return copied
|
||||
}
|
||||
|
||||
func normalizeReferenceMap(values map[string]string) map[string]string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(values))
|
||||
keys := make([]string, 0, len(values))
|
||||
rawByNormalized := make(map[string]string, len(values))
|
||||
for rawKey := range values {
|
||||
key := strings.TrimSpace(rawKey)
|
||||
rawByNormalized[key] = rawKey
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
out[key] = strings.TrimSpace(values[rawByNormalized[key]])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneProfile, options ResolveOptions) (map[string]ArtifactLaneProfile, []string, error) {
|
||||
lanesByID := make(map[string]ArtifactLaneProfile, len(artifacts))
|
||||
for rawLaneID, lane := range artifacts {
|
||||
@@ -290,17 +854,19 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
|
||||
|
||||
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
|
||||
withoutDigest := struct {
|
||||
ID string
|
||||
Input ModuleBinding
|
||||
Chunk ModuleBinding
|
||||
ArtifactLanes []ResolvedArtifactLane
|
||||
Output ModuleBinding
|
||||
ID string
|
||||
Input ModuleBinding
|
||||
Chunk ModuleBinding
|
||||
ChunkReferences ResolvedReferenceTarget
|
||||
ArtifactLanes []ResolvedArtifactLane
|
||||
Output ModuleBinding
|
||||
}{
|
||||
ID: resolved.ID,
|
||||
Input: resolved.Input,
|
||||
Chunk: resolved.Chunk,
|
||||
ArtifactLanes: resolved.ArtifactLanes,
|
||||
Output: resolved.Output,
|
||||
ID: resolved.ID,
|
||||
Input: resolved.Input,
|
||||
Chunk: resolved.Chunk,
|
||||
ChunkReferences: resolved.ChunkReferences,
|
||||
ArtifactLanes: resolved.ArtifactLanes,
|
||||
Output: resolved.Output,
|
||||
}
|
||||
encoded, err := json.Marshal(withoutDigest)
|
||||
if err != nil {
|
||||
@@ -331,10 +897,6 @@ func normalizerSpec(catalog ModuleCatalog, key string) (ModuleSpec, error) {
|
||||
return registrySpec(catalog.Normalizers, key)
|
||||
}
|
||||
|
||||
func validatorSpec(catalog ModuleCatalog, key string) (ModuleSpec, error) {
|
||||
return registrySpec(catalog.Validators, key)
|
||||
}
|
||||
|
||||
func outputSpec(catalog ModuleCatalog, key string) (ModuleSpec, error) {
|
||||
return registrySpec(catalog.Outputs, key)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package pipeline
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -18,8 +19,7 @@ func TestResolvePipelineWithExplicitModules(t *testing.T) {
|
||||
ModuleSpec{Key: "record-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}},
|
||||
ModuleSpec{Key: "dedupe", Stage: StageMerge, Requires: []string{"candidate"}, Provides: []string{"merged"}},
|
||||
ModuleSpec{Key: "canonical", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}},
|
||||
ModuleSpec{Key: "schema-check", Stage: StageValidate, Requires: []string{"normalized"}, Provides: []string{"validated"}},
|
||||
ModuleSpec{Key: "ndjson", Stage: StageOutput, Requires: []string{"validated"}, Provides: []string{"encoded"}},
|
||||
ModuleSpec{Key: "ndjson", Stage: StageOutput, Requires: []string{"normalized"}, Provides: []string{"encoded"}},
|
||||
)
|
||||
|
||||
resolved, err := ResolvePipeline(PipelineProfile{
|
||||
@@ -30,10 +30,9 @@ func TestResolvePipelineWithExplicitModules(t *testing.T) {
|
||||
}},
|
||||
Artifacts: map[string]ArtifactLaneProfile{
|
||||
" records ": {
|
||||
Extract: ModuleBinding{Module: " record-extractor ", LLMProfile: " careful "},
|
||||
Merge: Binding(" dedupe "),
|
||||
Normalize: Binding(" canonical "),
|
||||
Validators: []ModuleBinding{Binding(" schema-check ")},
|
||||
Extract: ModuleBinding{Module: " record-extractor ", LLMProfile: " careful "},
|
||||
Merge: Binding(" dedupe "),
|
||||
Normalize: Binding(" canonical "),
|
||||
},
|
||||
},
|
||||
Output: Binding(" ndjson "),
|
||||
@@ -48,8 +47,8 @@ func TestResolvePipelineWithExplicitModules(t *testing.T) {
|
||||
if !reflect.DeepEqual(resolved.Input, ModuleBinding{Module: "text", LLMProfile: "fast"}) {
|
||||
t.Fatalf("Input = %#v, want trimmed explicit input", resolved.Input)
|
||||
}
|
||||
if resolved.Chunk.Module != "window" || resolved.Chunk.LLMProfile != DefaultLLMProfile {
|
||||
t.Fatalf("Chunk = %#v, want explicit module and default LLM profile", resolved.Chunk)
|
||||
if resolved.Chunk.Module != "window" || resolved.Chunk.LLMProfile != "" {
|
||||
t.Fatalf("Chunk = %#v, want explicit module and empty LLM profile", resolved.Chunk)
|
||||
}
|
||||
if resolved.Chunk.Options["size"] != 10 {
|
||||
t.Fatalf("Chunk.Options = %#v, want size option", resolved.Chunk.Options)
|
||||
@@ -67,8 +66,8 @@ func TestResolvePipelineWithExplicitModules(t *testing.T) {
|
||||
if lane.Merge.Module != "dedupe" || lane.Normalize.Module != "canonical" {
|
||||
t.Fatalf("lane merge/normalize = %#v/%#v, want explicit modules", lane.Merge, lane.Normalize)
|
||||
}
|
||||
if len(lane.Validators) != 1 || lane.Validators[0].Module != "schema-check" {
|
||||
t.Fatalf("lane.Validators = %#v, want schema-check", lane.Validators)
|
||||
if len(lane.Validators) != 0 {
|
||||
t.Fatalf("lane.Validators = %#v, want none", lane.Validators)
|
||||
}
|
||||
if resolved.Output.Module != "ndjson" {
|
||||
t.Fatalf("Output.Module = %q, want ndjson", resolved.Output.Module)
|
||||
@@ -90,24 +89,233 @@ func TestResolvePipelineAppliesDefaults(t *testing.T) {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if resolved.Input.LLMProfile != DefaultLLMProfile {
|
||||
t.Fatalf("Input.LLMProfile = %q, want %q", resolved.Input.LLMProfile, DefaultLLMProfile)
|
||||
if resolved.Input.LLMProfile != "" {
|
||||
t.Fatalf("Input.LLMProfile = %q, want empty", resolved.Input.LLMProfile)
|
||||
}
|
||||
if !reflect.DeepEqual(resolved.Chunk, ModuleBinding{Module: DefaultChunkModule, LLMProfile: DefaultLLMProfile}) {
|
||||
if !reflect.DeepEqual(resolved.Chunk, ModuleBinding{Module: DefaultChunkModule}) {
|
||||
t.Fatalf("Chunk = %#v, want default chunk binding", resolved.Chunk)
|
||||
}
|
||||
if !reflect.DeepEqual(resolved.Output, ModuleBinding{Module: DefaultOutputModule, LLMProfile: DefaultLLMProfile}) {
|
||||
if !reflect.DeepEqual(resolved.Output, ModuleBinding{Module: DefaultOutputModule}) {
|
||||
t.Fatalf("Output = %#v, want default output binding", resolved.Output)
|
||||
}
|
||||
lane := resolved.ArtifactLanes[0]
|
||||
if !reflect.DeepEqual(lane.Merge, ModuleBinding{Module: DefaultMergeModule, LLMProfile: DefaultLLMProfile}) {
|
||||
if !reflect.DeepEqual(lane.Merge, ModuleBinding{Module: DefaultMergeModule}) {
|
||||
t.Fatalf("Merge = %#v, want default merge binding", lane.Merge)
|
||||
}
|
||||
if !reflect.DeepEqual(lane.Normalize, ModuleBinding{Module: DefaultNormalizeModule, LLMProfile: DefaultLLMProfile}) {
|
||||
if !reflect.DeepEqual(lane.Normalize, ModuleBinding{Module: DefaultNormalizeModule}) {
|
||||
t.Fatalf("Normalize = %#v, want default normalize binding", lane.Normalize)
|
||||
}
|
||||
if lane.Extract.LLMProfile != DefaultLLMProfile {
|
||||
t.Fatalf("Extract.LLMProfile = %q, want %q", lane.Extract.LLMProfile, DefaultLLMProfile)
|
||||
if lane.Extract.LLMProfile != "" {
|
||||
t.Fatalf("Extract.LLMProfile = %q, want empty", lane.Extract.LLMProfile)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRecordsValidatorChains(t *testing.T) {
|
||||
catalog := newProfileCatalog(t)
|
||||
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||
Stage: StageExtract,
|
||||
Module: "event-extractor",
|
||||
Validators: []ModuleBinding{Binding("grounded")},
|
||||
}); err != nil {
|
||||
t.Fatalf("register validator chain: %v", err)
|
||||
}
|
||||
|
||||
resolved, err := ResolvePipeline(PipelineProfile{
|
||||
ID: "validated",
|
||||
Input: Binding("text"),
|
||||
Artifacts: map[string]ArtifactLaneProfile{
|
||||
"events": {Extract: Binding("event-extractor")},
|
||||
},
|
||||
}, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if len(resolved.ValidatorChains) != 4 {
|
||||
t.Fatalf("len(ValidatorChains) = %d, want chunk plus lane extract/merge/normalize", len(resolved.ValidatorChains))
|
||||
}
|
||||
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
|
||||
if extractChain == nil {
|
||||
t.Fatal("extract validator chain not found")
|
||||
}
|
||||
if len(extractChain.Validators) != 1 {
|
||||
t.Fatalf("extract validators = %#v, want one validator", extractChain.Validators)
|
||||
}
|
||||
if extractChain.Validators[0].Binding.Module != "grounded" {
|
||||
t.Fatalf("extract validator key = %q, want grounded", extractChain.Validators[0].Binding.Module)
|
||||
}
|
||||
if extractChain.Validators[0].ExecutionClass != contracts.ExecutionClassDeterministic {
|
||||
t.Fatalf("extract validator execution class = %q, want deterministic", extractChain.Validators[0].ExecutionClass)
|
||||
}
|
||||
|
||||
chunkChain := findResolvedValidatorChain(resolved.ValidatorChains, StageChunk, "", DefaultChunkModule)
|
||||
if chunkChain == nil {
|
||||
t.Fatal("chunk validator chain not found")
|
||||
}
|
||||
if len(chunkChain.Validators) != 0 {
|
||||
t.Fatalf("chunk validators = %#v, want explicit empty chain", chunkChain.Validators)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsUnknownDefaultValidator(t *testing.T) {
|
||||
catalog := newProfileCatalog(t)
|
||||
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||
Stage: StageNormalize,
|
||||
Module: DefaultNormalizeModule,
|
||||
Validators: []ModuleBinding{Binding("missing-validator")},
|
||||
}); err != nil {
|
||||
t.Fatalf("register validator chain: %v", err)
|
||||
}
|
||||
|
||||
_, err := ResolvePipeline(PipelineProfile{
|
||||
ID: "invalid-chain",
|
||||
Input: Binding("text"),
|
||||
Artifacts: map[string]ArtifactLaneProfile{
|
||||
"events": {Extract: Binding("event-extractor")},
|
||||
},
|
||||
}, ResolveOptions{}, catalog)
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want unknown validator error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing-validator") {
|
||||
t.Fatalf("ResolvePipeline() error = %q, want missing validator context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineValidatorOverrideReplacesDefaultChain(t *testing.T) {
|
||||
catalog := newProfileCatalog(t)
|
||||
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "second-validator", ExecutionClass: contracts.ExecutionClassLLMBacked})
|
||||
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||
Stage: StageExtract,
|
||||
Module: "event-extractor",
|
||||
Validators: []ModuleBinding{Binding("grounded")},
|
||||
}); err != nil {
|
||||
t.Fatalf("register validator chain: %v", err)
|
||||
}
|
||||
|
||||
profile := PipelineProfile{
|
||||
ID: "validated",
|
||||
Input: Binding("text"),
|
||||
Artifacts: map[string]ArtifactLaneProfile{
|
||||
"events": {
|
||||
Extract: ModuleBinding{
|
||||
Module: "event-extractor",
|
||||
Validators: ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []ModuleBinding{
|
||||
{Module: "second-validator", LLMProfile: "careful"},
|
||||
Binding("grounded"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
|
||||
if extractChain == nil {
|
||||
t.Fatal("extract validator chain not found")
|
||||
}
|
||||
if len(extractChain.Validators) != 2 {
|
||||
t.Fatalf("extract validators = %#v, want explicit two-validator override", extractChain.Validators)
|
||||
}
|
||||
if extractChain.Validators[0].Binding.Module != "second-validator" || extractChain.Validators[0].Binding.LLMProfile != "careful" {
|
||||
t.Fatalf("first validator = %#v, want explicit LLM-backed validator first", extractChain.Validators[0])
|
||||
}
|
||||
if extractChain.Validators[1].Binding.Module != "grounded" {
|
||||
t.Fatalf("second validator = %#v, want grounded second", extractChain.Validators[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineExplicitEmptyValidatorOverrideSuppressesDefaultChain(t *testing.T) {
|
||||
catalog := newProfileCatalog(t)
|
||||
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||
Stage: StageExtract,
|
||||
Module: "event-extractor",
|
||||
Validators: []ModuleBinding{Binding("grounded")},
|
||||
}); err != nil {
|
||||
t.Fatalf("register validator chain: %v", err)
|
||||
}
|
||||
|
||||
profile := PipelineProfile{
|
||||
ID: "validated",
|
||||
Input: Binding("text"),
|
||||
Artifacts: map[string]ArtifactLaneProfile{
|
||||
"events": {
|
||||
Extract: ModuleBinding{
|
||||
Module: "event-extractor",
|
||||
Validators: ValidatorOverride{Set: true},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
|
||||
if extractChain == nil {
|
||||
t.Fatal("extract validator chain not found")
|
||||
}
|
||||
if len(extractChain.Validators) != 0 {
|
||||
t.Fatalf("extract validators = %#v, want explicit empty override", extractChain.Validators)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsUnknownOverrideValidator(t *testing.T) {
|
||||
_, err := ResolvePipeline(PipelineProfile{
|
||||
ID: "validated",
|
||||
Input: Binding("text"),
|
||||
Artifacts: map[string]ArtifactLaneProfile{
|
||||
"events": {
|
||||
Extract: ModuleBinding{
|
||||
Module: "event-extractor",
|
||||
Validators: ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []ModuleBinding{Binding("missing-validator")},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want unknown validator error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing-validator") {
|
||||
t.Fatalf("ResolvePipeline() error = %q, want missing validator context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsLLMProfileForDeterministicValidator(t *testing.T) {
|
||||
_, err := ResolvePipeline(PipelineProfile{
|
||||
ID: "validated",
|
||||
Input: Binding("text"),
|
||||
Artifacts: map[string]ArtifactLaneProfile{
|
||||
"events": {
|
||||
Extract: ModuleBinding{
|
||||
Module: "event-extractor",
|
||||
Validators: ValidatorOverride{
|
||||
Set: true,
|
||||
Validators: []ModuleBinding{
|
||||
{Module: "grounded", LLMProfile: "careful"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want deterministic validator profile error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "grounded") || !strings.Contains(err.Error(), "llm_profile") {
|
||||
t.Fatalf("ResolvePipeline() error = %q, want validator profile context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,6 +333,467 @@ func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineAppliesReferenceBindings(t *testing.T) {
|
||||
profile := multiLaneProfile()
|
||||
profile.References = map[string]string{
|
||||
" roster ": " ./shared-roster.yml ",
|
||||
}
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.References = map[string]string{
|
||||
"roster": "./lane-roster.yml",
|
||||
" lore ": " ./lore.md ",
|
||||
}
|
||||
profile.Artifacts["events"] = lane
|
||||
|
||||
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
|
||||
Key: "event-extractor",
|
||||
Stage: StageExtract,
|
||||
Requires: []string{"chunk"},
|
||||
Provides: []string{"candidate"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "roster", Required: true},
|
||||
{Name: "lore"},
|
||||
},
|
||||
})
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events", "summaries"}}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
events := resolvedLane(t, resolved.ArtifactLanes, "events")
|
||||
if events.ExtractReferences.Stage != StageExtract || events.ExtractReferences.LaneID != "events" || events.ExtractReferences.Module != "event-extractor" {
|
||||
t.Fatalf("extract reference target = %#v, want event extractor target", events.ExtractReferences)
|
||||
}
|
||||
if events.NormalizeReferences.Stage != StageNormalize || events.NormalizeReferences.LaneID != "events" || events.NormalizeReferences.Module != DefaultNormalizeModule {
|
||||
t.Fatalf("normalize reference target = %#v, want event normalizer target", events.NormalizeReferences)
|
||||
}
|
||||
if events.MergeReferences.Stage != StageMerge || events.MergeReferences.LaneID != "events" || events.MergeReferences.Module != DefaultMergeModule {
|
||||
t.Fatalf("merge reference target = %#v, want event merger target", events.MergeReferences)
|
||||
}
|
||||
if resolved.ChunkReferences.Stage != StageChunk || resolved.ChunkReferences.Module != DefaultChunkModule {
|
||||
t.Fatalf("chunk reference target = %#v, want chunk target", resolved.ChunkReferences)
|
||||
}
|
||||
want := []ReferenceBinding{
|
||||
{LaneID: "events", SlotName: "lore", Source: "./lore.md", BindingSource: contracts.ReferenceBindingSourceConfig},
|
||||
{LaneID: "events", SlotName: "roster", Source: "./lane-roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig},
|
||||
}
|
||||
if !reflect.DeepEqual(events.ExtractReferences.Bindings, want) {
|
||||
t.Fatalf("events references = %#v, want %#v", events.ExtractReferences.Bindings, want)
|
||||
}
|
||||
summaries := resolvedLane(t, resolved.ArtifactLanes, "summaries")
|
||||
if len(summaries.ExtractReferences.Bindings) != 0 {
|
||||
t.Fatalf("summaries references = %#v, want none", summaries.ExtractReferences.Bindings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineAppliesPipelineReferenceDefaultToChunkTarget(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.References = map[string]string{"scene_guide": "./scenes.md"}
|
||||
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||
Key: "generic",
|
||||
Stage: StageChunk,
|
||||
Requires: []string{"source"},
|
||||
Provides: []string{"chunk"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{{Name: "scene_guide"}},
|
||||
})
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
want := []ReferenceBinding{{SlotName: "scene_guide", Source: "./scenes.md", BindingSource: contracts.ReferenceBindingSourceConfig}}
|
||||
if !reflect.DeepEqual(resolved.ChunkReferences.Bindings, want) {
|
||||
t.Fatalf("chunk references = %#v, want %#v", resolved.ChunkReferences.Bindings, want)
|
||||
}
|
||||
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
|
||||
t.Fatalf("extract references = %#v, want none", refs)
|
||||
}
|
||||
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
|
||||
t.Fatalf("normalize references = %#v, want none", refs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineAppliesPipelineReferenceDefaultToExtractorTarget(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.References = map[string]string{"roster": "./roster.yml"}
|
||||
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||
Key: "event-extractor",
|
||||
Stage: StageExtract,
|
||||
Requires: []string{"chunk"},
|
||||
Provides: []string{"candidate"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}},
|
||||
})
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
want := []ReferenceBinding{{LaneID: "events", SlotName: "roster", Source: "./roster.yml", BindingSource: contracts.ReferenceBindingSourceConfig}}
|
||||
if !reflect.DeepEqual(resolved.ArtifactLanes[0].ExtractReferences.Bindings, want) {
|
||||
t.Fatalf("extract references = %#v, want %#v", resolved.ArtifactLanes[0].ExtractReferences.Bindings, want)
|
||||
}
|
||||
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
|
||||
t.Fatalf("chunk references = %#v, want none", refs)
|
||||
}
|
||||
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
|
||||
t.Fatalf("normalize references = %#v, want none", refs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineAppliesPipelineReferenceDefaultToNormalizerTarget(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.References = map[string]string{"normalization_notes": "./normalize.md"}
|
||||
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||
Key: "noop",
|
||||
Stage: StageNormalize,
|
||||
Requires: []string{"merged"},
|
||||
Provides: []string{"normalized"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}},
|
||||
})
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
want := []ReferenceBinding{{LaneID: "events", SlotName: "normalization_notes", Source: "./normalize.md", BindingSource: contracts.ReferenceBindingSourceConfig}}
|
||||
if !reflect.DeepEqual(resolved.ArtifactLanes[0].NormalizeReferences.Bindings, want) {
|
||||
t.Fatalf("normalize references = %#v, want %#v", resolved.ArtifactLanes[0].NormalizeReferences.Bindings, want)
|
||||
}
|
||||
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
|
||||
t.Fatalf("chunk references = %#v, want none", refs)
|
||||
}
|
||||
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
|
||||
t.Fatalf("extract references = %#v, want none", refs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineAppliesPipelineReferenceDefaultToMergeTarget(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.References = map[string]string{"merge_notes": "./merge.md"}
|
||||
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||
Key: "appendorder",
|
||||
Stage: StageMerge,
|
||||
Requires: []string{"candidate"},
|
||||
Provides: []string{"merged"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{{Name: "merge_notes"}},
|
||||
})
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
want := []ReferenceBinding{{LaneID: "events", SlotName: "merge_notes", Source: "./merge.md", BindingSource: contracts.ReferenceBindingSourceConfig}}
|
||||
if !reflect.DeepEqual(resolved.ArtifactLanes[0].MergeReferences.Bindings, want) {
|
||||
t.Fatalf("merge references = %#v, want %#v", resolved.ArtifactLanes[0].MergeReferences.Bindings, want)
|
||||
}
|
||||
if refs := resolved.ChunkReferences.Bindings; len(refs) != 0 {
|
||||
t.Fatalf("chunk references = %#v, want none", refs)
|
||||
}
|
||||
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
|
||||
t.Fatalf("extract references = %#v, want none", refs)
|
||||
}
|
||||
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
|
||||
t.Fatalf("normalize references = %#v, want none", refs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineAppliesOnePipelineReferenceDefaultToMultipleTargets(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.References = map[string]string{"context": "./context.md"}
|
||||
catalog := newProfileCatalogWithOverrides(t,
|
||||
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
|
||||
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
|
||||
ModuleSpec{Key: "appendorder", Stage: StageMerge, Requires: []string{"candidate"}, Provides: []string{"merged"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
|
||||
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
|
||||
)
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
assertBindingSource(t, resolved.ChunkReferences.Bindings, "context", "./context.md")
|
||||
assertBindingSource(t, resolved.ArtifactLanes[0].ExtractReferences.Bindings, "context", "./context.md")
|
||||
assertBindingSource(t, resolved.ArtifactLanes[0].MergeReferences.Bindings, "context", "./context.md")
|
||||
assertBindingSource(t, resolved.ArtifactLanes[0].NormalizeReferences.Bindings, "context", "./context.md")
|
||||
}
|
||||
|
||||
func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedLane(t *testing.T) {
|
||||
profile := multiLaneProfile()
|
||||
profile.References = map[string]string{"notes_context": "./notes.md"}
|
||||
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
|
||||
Key: "note-extractor",
|
||||
Stage: StageExtract,
|
||||
Requires: []string{"chunk"},
|
||||
Provides: []string{"candidate"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "notes_context"},
|
||||
},
|
||||
})
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events"}}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
|
||||
t.Fatalf("selected lane references = %#v, want none", refs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineAllowsPipelineReferenceDeclaredOnlyByUnselectedNormalizer(t *testing.T) {
|
||||
profile := multiLaneProfile()
|
||||
profile.References = map[string]string{"notes_context": "./notes.md"}
|
||||
lane := profile.Artifacts["notes"]
|
||||
lane.Normalize = Binding("note-normalizer")
|
||||
profile.Artifacts["notes"] = lane
|
||||
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||
Key: "note-normalizer",
|
||||
Stage: StageNormalize,
|
||||
Requires: []string{"merged"},
|
||||
Provides: []string{"normalized"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "notes_context"},
|
||||
},
|
||||
})
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events"}}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
if refs := resolved.ArtifactLanes[0].ExtractReferences.Bindings; len(refs) != 0 {
|
||||
t.Fatalf("selected extract references = %#v, want none", refs)
|
||||
}
|
||||
if refs := resolved.ArtifactLanes[0].NormalizeReferences.Bindings; len(refs) != 0 {
|
||||
t.Fatalf("selected normalize references = %#v, want none", refs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsPipelineReferenceNotDeclaredByAnyLane(t *testing.T) {
|
||||
profile := multiLaneProfile()
|
||||
profile.References = map[string]string{"missing": "./missing.md"}
|
||||
|
||||
_, err := ResolvePipeline(profile, ResolveOptions{Only: []string{"events"}}, newProfileCatalog(t))
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||
}
|
||||
assertErrorContains(t, err, "multi", "reference slot", "missing", "not declared")
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsUndeclaredReferenceSlot(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.References = map[string]string{"missing": "./missing.yml"}
|
||||
profile.Artifacts["events"] = lane
|
||||
|
||||
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||
}
|
||||
assertErrorContains(t, err, "events", "missing", "not declared")
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsExtractLocalReferenceDeclaredOnlyByNormalizer(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.References = map[string]string{"normalization_notes": "./normalize.md"}
|
||||
profile.Artifacts["events"] = lane
|
||||
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||
Key: "noop",
|
||||
Stage: StageNormalize,
|
||||
Requires: []string{"merged"},
|
||||
Provides: []string{"normalized"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}},
|
||||
})
|
||||
|
||||
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||
}
|
||||
assertErrorContains(t, err, "baseline", "events", "extract", "event-extractor", "normalization_notes", "not declared")
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsMergeLocalReferenceDeclaredOnlyByNormalizer(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Merge.References = map[string]string{"normalization_notes": "./normalize.md"}
|
||||
profile.Artifacts["events"] = lane
|
||||
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||
Key: "noop",
|
||||
Stage: StageNormalize,
|
||||
Requires: []string{"merged"},
|
||||
Provides: []string{"normalized"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "normalization_notes"},
|
||||
},
|
||||
})
|
||||
|
||||
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||
}
|
||||
assertErrorContains(t, err, "merge", "normalization_notes", "not declared")
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsNormalizeLocalReferenceDeclaredOnlyByExtractor(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Normalize.References = map[string]string{"roster": "./roster.yml"}
|
||||
profile.Artifacts["events"] = lane
|
||||
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||
Key: "event-extractor",
|
||||
Stage: StageExtract,
|
||||
Requires: []string{"chunk"},
|
||||
Provides: []string{"candidate"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}},
|
||||
})
|
||||
|
||||
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||
}
|
||||
assertErrorContains(t, err, "baseline", "events", "normalize", "noop", "roster", "not declared")
|
||||
}
|
||||
|
||||
func TestResolvePipelineRequiresBoundChunkReference(t *testing.T) {
|
||||
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||
Key: "generic",
|
||||
Stage: StageChunk,
|
||||
Requires: []string{"source"},
|
||||
Provides: []string{"chunk"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{{Name: "scene_guide", Required: true}},
|
||||
})
|
||||
|
||||
_, err := ResolvePipeline(baselineProfile(), ResolveOptions{}, catalog)
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||
}
|
||||
assertErrorContains(t, err, "baseline", "chunk", "generic", "required", "scene_guide", "not bound")
|
||||
}
|
||||
|
||||
func TestResolvePipelineRequiresBoundNormalizeReference(t *testing.T) {
|
||||
catalog := newProfileCatalogWithOverrides(t, ModuleSpec{
|
||||
Key: "noop",
|
||||
Stage: StageNormalize,
|
||||
Requires: []string{"merged"},
|
||||
Provides: []string{"normalized"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes", Required: true}},
|
||||
})
|
||||
|
||||
_, err := ResolvePipeline(baselineProfile(), ResolveOptions{}, catalog)
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||
}
|
||||
assertErrorContains(t, err, "baseline", "events", "normalize", "noop", "required", "normalization_notes", "not bound")
|
||||
}
|
||||
|
||||
func TestResolvePipelineLocalReferencesOverridePipelineDefaultsForEligibleTargets(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.References = map[string]string{
|
||||
"context": "./shared-context.md",
|
||||
"roster": "./shared-roster.yml",
|
||||
"normalization_notes": "./shared-normalize.md",
|
||||
}
|
||||
profile.Chunk.References = map[string]string{"context": "./chunk-context.md"}
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.References = map[string]string{"roster": "./extract-roster.yml"}
|
||||
lane.Normalize.References = map[string]string{"normalization_notes": "./local-normalize.md"}
|
||||
profile.Artifacts["events"] = lane
|
||||
catalog := newProfileCatalogWithOverrides(t,
|
||||
ModuleSpec{Key: "generic", Stage: StageChunk, Requires: []string{"source"}, Provides: []string{"chunk"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "context"}}},
|
||||
ModuleSpec{Key: "event-extractor", Stage: StageExtract, Requires: []string{"chunk"}, Provides: []string{"candidate"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "roster"}}},
|
||||
ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, ReferenceSlots: []contracts.ReferenceSlot{{Name: "normalization_notes"}}},
|
||||
)
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
assertBindingSource(t, resolved.ChunkReferences.Bindings, "context", "./chunk-context.md")
|
||||
assertBindingSource(t, resolved.ArtifactLanes[0].ExtractReferences.Bindings, "roster", "./extract-roster.yml")
|
||||
assertBindingSource(t, resolved.ArtifactLanes[0].NormalizeReferences.Bindings, "normalization_notes", "./local-normalize.md")
|
||||
}
|
||||
|
||||
func TestResolvePipelineRequiresBoundReferenceSlotsForSelectedLanes(t *testing.T) {
|
||||
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
|
||||
Key: "event-extractor",
|
||||
Stage: StageExtract,
|
||||
Requires: []string{"chunk"},
|
||||
Provides: []string{"candidate"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "roster", Required: true},
|
||||
},
|
||||
})
|
||||
|
||||
if _, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"notes"}}, catalog); err != nil {
|
||||
t.Fatalf("ResolvePipeline(unselected required slot) error = %v, want nil", err)
|
||||
}
|
||||
|
||||
_, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"events"}}, catalog)
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline(selected required slot) error = nil, want error")
|
||||
}
|
||||
assertErrorContains(t, err, "events", "required", "roster", "not bound")
|
||||
}
|
||||
|
||||
func TestResolvePipelineReferenceUnbindCanLeaveRequiredSlotMissing(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.References = map[string]string{"roster": "./roster.yml"}
|
||||
catalog := newProfileCatalogWithOverride(t, ModuleSpec{
|
||||
Key: "event-extractor",
|
||||
Stage: StageExtract,
|
||||
Requires: []string{"chunk"},
|
||||
Provides: []string{"candidate"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "roster", Required: true},
|
||||
},
|
||||
})
|
||||
|
||||
_, err := ResolvePipeline(profile, ResolveOptions{
|
||||
ReferenceUnbinds: []ReferenceUnbind{{LaneID: "events", SlotName: "roster"}},
|
||||
}, catalog)
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||
}
|
||||
assertErrorContains(t, err, "events", "required", "roster", "not bound")
|
||||
}
|
||||
|
||||
func TestResolvePipelineUsesReferenceSlotsFromSpecWithoutConstructingExtractor(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.References = map[string]string{"roster": "./roster.yml"}
|
||||
catalog := emptyProfileCatalog()
|
||||
for _, spec := range defaultProfileSpecs() {
|
||||
if spec.Key != "event-extractor" {
|
||||
registerProfileSpecs(t, catalog, spec)
|
||||
}
|
||||
}
|
||||
if err := catalog.Extractors.RegisterWithSpec(ModuleSpec{
|
||||
Key: "event-extractor",
|
||||
Stage: StageExtract,
|
||||
Requires: []string{"chunk"},
|
||||
Provides: []string{"candidate"},
|
||||
ReferenceSlots: []contracts.ReferenceSlot{
|
||||
{Name: "roster", Required: true},
|
||||
},
|
||||
}, func() (contracts.Extractor, error) {
|
||||
return nil, errors.New("constructor should not run")
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterWithSpec() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
if got := resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].Source; got != "./roster.yml" {
|
||||
t.Fatalf("reference source = %q, want ./roster.yml", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsUnknownOnlyLane(t *testing.T) {
|
||||
_, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{Only: []string{"missing"}}, newProfileCatalog(t))
|
||||
if err == nil {
|
||||
@@ -235,16 +904,6 @@ func TestResolvePipelineRejectsUnknownModuleKeys(t *testing.T) {
|
||||
}),
|
||||
want: []string{"baseline", "events", "normalize", "missing-normalize"},
|
||||
},
|
||||
{
|
||||
name: "validate",
|
||||
profile: withProfileChange(func(profile PipelineProfile) PipelineProfile {
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Validators = []ModuleBinding{Binding("missing-validator")}
|
||||
profile.Artifacts["events"] = lane
|
||||
return profile
|
||||
}),
|
||||
want: []string{"baseline", "events", "validate", "missing-validator"},
|
||||
},
|
||||
{
|
||||
name: "output",
|
||||
profile: withProfileChange(func(profile PipelineProfile) PipelineProfile {
|
||||
@@ -297,11 +956,6 @@ func TestResolvePipelineRejectsMissingCapabilities(t *testing.T) {
|
||||
spec: ModuleSpec{Key: "noop", Stage: StageNormalize, Requires: []string{"missing"}},
|
||||
want: []string{"baseline", "events", "normalize", "noop", "missing"},
|
||||
},
|
||||
{
|
||||
name: "validate",
|
||||
spec: ModuleSpec{Key: "grounded", Stage: StageValidate, Requires: []string{"missing"}},
|
||||
want: []string{"baseline", "events", "validate", "grounded", "missing"},
|
||||
},
|
||||
{
|
||||
name: "output",
|
||||
spec: ModuleSpec{Key: "json", Stage: StageOutput, Requires: []string{"missing"}},
|
||||
@@ -313,9 +967,6 @@ func TestResolvePipelineRejectsMissingCapabilities(t *testing.T) {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
catalog := newProfileCatalogWithOverride(t, test.spec)
|
||||
profile := baselineProfile()
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Validators = []ModuleBinding{Binding("grounded")}
|
||||
profile.Artifacts["events"] = lane
|
||||
|
||||
_, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err == nil {
|
||||
@@ -326,6 +977,19 @@ func TestResolvePipelineRejectsMissingCapabilities(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsConfiguredValidators(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Validators = []ModuleBinding{Binding("grounded")}
|
||||
profile.Artifacts["events"] = lane
|
||||
|
||||
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err == nil {
|
||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||
}
|
||||
assertErrorContains(t, err, "baseline", "events", "validators", "extract.validators")
|
||||
}
|
||||
|
||||
func TestResolvePipelineOrdersLanesDeterministically(t *testing.T) {
|
||||
resolved, err := ResolvePipeline(multiLaneProfile(), ResolveOptions{}, newProfileCatalog(t))
|
||||
if err != nil {
|
||||
@@ -463,6 +1127,17 @@ func laneIDs(lanes []ResolvedArtifactLane) []string {
|
||||
return ids
|
||||
}
|
||||
|
||||
func resolvedLane(t *testing.T, lanes []ResolvedArtifactLane, laneID string) ResolvedArtifactLane {
|
||||
t.Helper()
|
||||
for _, lane := range lanes {
|
||||
if lane.ID == laneID {
|
||||
return lane
|
||||
}
|
||||
}
|
||||
t.Fatalf("lane %q not found in %#v", laneID, laneIDs(lanes))
|
||||
return ResolvedArtifactLane{}
|
||||
}
|
||||
|
||||
func assertErrorContains(t *testing.T, err error, values ...string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -474,6 +1149,30 @@ func assertErrorContains(t *testing.T, err error, values ...string) {
|
||||
}
|
||||
}
|
||||
|
||||
func assertBindingSource(t *testing.T, bindings []ReferenceBinding, slotName string, source string) {
|
||||
t.Helper()
|
||||
|
||||
for _, binding := range bindings {
|
||||
if binding.SlotName != slotName {
|
||||
continue
|
||||
}
|
||||
if binding.Source != source {
|
||||
t.Fatalf("binding %q source = %q, want %q in %#v", slotName, binding.Source, source, bindings)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatalf("binding %q not found in %#v", slotName, bindings)
|
||||
}
|
||||
|
||||
func findResolvedValidatorChain(chains []ResolvedValidatorChain, stage ModuleStage, laneID string, module string) *ResolvedValidatorChain {
|
||||
for i := range chains {
|
||||
if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module {
|
||||
return &chains[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newProfileCatalog(t *testing.T) ModuleCatalog {
|
||||
t.Helper()
|
||||
|
||||
@@ -485,31 +1184,42 @@ func newProfileCatalog(t *testing.T) ModuleCatalog {
|
||||
func newProfileCatalogWithOverride(t *testing.T, override ModuleSpec) ModuleCatalog {
|
||||
t.Helper()
|
||||
|
||||
return newProfileCatalogWithOverrides(t, override)
|
||||
}
|
||||
|
||||
func newProfileCatalogWithOverrides(t *testing.T, overrides ...ModuleSpec) ModuleCatalog {
|
||||
t.Helper()
|
||||
|
||||
specs := defaultProfileSpecs()
|
||||
for index, spec := range specs {
|
||||
if spec.Stage == override.Stage && spec.Key == override.Key {
|
||||
specs[index] = override
|
||||
catalog := emptyProfileCatalog()
|
||||
registerProfileSpecs(t, catalog, specs...)
|
||||
return catalog
|
||||
for _, override := range overrides {
|
||||
replaced := false
|
||||
for index, spec := range specs {
|
||||
if spec.Stage == override.Stage && spec.Key == override.Key {
|
||||
specs[index] = override
|
||||
replaced = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !replaced {
|
||||
specs = append(specs, override)
|
||||
}
|
||||
}
|
||||
|
||||
catalog := emptyProfileCatalog()
|
||||
registerProfileSpecs(t, catalog, specs...)
|
||||
registerProfileSpecs(t, catalog, override)
|
||||
return catalog
|
||||
}
|
||||
|
||||
func emptyProfileCatalog() ModuleCatalog {
|
||||
return ModuleCatalog{
|
||||
Inputs: NewInputAdapterRegistry(),
|
||||
Chunkers: NewChunkerRegistry(),
|
||||
Extractors: NewExtractorRegistry(),
|
||||
Mergers: NewMergerRegistry(),
|
||||
Normalizers: NewNormalizerRegistry(),
|
||||
Validators: NewValidatorRegistry(),
|
||||
Outputs: NewOutputEncoderRegistry(),
|
||||
Inputs: NewInputAdapterRegistry(),
|
||||
Chunkers: NewChunkerRegistry(),
|
||||
Extractors: NewExtractorRegistry(),
|
||||
Mergers: NewMergerRegistry(),
|
||||
Normalizers: NewNormalizerRegistry(),
|
||||
Validators: NewValidatorRegistry(),
|
||||
ValidatorChains: NewValidatorChainRegistry(),
|
||||
Outputs: NewOutputEncoderRegistry(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -552,7 +1262,8 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
|
||||
t.Fatalf("register normalizer spec %#v: %v", spec, err)
|
||||
}
|
||||
case StageValidate:
|
||||
if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
|
||||
validatorSpec := ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
if err := catalog.Validators.RegisterWithSpec(validatorSpec, profileValidatorConstructor(spec.Key)); err != nil {
|
||||
t.Fatalf("register validator spec %#v: %v", spec, err)
|
||||
}
|
||||
case StageOutput:
|
||||
@@ -565,6 +1276,13 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
|
||||
}
|
||||
}
|
||||
|
||||
func registerProfileValidatorSpec(t *testing.T, catalog ModuleCatalog, spec ValidatorSpec) {
|
||||
t.Helper()
|
||||
if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
|
||||
t.Fatalf("register validator spec %#v: %v", spec, err)
|
||||
}
|
||||
}
|
||||
|
||||
func profileInputConstructor(key string) InputAdapterConstructor {
|
||||
return func() (contracts.InputAdapter, error) {
|
||||
return profileInputAdapter{key: key}, nil
|
||||
|
||||
337
internal/framework/pipeline/references.go
Normal file
337
internal/framework/pipeline/references.go
Normal file
@@ -0,0 +1,337 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
const (
|
||||
referenceOriginFile = "file"
|
||||
referenceMediaType = "text/plain"
|
||||
unknownMediaType = "application/octet-stream"
|
||||
)
|
||||
|
||||
type ReferenceMaterializationOptions struct {
|
||||
ConfigPath string
|
||||
WorkingDir string
|
||||
}
|
||||
|
||||
func MaterializeReferences(resolved ResolvedPipeline, catalog ModuleCatalog, options ReferenceMaterializationOptions) (ResolvedPipeline, []contracts.Warning, error) {
|
||||
out := resolved
|
||||
out.ChunkReferences = CloneReferenceTarget(resolved.ChunkReferences)
|
||||
chunkReferenceSet, chunkWarnings, err := materializeReferenceTarget(resolved.ID, resolved.ChunkReferences, catalog, options)
|
||||
if err != nil {
|
||||
return ResolvedPipeline{}, nil, err
|
||||
}
|
||||
out.ChunkReferences.ReferenceSet = chunkReferenceSet
|
||||
warnings := append([]contracts.Warning(nil), chunkWarnings...)
|
||||
if len(resolved.ArtifactLanes) == 0 {
|
||||
return out, warnings, nil
|
||||
}
|
||||
|
||||
out.ArtifactLanes = make([]ResolvedArtifactLane, len(resolved.ArtifactLanes))
|
||||
for i, lane := range resolved.ArtifactLanes {
|
||||
materializedLane := lane
|
||||
materializedLane.ExtractReferences = CloneReferenceTarget(lane.ExtractReferences)
|
||||
materializedLane.MergeReferences = CloneReferenceTarget(lane.MergeReferences)
|
||||
materializedLane.NormalizeReferences = CloneReferenceTarget(lane.NormalizeReferences)
|
||||
extractReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.ExtractReferences, catalog, options)
|
||||
if err != nil {
|
||||
return ResolvedPipeline{}, nil, err
|
||||
}
|
||||
materializedLane.ExtractReferences.ReferenceSet = extractReferenceSet
|
||||
warnings = append(warnings, laneWarnings...)
|
||||
|
||||
mergeReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.MergeReferences, catalog, options)
|
||||
if err != nil {
|
||||
return ResolvedPipeline{}, nil, err
|
||||
}
|
||||
materializedLane.MergeReferences.ReferenceSet = mergeReferenceSet
|
||||
warnings = append(warnings, laneWarnings...)
|
||||
|
||||
normalizeReferenceSet, laneWarnings, err := materializeReferenceTarget(resolved.ID, lane.NormalizeReferences, catalog, options)
|
||||
if err != nil {
|
||||
return ResolvedPipeline{}, nil, err
|
||||
}
|
||||
materializedLane.NormalizeReferences.ReferenceSet = normalizeReferenceSet
|
||||
warnings = append(warnings, laneWarnings...)
|
||||
out.ArtifactLanes[i] = materializedLane
|
||||
}
|
||||
return out, warnings, nil
|
||||
}
|
||||
|
||||
func materializeReferenceTarget(
|
||||
pipelineID string,
|
||||
target ResolvedReferenceTarget,
|
||||
catalog ModuleCatalog,
|
||||
options ReferenceMaterializationOptions,
|
||||
) (contracts.ReferenceSet, []contracts.Warning, error) {
|
||||
if len(target.Bindings) == 0 {
|
||||
return contracts.ReferenceSet{}, nil, nil
|
||||
}
|
||||
spec, err := referenceTargetSpec(target, catalog)
|
||||
if err != nil {
|
||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s: %w", referenceTargetContext(pipelineID, target), err)
|
||||
}
|
||||
|
||||
slotByName := make(map[string]contracts.ReferenceSlot, len(spec.ReferenceSlots))
|
||||
for _, slot := range spec.ReferenceSlots {
|
||||
slotByName[slot.Name] = slot
|
||||
}
|
||||
|
||||
set := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(target.Bindings))}
|
||||
var warnings []contracts.Warning
|
||||
for _, binding := range target.Bindings {
|
||||
slotName := strings.TrimSpace(binding.SlotName)
|
||||
slot, ok := slotByName[slotName]
|
||||
if !ok {
|
||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q is not declared by %s module %q", referenceTargetContext(pipelineID, target), slotName, target.Stage, target.Module)
|
||||
}
|
||||
|
||||
path, err := referencePath(binding, options)
|
||||
if err != nil {
|
||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q: %w", referenceTargetContext(pipelineID, target), slotName, binding.Source, err)
|
||||
}
|
||||
content, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q read %q: %w", referenceTargetContext(pipelineID, target), slotName, path, err)
|
||||
}
|
||||
if !utf8.Valid(content) {
|
||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q must be UTF-8 text", referenceTargetContext(pipelineID, target), slotName, path)
|
||||
}
|
||||
mediaType := referenceMediaTypeForPath(path)
|
||||
if !referenceMediaTypeAccepted(mediaType, slot.AcceptedMediaTypes) {
|
||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q media type %q is not accepted", referenceTargetContext(pipelineID, target), slotName, path, mediaType)
|
||||
}
|
||||
if slot.MaxBytes > 0 && int64(len(content)) > slot.MaxBytes {
|
||||
return contracts.ReferenceSet{}, nil, fmt.Errorf("%s reference slot %q path %q is %d bytes, limit %d", referenceTargetContext(pipelineID, target), slotName, path, len(content), slot.MaxBytes)
|
||||
}
|
||||
if len(content) == 0 {
|
||||
warnings = append(warnings, contracts.Warning{
|
||||
Scope: referenceWarningScope(pipelineID, target, slotName),
|
||||
ReasonCode: "empty_reference",
|
||||
Message: fmt.Sprintf("reference slot %q for %s is bound to an empty file", slotName, referenceTargetLabel(target)),
|
||||
})
|
||||
}
|
||||
|
||||
item := contracts.ReferenceItem{
|
||||
SlotName: slotName,
|
||||
MediaType: mediaType,
|
||||
Content: append([]byte(nil), content...),
|
||||
Digest: referenceDigest(content),
|
||||
Origin: contracts.ReferenceOrigin{Type: referenceOriginFile, URI: fileURI(path)},
|
||||
SizeBytes: int64(len(content)),
|
||||
BindingSource: strings.TrimSpace(binding.BindingSource),
|
||||
}
|
||||
set.Slots[slotName] = contracts.ResolvedReferenceSlot{
|
||||
Slot: cloneReferenceSlot(slot),
|
||||
Items: []contracts.ReferenceItem{item},
|
||||
}
|
||||
}
|
||||
return set, warnings, nil
|
||||
}
|
||||
|
||||
func referenceTargetSpec(target ResolvedReferenceTarget, catalog ModuleCatalog) (ModuleSpec, error) {
|
||||
switch target.Stage {
|
||||
case StageChunk:
|
||||
return registrySpec(catalog.Chunkers, target.Module)
|
||||
case StageExtract:
|
||||
return registrySpec(catalog.Extractors, target.Module)
|
||||
case StageMerge:
|
||||
return registrySpec(catalog.Mergers, target.Module)
|
||||
case StageNormalize:
|
||||
return registrySpec(catalog.Normalizers, target.Module)
|
||||
default:
|
||||
return ModuleSpec{}, fmt.Errorf("reference target stage %q is not supported", target.Stage)
|
||||
}
|
||||
}
|
||||
|
||||
func referenceTargetContext(pipelineID string, target ResolvedReferenceTarget) string {
|
||||
if target.LaneID != "" {
|
||||
return fmt.Sprintf("pipeline %q lane %q %s module %q", pipelineID, target.LaneID, target.Stage, target.Module)
|
||||
}
|
||||
return fmt.Sprintf("pipeline %q %s module %q", pipelineID, target.Stage, target.Module)
|
||||
}
|
||||
|
||||
func referenceTargetLabel(target ResolvedReferenceTarget) string {
|
||||
if target.LaneID != "" {
|
||||
return fmt.Sprintf("lane %q %s target", target.LaneID, target.Stage)
|
||||
}
|
||||
return fmt.Sprintf("%s target", target.Stage)
|
||||
}
|
||||
|
||||
func referenceWarningScope(pipelineID string, target ResolvedReferenceTarget, slotName string) string {
|
||||
if target.LaneID != "" {
|
||||
return fmt.Sprintf("pipeline.%s.lane.%s.%s.reference.%s", pipelineID, target.LaneID, target.Stage, slotName)
|
||||
}
|
||||
return fmt.Sprintf("pipeline.%s.%s.reference.%s", pipelineID, target.Stage, slotName)
|
||||
}
|
||||
|
||||
func referenceMediaTypeForPath(path string) string {
|
||||
extension := strings.ToLower(filepath.Ext(path))
|
||||
mediaType := mime.TypeByExtension(extension)
|
||||
if strings.TrimSpace(mediaType) == "" {
|
||||
if extension == ".md" || extension == ".markdown" {
|
||||
return "text/markdown"
|
||||
}
|
||||
if extension == ".yaml" || extension == ".yml" {
|
||||
return "application/yaml"
|
||||
}
|
||||
return unknownMediaType
|
||||
}
|
||||
return canonicalMediaType(mediaType)
|
||||
}
|
||||
|
||||
func referenceMediaTypeAccepted(mediaType string, accepted []string) bool {
|
||||
if len(accepted) == 0 {
|
||||
return true
|
||||
}
|
||||
mediaType = canonicalMediaType(mediaType)
|
||||
for _, value := range accepted {
|
||||
if strings.EqualFold(mediaType, canonicalMediaType(value)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func canonicalMediaType(mediaType string) string {
|
||||
trimmed := strings.TrimSpace(mediaType)
|
||||
if trimmed == "" {
|
||||
return ""
|
||||
}
|
||||
parsed, _, err := mime.ParseMediaType(trimmed)
|
||||
if err != nil {
|
||||
return strings.ToLower(trimmed)
|
||||
}
|
||||
return strings.ToLower(parsed)
|
||||
}
|
||||
|
||||
func referencePath(binding ReferenceBinding, options ReferenceMaterializationOptions) (string, error) {
|
||||
source := strings.TrimSpace(binding.Source)
|
||||
if source == "" {
|
||||
return "", fmt.Errorf("must not be empty")
|
||||
}
|
||||
if filepath.IsAbs(source) {
|
||||
return filepath.Clean(source), nil
|
||||
}
|
||||
|
||||
base := strings.TrimSpace(options.WorkingDir)
|
||||
if strings.TrimSpace(binding.BindingSource) == contracts.ReferenceBindingSourceConfig {
|
||||
base = filepath.Dir(strings.TrimSpace(options.ConfigPath))
|
||||
}
|
||||
if base == "" {
|
||||
var err error
|
||||
base, err = os.Getwd()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve working directory: %w", err)
|
||||
}
|
||||
}
|
||||
return filepath.Clean(filepath.Join(base, source)), nil
|
||||
}
|
||||
|
||||
func referenceDigest(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func fileURI(path string) string {
|
||||
absolute, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
absolute = path
|
||||
}
|
||||
absolute = filepath.ToSlash(filepath.Clean(absolute))
|
||||
if strings.HasPrefix(absolute, "/") {
|
||||
return "file://" + (&url.URL{Path: absolute}).EscapedPath()
|
||||
}
|
||||
return "file:///" + (&url.URL{Path: absolute}).EscapedPath()
|
||||
}
|
||||
|
||||
func cloneReferenceSlot(slot contracts.ReferenceSlot) contracts.ReferenceSlot {
|
||||
slot.AcceptedMediaTypes = append([]string(nil), slot.AcceptedMediaTypes...)
|
||||
return slot
|
||||
}
|
||||
|
||||
func CloneReferenceSet(in contracts.ReferenceSet) contracts.ReferenceSet {
|
||||
if len(in.Slots) == 0 {
|
||||
return contracts.ReferenceSet{}
|
||||
}
|
||||
out := contracts.ReferenceSet{Slots: make(map[string]contracts.ResolvedReferenceSlot, len(in.Slots))}
|
||||
keys := make([]string, 0, len(in.Slots))
|
||||
for key := range in.Slots {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
slot := in.Slots[key]
|
||||
slot.Slot = cloneReferenceSlot(slot.Slot)
|
||||
if len(slot.Items) > 0 {
|
||||
items := make([]contracts.ReferenceItem, len(slot.Items))
|
||||
for i, item := range slot.Items {
|
||||
item.Content = append([]byte(nil), item.Content...)
|
||||
items[i] = item
|
||||
}
|
||||
slot.Items = items
|
||||
}
|
||||
out.Slots[key] = slot
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func CloneReferenceTarget(in ResolvedReferenceTarget) ResolvedReferenceTarget {
|
||||
out := in
|
||||
out.Bindings = append([]ReferenceBinding(nil), in.Bindings...)
|
||||
out.ReferenceSet = CloneReferenceSet(in.ReferenceSet)
|
||||
return out
|
||||
}
|
||||
|
||||
func ReferenceProvenance(resolved ResolvedPipeline) []artifacts.ReferenceProvenance {
|
||||
provenance := []artifacts.ReferenceProvenance{}
|
||||
provenance = append(provenance, referenceTargetProvenance(resolved.ChunkReferences)...)
|
||||
for _, lane := range resolved.ArtifactLanes {
|
||||
provenance = append(provenance, referenceTargetProvenance(lane.ExtractReferences)...)
|
||||
provenance = append(provenance, referenceTargetProvenance(lane.MergeReferences)...)
|
||||
provenance = append(provenance, referenceTargetProvenance(lane.NormalizeReferences)...)
|
||||
}
|
||||
return provenance
|
||||
}
|
||||
|
||||
func referenceTargetProvenance(target ResolvedReferenceTarget) []artifacts.ReferenceProvenance {
|
||||
if len(target.ReferenceSet.Slots) == 0 {
|
||||
return nil
|
||||
}
|
||||
provenance := []artifacts.ReferenceProvenance{}
|
||||
slotNames := make([]string, 0, len(target.ReferenceSet.Slots))
|
||||
for slotName := range target.ReferenceSet.Slots {
|
||||
slotNames = append(slotNames, slotName)
|
||||
}
|
||||
sort.Strings(slotNames)
|
||||
for _, slotName := range slotNames {
|
||||
slot := target.ReferenceSet.Slots[slotName]
|
||||
for _, item := range slot.Items {
|
||||
provenance = append(provenance, artifacts.ReferenceProvenance{
|
||||
Stage: string(target.Stage),
|
||||
LaneID: target.LaneID,
|
||||
SlotName: item.SlotName,
|
||||
OriginType: item.Origin.Type,
|
||||
OriginURI: item.Origin.URI,
|
||||
Digest: item.Digest,
|
||||
MediaType: item.MediaType,
|
||||
SizeBytes: item.SizeBytes,
|
||||
BindingSource: item.BindingSource,
|
||||
})
|
||||
}
|
||||
}
|
||||
return provenance
|
||||
}
|
||||
522
internal/framework/pipeline/references_test.go
Normal file
522
internal/framework/pipeline/references_test.go
Normal file
@@ -0,0 +1,522 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestMaterializeReferencesResolvesPathsAndDigestsContent(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
workingDir := t.TempDir()
|
||||
configReference := filepath.Join(configDir, "config-reference.txt")
|
||||
cliReference := filepath.Join(workingDir, "cli-reference.txt")
|
||||
writeReferenceFile(t, configReference, []byte("config text"))
|
||||
writeReferenceFile(t, cliReference, []byte("cli text"))
|
||||
|
||||
pipeline := baselineProfile()
|
||||
pipeline.References = map[string]string{"roster": "config-reference.txt"}
|
||||
lane := pipeline.Artifacts["events"]
|
||||
lane.References = map[string]string{"glossary": "cli-reference.txt"}
|
||||
pipeline.Artifacts["events"] = lane
|
||||
catalog := referenceCatalog(t, []contracts.ReferenceSlot{
|
||||
{Name: "roster"},
|
||||
{Name: "glossary"},
|
||||
})
|
||||
resolved, err := ResolvePipeline(pipeline, ResolveOptions{
|
||||
ReferenceOverrides: []ReferenceBinding{
|
||||
{LaneID: "events", SlotName: "glossary", Source: "cli-reference.txt", BindingSource: contracts.ReferenceBindingSourceCLI},
|
||||
},
|
||||
}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
first, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
WorkingDir: workingDir,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||
}
|
||||
if len(warnings) != 0 {
|
||||
t.Fatalf("warnings = %#v, want none", warnings)
|
||||
}
|
||||
second, _, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
WorkingDir: workingDir,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences(second) error = %v, want nil", err)
|
||||
}
|
||||
|
||||
referenceSet := first.ArtifactLanes[0].ExtractReferences.ReferenceSet
|
||||
roster := referenceSet.Slots["roster"].Items[0]
|
||||
if string(roster.Content) != "config text" {
|
||||
t.Fatalf("roster content = %q, want config text", roster.Content)
|
||||
}
|
||||
if roster.Digest != referenceDigest([]byte("config text")) || roster.Digest != second.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0].Digest {
|
||||
t.Fatalf("roster digest = %q, want stable digest", roster.Digest)
|
||||
}
|
||||
if roster.BindingSource != contracts.ReferenceBindingSourceConfig {
|
||||
t.Fatalf("roster binding source = %q, want config", roster.BindingSource)
|
||||
}
|
||||
if roster.MediaType != referenceMediaType || roster.Origin.Type != referenceOriginFile || roster.SizeBytes != int64(len("config text")) {
|
||||
t.Fatalf("roster metadata = %#v, want text file metadata", roster)
|
||||
}
|
||||
if !strings.Contains(roster.Origin.URI, "config-reference.txt") {
|
||||
t.Fatalf("roster origin URI = %q, want config reference path", roster.Origin.URI)
|
||||
}
|
||||
|
||||
glossary := referenceSet.Slots["glossary"].Items[0]
|
||||
if string(glossary.Content) != "cli text" {
|
||||
t.Fatalf("glossary content = %q, want cli text", glossary.Content)
|
||||
}
|
||||
if glossary.BindingSource != contracts.ReferenceBindingSourceCLI {
|
||||
t.Fatalf("glossary binding source = %q, want cli", glossary.BindingSource)
|
||||
}
|
||||
if !strings.Contains(glossary.Origin.URI, "cli-reference.txt") {
|
||||
t.Fatalf("glossary origin URI = %q, want cli reference path", glossary.Origin.URI)
|
||||
}
|
||||
|
||||
provenance := ReferenceProvenance(first)
|
||||
if len(provenance) != 2 {
|
||||
t.Fatalf("ReferenceProvenance() = %#v, want two entries", provenance)
|
||||
}
|
||||
if provenance[0].Stage != string(StageExtract) || provenance[0].LaneID != "events" || provenance[0].SlotName != "glossary" || provenance[0].Digest != glossary.Digest {
|
||||
t.Fatalf("ReferenceProvenance()[0] = %#v, want sorted glossary provenance", provenance[0])
|
||||
}
|
||||
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != roster.Digest {
|
||||
t.Fatalf("ReferenceProvenance()[1] = %#v, want roster provenance", provenance[1])
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(first)
|
||||
if err != nil {
|
||||
t.Fatalf("json.Marshal(materialized) error = %v, want nil", err)
|
||||
}
|
||||
if strings.Contains(string(encoded), "config text") || strings.Contains(string(encoded), "cli text") {
|
||||
t.Fatalf("materialized pipeline JSON contains reference content: %s", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesStoresSetsAndProvenanceForAllTargets(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), []byte("chunk text"))
|
||||
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), []byte("extract text"))
|
||||
writeReferenceFile(t, filepath.Join(configDir, "merge.txt"), []byte("merge text"))
|
||||
writeReferenceFile(t, filepath.Join(configDir, "normalize.txt"), []byte("normalize text"))
|
||||
|
||||
profile := baselineProfile()
|
||||
profile.References = map[string]string{
|
||||
"scene_guide": "chunk.txt",
|
||||
"roster": "extract.txt",
|
||||
"merge_notes": "merge.txt",
|
||||
"normalization_notes": "normalize.txt",
|
||||
}
|
||||
catalog := referenceCatalogForTargets(t,
|
||||
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
|
||||
[]contracts.ReferenceSlot{{Name: "roster"}},
|
||||
[]contracts.ReferenceSlot{{Name: "merge_notes"}},
|
||||
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
|
||||
)
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
materialized, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||
}
|
||||
if len(warnings) != 0 {
|
||||
t.Fatalf("warnings = %#v, want none", warnings)
|
||||
}
|
||||
|
||||
chunkItem := materialized.ChunkReferences.ReferenceSet.Slots["scene_guide"].Items[0]
|
||||
if string(chunkItem.Content) != "chunk text" {
|
||||
t.Fatalf("chunk content = %q, want chunk text", chunkItem.Content)
|
||||
}
|
||||
extractItem := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
|
||||
if string(extractItem.Content) != "extract text" {
|
||||
t.Fatalf("extract content = %q, want extract text", extractItem.Content)
|
||||
}
|
||||
mergeItem := materialized.ArtifactLanes[0].MergeReferences.ReferenceSet.Slots["merge_notes"].Items[0]
|
||||
if string(mergeItem.Content) != "merge text" {
|
||||
t.Fatalf("merge content = %q, want merge text", mergeItem.Content)
|
||||
}
|
||||
normalizeItem := materialized.ArtifactLanes[0].NormalizeReferences.ReferenceSet.Slots["normalization_notes"].Items[0]
|
||||
if string(normalizeItem.Content) != "normalize text" {
|
||||
t.Fatalf("normalize content = %q, want normalize text", normalizeItem.Content)
|
||||
}
|
||||
|
||||
provenance := ReferenceProvenance(materialized)
|
||||
if len(provenance) != 4 {
|
||||
t.Fatalf("ReferenceProvenance() = %#v, want four entries", provenance)
|
||||
}
|
||||
if provenance[0].Stage != string(StageChunk) || provenance[0].LaneID != "" || provenance[0].SlotName != "scene_guide" || provenance[0].Digest != chunkItem.Digest {
|
||||
t.Fatalf("ReferenceProvenance()[0] = %#v, want chunk scene guide provenance", provenance[0])
|
||||
}
|
||||
if provenance[1].Stage != string(StageExtract) || provenance[1].LaneID != "events" || provenance[1].SlotName != "roster" || provenance[1].Digest != extractItem.Digest {
|
||||
t.Fatalf("ReferenceProvenance()[1] = %#v, want extract roster provenance", provenance[1])
|
||||
}
|
||||
if provenance[2].Stage != string(StageMerge) || provenance[2].LaneID != "events" || provenance[2].SlotName != "merge_notes" || provenance[2].Digest != mergeItem.Digest {
|
||||
t.Fatalf("ReferenceProvenance()[2] = %#v, want merge notes provenance", provenance[2])
|
||||
}
|
||||
if provenance[3].Stage != string(StageNormalize) || provenance[3].LaneID != "events" || provenance[3].SlotName != "normalization_notes" || provenance[3].Digest != normalizeItem.Digest {
|
||||
t.Fatalf("ReferenceProvenance()[2] = %#v, want normalize notes provenance", provenance[2])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesRejectsNonUTF8Content(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
path := filepath.Join(configDir, "bad.txt")
|
||||
writeReferenceFile(t, path, []byte{0xff, 0xfe})
|
||||
|
||||
resolved := resolvedPipelineWithReference(t, "roster", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
|
||||
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "roster") || !strings.Contains(err.Error(), path) {
|
||||
t.Fatalf("error = %v, want UTF-8 path error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesRejectsNonUTF8ContentForChunkTarget(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
path := filepath.Join(configDir, "bad.txt")
|
||||
writeReferenceFile(t, path, []byte{0xff, 0xfe})
|
||||
|
||||
resolved := resolvedPipelineWithTargetReference(t, StageChunk, "", "scene_guide", "bad.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "scene_guide"})
|
||||
_, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, []contracts.ReferenceSlot{{Name: "scene_guide"}}, nil, nil, nil), ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "chunk") || !strings.Contains(err.Error(), "UTF-8") || !strings.Contains(err.Error(), "scene_guide") || !strings.Contains(err.Error(), path) {
|
||||
t.Fatalf("error = %v, want chunk UTF-8 path error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesAllowsAnyMediaTypeWhenSlotDoesNotRestrictIt(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
path := filepath.Join(configDir, "roster.reference")
|
||||
writeReferenceFile(t, path, []byte("plain text"))
|
||||
|
||||
resolved := resolvedPipelineWithReference(t, "roster", "roster.reference", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
|
||||
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||
}
|
||||
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
|
||||
if item.MediaType != unknownMediaType {
|
||||
t.Fatalf("MediaType = %q, want %q", item.MediaType, unknownMediaType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesAcceptsDeclaredMarkdownMediaType(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
path := filepath.Join(configDir, "glossary.md")
|
||||
writeReferenceFile(t, path, []byte("# Terms\n"))
|
||||
|
||||
slot := contracts.ReferenceSlot{Name: "glossary", AcceptedMediaTypes: []string{"text/markdown"}}
|
||||
resolved := resolvedPipelineWithReference(t, "glossary", "glossary.md", contracts.ReferenceBindingSourceConfig, slot)
|
||||
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||
}
|
||||
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["glossary"].Items[0]
|
||||
if item.MediaType != "text/markdown" {
|
||||
t.Fatalf("MediaType = %q, want text/markdown", item.MediaType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesAcceptsDeclaredJSONMediaType(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
path := filepath.Join(configDir, "roster.json")
|
||||
writeReferenceFile(t, path, []byte(`{"aria":"cleric"}`))
|
||||
|
||||
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"application/json"}}
|
||||
resolved := resolvedPipelineWithReference(t, "roster", "roster.json", contracts.ReferenceBindingSourceConfig, slot)
|
||||
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||
}
|
||||
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
|
||||
if item.MediaType != "application/json" {
|
||||
t.Fatalf("MediaType = %q, want application/json", item.MediaType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesAcceptsDeclaredYAMLMediaType(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
path := filepath.Join(configDir, "roster.yaml")
|
||||
writeReferenceFile(t, path, []byte("aria: cleric\n"))
|
||||
|
||||
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"application/yaml"}}
|
||||
resolved := resolvedPipelineWithReference(t, "roster", "roster.yaml", contracts.ReferenceBindingSourceConfig, slot)
|
||||
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||
}
|
||||
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
|
||||
if item.MediaType != "application/yaml" {
|
||||
t.Fatalf("MediaType = %q, want application/yaml", item.MediaType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesRejectsUnacceptedMediaType(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
path := filepath.Join(configDir, "roster.json")
|
||||
writeReferenceFile(t, path, []byte(`{"aria":"cleric"}`))
|
||||
|
||||
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"text/markdown"}}
|
||||
resolved := resolvedPipelineWithReference(t, "roster", "roster.json", contracts.ReferenceBindingSourceConfig, slot)
|
||||
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "media type") || !strings.Contains(err.Error(), "application/json") || !strings.Contains(err.Error(), "roster") {
|
||||
t.Fatalf("error = %v, want media type rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesMatchesAcceptedMediaTypesIgnoringParameters(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
path := filepath.Join(configDir, "roster.txt")
|
||||
writeReferenceFile(t, path, []byte("Aria\n"))
|
||||
|
||||
slot := contracts.ReferenceSlot{Name: "roster", AcceptedMediaTypes: []string{"text/plain; charset=utf-8"}}
|
||||
resolved := resolvedPipelineWithReference(t, "roster", "roster.txt", contracts.ReferenceBindingSourceConfig, slot)
|
||||
materialized, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||
}
|
||||
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
|
||||
if item.MediaType != referenceMediaType {
|
||||
t.Fatalf("MediaType = %q, want %q", item.MediaType, referenceMediaType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesRejectsUnacceptedMediaTypeForNormalizeTarget(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
path := filepath.Join(configDir, "notes.json")
|
||||
writeReferenceFile(t, path, []byte(`{"notes":true}`))
|
||||
|
||||
slot := contracts.ReferenceSlot{Name: "normalization_notes", AcceptedMediaTypes: []string{"text/markdown"}}
|
||||
resolved := resolvedPipelineWithTargetReference(t, StageNormalize, "events", "normalization_notes", "notes.json", contracts.ReferenceBindingSourceConfig, slot)
|
||||
_, _, err := MaterializeReferences(resolved, referenceCatalogForTargets(t, nil, nil, nil, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "normalize") || !strings.Contains(err.Error(), "media type") || !strings.Contains(err.Error(), "application/json") || !strings.Contains(err.Error(), "normalization_notes") {
|
||||
t.Fatalf("error = %v, want normalize media type rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesWarnsForEmptyFiles(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
path := filepath.Join(configDir, "empty.txt")
|
||||
writeReferenceFile(t, path, nil)
|
||||
|
||||
resolved := resolvedPipelineWithReference(t, "roster", "empty.txt", contracts.ReferenceBindingSourceConfig, contracts.ReferenceSlot{Name: "roster"})
|
||||
materialized, warnings, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{{Name: "roster"}}), ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||
}
|
||||
if len(warnings) != 1 || warnings[0].ReasonCode != "empty_reference" {
|
||||
t.Fatalf("warnings = %#v, want empty reference warning", warnings)
|
||||
}
|
||||
item := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["roster"].Items[0]
|
||||
if item.SizeBytes != 0 || item.Digest != referenceDigest(nil) {
|
||||
t.Fatalf("empty item = %#v, want zero size and empty digest", item)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesWarningScopesIncludeTargetContext(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
writeReferenceFile(t, filepath.Join(configDir, "chunk.txt"), nil)
|
||||
writeReferenceFile(t, filepath.Join(configDir, "extract.txt"), nil)
|
||||
writeReferenceFile(t, filepath.Join(configDir, "normalize.txt"), nil)
|
||||
|
||||
profile := baselineProfile()
|
||||
profile.References = map[string]string{
|
||||
"scene_guide": "chunk.txt",
|
||||
"roster": "extract.txt",
|
||||
"normalization_notes": "normalize.txt",
|
||||
}
|
||||
catalog := referenceCatalogForTargets(t,
|
||||
[]contracts.ReferenceSlot{{Name: "scene_guide"}},
|
||||
[]contracts.ReferenceSlot{{Name: "roster"}},
|
||||
nil,
|
||||
[]contracts.ReferenceSlot{{Name: "normalization_notes"}},
|
||||
)
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
_, warnings, err := MaterializeReferences(resolved, catalog, ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("MaterializeReferences() error = %v, want nil", err)
|
||||
}
|
||||
got := warningScopes(warnings)
|
||||
want := []string{
|
||||
"pipeline.baseline.chunk.reference.scene_guide",
|
||||
"pipeline.baseline.lane.events.extract.reference.roster",
|
||||
"pipeline.baseline.lane.events.normalize.reference.normalization_notes",
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("warning scopes = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaterializeReferencesEnforcesMaxBytes(t *testing.T) {
|
||||
configDir := t.TempDir()
|
||||
path := filepath.Join(configDir, "large.txt")
|
||||
writeReferenceFile(t, path, []byte("too large"))
|
||||
|
||||
slot := contracts.ReferenceSlot{Name: "roster", MaxBytes: 3}
|
||||
resolved := resolvedPipelineWithReference(t, "roster", "large.txt", contracts.ReferenceBindingSourceConfig, slot)
|
||||
_, _, err := MaterializeReferences(resolved, referenceCatalog(t, []contracts.ReferenceSlot{slot}), ReferenceMaterializationOptions{
|
||||
ConfigPath: filepath.Join(configDir, "config.yml"),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "9 bytes") || !strings.Contains(err.Error(), "limit 3") || !strings.Contains(err.Error(), "roster") {
|
||||
t.Fatalf("error = %v, want max bytes error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func resolvedPipelineWithReference(t *testing.T, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
|
||||
t.Helper()
|
||||
return resolvedPipelineWithTargetReference(t, StageExtract, "events", slotName, source, bindingSource, slot)
|
||||
}
|
||||
|
||||
func resolvedPipelineWithTargetReference(t *testing.T, stage ModuleStage, laneID string, slotName, source, bindingSource string, slot contracts.ReferenceSlot) ResolvedPipeline {
|
||||
t.Helper()
|
||||
profile := baselineProfile()
|
||||
switch stage {
|
||||
case StageChunk:
|
||||
profile.Chunk.References = map[string]string{slotName: source}
|
||||
case StageExtract:
|
||||
lane := profile.Artifacts[laneID]
|
||||
lane.References = map[string]string{slotName: source}
|
||||
profile.Artifacts[laneID] = lane
|
||||
case StageMerge:
|
||||
lane := profile.Artifacts[laneID]
|
||||
lane.Merge.References = map[string]string{slotName: source}
|
||||
profile.Artifacts[laneID] = lane
|
||||
case StageNormalize:
|
||||
lane := profile.Artifacts[laneID]
|
||||
lane.Normalize.References = map[string]string{slotName: source}
|
||||
profile.Artifacts[laneID] = lane
|
||||
default:
|
||||
t.Fatalf("unsupported reference target stage %q", stage)
|
||||
}
|
||||
catalog := referenceCatalogForStage(t, stage, []contracts.ReferenceSlot{slot})
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||
}
|
||||
if bindingSource != contracts.ReferenceBindingSourceConfig {
|
||||
switch stage {
|
||||
case StageChunk:
|
||||
resolved.ChunkReferences.Bindings[0].BindingSource = bindingSource
|
||||
case StageExtract:
|
||||
resolved.ArtifactLanes[0].ExtractReferences.Bindings[0].BindingSource = bindingSource
|
||||
case StageMerge:
|
||||
resolved.ArtifactLanes[0].MergeReferences.Bindings[0].BindingSource = bindingSource
|
||||
case StageNormalize:
|
||||
resolved.ArtifactLanes[0].NormalizeReferences.Bindings[0].BindingSource = bindingSource
|
||||
}
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
|
||||
func referenceCatalog(t *testing.T, slots []contracts.ReferenceSlot) ModuleCatalog {
|
||||
t.Helper()
|
||||
return referenceCatalogForStage(t, StageExtract, slots)
|
||||
}
|
||||
|
||||
func referenceCatalogForStage(t *testing.T, stage ModuleStage, slots []contracts.ReferenceSlot) ModuleCatalog {
|
||||
t.Helper()
|
||||
switch stage {
|
||||
case StageChunk:
|
||||
return referenceCatalogForTargets(t, slots, nil, nil, nil)
|
||||
case StageExtract:
|
||||
return referenceCatalogForTargets(t, nil, slots, nil, nil)
|
||||
case StageMerge:
|
||||
return referenceCatalogForTargets(t, nil, nil, slots, nil)
|
||||
case StageNormalize:
|
||||
return referenceCatalogForTargets(t, nil, nil, nil, slots)
|
||||
default:
|
||||
t.Fatalf("unsupported reference target stage %q", stage)
|
||||
return ModuleCatalog{}
|
||||
}
|
||||
}
|
||||
|
||||
func referenceCatalogForTargets(t *testing.T, chunkSlots, extractSlots, mergeSlots, normalizeSlots []contracts.ReferenceSlot) ModuleCatalog {
|
||||
t.Helper()
|
||||
return newProfileCatalogWithOverrides(t,
|
||||
ModuleSpec{
|
||||
Key: "generic",
|
||||
Stage: StageChunk,
|
||||
Requires: []string{"source"},
|
||||
Provides: []string{"chunk"},
|
||||
ReferenceSlots: chunkSlots,
|
||||
},
|
||||
ModuleSpec{
|
||||
Key: "event-extractor",
|
||||
Stage: StageExtract,
|
||||
Requires: []string{"chunk"},
|
||||
Provides: []string{"candidate"},
|
||||
ReferenceSlots: extractSlots,
|
||||
},
|
||||
ModuleSpec{
|
||||
Key: "appendorder",
|
||||
Stage: StageMerge,
|
||||
Requires: []string{"candidate"},
|
||||
Provides: []string{"merged"},
|
||||
ReferenceSlots: mergeSlots,
|
||||
},
|
||||
ModuleSpec{
|
||||
Key: "noop",
|
||||
Stage: StageNormalize,
|
||||
Requires: []string{"merged"},
|
||||
Provides: []string{"normalized"},
|
||||
ReferenceSlots: normalizeSlots,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func warningScopes(warnings []contracts.Warning) []string {
|
||||
scopes := make([]string, 0, len(warnings))
|
||||
for _, warning := range warnings {
|
||||
scopes = append(scopes, warning.Scope)
|
||||
}
|
||||
sort.Strings(scopes)
|
||||
return scopes
|
||||
}
|
||||
|
||||
func writeReferenceFile(t *testing.T, path string, content []byte) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, content, 0o644); err != nil {
|
||||
t.Fatalf("write reference %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
@@ -5,11 +5,8 @@ import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
|
||||
validate "gitea.maximumdirect.net/eric/notarius/internal/framework/validate"
|
||||
)
|
||||
|
||||
func TestRunnerUsesRegistries(t *testing.T) {
|
||||
@@ -33,11 +30,11 @@ func TestRunnerUsesRegistries(t *testing.T) {
|
||||
if !reflect.DeepEqual(executed, []string{"extract-first:chunk-0", "extract-second:chunk-0"}) {
|
||||
t.Fatalf("executed = %#v, want extractor chunk execution", executed)
|
||||
}
|
||||
if got := artifactKeys(output.Approved); !reflect.DeepEqual(got, []string{"extract-first"}) {
|
||||
t.Fatalf("approved keys = %#v, want [extract-first]", got)
|
||||
if got := normalizeOutputKeys(output.NormalizeOutputs); !reflect.DeepEqual(got, []string{"normalize", "normalize"}) {
|
||||
t.Fatalf("normalize output keys = %#v, want one output from each lane", got)
|
||||
}
|
||||
if got := rejectedKeys(output.Rejected); !reflect.DeepEqual(got, []string{"extract-second"}) {
|
||||
t.Fatalf("rejected keys = %#v, want [extract-second]", got)
|
||||
if len(output.Rejected) != 0 {
|
||||
t.Fatalf("len(Rejected) = %d, want none", len(output.Rejected))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,12 +61,8 @@ func integrationRegistries(t *testing.T, built, executed *[]string) Registries {
|
||||
}); err != nil {
|
||||
t.Fatalf("register chunker: %v", err)
|
||||
}
|
||||
registerIntegrationExtractor(t, registries.Extractors, "extract-first", built, executed, []contracts.Validator{
|
||||
integrationValidator{name: "approve-first", approve: true},
|
||||
})
|
||||
registerIntegrationExtractor(t, registries.Extractors, "extract-second", built, executed, []contracts.Validator{
|
||||
integrationValidator{name: "reject-second", approve: false},
|
||||
})
|
||||
registerIntegrationExtractor(t, registries.Extractors, "extract-first", built, executed)
|
||||
registerIntegrationExtractor(t, registries.Extractors, "extract-second", built, executed)
|
||||
if err := registries.Mergers.Register("merge", func() (contracts.Merger, error) {
|
||||
*built = append(*built, "merge")
|
||||
return integrationMerger{}, nil
|
||||
@@ -91,12 +84,12 @@ func integrationRegistries(t *testing.T, built, executed *[]string) Registries {
|
||||
return registries
|
||||
}
|
||||
|
||||
func registerIntegrationExtractor(t *testing.T, registry *ExtractorRegistry, key string, built, executed *[]string, validators []contracts.Validator) {
|
||||
func registerIntegrationExtractor(t *testing.T, registry *ExtractorRegistry, key string, built, executed *[]string) {
|
||||
t.Helper()
|
||||
|
||||
if err := registry.Register(key, func() (contracts.Extractor, error) {
|
||||
*built = append(*built, key)
|
||||
return integrationExtractor{key: key, executed: executed, validators: validators}, nil
|
||||
return integrationExtractor{key: key, executed: executed}, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("Register(%q) error = %v, want nil", key, err)
|
||||
}
|
||||
@@ -118,46 +111,49 @@ func (chunker integrationChunker) Key() string {
|
||||
return "chunk"
|
||||
}
|
||||
|
||||
func (chunker integrationChunker) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (chunker integrationChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
|
||||
return contracts.ChunkResult{
|
||||
Chunks: []contracts.SourceChunk{
|
||||
{
|
||||
ID: "chunk-0",
|
||||
SourceID: req.Source.ID,
|
||||
Index: 0,
|
||||
Units: req.Source.Units,
|
||||
ID: "chunk-0",
|
||||
SourceID: req.Source.ID,
|
||||
Index: 0,
|
||||
StartUnitID: req.Source.Units[0].ID,
|
||||
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
|
||||
Content: []byte(`{"units":[1]}`),
|
||||
MediaType: "application/json",
|
||||
Units: req.Source.Units,
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type integrationExtractor struct {
|
||||
key string
|
||||
executed *[]string
|
||||
validators []contracts.Validator
|
||||
key string
|
||||
executed *[]string
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) Key() string {
|
||||
return extractor.key
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) ArtifactType() string {
|
||||
return "generic-artifact"
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) SchemaVersion() string {
|
||||
return "v1"
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) Validators() []contracts.Validator {
|
||||
return extractor.validators
|
||||
func (extractor integrationExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (extractor integrationExtractor) Extract(ctx context.Context, req contracts.ExtractionRequest) (contracts.ExtractionResult, error) {
|
||||
*extractor.executed = append(*extractor.executed, extractor.key+":"+req.Chunk.ID)
|
||||
return contracts.ExtractionResult{
|
||||
Candidates: []artifacts.ArtifactCandidate{
|
||||
{Payload: []byte(`{"value":true}`)},
|
||||
Output: contracts.ExtractOutput{
|
||||
Schema: contracts.ResponseSchema{ID: "integration", Name: "integration", Version: "v1"},
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"value":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -171,19 +167,39 @@ func (merger integrationMerger) Key() string {
|
||||
}
|
||||
|
||||
func (merger integrationMerger) Merge(ctx context.Context, req contracts.MergeRequest) (contracts.MergeResult, error) {
|
||||
var candidates []artifacts.ArtifactCandidate
|
||||
for _, chunkArtifacts := range req.ChunkArtifacts {
|
||||
candidates = append(candidates, chunkArtifacts.Candidates...)
|
||||
output := contracts.MergeOutput{
|
||||
LaneID: req.LaneID,
|
||||
Schema: contracts.ResponseSchema{ID: "integration", Name: "integration", Version: "v1"},
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"merged":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
return contracts.MergeResult{Candidates: candidates}, nil
|
||||
if len(req.ExtractOutputs) > 0 {
|
||||
output.SourceID = req.ExtractOutputs[0].SourceID
|
||||
output.Schema = req.ExtractOutputs[0].Schema
|
||||
output.Payload = req.ExtractOutputs[0].Payload
|
||||
}
|
||||
return contracts.MergeResult{Output: output}, nil
|
||||
}
|
||||
|
||||
func (normalizer integrationNormalizer) Key() string {
|
||||
return "normalize"
|
||||
}
|
||||
|
||||
func (normalizer integrationNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (normalizer integrationNormalizer) Normalize(ctx context.Context, req contracts.NormalizeRequest) (contracts.NormalizeResult, error) {
|
||||
return contracts.NormalizeResult{Candidates: req.Candidates}, nil
|
||||
return contracts.NormalizeResult{
|
||||
Output: contracts.NormalizeOutput{
|
||||
LaneID: req.LaneID,
|
||||
SourceID: req.MergeOutput.SourceID,
|
||||
Schema: req.MergeOutput.Schema,
|
||||
Payload: req.MergeOutput.Payload,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type integrationOutput struct{}
|
||||
@@ -200,30 +216,6 @@ func (output integrationOutput) Encode(ctx context.Context, req contracts.Output
|
||||
}, nil
|
||||
}
|
||||
|
||||
type integrationValidator struct {
|
||||
name string
|
||||
approve bool
|
||||
}
|
||||
|
||||
func (validator integrationValidator) Name() string {
|
||||
return validator.name
|
||||
}
|
||||
|
||||
func (validator integrationValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (contracts.ValidationResult, error) {
|
||||
decisions := make([]contracts.ValidationDecision, 0, len(req.Candidates))
|
||||
for _, candidate := range req.Candidates {
|
||||
if validator.approve {
|
||||
decisions = append(decisions, validate.Approved(candidate.Index))
|
||||
} else {
|
||||
decisions = append(decisions, validate.Rejected(candidate.Index, "invalid", "not accepted"))
|
||||
}
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
ValidatorName: validator.name,
|
||||
Decisions: decisions,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func integrationPipeline() ResolvedPipeline {
|
||||
return ResolvedPipeline{
|
||||
ID: "pipeline-1",
|
||||
@@ -255,23 +247,15 @@ func integrationSourceDocument() *source.SourceDocument {
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:abc123",
|
||||
Units: []source.SourceUnit{
|
||||
{ID: "u1", Kind: "unit", Text: "Source unit."},
|
||||
{ID: 1, Kind: "unit", Text: "Source unit."},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func artifactKeys(approved []artifacts.Artifact) []string {
|
||||
keys := make([]string, 0, len(approved))
|
||||
for _, artifact := range approved {
|
||||
keys = append(keys, artifact.ExtractorKey)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
func rejectedKeys(rejected []artifacts.RejectedArtifact) []string {
|
||||
keys := make([]string, 0, len(rejected))
|
||||
for _, artifact := range rejected {
|
||||
keys = append(keys, artifact.Candidate.ExtractorKey)
|
||||
func normalizeOutputKeys(outputs []contracts.NormalizeOutput) []string {
|
||||
keys := make([]string, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
keys = append(keys, output.NormalizerKey)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -2,15 +2,15 @@
|
||||
"id": "fixture-source",
|
||||
"units": [
|
||||
{
|
||||
"id": "u1",
|
||||
"id": 1,
|
||||
"text": "First event."
|
||||
},
|
||||
{
|
||||
"id": "u2",
|
||||
"id": 2,
|
||||
"text": "Second event."
|
||||
},
|
||||
{
|
||||
"id": "u3",
|
||||
"id": 3,
|
||||
"text": "Third event."
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"manifest": {
|
||||
"pipeline_id": "walking-skeleton",
|
||||
"pipeline_digest": "sha256:5df1e501a2307ef75bbfeb59d315b3710571d52e5466a9c7f8320248740e6fca",
|
||||
"pipeline_digest": "sha256:75c1f6d64d86666734ccf175cdb82ae90e0ba5be3194be952850d7c00c66e615",
|
||||
"validation_status": "approved",
|
||||
"artifact_lanes": [
|
||||
{
|
||||
@@ -12,40 +12,31 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"approved": [
|
||||
"normalize_outputs": [
|
||||
{
|
||||
"extractor_key": "fake/extract",
|
||||
"artifact_type": "fake_event",
|
||||
"schema_version": "v1",
|
||||
"payload": {
|
||||
"chunk_id": "fixture-source:chunk:0",
|
||||
"llm_call": 1,
|
||||
"text": "First event. Second event."
|
||||
"lane_id": "events",
|
||||
"normalizer_key": "noop",
|
||||
"source_id": "fixture-source",
|
||||
"schema": {
|
||||
"id": "fake_event",
|
||||
"name": "fake_event",
|
||||
"version": "v1"
|
||||
},
|
||||
"source_refs": [
|
||||
{
|
||||
"source_id": "fixture-source",
|
||||
"start_unit_id": "u1",
|
||||
"end_unit_id": "u2"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"extractor_key": "fake/extract",
|
||||
"artifact_type": "fake_event",
|
||||
"schema_version": "v1",
|
||||
"payload": {
|
||||
"chunk_id": "fixture-source:chunk:1",
|
||||
"llm_call": 2,
|
||||
"text": "Third event."
|
||||
},
|
||||
"source_refs": [
|
||||
{
|
||||
"source_id": "fixture-source",
|
||||
"start_unit_id": "u3",
|
||||
"end_unit_id": "u3"
|
||||
}
|
||||
]
|
||||
"media_type": "application/json",
|
||||
"content": {
|
||||
"outputs": [
|
||||
{
|
||||
"chunk_id": "fixture-source:chunk:0",
|
||||
"llm_call": 1,
|
||||
"text": "First event. Second event."
|
||||
},
|
||||
{
|
||||
"chunk_id": "fixture-source:chunk:1",
|
||||
"llm_call": 2,
|
||||
"text": "Third event."
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
110
internal/framework/pipeline/validator_chain_registry.go
Normal file
110
internal/framework/pipeline/validator_chain_registry.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type validatorChainKey struct {
|
||||
stage ModuleStage
|
||||
module string
|
||||
}
|
||||
|
||||
type ValidatorChainMapping struct {
|
||||
Stage ModuleStage `json:"stage"`
|
||||
Module string `json:"module"`
|
||||
Validators []ModuleBinding `json:"validators,omitempty"`
|
||||
}
|
||||
|
||||
type ValidatorChainRegistry struct {
|
||||
chains map[validatorChainKey][]ModuleBinding
|
||||
}
|
||||
|
||||
func NewValidatorChainRegistry() *ValidatorChainRegistry {
|
||||
return &ValidatorChainRegistry{
|
||||
chains: make(map[validatorChainKey][]ModuleBinding),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *ValidatorChainRegistry) Register(mapping ValidatorChainMapping) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("validator chain registry must not be nil")
|
||||
}
|
||||
normalized, err := normalizeValidatorChainMapping(mapping)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if r.chains == nil {
|
||||
r.chains = make(map[validatorChainKey][]ModuleBinding)
|
||||
}
|
||||
key := validatorChainKey{stage: normalized.Stage, module: normalized.Module}
|
||||
if _, exists := r.chains[key]; exists {
|
||||
return fmt.Errorf("validator chain for %q %q is already registered", normalized.Stage, normalized.Module)
|
||||
}
|
||||
r.chains[key] = cloneModuleBindings(normalized.Validators)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ValidatorChainRegistry) Validators(stage ModuleStage, module string) []ModuleBinding {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
chain := r.chains[validatorChainKey{stage: stage, module: strings.TrimSpace(module)}]
|
||||
return cloneModuleBindings(chain)
|
||||
}
|
||||
|
||||
func normalizeValidatorChainMapping(mapping ValidatorChainMapping) (ValidatorChainMapping, error) {
|
||||
normalized := ValidatorChainMapping{
|
||||
Stage: mapping.Stage,
|
||||
Module: strings.TrimSpace(mapping.Module),
|
||||
Validators: cloneModuleBindings(mapping.Validators),
|
||||
}
|
||||
switch normalized.Stage {
|
||||
case StageChunk, StageExtract, StageMerge, StageNormalize:
|
||||
default:
|
||||
return ValidatorChainMapping{}, fmt.Errorf("validator chain stage %q is not supported", normalized.Stage)
|
||||
}
|
||||
if normalized.Module == "" {
|
||||
return ValidatorChainMapping{}, fmt.Errorf("validator chain module key must not be empty")
|
||||
}
|
||||
for i, validator := range normalized.Validators {
|
||||
if strings.TrimSpace(validator.Module) == "" {
|
||||
return ValidatorChainMapping{}, fmt.Errorf("validator chain for %q %q has empty validator key at index %d", normalized.Stage, normalized.Module, i)
|
||||
}
|
||||
normalized.Validators[i] = resolveBinding(validator, "")
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func cloneModuleBindings(bindings []ModuleBinding) []ModuleBinding {
|
||||
if len(bindings) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]ModuleBinding, len(bindings))
|
||||
for i, binding := range bindings {
|
||||
out[i] = cloneModuleBinding(binding)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
|
||||
binding.Module = strings.TrimSpace(binding.Module)
|
||||
binding.LLMProfile = strings.TrimSpace(binding.LLMProfile)
|
||||
binding.Options = cloneOptions(binding.Options)
|
||||
if len(binding.References) > 0 {
|
||||
references := make(map[string]string, len(binding.References))
|
||||
for key, value := range binding.References {
|
||||
references[key] = value
|
||||
}
|
||||
binding.References = references
|
||||
}
|
||||
binding.Validators = cloneValidatorOverride(binding.Validators)
|
||||
return binding
|
||||
}
|
||||
|
||||
func cloneValidatorOverride(override ValidatorOverride) ValidatorOverride {
|
||||
return ValidatorOverride{
|
||||
Set: override.Set,
|
||||
Validators: cloneModuleBindings(override.Validators),
|
||||
}
|
||||
}
|
||||
74
internal/framework/pipeline/validator_chain_registry_test.go
Normal file
74
internal/framework/pipeline/validator_chain_registry_test.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package pipeline
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestValidatorChainRegistryRegistersAndLooksUpChains(t *testing.T) {
|
||||
registry := NewValidatorChainRegistry()
|
||||
err := registry.Register(ValidatorChainMapping{
|
||||
Stage: StageExtract,
|
||||
Module: " extractor ",
|
||||
Validators: []ModuleBinding{{Module: " first "}, {Module: "second"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
got := registry.Validators(StageExtract, " extractor ")
|
||||
if len(got) != 2 || got[0].Module != "first" || got[1].Module != "second" {
|
||||
t.Fatalf("Validators() = %#v, want trimmed chain", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorChainRegistryRejectsDuplicateMappings(t *testing.T) {
|
||||
registry := NewValidatorChainRegistry()
|
||||
mapping := ValidatorChainMapping{Stage: StageMerge, Module: "merge", Validators: []ModuleBinding{Binding("validator")}}
|
||||
if err := registry.Register(mapping); err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if err := registry.Register(mapping); err == nil {
|
||||
t.Fatal("Register() error = nil, want duplicate mapping error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorChainRegistryRejectsUnsupportedStage(t *testing.T) {
|
||||
registry := NewValidatorChainRegistry()
|
||||
err := registry.Register(ValidatorChainMapping{Stage: StageInput, Module: "input"})
|
||||
if err == nil {
|
||||
t.Fatal("Register() error = nil, want unsupported stage error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorChainRegistryAllowsAbsentAndEmptyChains(t *testing.T) {
|
||||
registry := NewValidatorChainRegistry()
|
||||
if got := registry.Validators(StageNormalize, "normalize"); got != nil {
|
||||
t.Fatalf("absent chain = %#v, want nil", got)
|
||||
}
|
||||
if err := registry.Register(ValidatorChainMapping{Stage: StageNormalize, Module: "normalize"}); err != nil {
|
||||
t.Fatalf("Register(empty) error = %v, want nil", err)
|
||||
}
|
||||
if got := registry.Validators(StageNormalize, "normalize"); got != nil {
|
||||
t.Fatalf("empty chain = %#v, want nil", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidatorChainRegistryReturnsDefensiveCopies(t *testing.T) {
|
||||
registry := NewValidatorChainRegistry()
|
||||
err := registry.Register(ValidatorChainMapping{
|
||||
Stage: StageChunk,
|
||||
Module: "chunk",
|
||||
Validators: []ModuleBinding{{Module: "validator", Options: map[string]any{"level": "strict"}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Register() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
got := registry.Validators(StageChunk, "chunk")
|
||||
got[0].Module = "changed"
|
||||
got[0].Options["level"] = "changed"
|
||||
|
||||
again := registry.Validators(StageChunk, "chunk")
|
||||
if again[0].Module != "validator" || again[0].Options["level"] != "strict" {
|
||||
t.Fatalf("Validators() after caller mutation = %#v, want original chain", again)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user