Compare commits
8 Commits
304c68f9fc
...
ae65b95374
| Author | SHA1 | Date | |
|---|---|---|---|
| ae65b95374 | |||
| a5bbfea9b9 | |||
| ae9c2e1d5e | |||
| 1d3a444df8 | |||
| f044c00a7c | |||
| 7d89c2702b | |||
| 93653cccb8 | |||
| a024492dbf |
21
docs/cli.md
21
docs/cli.md
@@ -21,7 +21,7 @@ ID in config or with `--llm-profile`.
|
||||
|
||||
```text
|
||||
notarius help
|
||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--session-id id] [--reference selector=path] [--without-reference selector]
|
||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
|
||||
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||
notarius pipelines list --config path/to/config.yml [--json]
|
||||
```
|
||||
@@ -41,10 +41,12 @@ Flags:
|
||||
`NOTARIUS_CONFIG`, then `/usr/local/etc/notarius/config.yml`.
|
||||
- `--only lane-a,lane-b`: run only the named artifact lanes. Values are
|
||||
comma-separated and must be non-empty.
|
||||
- `--resume`: reuse valid workspace checkpoints for this invocation. Requires
|
||||
an effective workspace directory and `workspace.resume.enabled: true`.
|
||||
- `--output-dir path`: output root. The run writes to `<path>/<run-id>/`.
|
||||
Defaults to `./notarius-output`.
|
||||
- `--diagnostics-dir path`: diagnostics work directory override for this
|
||||
invocation.
|
||||
invocation. It does not change the workspace directory.
|
||||
- `--llm-profile id`: override every effective LLM-capable pipeline module
|
||||
binding to use one Scriptorium profile ID. Validator-specific profiles are
|
||||
not overridden. Configured LLM-backed validators with explicit profiles are
|
||||
@@ -140,6 +142,21 @@ go run ./cmd/notarius run dnd-session \
|
||||
--session-id campaign-17-session-04
|
||||
```
|
||||
|
||||
Use `--resume` to reuse valid checkpoints from a previous compatible
|
||||
invocation:
|
||||
|
||||
```sh
|
||||
go run ./cmd/notarius run dnd-session \
|
||||
--config examples/dnd-spells.config.yml \
|
||||
--input examples/seriatim-minimal-transcript.json \
|
||||
--resume
|
||||
```
|
||||
|
||||
Plain `run` does not skip completed work. It executes the pipeline normally and
|
||||
refreshes checkpoints when checkpointing is enabled. `--resume` verifies each
|
||||
checkpoint before reuse and executes any missing, corrupt, or incompatible step
|
||||
normally.
|
||||
|
||||
For durable output, diagnostics, retention, and failure inspection, see
|
||||
[Operations](operations.md).
|
||||
|
||||
|
||||
102
docs/config.md
102
docs/config.md
@@ -42,6 +42,7 @@ The maintained fixture is [examples/dnd-spells.config.yml](../examples/dnd-spell
|
||||
- `scriptorium`: optional Scriptorium profile source settings.
|
||||
- `pipelines`: optional map of pipeline IDs to pipeline definitions.
|
||||
- `concurrency`: optional global concurrency settings.
|
||||
- `workspace`: optional workspace settings for Notarius-owned local state.
|
||||
- `diagnostics`: optional diagnostics settings.
|
||||
|
||||
Unknown YAML fields are rejected. The removed top-level `llm_profiles` field is
|
||||
@@ -57,8 +58,19 @@ concurrency:
|
||||
diagnostics:
|
||||
work_dir: /tmp/notarius
|
||||
retention: auto
|
||||
workspace:
|
||||
diagnostics:
|
||||
enabled: true
|
||||
resume:
|
||||
enabled: false
|
||||
debug:
|
||||
enabled: false
|
||||
```
|
||||
|
||||
`workspace.directory` is unset by default. Without a workspace directory,
|
||||
diagnostics continue to use `/tmp/notarius`, and checkpoint and debug workspace
|
||||
features have no storage root.
|
||||
|
||||
No pipelines are built in. A run requires a configured pipeline.
|
||||
|
||||
If `scriptorium` is omitted, Notarius uses Scriptorium's built-in profile
|
||||
@@ -98,10 +110,20 @@ These environment variables are applied after the config file:
|
||||
|
||||
- `NOTARIUS_CONFIG`: config discovery path.
|
||||
- `NOTARIUS_TOTAL_LLM_CONCURRENCY`: integer global LLM concurrency.
|
||||
- `NOTARIUS_WORK_DIR`: diagnostics work directory.
|
||||
- `NOTARIUS_DIAGNOSTICS_RETENTION`: diagnostics retention mode.
|
||||
- `NOTARIUS_WORKSPACE_DIR`: workspace directory.
|
||||
- `NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED`: boolean diagnostics enablement.
|
||||
- `NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION`: workspace diagnostics retention
|
||||
mode.
|
||||
- `NOTARIUS_WORKSPACE_RESUME_ENABLED`: boolean resume checkpointing
|
||||
enablement.
|
||||
- `NOTARIUS_WORKSPACE_DEBUG_ENABLED`: boolean debug artifact enablement.
|
||||
- `NOTARIUS_WORK_DIR`: deprecated diagnostics work directory compatibility
|
||||
override.
|
||||
- `NOTARIUS_DIAGNOSTICS_RETENTION`: deprecated diagnostics retention
|
||||
compatibility override.
|
||||
|
||||
Integer environment values must parse as base-10 integers.
|
||||
Integer environment values must parse as base-10 integers. Boolean environment
|
||||
values must parse as Go booleans such as `true`, `false`, `1`, or `0`.
|
||||
|
||||
The removed `NOTARIUS_LLM_DEFAULT_*` variables are not read. Configure provider
|
||||
endpoint, model, and credential environment variable names through Scriptorium
|
||||
@@ -323,19 +345,85 @@ Both modules accept UTF-8 plain text, Markdown, YAML, or JSON reference files.
|
||||
The extractor uses references only as supporting disambiguation material; spell
|
||||
casts still must be present in the source transcript.
|
||||
|
||||
## Workspace
|
||||
|
||||
`workspace` fields:
|
||||
|
||||
- `directory`: optional workspace root for Notarius-owned local state.
|
||||
- `diagnostics.enabled`: set to `false` to skip diagnostics run directories and
|
||||
diagnostics artifact writes. Default: `true`.
|
||||
- `diagnostics.retention`: `auto`, `always`, or `never`.
|
||||
- `resume.enabled`: boolean resume checkpointing setting. Default: `false`.
|
||||
- `debug.enabled`: boolean debug artifact setting. Default: `false`.
|
||||
|
||||
Use `/var/lib/notarius` as the standard production workspace directory. For
|
||||
local development, prefer a project-local ignored path such as
|
||||
`./.notarius/workspace`.
|
||||
|
||||
```yaml
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
enabled: true
|
||||
retention: auto
|
||||
resume:
|
||||
enabled: false
|
||||
debug:
|
||||
enabled: false
|
||||
```
|
||||
|
||||
When `workspace.directory` is set, diagnostics are written under
|
||||
`<workspace.directory>/diagnostics/`.
|
||||
|
||||
When both `workspace.directory` and `workspace.resume.enabled` are set, runs
|
||||
write stage-owned checkpoint artifacts under
|
||||
`<workspace.directory>/checkpoints/`. `notarius run --resume` can reuse valid
|
||||
checkpoints from a compatible invocation. Checkpoints may contain source text,
|
||||
intermediate raw outputs, rejected outputs, metadata, and warnings. Protect the
|
||||
workspace as sensitive local state.
|
||||
|
||||
When both `workspace.directory` and `workspace.debug.enabled` are set, runs
|
||||
write per-invocation debug artifacts under
|
||||
`<workspace.directory>/debug/<run-id>/`. Debug artifacts may contain source
|
||||
material, reference material, prompt inputs, model outputs, validation payloads,
|
||||
and other sensitive content. Debug is disabled by default.
|
||||
|
||||
`workspace.resume.enabled` and `workspace.debug.enabled` are independent.
|
||||
Enabling one does not enable the other.
|
||||
|
||||
## Diagnostics
|
||||
|
||||
Preferred workspace diagnostics fields:
|
||||
|
||||
- `workspace.directory`: workspace root for Notarius-owned local state.
|
||||
- `workspace.diagnostics.enabled`: set to `false` to skip creating diagnostics
|
||||
run directories and diagnostics artifacts. Default: `true`.
|
||||
- `workspace.diagnostics.retention`: `auto`, `always`, or `never`.
|
||||
|
||||
When `workspace.directory` is set, diagnostics use
|
||||
`<workspace.directory>/diagnostics` as their work directory.
|
||||
`workspace.diagnostics.retention` overrides legacy diagnostics retention when
|
||||
set.
|
||||
|
||||
`diagnostics` fields:
|
||||
|
||||
- `work_dir`: directory for per-run diagnostics. Default: `/tmp/notarius`.
|
||||
- `retention`: `auto`, `always`, or `never`. Empty uses `auto`.
|
||||
- `work_dir`: deprecated compatibility directory for per-run diagnostics.
|
||||
Default: `/tmp/notarius`.
|
||||
- `retention`: deprecated compatibility retention mode. `auto`, `always`, or
|
||||
`never`. Empty uses `auto`.
|
||||
|
||||
Existing `diagnostics.work_dir`, `diagnostics.retention`, `NOTARIUS_WORK_DIR`,
|
||||
and `NOTARIUS_DIAGNOSTICS_RETENTION` inputs remain supported for compatibility.
|
||||
New configuration should use `workspace.directory` and
|
||||
`workspace.diagnostics.retention` instead.
|
||||
|
||||
`auto` retains diagnostics for failed runs and successful runs with warnings.
|
||||
`always` retains diagnostics for every run. `never` removes diagnostics for
|
||||
successful runs without regard to warnings; failed runs are retained.
|
||||
|
||||
The `--diagnostics-dir` run flag overrides `diagnostics.work_dir` for that
|
||||
invocation.
|
||||
The `--diagnostics-dir` run flag overrides the effective diagnostics work
|
||||
directory for that invocation. It affects diagnostics only and does not change
|
||||
the workspace directory.
|
||||
|
||||
## Validation
|
||||
|
||||
|
||||
@@ -22,6 +22,11 @@ Diagnostics must not expose secrets.
|
||||
If `workDir` is empty, it defaults to `/tmp/notarius`. Empty retention defaults
|
||||
to `auto`.
|
||||
|
||||
The CLI passes the effective diagnostics root from workspace configuration.
|
||||
When `workspace.directory` is set and diagnostics are enabled, that root is
|
||||
`<workspace.directory>/diagnostics`. The legacy diagnostics work directory and
|
||||
`--diagnostics-dir` still pass a diagnostics-only root to this constructor.
|
||||
|
||||
The writer makes the work directory if needed, then attempts to create a unique
|
||||
run directory. It retries run ID creation a bounded number of times if a
|
||||
collision occurs.
|
||||
@@ -34,6 +39,7 @@ Implemented artifact names:
|
||||
- `effective-config.json`
|
||||
- `resolved-pipeline.json`
|
||||
- `resolved-references.json`
|
||||
- `checkpoint-events.json`
|
||||
- `source-document.json`
|
||||
- `run-manifest.json`
|
||||
- `run-report.json`
|
||||
@@ -69,8 +75,13 @@ Retention is decided by `ShouldRetainRunDirectory`.
|
||||
|
||||
## CLI Failure Behavior
|
||||
|
||||
The CLI creates the diagnostics run directory after config loading and before
|
||||
pipeline resolution. Failures before that point do not have diagnostics.
|
||||
When diagnostics are enabled, the CLI creates the diagnostics run directory
|
||||
after config loading and before pipeline resolution. Failures before that point
|
||||
do not have diagnostics.
|
||||
|
||||
When workspace diagnostics are explicitly disabled, the CLI does not create a
|
||||
diagnostics run directory and skips diagnostics artifact writes. Failures are
|
||||
still printed to stderr.
|
||||
|
||||
After diagnostics creation, run failures call `WriteErrorLog` and apply
|
||||
retention with `RunSucceeded: false`, so the run directory remains available.
|
||||
@@ -87,3 +98,5 @@ manifest before logging the failure.
|
||||
information needed for recovery.
|
||||
- Durable output file contracts belong to output modules and integration docs,
|
||||
not to diagnostics.
|
||||
- Checkpoint and debug workspace files are separate framework-owned artifacts,
|
||||
not diagnostics artifacts.
|
||||
|
||||
@@ -29,6 +29,10 @@ belongs in modules, not in command handlers.
|
||||
diagnostics artifact writers, atomic writes, and retention decisions.
|
||||
- `internal/core/source`: source documents, source units, source references, and
|
||||
validation.
|
||||
- `internal/core/workspace`: effective workspace roots, enabled-state helpers,
|
||||
safe workspace-relative path construction, atomic workspace artifact writes,
|
||||
checkpoint identities, checkpoint path construction, and checkpoint manifest
|
||||
types.
|
||||
|
||||
Core packages should remain deterministic and concrete. They should not import
|
||||
production modules.
|
||||
@@ -38,9 +42,12 @@ production modules.
|
||||
- `internal/framework/contracts`: interfaces and request/result structs for
|
||||
input adapters, chunkers, extractors, mergers, normalizers, validators, output
|
||||
encoders, and structured LLM clients.
|
||||
- `internal/framework/checkpoint`: workspace-backed checkpoint recorder and
|
||||
checkpoint payload envelope serialization.
|
||||
- `internal/framework/debug`: workspace-backed debug artifact writer.
|
||||
- `internal/framework/pipeline`: module registries, module specs, profile
|
||||
resolution, capability checks, run orchestration, warnings, validation, and
|
||||
manifest population.
|
||||
resolution, capability checks, run orchestration, checkpoint and debug
|
||||
recorder boundaries, warnings, validation, and manifest population.
|
||||
- `internal/framework/llm`: Scriptorium-backed structured-output client,
|
||||
prompt/schema asset registry, scheduler, schema registry, and secret
|
||||
redaction.
|
||||
|
||||
@@ -71,6 +71,27 @@ to chunk, extract, merge, and normalize requests; LLM-backed modules forward it
|
||||
through their structured completion requests so Scriptorium can include it in
|
||||
prompt execution metadata.
|
||||
|
||||
When workspace resume checkpointing is enabled, the CLI constructs a checkpoint
|
||||
recorder after pipeline resolution and reference materialization and passes it
|
||||
through `pipeline.RunInput`. The runner records source, chunk, extract, merge,
|
||||
and normalize outcomes through that interface. Concrete modules do not receive
|
||||
workspace paths and do not write checkpoint files directly.
|
||||
|
||||
For `run --resume`, the CLI also passes a checkpoint loader. The runner consults
|
||||
the loader in workflow order and reuses only checkpoints whose manifest schema,
|
||||
status, identity digest, dependency fingerprints, payload files, and payload
|
||||
digests validate for the current invocation. The identity includes the resolved
|
||||
pipeline, selected lanes, source/input digest, runtime overrides that affect
|
||||
execution, and materialized reference digests. Missing or invalid checkpoints
|
||||
fall back to normal execution and are refreshed by the recorder.
|
||||
|
||||
When workspace debug output is enabled, the CLI passes a debug recorder for the
|
||||
current run ID. The runner writes framework-boundary inputs, outputs,
|
||||
structured LLM calls, validator calls, timing, and retry attempt metadata
|
||||
through that interface. Debug output is not used for resume and can contain
|
||||
sensitive source, reference, prompt, and model-output material. Concrete modules
|
||||
still do not receive workspace paths.
|
||||
|
||||
## Registries And Module Specs
|
||||
|
||||
`pipeline.Registries` holds concrete constructors for execution. A
|
||||
|
||||
@@ -19,6 +19,10 @@ go run ./cmd/notarius run dnd-session \
|
||||
The command prints a success line with the pipeline ID, normalized output count,
|
||||
rejected output count, and the output path.
|
||||
|
||||
For production, configure a workspace such as `/var/lib/notarius` and ensure the
|
||||
Notarius process can create files below it. For local development, prefer an
|
||||
ignored project-local workspace such as `./.notarius/workspace`.
|
||||
|
||||
## Output Directory
|
||||
|
||||
Durable output is written to:
|
||||
@@ -54,7 +58,16 @@ Diagnostics are written under:
|
||||
```
|
||||
|
||||
The default diagnostics work directory is `/tmp/notarius`. It can be set with
|
||||
`diagnostics.work_dir`, `NOTARIUS_WORK_DIR`, or `--diagnostics-dir`.
|
||||
`workspace.directory`, `NOTARIUS_WORKSPACE_DIR`, legacy
|
||||
`diagnostics.work_dir`, legacy `NOTARIUS_WORK_DIR`, or `--diagnostics-dir`.
|
||||
When a workspace directory is set, diagnostics are written under
|
||||
`<workspace.directory>/diagnostics/<run-id>/` unless `--diagnostics-dir`
|
||||
overrides the diagnostics work directory for that invocation.
|
||||
|
||||
Set `workspace.diagnostics.enabled: false` or
|
||||
`NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED=false` to skip diagnostics directory
|
||||
creation and diagnostics artifact writes. Concise failures are still printed to
|
||||
stderr.
|
||||
|
||||
Implemented diagnostics artifacts:
|
||||
|
||||
@@ -65,6 +78,8 @@ Implemented diagnostics artifacts:
|
||||
- `resolved-references.json`: resolved reference provenance, including target
|
||||
stage, lane ID when present, origin, digest, media type, byte size, and
|
||||
binding source, without reference content.
|
||||
- `checkpoint-events.json`: checkpoint steps that were reused or executed
|
||||
during an explicit resume invocation.
|
||||
- `run-manifest.json`: the same run manifest written to durable output when it
|
||||
is available, including top-level module metadata when present.
|
||||
- `warnings.json`: warning list.
|
||||
@@ -75,10 +90,63 @@ Implemented diagnostics artifacts:
|
||||
`source-document.json` is supported by the diagnostics writer but is not written
|
||||
by the current CLI run workflow.
|
||||
|
||||
## Checkpoints
|
||||
|
||||
When `workspace.resume.enabled: true` and `workspace.directory` is set, runs
|
||||
write checkpoints under:
|
||||
|
||||
```text
|
||||
<workspace.directory>/checkpoints/<pipeline-id>/<input-key>-<source-digest>/<pipeline-digest>/
|
||||
```
|
||||
|
||||
Each workflow step owns its own manifest and payload files. There is no
|
||||
root-level checkpoint summary. Ordinary `notarius run` invocations execute the
|
||||
pipeline normally and refresh checkpoints. `notarius run --resume` reuses valid
|
||||
checkpoints and executes any missing, invalid, or incompatible step normally.
|
||||
|
||||
Checkpoint payloads preserve byte content with base64 envelopes, media type,
|
||||
metadata, warnings, and content digests where applicable. Checkpoints do not
|
||||
include raw prompts, raw reference contents, raw LLM request payloads, or debug
|
||||
traces. They can still contain source text, intermediate extracted content,
|
||||
rejected outputs, metadata, and warnings. Treat checkpoint directories as
|
||||
sensitive local state.
|
||||
|
||||
A checkpoint is reused only when its workspace schema version, checkpoint
|
||||
identity digest, step status, dependency fingerprints, payload files, and
|
||||
payload digests match the current invocation. Changes to input bytes, resolved
|
||||
pipeline digest, selected lanes, runtime LLM profile override, or materialized
|
||||
reference digests invalidate reuse.
|
||||
|
||||
Plain `notarius run` does not reuse checkpoints. It executes the workflow and
|
||||
refreshes checkpoint files when checkpointing is enabled. `notarius run
|
||||
--resume` is the explicit reuse path.
|
||||
|
||||
## Debug
|
||||
|
||||
When `workspace.debug.enabled: true` and `workspace.directory` is set, runs
|
||||
write debug artifacts under:
|
||||
|
||||
```text
|
||||
<workspace.directory>/debug/<run-id>/
|
||||
```
|
||||
|
||||
Debug output is per invocation. It is independent of checkpointing and is not
|
||||
used for resume. Enabling debug does not write checkpoints, and enabling resume
|
||||
checkpointing does not write debug output.
|
||||
|
||||
Debug artifacts include framework-boundary inputs and outputs for source,
|
||||
chunk, extract, merge, normalize, and output work, structured LLM request and
|
||||
response data from Notarius contracts, validator requests and results, timing,
|
||||
and retry attempt metadata. Debug artifacts may contain source material,
|
||||
reference material, prompt inputs, model outputs, and other sensitive data.
|
||||
Obvious credential-shaped values and sensitive map keys are redacted, but debug
|
||||
directories should still be protected as sensitive local state.
|
||||
|
||||
## Retention
|
||||
|
||||
Diagnostics retention is configured with `diagnostics.retention`,
|
||||
`NOTARIUS_DIAGNOSTICS_RETENTION`, or the default `auto`.
|
||||
Diagnostics retention is configured with `workspace.diagnostics.retention`,
|
||||
`NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION`, legacy `diagnostics.retention`,
|
||||
legacy `NOTARIUS_DIAGNOSTICS_RETENTION`, or the default `auto`.
|
||||
|
||||
- `auto`: keep failed runs and successful runs with warnings; remove successful
|
||||
warning-free runs.
|
||||
@@ -123,13 +191,21 @@ rm -rf /tmp/notarius/run-1234567890
|
||||
rm -rf ./notarius-output/run-1234567890
|
||||
```
|
||||
|
||||
Workspace checkpoint and debug directories can also be removed when no longer
|
||||
needed. Remove exact identity or run directories, for example:
|
||||
|
||||
```sh
|
||||
rm -rf /var/lib/notarius/checkpoints/dnd-session/seriatim-abcdef123456/7890abcd1234
|
||||
rm -rf /var/lib/notarius/debug/run-1234567890
|
||||
```
|
||||
|
||||
Use exact run-directory paths. Avoid broad cleanup commands against parent
|
||||
directories unless they are part of your own operational policy.
|
||||
|
||||
## Operational Limits
|
||||
|
||||
There is no command to resume a failed run. Re-run `notarius run` after fixing
|
||||
the cause.
|
||||
If `--resume` cannot reuse a checkpoint, Notarius executes that step and writes
|
||||
a fresh checkpoint when checkpointing is enabled.
|
||||
|
||||
Provider retries and timeouts are handled by Scriptorium according to the
|
||||
selected execution profile. Pipeline module retries are controlled by module
|
||||
|
||||
@@ -1,394 +1,26 @@
|
||||
# Workspace Implementation Plan
|
||||
|
||||
This plan implements the workspace target described in
|
||||
[Workspace Roadmap](workspace.md). Follow the stages in order. Keep code changes
|
||||
focused on the current stage, and update tests and canonical docs in the same
|
||||
stage when behavior changes.
|
||||
|
||||
The intended end state is:
|
||||
|
||||
- one configurable workspace root for Notarius-owned local state;
|
||||
- diagnostics written under `workspace/diagnostics/<run-id>/` when workspace
|
||||
diagnostics are enabled;
|
||||
- resumability checkpoints written under deterministic `workspace/checkpoints/`
|
||||
paths only when resume checkpointing is enabled;
|
||||
- debug artifacts written under `workspace/debug/<run-id>/` only when debug is
|
||||
enabled;
|
||||
- stage-owned checkpoint manifests only, with no root-level checkpoint summary;
|
||||
- explicit resume behavior through `run --resume`, not implicit skipping during
|
||||
ordinary `run`.
|
||||
|
||||
## Stage 1: Add Workspace Configuration
|
||||
|
||||
Add workspace configuration while preserving legacy diagnostics configuration
|
||||
long enough for backward compatibility.
|
||||
|
||||
Implement in `internal/core/config`:
|
||||
|
||||
- Add `WorkspaceConfig` to `Config`.
|
||||
- Add nested structs:
|
||||
- `WorkspaceDiagnosticsConfig`
|
||||
- `WorkspaceResumeConfig`
|
||||
- `WorkspaceDebugConfig`
|
||||
- Add file config parsing for:
|
||||
|
||||
```yaml
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
enabled: true
|
||||
retention: auto
|
||||
resume:
|
||||
enabled: false
|
||||
debug:
|
||||
enabled: false
|
||||
```
|
||||
|
||||
- Preserve existing `diagnostics.work_dir` and `diagnostics.retention` parsing as
|
||||
deprecated compatibility input.
|
||||
- Resolve effective diagnostics behavior as follows:
|
||||
- if `workspace.directory` is set, diagnostics root is
|
||||
`<workspace.directory>/diagnostics`;
|
||||
- if `workspace.directory` is unset and legacy `diagnostics.work_dir` is set,
|
||||
use legacy diagnostics behavior unchanged;
|
||||
- if neither is set, preserve the current default diagnostics behavior for
|
||||
migration compatibility;
|
||||
- `workspace.diagnostics.retention` overrides legacy diagnostics retention
|
||||
when set;
|
||||
- legacy diagnostics retention remains accepted when workspace diagnostics
|
||||
retention is unset.
|
||||
- If `workspace.diagnostics.enabled` is explicitly false, do not create a
|
||||
diagnostics run directory and do not write diagnostics artifacts. CLI failure
|
||||
handling must still print concise errors to stderr without assuming a
|
||||
diagnostics directory exists.
|
||||
- Add environment support:
|
||||
- `NOTARIUS_WORKSPACE_DIR` sets `workspace.directory`;
|
||||
- `NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED` parses a boolean;
|
||||
- `NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION` sets workspace diagnostics
|
||||
retention;
|
||||
- `NOTARIUS_WORKSPACE_RESUME_ENABLED` parses a boolean;
|
||||
- `NOTARIUS_WORKSPACE_DEBUG_ENABLED` parses a boolean;
|
||||
- keep `NOTARIUS_WORK_DIR` and `NOTARIUS_DIAGNOSTICS_RETENTION` as deprecated
|
||||
compatibility overrides for legacy diagnostics config.
|
||||
- Keep `--diagnostics-dir` as a backwards-compatible per-invocation diagnostics
|
||||
root override. It should affect diagnostics only and should not change
|
||||
checkpoint or debug roots.
|
||||
- Update redaction/effective-config diagnostics so workspace config is included
|
||||
and no secrets are introduced.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add config default tests for workspace defaults.
|
||||
- Add file config parsing tests for nested workspace fields.
|
||||
- Add env override tests for all new env vars.
|
||||
- Add precedence tests covering workspace config, legacy diagnostics config,
|
||||
env overrides, and `--diagnostics-dir`.
|
||||
- Update redacted/effective config tests.
|
||||
|
||||
## Stage 2: Introduce Workspace Filesystem Helpers
|
||||
|
||||
Create reusable filesystem helpers for workspace state. Keep concrete module
|
||||
packages out of this layer.
|
||||
|
||||
Implement a new package, recommended path `internal/core/workspace`, with:
|
||||
|
||||
- `Config` or `Settings` describing effective workspace roots:
|
||||
- root directory;
|
||||
- diagnostics root;
|
||||
- checkpoints root;
|
||||
- debug root;
|
||||
- enabled flags.
|
||||
- path construction helpers for:
|
||||
- diagnostics run directories;
|
||||
- checkpoint identity directories;
|
||||
- debug run directories.
|
||||
- safe path helpers that reject absolute artifact names, `..`, backslashes, and
|
||||
paths that escape their intended root.
|
||||
- atomic JSON and byte-file writes using the existing diagnostics atomic-write
|
||||
behavior as the model.
|
||||
- optional shared internal helper for atomic file writes so diagnostics and
|
||||
workspace writers do not duplicate low-level write logic.
|
||||
|
||||
Do not add resume behavior in this stage.
|
||||
|
||||
Tests:
|
||||
|
||||
- Unit-test path construction and path-safety failures.
|
||||
- Unit-test atomic JSON/byte writes.
|
||||
- Unit-test disabled workspace settings returning no-op or empty roots as
|
||||
appropriate.
|
||||
- Confirm no helper permits writes outside the configured root.
|
||||
|
||||
## Stage 3: Move Diagnostics Under Workspace
|
||||
|
||||
Update diagnostics creation to use the effective diagnostics root from workspace
|
||||
configuration when workspace is configured.
|
||||
|
||||
Implementation requirements:
|
||||
|
||||
- Keep diagnostics artifacts and retention behavior unchanged.
|
||||
- Honor disabled diagnostics by using a no-op diagnostics writer or nil-safe
|
||||
diagnostics path through the CLI failure and success paths.
|
||||
- Preserve the existing `diagnostics.RunDirectory` contract unless a narrow
|
||||
constructor addition is cleaner.
|
||||
- Route normal workspace diagnostics to:
|
||||
|
||||
```text
|
||||
<workspace.directory>/diagnostics/<run-id>/
|
||||
```
|
||||
|
||||
- Preserve legacy behavior when only legacy diagnostics config is present.
|
||||
- Preserve `--diagnostics-dir` behavior as diagnostics-only override.
|
||||
- Keep diagnostics run IDs run-based and non-deterministic.
|
||||
- Do not write checkpoint or debug output in this stage.
|
||||
|
||||
Docs to update after behavior exists:
|
||||
|
||||
- `docs/config.md`
|
||||
- `docs/operations.md`
|
||||
- `docs/internal/diagnostics.md`
|
||||
- `docs/cli.md` if CLI help text changes.
|
||||
|
||||
Tests:
|
||||
|
||||
- Update diagnostics run directory tests for workspace diagnostics roots.
|
||||
- Update CLI tests that inspect diagnostics paths.
|
||||
- Add migration tests proving legacy diagnostics config still works.
|
||||
- Run focused checks:
|
||||
- `go test ./internal/core/config`
|
||||
- `go test ./internal/core/diagnostics`
|
||||
- `go test ./internal/cli`
|
||||
|
||||
## Stage 4: Define Checkpoint Identity And Stage Manifest Types
|
||||
|
||||
Add checkpoint identity and manifest data structures before writing stage
|
||||
payloads.
|
||||
|
||||
Implement in `internal/core/workspace` or a closely related core package:
|
||||
|
||||
- `CheckpointIdentity`, derived from:
|
||||
- resolved pipeline ID;
|
||||
- resolved pipeline digest;
|
||||
- input adapter key;
|
||||
- raw input digest or source digest;
|
||||
- selected lanes;
|
||||
- runtime overrides that affect execution;
|
||||
- materialized reference digests;
|
||||
- prompt/schema/profile provenance not already represented by the pipeline
|
||||
digest.
|
||||
- A deterministic, filesystem-safe checkpoint path:
|
||||
|
||||
```text
|
||||
<workspace.directory>/checkpoints/<pipeline-id>/<input-key>-<source-digest-prefix>/<pipeline-digest-prefix>/
|
||||
```
|
||||
|
||||
- Stage manifest structs for:
|
||||
- source;
|
||||
- chunk;
|
||||
- extract lane;
|
||||
- merge lane;
|
||||
- normalize lane.
|
||||
- Shared manifest fields:
|
||||
- workspace schema version;
|
||||
- stage name;
|
||||
- lane ID when applicable;
|
||||
- module key;
|
||||
- dependency fingerprints;
|
||||
- status;
|
||||
- output digests;
|
||||
- validation status and rejection summaries where applicable;
|
||||
- started/completed timestamps where useful.
|
||||
- Status values:
|
||||
- `pending`;
|
||||
- `running`;
|
||||
- `succeeded`;
|
||||
- `succeeded_with_rejections`;
|
||||
- `failed`;
|
||||
- `invalidated`.
|
||||
|
||||
Do not add a root-level checkpoint manifest.
|
||||
|
||||
Tests:
|
||||
|
||||
- Unit-test deterministic identity generation.
|
||||
- Unit-test identity changes when pipeline digest, input digest, selected lanes,
|
||||
or reference digests change.
|
||||
- Unit-test manifest JSON round trips.
|
||||
- Unit-test filesystem-safe path generation.
|
||||
|
||||
## Stage 5: Write Checkpoints Without Resuming
|
||||
|
||||
Add write-only checkpoint support behind `workspace.resume.enabled`.
|
||||
|
||||
Implementation requirements:
|
||||
|
||||
- Add a framework-owned checkpoint recorder to pipeline execution. Recommended
|
||||
shape:
|
||||
- CLI constructs the effective workspace/checkpoint recorder after pipeline
|
||||
resolution and reference materialization.
|
||||
- `pipeline.RunInput` receives a recorder interface or no-op recorder.
|
||||
- concrete modules do not receive workspace paths and do not write directly to
|
||||
workspace.
|
||||
- Write checkpoint artifacts only when `workspace.resume.enabled` is true.
|
||||
- Use stage-owned manifests and no root summary.
|
||||
- Use JSON envelope types for checkpointed payloads that preserve byte content,
|
||||
such as `content_base64`, media type, metadata, warnings, and content digest.
|
||||
Do not rely on existing runtime structs whose byte fields are tagged
|
||||
`json:"-"`.
|
||||
- Recommended checkpoint files:
|
||||
|
||||
```text
|
||||
checkpoints/<identity>/source/manifest.json
|
||||
checkpoints/<identity>/source/source-document.json
|
||||
checkpoints/<identity>/chunk/manifest.json
|
||||
checkpoints/<identity>/chunk/chunks.json
|
||||
checkpoints/<identity>/extract/<lane-id>/manifest.json
|
||||
checkpoints/<identity>/extract/<lane-id>/outputs.json
|
||||
checkpoints/<identity>/merge/<lane-id>/manifest.json
|
||||
checkpoints/<identity>/merge/<lane-id>/output.json
|
||||
checkpoints/<identity>/normalize/<lane-id>/manifest.json
|
||||
checkpoints/<identity>/normalize/<lane-id>/output.json
|
||||
```
|
||||
|
||||
- Record rejected extract outputs as checkpointed stage outcomes. Under current
|
||||
pipeline policy they do not pass downstream, but they are not framework errors.
|
||||
- Mark a stage `running` before writing its payloads, then atomically replace the
|
||||
manifest with a final success/failure status after payload writes complete.
|
||||
- Ensure interrupted or partial writes cannot be mistaken for successful
|
||||
checkpoints.
|
||||
- Do not write raw prompts, raw references, raw LLM request payloads, or debug
|
||||
traces in checkpoint output.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add runner/CLI tests proving no checkpoint files are written when resume is
|
||||
disabled.
|
||||
- Add tests proving checkpoint files are written when resume is enabled.
|
||||
- Add tests for successful runs, rejected extract outputs, failed stages, and
|
||||
warning-only validation.
|
||||
- Add tests proving checkpoint payload content digests match written content.
|
||||
- Add tests proving modules do not receive filesystem paths.
|
||||
|
||||
## Stage 6: Add Explicit Resume Reads
|
||||
|
||||
Add explicit resume behavior after write-only checkpoints are stable.
|
||||
|
||||
CLI behavior:
|
||||
|
||||
- Add `run --resume`.
|
||||
- `--resume` should require `workspace.resume.enabled: true`; otherwise return a
|
||||
clear configuration error.
|
||||
- Plain `run` should continue to execute stages normally and should not silently
|
||||
skip completed stages.
|
||||
|
||||
Resume behavior:
|
||||
|
||||
- Load and validate checkpoint stage manifests in workflow order.
|
||||
- Reuse a checkpoint only when:
|
||||
- workspace schema version is supported;
|
||||
- stage status is successful for the current purpose;
|
||||
- dependency fingerprints match the current invocation;
|
||||
- referenced checkpoint payload files exist;
|
||||
- payload digests match manifest digests;
|
||||
- selected lanes and runtime overrides are compatible.
|
||||
- If a checkpoint is missing or invalid, execute that stage normally and write a
|
||||
fresh checkpoint if resume checkpointing remains enabled.
|
||||
- If a source or chunk checkpoint is reused, downstream dependency fingerprints
|
||||
must still be validated before downstream reuse.
|
||||
- If an extract checkpoint includes rejected outputs, preserve those rejection
|
||||
records and continue to omit rejected outputs from merge input.
|
||||
- If merge is reused, skip merge execution only for the matching lane.
|
||||
- If normalize is reused, pass the reused normalized output to the output stage.
|
||||
- Always create fresh diagnostics for the current invocation, including records
|
||||
showing which stages were reused versus executed.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add CLI tests for `--resume` without workspace resume enabled.
|
||||
- Add tests for reusing source, chunk, extract, merge, and normalize
|
||||
checkpoints.
|
||||
- Add tests for invalidation when input, pipeline digest, references, selected
|
||||
lanes, or runtime LLM profile override changes.
|
||||
- Add tests for corrupt or missing checkpoint payloads.
|
||||
- Add tests proving fresh diagnostics are written for resumed invocations.
|
||||
|
||||
## Stage 7: Add Debug Output
|
||||
|
||||
Add debug output behind `workspace.debug.enabled`.
|
||||
|
||||
Implementation requirements:
|
||||
|
||||
- Write debug artifacts under:
|
||||
|
||||
```text
|
||||
<workspace.directory>/debug/<run-id>/
|
||||
```
|
||||
|
||||
- Debug is per-invocation and should not be used for resume.
|
||||
- Debug and resume are independent:
|
||||
- debug enabled does not imply checkpoint writing;
|
||||
- resume enabled does not imply debug writing.
|
||||
- Start with artifacts available from Notarius framework boundaries:
|
||||
- source/checkpoint-like stage inputs and outputs;
|
||||
- structured LLM request inputs and response content from Notarius contracts;
|
||||
- validator requests and results;
|
||||
- stage timing and attempt metadata.
|
||||
- Do not depend on Scriptorium internals for rendered upstream provider
|
||||
requests. If Scriptorium later exposes rendered prompt traces safely, add them
|
||||
in a separate pass.
|
||||
- Never write API keys, bearer tokens, or raw provider credentials.
|
||||
- Document clearly that debug output may contain source material, references,
|
||||
prompts, model outputs, and other sensitive content.
|
||||
|
||||
Tests:
|
||||
|
||||
- Add tests proving no debug files are written when debug is disabled.
|
||||
- Add tests proving debug files are written under `debug/<run-id>/` when enabled.
|
||||
- Add tests proving debug and resume can be enabled independently.
|
||||
- Add tests proving obvious secrets are not written.
|
||||
- Add diagnostics/debug path tests to ensure path roots do not overlap
|
||||
accidentally.
|
||||
|
||||
## Stage 8: Documentation, Examples, And Cleanup
|
||||
|
||||
After behavior is implemented, update canonical current-behavior docs.
|
||||
|
||||
Update:
|
||||
|
||||
- `docs/config.md`
|
||||
- `docs/cli.md`
|
||||
- `docs/operations.md`
|
||||
- `docs/troubleshooting.md`
|
||||
- `docs/internal/diagnostics.md`
|
||||
- `docs/internal/pipeline.md`
|
||||
- maintained examples under `examples/` when practical.
|
||||
|
||||
Documentation requirements:
|
||||
|
||||
- Document workspace config and defaults.
|
||||
- Document legacy diagnostics config compatibility and any deprecation language.
|
||||
- Document `/var/lib/notarius` as the production recommendation.
|
||||
- Document local development recommendations.
|
||||
- Document checkpoint sensitivity and debug sensitivity.
|
||||
- Document explicit resume behavior and invalidation rules.
|
||||
- Keep future or deferred behavior only in roadmap files.
|
||||
|
||||
Cleanup:
|
||||
|
||||
- Update `docs/roadmap/workspace.md` to a completed-status note after the
|
||||
implementation lands.
|
||||
- Remove or replace this implementation plan with a completed note after the
|
||||
implementation lands.
|
||||
|
||||
Validation:
|
||||
|
||||
```sh
|
||||
go test ./internal/core/config
|
||||
go test ./internal/core/diagnostics
|
||||
go test ./internal/core/workspace
|
||||
go test ./internal/framework/pipeline
|
||||
go test ./internal/cli
|
||||
go test ./...
|
||||
go vet ./...
|
||||
go build ./cmd/notarius
|
||||
```
|
||||
# Workspace Implementation Status
|
||||
|
||||
The workspace implementation described by this roadmap has landed. Current
|
||||
behavior is documented in the canonical current-behavior docs:
|
||||
|
||||
- [Configuration](../config.md)
|
||||
- [CLI Reference](../cli.md)
|
||||
- [Operations](../operations.md)
|
||||
- [Troubleshooting](../troubleshooting.md)
|
||||
- [Diagnostics Internals](../internal/diagnostics.md)
|
||||
- [Pipeline Internals](../internal/pipeline.md)
|
||||
|
||||
Implemented behavior includes:
|
||||
|
||||
- `workspace.directory` as the root for Notarius-owned local state;
|
||||
- workspace diagnostics under `<workspace.directory>/diagnostics/<run-id>/`;
|
||||
- compatibility for legacy `diagnostics.work_dir`, `diagnostics.retention`,
|
||||
`NOTARIUS_WORK_DIR`, and `NOTARIUS_DIAGNOSTICS_RETENTION`;
|
||||
- checkpoint writes under `<workspace.directory>/checkpoints/` when resume
|
||||
checkpointing is enabled;
|
||||
- explicit checkpoint reuse through `notarius run --resume`;
|
||||
- debug artifacts under `<workspace.directory>/debug/<run-id>/` when debug
|
||||
output is enabled;
|
||||
- independent resume and debug settings.
|
||||
|
||||
Deferred workspace ideas remain in [Workspace Roadmap](workspace.md).
|
||||
|
||||
@@ -1,55 +1,11 @@
|
||||
# Workspace Roadmap
|
||||
# Workspace Roadmap Status
|
||||
|
||||
Notarius should gain a single configurable workspace root for application-owned
|
||||
local state. The workspace exists only to support enabled workspace features;
|
||||
ordinary runs should not write workspace files by default.
|
||||
The local workspace feature has been implemented. Current behavior is documented
|
||||
in [Configuration](../config.md), [CLI Reference](../cli.md),
|
||||
[Operations](../operations.md), and the relevant internal docs.
|
||||
|
||||
The workspace is distinct from durable output:
|
||||
|
||||
- durable output is the user-requested final product of a run;
|
||||
- the workspace is application-owned local state for diagnostics, resumable
|
||||
checkpoints, and development/debug workflows.
|
||||
|
||||
Within the workspace, diagnostics, checkpoints, and debug artifacts should use
|
||||
separate subdirectories because they have different identity and lifecycle
|
||||
models.
|
||||
|
||||
## Target Configuration
|
||||
|
||||
Workspace behavior should be configured under a dedicated top-level object:
|
||||
|
||||
```yaml
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
enabled: true
|
||||
retention: auto
|
||||
resume:
|
||||
enabled: false
|
||||
debug:
|
||||
enabled: false
|
||||
```
|
||||
|
||||
Policy:
|
||||
|
||||
- `workspace.diagnostics.enabled` defaults to the current diagnostics behavior
|
||||
during migration.
|
||||
- `workspace.resume.enabled` defaults to `false`.
|
||||
- `workspace.debug.enabled` defaults to `false`.
|
||||
- no checkpoint or debug files are written when those features are disabled.
|
||||
- `/var/lib/notarius` should be the standard production recommendation in docs
|
||||
and examples.
|
||||
- local development docs may recommend a project-local path such as
|
||||
`./.notarius/workspace`.
|
||||
|
||||
`workspace.directory` should become the single configured root for Notarius local
|
||||
state. Existing `diagnostics.work_dir` behavior should be migrated carefully for
|
||||
backward compatibility, but the long-term configuration model should avoid two
|
||||
separate roots for application-owned local state.
|
||||
|
||||
## Workspace Layout
|
||||
|
||||
The workspace root should contain separate subtrees:
|
||||
The implemented workspace provides one configurable root for Notarius-owned
|
||||
local state:
|
||||
|
||||
```text
|
||||
<workspace.directory>/
|
||||
@@ -58,176 +14,13 @@ The workspace root should contain separate subtrees:
|
||||
debug/
|
||||
```
|
||||
|
||||
Diagnostics remain run-id based. They are invocation history and should preserve
|
||||
the existing diagnostics retention model:
|
||||
|
||||
```text
|
||||
<workspace.directory>/
|
||||
diagnostics/
|
||||
<run-id>/
|
||||
invocation.json
|
||||
effective-config.json
|
||||
resolved-pipeline.json
|
||||
resolved-references.json
|
||||
run-manifest.json
|
||||
warnings.json
|
||||
run-report.json
|
||||
error.log
|
||||
```
|
||||
|
||||
Checkpoints should be deterministic and should include enough identity material
|
||||
to prevent unsafe reuse across incompatible runs:
|
||||
|
||||
```text
|
||||
<workspace.directory>/
|
||||
checkpoints/
|
||||
<pipeline-id>/
|
||||
<input-key>-<source-digest-prefix>/
|
||||
<pipeline-digest-prefix>/
|
||||
source/
|
||||
chunk/
|
||||
extract/<lane-id>/
|
||||
merge/<lane-id>/
|
||||
normalize/<lane-id>/
|
||||
```
|
||||
|
||||
Checkpoint identity should account for:
|
||||
|
||||
- resolved pipeline ID;
|
||||
- resolved pipeline digest;
|
||||
- input adapter key;
|
||||
- source/input digest;
|
||||
- selected lanes and runtime overrides that affect execution;
|
||||
- reference digests;
|
||||
- prompt, schema, and profile provenance when not already captured by the
|
||||
resolved pipeline digest.
|
||||
|
||||
Debug output should be run-id based like diagnostics, because debug traces are
|
||||
development artifacts from a specific invocation rather than resumable state:
|
||||
|
||||
```text
|
||||
<workspace.directory>/
|
||||
debug/
|
||||
<run-id>/
|
||||
```
|
||||
|
||||
Human-readable path segments are useful, but content hashes should be
|
||||
authoritative for correctness. The exact path shape may change during
|
||||
implementation, but checkpoint identity must be stable, deterministic, and safe
|
||||
to compare across runs. Diagnostics and debug output do not need deterministic
|
||||
resume identity because they are per-invocation artifacts.
|
||||
|
||||
## Stage Ownership
|
||||
|
||||
Stage manifests should be stage-owned. There should not be a root-level manifest
|
||||
that summarizes all stages.
|
||||
|
||||
Rationale:
|
||||
|
||||
- stage manifests are the authoritative state for the stage that wrote them;
|
||||
- avoiding a root summary prevents duplicate state from drifting;
|
||||
- stage-local manifests make partial writes, failure recovery, and future stage
|
||||
invalidation easier to reason about.
|
||||
|
||||
Each stage manifest should record enough information to determine whether its
|
||||
outputs can be trusted for resume:
|
||||
|
||||
- workspace schema/version;
|
||||
- stage name and lane ID when applicable;
|
||||
- module key and relevant module provenance;
|
||||
- dependency fingerprints and input digests;
|
||||
- status such as pending, running, succeeded, succeeded with rejections, failed,
|
||||
or invalidated;
|
||||
- output content digests;
|
||||
- validation status and rejection summaries where relevant;
|
||||
- timing metadata when useful and non-sensitive.
|
||||
|
||||
## Checkpoint Output
|
||||
|
||||
When `workspace.resume.enabled` is enabled, Notarius should write under the
|
||||
`checkpoints/` subtree. Checkpoints should contain only artifacts required to
|
||||
resume safely.
|
||||
|
||||
Checkpoint artifacts may include:
|
||||
|
||||
- source document checkpoint;
|
||||
- chunk collection checkpoint;
|
||||
- accepted extract outputs;
|
||||
- rejected extract records;
|
||||
- merge output;
|
||||
- normalize output;
|
||||
- stage manifests with dependency fingerprints and output digests.
|
||||
|
||||
Checkpoint output should avoid raw prompts, raw references, raw LLM request
|
||||
payloads, and other sensitive development artifacts unless they are strictly
|
||||
required for safe resume. Prefer digests and provenance over copying sensitive
|
||||
inputs.
|
||||
|
||||
Checkpoint writes should be atomic at the file level. Partial or interrupted
|
||||
writes must not be mistaken for successful stage completion.
|
||||
|
||||
## Debug Output
|
||||
|
||||
When `workspace.debug.enabled` is enabled, Notarius should write under the
|
||||
`debug/<run-id>/` subtree. Debug artifacts are useful for inspection but not
|
||||
required for resume.
|
||||
|
||||
Debug artifacts may include:
|
||||
|
||||
- rendered prompt inputs;
|
||||
- structured LLM request and response traces;
|
||||
- raw model responses;
|
||||
- copied reference content;
|
||||
- copied source snippets;
|
||||
- intermediate raw payloads;
|
||||
- validator request/response details.
|
||||
|
||||
Debug output is explicitly sensitive. Documentation should warn that debug
|
||||
workspace data may contain source material, references, prompts, and model
|
||||
outputs. Debug output should remain disabled by default.
|
||||
|
||||
`workspace.debug.enabled` and `workspace.resume.enabled` should be independent.
|
||||
Enabling debug should not imply resume, and enabling resume should not imply
|
||||
debug.
|
||||
|
||||
## Resume Semantics
|
||||
|
||||
Resume should be explicit at first, for example through a future `resume` command
|
||||
or a `run --resume` flag. Plain `run` should not silently skip completed stages
|
||||
in the initial workspace feature.
|
||||
|
||||
Before reusing a checkpoint, Notarius should verify that:
|
||||
|
||||
- the workspace schema/version is supported;
|
||||
- the checkpoint stage manifest is complete and successful;
|
||||
- dependency fingerprints match the current invocation;
|
||||
- referenced checkpoint files exist and match recorded digests;
|
||||
- selected lanes and runtime overrides are compatible with the checkpoint.
|
||||
|
||||
Rejected extract outputs should be treated as recorded stage outcomes, not
|
||||
framework failures. Under current pipeline policy, rejected outputs do not pass
|
||||
to downstream stages.
|
||||
|
||||
## Boundaries
|
||||
|
||||
Workspace writing should be framework-owned. Concrete modules should not write
|
||||
directly to workspace paths.
|
||||
|
||||
If modules need to expose debug material later, they should return logical
|
||||
artifacts through framework contracts and let framework or CLI code perform path
|
||||
validation, redaction policy, and writes.
|
||||
|
||||
Workspace state should preserve existing Notarius boundaries:
|
||||
|
||||
- provider plumbing remains behind LLM runtime contracts;
|
||||
- prompt ownership remains with modules and prompt asset helpers;
|
||||
- diagnostics remain per-run invocation artifacts under the workspace root;
|
||||
- output modules continue to own final durable output encoding;
|
||||
- secrets and sensitive payloads are not written unless an explicit debug policy
|
||||
enables them.
|
||||
Implemented behavior includes workspace-backed diagnostics, checkpoint writing,
|
||||
explicit checkpoint reuse through `notarius run --resume`, workspace debug
|
||||
artifacts, safe workspace-relative writes, and compatibility for legacy
|
||||
diagnostics configuration.
|
||||
|
||||
## Deferred Work
|
||||
|
||||
Default-idempotent `run` behavior with a `--force` override, remote workspace
|
||||
Default-idempotent `run` behavior with a force override, remote workspace
|
||||
storage, workspace garbage collection, archival policy, and cross-machine resume
|
||||
are deferred.
|
||||
remain deferred.
|
||||
|
||||
@@ -280,6 +280,50 @@ Fix:
|
||||
- Pass `--session-id <id>` to `notarius run`.
|
||||
- Use a stable, non-secret identifier from the external orchestrator.
|
||||
|
||||
## Resume Or Checkpoint Reuse Failure
|
||||
|
||||
Symptoms include:
|
||||
|
||||
- `--resume requires workspace.resume.enabled: true`
|
||||
- `checkpoint artifact is missing`
|
||||
- `checkpoint workspace schema version`
|
||||
- `checkpoint dependency fingerprints do not match`
|
||||
- a resumed run executes work instead of reusing a checkpoint
|
||||
|
||||
Fix:
|
||||
|
||||
- Set both `workspace.directory` and `workspace.resume.enabled: true`.
|
||||
- Use `--resume`; plain `notarius run` executes normally and refreshes
|
||||
checkpoints.
|
||||
- Confirm the current run uses the same input bytes, resolved pipeline, selected
|
||||
lanes, runtime LLM profile override, and materialized references as the run
|
||||
that wrote the checkpoint.
|
||||
- Inspect retained diagnostics `checkpoint-events.json` to see which workflow
|
||||
steps were reused or executed.
|
||||
- If a checkpoint payload is missing or corrupt, rerun without relying on that
|
||||
checkpoint. Notarius executes invalidated steps normally and writes fresh
|
||||
checkpoints when checkpointing remains enabled.
|
||||
|
||||
Checkpoint files can contain source text, intermediate outputs, rejected
|
||||
outputs, metadata, and warnings. Protect the workspace directory accordingly.
|
||||
|
||||
## Debug Output Missing Or Too Verbose
|
||||
|
||||
Symptoms:
|
||||
|
||||
- no files appear under `<workspace.directory>/debug/<run-id>/`;
|
||||
- debug files contain more source, reference, prompt, or model-output material
|
||||
than expected.
|
||||
|
||||
Fix:
|
||||
|
||||
- Set both `workspace.directory` and `workspace.debug.enabled: true`.
|
||||
- Confirm you are inspecting the current run ID. Debug output is per invocation
|
||||
and is not used for resume.
|
||||
- Disable `workspace.debug.enabled` after the inspection run. Debug output may
|
||||
contain sensitive source material, reference material, prompt inputs, model
|
||||
outputs, and validation payloads.
|
||||
|
||||
## Output Write Failure
|
||||
|
||||
Symptoms include:
|
||||
|
||||
@@ -1,4 +1,19 @@
|
||||
version: 2
|
||||
# For production runs, use a writable application-owned workspace such as:
|
||||
#
|
||||
# workspace:
|
||||
# directory: /var/lib/notarius
|
||||
# diagnostics:
|
||||
# retention: auto
|
||||
# resume:
|
||||
# enabled: false
|
||||
# debug:
|
||||
# enabled: false
|
||||
#
|
||||
# For local development, use a project-local ignored path such as:
|
||||
#
|
||||
# workspace:
|
||||
# directory: ./.notarius/workspace
|
||||
pipelines:
|
||||
dnd-session:
|
||||
input: seriatim
|
||||
|
||||
@@ -2,6 +2,8 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
@@ -16,7 +18,10 @@ import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
@@ -25,7 +30,7 @@ const defaultOutputRoot = "./notarius-output"
|
||||
|
||||
const usage = `Usage:
|
||||
notarius help
|
||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--session-id id] [--reference selector=path] [--without-reference selector]
|
||||
notarius run <pipeline-id> --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector]
|
||||
notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
|
||||
notarius pipelines list --config path/to/config.yml [--json]
|
||||
`
|
||||
@@ -95,6 +100,7 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
outputDir := fs.String("output-dir", "", "output directory")
|
||||
diagnosticsDir := fs.String("diagnostics-dir", "", "diagnostics directory")
|
||||
llmProfile := fs.String("llm-profile", "", "LLM profile override")
|
||||
resume := fs.Bool("resume", false, "reuse valid workspace checkpoints")
|
||||
sessionID := sessionIDFlag{}
|
||||
referenceFlags := stringListFlag{}
|
||||
withoutReferenceFlags := stringListFlag{}
|
||||
@@ -151,15 +157,22 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
workspaceSettings := workspace.FromConfig(cfg)
|
||||
if dir := strings.TrimSpace(*diagnosticsDir); dir != "" {
|
||||
cfg.Diagnostics.WorkDir = dir
|
||||
workspaceSettings.DiagnosticsRoot = dir
|
||||
}
|
||||
|
||||
startedAt := opts.Now().UTC()
|
||||
runDir, err := diagnostics.NewRunDirectory(cfg.Diagnostics.WorkDir, cfg.Diagnostics.Retention)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
runID := fmt.Sprintf("run-%d", startedAt.UnixNano())
|
||||
var runDir *diagnostics.RunDirectory
|
||||
if workspaceSettings.DiagnosticsEnabled {
|
||||
var err error
|
||||
runDir, err = diagnostics.NewRunDirectory(workspaceSettings.DiagnosticsRoot, cfg.Diagnostics.Retention)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "notarius: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
runID = runDir.RunID()
|
||||
}
|
||||
invocation := diagnostics.InvocationMetadata{
|
||||
Operation: "run",
|
||||
@@ -168,12 +181,20 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
ConfigPath: loadedConfigPath,
|
||||
ConfigSource: configSource(*configPath),
|
||||
OnlyLanes: append([]string(nil), only...),
|
||||
RunID: runDir.RunID(),
|
||||
Resume: *resume,
|
||||
RunID: runID,
|
||||
StartedAt: startedAt,
|
||||
}
|
||||
if err := runDir.WriteInvocationMetadata(invocation); err != nil {
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteInvocationMetadata(invocation) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
|
||||
}
|
||||
if *resume && !workspaceSettings.ResumeEnabled {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("--resume requires workspace.resume.enabled: true"))
|
||||
}
|
||||
debugRecorder, err := frameworkdebug.NewWorkspaceRecorder(workspaceSettings, runID)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create debug recorder: %w", err))
|
||||
}
|
||||
|
||||
catalog, err := effectiveCatalog(opts)
|
||||
if err != nil {
|
||||
@@ -211,16 +232,18 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
}
|
||||
effective.ResolvedPipeline = materialized
|
||||
invocation.PipelineDigest = effective.ResolvedPipeline.Digest
|
||||
if err := runDir.WriteInvocationMetadata(invocation); err != nil {
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteInvocationMetadata(invocation) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics invocation metadata: %w", err))
|
||||
}
|
||||
if err := runDir.WriteRedactedEffectiveConfig(effective); err != nil {
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteRedactedEffectiveConfig(effective) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics effective config: %w", err))
|
||||
}
|
||||
if err := runDir.WriteResolvedPipeline(effective.ResolvedPipeline); err != nil {
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteResolvedPipeline(effective.ResolvedPipeline) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved pipeline: %w", err))
|
||||
}
|
||||
if err := runDir.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline)); err != nil {
|
||||
if err := writeDiagnostics(runDir, func() error {
|
||||
return runDir.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline))
|
||||
}); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics resolved references: %w", err))
|
||||
}
|
||||
|
||||
@@ -243,6 +266,10 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
|
||||
}
|
||||
checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(workspaceSettings, effective.ResolvedPipeline, rawInput, only, llmProfiles, strings.TrimSpace(*llmProfile), strings.TrimSpace(sessionID.value), *resume)
|
||||
if err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||
}
|
||||
|
||||
output, err := pipeline.New(registries).Run(ctx, pipeline.RunInput{
|
||||
Pipeline: effective.ResolvedPipeline,
|
||||
@@ -250,45 +277,56 @@ func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) i
|
||||
RawInput: rawInput,
|
||||
LLMClient: llmClient,
|
||||
SessionID: strings.TrimSpace(sessionID.value),
|
||||
RunID: runDir.RunID(),
|
||||
RunID: runID,
|
||||
StartedAt: startedAt,
|
||||
LLMProfiles: llmProfiles,
|
||||
Metadata: runMetadata(*outputDir, *diagnosticsDir),
|
||||
Warnings: referenceWarnings,
|
||||
Checkpoints: checkpointRecorder,
|
||||
Checkpoint: checkpointLoader,
|
||||
Debug: debugRecorder,
|
||||
})
|
||||
if err != nil {
|
||||
if output.Manifest.PipelineID != "" {
|
||||
if output.Manifest.PipelineID != "" && runDir != nil {
|
||||
_ = runDir.WriteRunManifest(output.Manifest)
|
||||
_ = runDir.WriteCheckpointEvents(output.CheckpointEvents)
|
||||
}
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("run pipeline %q: %w", pipelineID, err))
|
||||
}
|
||||
|
||||
runOutputDir := filepath.Join(outputRoot(*outputDir), runDir.RunID())
|
||||
if err := runDir.WriteRunManifest(output.Manifest); err != nil {
|
||||
runOutputDir := filepath.Join(outputRoot(*outputDir), runID)
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteRunManifest(output.Manifest) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics run manifest: %w", err))
|
||||
}
|
||||
if err := runDir.WriteWarnings(output.Warnings); err != nil {
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteWarnings(output.Warnings) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics warnings: %w", err))
|
||||
}
|
||||
if err := runDir.WriteRunReport(runReport{
|
||||
RunID: runDir.RunID(),
|
||||
PipelineID: effective.PipelineID,
|
||||
OutputPath: runOutputDir,
|
||||
DiagnosticsPath: runDir.Path(),
|
||||
OutputCount: len(output.NormalizeOutputs),
|
||||
RejectedCount: len(output.Rejected),
|
||||
WarningCount: len(output.Warnings),
|
||||
ValidationStatus: output.Manifest.ValidationStatus,
|
||||
if err := writeDiagnostics(runDir, func() error { return runDir.WriteCheckpointEvents(output.CheckpointEvents) }); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics checkpoint events: %w", err))
|
||||
}
|
||||
if err := writeDiagnostics(runDir, func() error {
|
||||
return runDir.WriteRunReport(runReport{
|
||||
RunID: runDir.RunID(),
|
||||
PipelineID: effective.PipelineID,
|
||||
OutputPath: runOutputDir,
|
||||
DiagnosticsPath: runDir.Path(),
|
||||
OutputCount: len(output.NormalizeOutputs),
|
||||
RejectedCount: len(output.Rejected),
|
||||
WarningCount: len(output.Warnings),
|
||||
ValidationStatus: output.Manifest.ValidationStatus,
|
||||
})
|
||||
}); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("write diagnostics run report: %w", err))
|
||||
}
|
||||
if err := writeOutputFiles(runOutputDir, output.OutputFiles); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, err)
|
||||
}
|
||||
if err := runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RetentionMode: cfg.Diagnostics.Retention,
|
||||
RunSucceeded: true,
|
||||
HasWarnings: len(output.Warnings) > 0,
|
||||
if err := writeDiagnostics(runDir, func() error {
|
||||
return runDir.ApplyRetention(diagnostics.RetentionDecisionInput{
|
||||
RetentionMode: cfg.Diagnostics.Retention,
|
||||
RunSucceeded: true,
|
||||
HasWarnings: len(output.Warnings) > 0,
|
||||
})
|
||||
}); err != nil {
|
||||
return failPipelineCommand(stderr, runDir, cfg.Diagnostics.Retention, fmt.Errorf("apply diagnostics retention: %w", err))
|
||||
}
|
||||
@@ -327,6 +365,83 @@ func failPipelineCommand(stderr io.Writer, runDir *diagnostics.RunDirectory, ret
|
||||
return 1
|
||||
}
|
||||
|
||||
func writeDiagnostics(runDir *diagnostics.RunDirectory, write func() error) error {
|
||||
if runDir == nil {
|
||||
return nil
|
||||
}
|
||||
return write()
|
||||
}
|
||||
|
||||
func checkpointHandlersForRun(
|
||||
settings workspace.Settings,
|
||||
resolved pipeline.ResolvedPipeline,
|
||||
rawInput []byte,
|
||||
only []string,
|
||||
llmProfiles []artifacts.LLMProfileManifest,
|
||||
llmProfileOverride string,
|
||||
sessionID string,
|
||||
resume bool,
|
||||
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
|
||||
identity, err := workspace.NewCheckpointIdentity(workspace.CheckpointIdentityInput{
|
||||
Pipeline: resolved,
|
||||
InputKey: resolved.Input.Module,
|
||||
RawInputDigest: rawInputDigest(rawInput),
|
||||
SelectedLanes: only,
|
||||
RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID),
|
||||
References: pipeline.ReferenceProvenance(resolved),
|
||||
ProvenanceFingerprints: llmProfileFingerprints(llmProfiles),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
|
||||
}
|
||||
recorder, err := checkpoint.NewWorkspaceRecorder(settings, identity)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create checkpoint recorder: %w", err)
|
||||
}
|
||||
loader := pipeline.NoopCheckpointLoader()
|
||||
if resume {
|
||||
loader, err = checkpoint.NewWorkspaceLoader(settings, identity)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("create checkpoint loader: %w", err)
|
||||
}
|
||||
}
|
||||
return recorder, loader, nil
|
||||
}
|
||||
|
||||
func rawInputDigest(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string) []workspace.Fingerprint {
|
||||
var values []workspace.Fingerprint
|
||||
if strings.TrimSpace(llmProfileOverride) != "" {
|
||||
values = append(values, workspace.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)})
|
||||
}
|
||||
if strings.TrimSpace(sessionID) != "" {
|
||||
values = append(values, workspace.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)})
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func llmProfileFingerprints(profiles []artifacts.LLMProfileManifest) []workspace.Fingerprint {
|
||||
if len(profiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
values := make([]workspace.Fingerprint, 0, len(profiles))
|
||||
for _, profile := range profiles {
|
||||
id := strings.TrimSpace(profile.ID)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
values = append(values, workspace.Fingerprint{
|
||||
Name: "llm_profile:" + id,
|
||||
Value: strings.TrimSpace(profile.Provider) + ":" + strings.TrimSpace(profile.Model),
|
||||
})
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func configSource(configPath string) string {
|
||||
if strings.TrimSpace(configPath) != "" {
|
||||
return "flag"
|
||||
|
||||
@@ -3,12 +3,14 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -2148,6 +2150,363 @@ func TestRunPipelineWritesDiagnosticsArtifactsOnSuccess(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineWritesWorkspaceDiagnosticsArtifactsOnSuccess(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
outputDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnostics("dnd-session", workspaceDir, "always"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
|
||||
})
|
||||
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
runDir := onlyChildDir(t, filepath.Join(workspaceDir, "diagnostics"))
|
||||
for _, name := range []string{
|
||||
diagnostics.ArtifactInvocationMetadata,
|
||||
diagnostics.ArtifactEffectiveConfig,
|
||||
diagnostics.ArtifactResolvedPipeline,
|
||||
diagnostics.ArtifactRunManifest,
|
||||
diagnostics.ArtifactRunReport,
|
||||
diagnostics.ArtifactWarnings,
|
||||
} {
|
||||
if _, err := os.Stat(filepath.Join(runDir, name)); err != nil {
|
||||
t.Fatalf("expected workspace diagnostics artifact %q: %v", name, err)
|
||||
}
|
||||
}
|
||||
assertPathNotExist(t, filepath.Join(workspaceDir, "checkpoints"))
|
||||
assertPathNotExist(t, filepath.Join(workspaceDir, "debug"))
|
||||
}
|
||||
|
||||
func TestRunPipelineWritesCheckpointsWhenWorkspaceResumeEnabled(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
outputDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
client := newFakeRunLLMClient(false)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(client, nil),
|
||||
})
|
||||
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
if client.calls != 1 {
|
||||
t.Fatalf("LLM calls = %d, want ordinary run to execute despite checkpoint writing", client.calls)
|
||||
}
|
||||
checkpointDir := onlyCheckpointIdentityDir(t, workspaceDir)
|
||||
for _, name := range []string{
|
||||
"source/manifest.json",
|
||||
"source/source-document.json",
|
||||
"chunk/manifest.json",
|
||||
"chunk/chunks.json",
|
||||
"extract/spells/manifest.json",
|
||||
"extract/spells/outputs.json",
|
||||
"merge/spells/manifest.json",
|
||||
"merge/spells/output.json",
|
||||
"normalize/spells/manifest.json",
|
||||
"normalize/spells/output.json",
|
||||
} {
|
||||
if _, err := os.Stat(filepath.Join(checkpointDir, name)); err != nil {
|
||||
t.Fatalf("expected checkpoint artifact %q: %v", name, err)
|
||||
}
|
||||
}
|
||||
assertPathNotExist(t, filepath.Join(workspaceDir, "debug"))
|
||||
}
|
||||
|
||||
func TestRunPipelineWritesDebugWhenWorkspaceDebugEnabled(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
outputDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDebugEnabled("dnd-session", workspaceDir, "always"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
|
||||
})
|
||||
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
debugDir := onlyChildDir(t, filepath.Join(workspaceDir, "debug"))
|
||||
for _, name := range []string{
|
||||
"run.json",
|
||||
"source/input.json",
|
||||
"source/output.json",
|
||||
"chunk/input.json",
|
||||
"chunk/output.json",
|
||||
"extract/spells/input.json",
|
||||
"extract/spells/output.json",
|
||||
"merge/spells/input.json",
|
||||
"merge/spells/output.json",
|
||||
"normalize/spells/input.json",
|
||||
"normalize/spells/output.json",
|
||||
"output/input.json",
|
||||
"output/output.json",
|
||||
"llm/call-0001.json",
|
||||
} {
|
||||
if _, err := os.Stat(filepath.Join(debugDir, name)); err != nil {
|
||||
t.Fatalf("expected debug artifact %q: %v", name, err)
|
||||
}
|
||||
}
|
||||
assertPathNotExist(t, filepath.Join(workspaceDir, "checkpoints"))
|
||||
}
|
||||
|
||||
func TestRunPipelineDebugAndResumeCanBeEnabledIndependently(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
outputDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnosticsAndStateEnabled("dnd-session", workspaceDir, "always"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
|
||||
})
|
||||
if code != 0 {
|
||||
t.Fatalf("seed RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
code = RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--resume"}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
|
||||
})
|
||||
if code != 0 {
|
||||
t.Fatalf("resume RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
if entries := childDirs(t, filepath.Join(workspaceDir, "checkpoints")); len(entries) != 1 {
|
||||
t.Fatalf("workspace checkpoint pipeline dirs = %v, want one", entries)
|
||||
}
|
||||
if entries := childDirs(t, filepath.Join(workspaceDir, "debug")); len(entries) != 2 {
|
||||
t.Fatalf("workspace debug run dirs = %v, want two", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineDebugRedactsObviousSecrets(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
outputDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDebugEnabled("dnd-session", workspaceDir, "always"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
client := newFakeRunLLMClient(false)
|
||||
client.payload = map[string]any{
|
||||
"spell_casts": []map[string]any{
|
||||
{
|
||||
"caster": "Aria",
|
||||
"spell": "sk-secretvalue",
|
||||
"effect": "Bearer secretvalue",
|
||||
"narrative_description": "Aria casts a spell.",
|
||||
"source_refs": []map[string]any{
|
||||
{"source_id": "session-alpha", "start_unit_id": 1, "end_unit_id": 1},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(client, nil),
|
||||
})
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
assertDebugTreeDoesNotContain(t, onlyChildDir(t, filepath.Join(workspaceDir, "debug")), "sk-secretvalue", "Bearer secretvalue")
|
||||
}
|
||||
|
||||
func TestWorkspaceStateRootsDoNotOverlap(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
outputDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnosticsAndStateEnabled("dnd-session", workspaceDir, "always"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
|
||||
})
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
assertDistinctRoots(t, filepath.Join(workspaceDir, "diagnostics"), filepath.Join(workspaceDir, "checkpoints"), filepath.Join(workspaceDir, "debug"))
|
||||
}
|
||||
|
||||
func TestRunPipelineResumeRequiresWorkspaceResumeEnabled(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
outputDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnostics("dnd-session", workspaceDir, "always"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--resume"}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
|
||||
})
|
||||
|
||||
if code != 1 {
|
||||
t.Fatalf("RunWithOptions() code = %d, want 1", code)
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "--resume requires workspace.resume.enabled: true") {
|
||||
t.Fatalf("stderr = %q, want resume configuration error", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineResumeReusesWorkspaceCheckpoints(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
outputDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
firstClient := newFakeRunLLMClient(false)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(firstClient, nil),
|
||||
})
|
||||
if code != 0 {
|
||||
t.Fatalf("first RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
if firstClient.calls != 1 {
|
||||
t.Fatalf("first LLM calls = %d, want checkpoint seed run to execute", firstClient.calls)
|
||||
}
|
||||
|
||||
secondClient := newFakeRunLLMClient(false)
|
||||
stdout.Reset()
|
||||
stderr.Reset()
|
||||
code = RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--resume"}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(secondClient, nil),
|
||||
})
|
||||
if code != 0 {
|
||||
t.Fatalf("resume RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
if secondClient.calls != 0 {
|
||||
t.Fatalf("resume LLM calls = %d, want checkpoint reuse", secondClient.calls)
|
||||
}
|
||||
runDirs := childDirs(t, filepath.Join(workspaceDir, "diagnostics"))
|
||||
if len(runDirs) != 2 {
|
||||
t.Fatalf("diagnostics run dirs = %v, want fresh diagnostics for each invocation", runDirs)
|
||||
}
|
||||
if !anyDiagnosticsFileContains(t, runDirs, diagnostics.ArtifactCheckpointEvents, `"action": "reused"`) {
|
||||
t.Fatalf("checkpoint event diagnostics under %v did not record reuse", runDirs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineResumeInvalidatesWhenInvocationIdentityChanges(t *testing.T) {
|
||||
t.Run("input", func(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always"))
|
||||
seedInput := writeSeriatimInput(t)
|
||||
changedInput := writeFile(t, "source-changed.json", `{
|
||||
"metadata": {"id": "session-alpha"},
|
||||
"segments": [{"id": 1, "start": 0, "end": 1, "speaker": "Aria", "text": "Aria casts Shield."}]
|
||||
}`)
|
||||
seedWorkspaceCheckpoint(t, configPath, seedInput, nil)
|
||||
client := runResumeWithClient(t, configPath, changedInput, nil)
|
||||
if client.calls == 0 {
|
||||
t.Fatal("resume LLM calls = 0, want execution after input identity change")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("pipeline digest", func(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
seedConfig := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always"))
|
||||
changedConfig := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeAndChunkOptions("dnd-session", workspaceDir, "always"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
seedWorkspaceCheckpoint(t, seedConfig, inputPath, nil)
|
||||
client := runResumeWithClient(t, changedConfig, inputPath, nil)
|
||||
if client.calls == 0 {
|
||||
t.Fatal("resume LLM calls = 0, want execution after pipeline digest change")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("selected lanes", func(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeLanes("dnd-session", workspaceDir, "always", "spells", "items"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
seedWorkspaceCheckpoint(t, configPath, inputPath, nil)
|
||||
client := runResumeWithClient(t, configPath, inputPath, []string{"--only", "spells"})
|
||||
if client.calls == 0 {
|
||||
t.Fatal("resume LLM calls = 0, want execution after selected lane change")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("references", func(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeEnabled("dnd-session", workspaceDir, "always"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
referencePath := writeFile(t, "players.md", "Aria is a cleric.\n")
|
||||
seedWorkspaceCheckpoint(t, configPath, inputPath, []string{"--reference", "players=" + referencePath})
|
||||
if err := os.WriteFile(referencePath, []byte("Aria is a wizard.\n"), 0o644); err != nil {
|
||||
t.Fatalf("update reference: %v", err)
|
||||
}
|
||||
client := runResumeWithClient(t, configPath, inputPath, []string{"--reference", "players=" + referencePath})
|
||||
if client.calls == 0 {
|
||||
t.Fatal("resume LLM calls = 0, want execution after reference digest change")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("llm profile override", func(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
profilePath := writeScriptoriumProfileFile(t, "runtime", "http://profile.test/v1", "test-model")
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceResumeAndProfileFile("dnd-session", workspaceDir, "always", profilePath))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
seedWorkspaceCheckpoint(t, configPath, inputPath, nil)
|
||||
client := runResumeWithClient(t, configPath, inputPath, []string{"--llm-profile", "runtime"})
|
||||
if client.calls == 0 {
|
||||
t.Fatal("resume LLM calls = 0, want execution after LLM profile override change")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunPipelineSkipsDiagnosticsWhenWorkspaceDiagnosticsDisabled(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
outputDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnosticsDisabled("dnd-session", workspaceDir))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
|
||||
})
|
||||
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(workspaceDir, "diagnostics")); !os.IsNotExist(err) {
|
||||
t.Fatalf("workspace diagnostics dir stat err = %v, want not exist", err)
|
||||
}
|
||||
if entries := childDirs(t, outputDir); len(entries) != 1 {
|
||||
t.Fatalf("output run dirs = %v, want one run dir", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineLegacyDiagnosticsConfigStillWritesDiagnostics(t *testing.T) {
|
||||
diagnosticsDir := t.TempDir()
|
||||
outputDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "always"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
|
||||
})
|
||||
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
runDir := onlyChildDir(t, diagnosticsDir)
|
||||
if _, err := os.Stat(filepath.Join(runDir, diagnostics.ArtifactRunReport)); err != nil {
|
||||
t.Fatalf("expected legacy diagnostics run report: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineWritesErrorLogAfterDiagnosticsCreation(t *testing.T) {
|
||||
diagnosticsDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithDiagnostics("dnd-session", diagnosticsDir, "always"))
|
||||
@@ -2239,6 +2598,34 @@ func TestRunPipelineDiagnosticsDirFlagOverridesConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPipelineDiagnosticsDirFlagOverridesWorkspaceDiagnosticsOnly(t *testing.T) {
|
||||
workspaceDir := filepath.Join(t.TempDir(), "workspace")
|
||||
overrideDiagnosticsDir := t.TempDir()
|
||||
outputDir := t.TempDir()
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithWorkspaceDiagnosticsAndStateEnabled("dnd-session", workspaceDir, "always"))
|
||||
inputPath := writeSeriatimInput(t)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", overrideDiagnosticsDir}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil),
|
||||
})
|
||||
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
if entries := childDirs(t, overrideDiagnosticsDir); len(entries) != 1 {
|
||||
t.Fatalf("override diagnostics dir entries = %v, want one run dir", entries)
|
||||
}
|
||||
assertPathNotExist(t, filepath.Join(workspaceDir, "diagnostics"))
|
||||
if entries := childDirs(t, filepath.Join(workspaceDir, "checkpoints")); len(entries) != 1 {
|
||||
t.Fatalf("workspace checkpoint pipeline dirs = %v, want one", entries)
|
||||
}
|
||||
if entries := childDirs(t, filepath.Join(workspaceDir, "debug")); len(entries) != 1 {
|
||||
t.Fatalf("workspace debug run dirs = %v, want one", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExampleFixtureConfigValidateAndPipelinesList(t *testing.T) {
|
||||
configPath := fixturePath(t, "examples/dnd-spells.config.yml")
|
||||
|
||||
@@ -2732,6 +3119,175 @@ pipelines:
|
||||
`
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithWorkspaceDiagnostics(pipelineID, workspaceDir, retention string) string {
|
||||
return `version: 2
|
||||
workspace:
|
||||
directory: ` + workspaceDir + `
|
||||
diagnostics:
|
||||
enabled: true
|
||||
retention: ` + retention + `
|
||||
pipelines:
|
||||
` + pipelineID + `:
|
||||
input: seriatim
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithWorkspaceDiagnosticsAndStateEnabled(pipelineID, workspaceDir, retention string) string {
|
||||
return `version: 2
|
||||
workspace:
|
||||
directory: ` + workspaceDir + `
|
||||
diagnostics:
|
||||
enabled: true
|
||||
retention: ` + retention + `
|
||||
resume:
|
||||
enabled: true
|
||||
debug:
|
||||
enabled: true
|
||||
pipelines:
|
||||
` + pipelineID + `:
|
||||
input: seriatim
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithWorkspaceResumeEnabled(pipelineID, workspaceDir, retention string) string {
|
||||
return `version: 2
|
||||
workspace:
|
||||
directory: ` + workspaceDir + `
|
||||
diagnostics:
|
||||
enabled: true
|
||||
retention: ` + retention + `
|
||||
resume:
|
||||
enabled: true
|
||||
pipelines:
|
||||
` + pipelineID + `:
|
||||
input: seriatim
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithWorkspaceDebugEnabled(pipelineID, workspaceDir, retention string) string {
|
||||
return `version: 2
|
||||
workspace:
|
||||
directory: ` + workspaceDir + `
|
||||
diagnostics:
|
||||
enabled: true
|
||||
retention: ` + retention + `
|
||||
debug:
|
||||
enabled: true
|
||||
pipelines:
|
||||
` + pipelineID + `:
|
||||
input: seriatim
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithWorkspaceResumeAndChunker(pipelineID, workspaceDir, retention, chunker string) string {
|
||||
return `version: 2
|
||||
workspace:
|
||||
directory: ` + workspaceDir + `
|
||||
diagnostics:
|
||||
enabled: true
|
||||
retention: ` + retention + `
|
||||
resume:
|
||||
enabled: true
|
||||
pipelines:
|
||||
` + pipelineID + `:
|
||||
input: seriatim
|
||||
chunk: ` + chunker + `
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithWorkspaceResumeAndChunkOptions(pipelineID, workspaceDir, retention string) string {
|
||||
return `version: 2
|
||||
workspace:
|
||||
directory: ` + workspaceDir + `
|
||||
diagnostics:
|
||||
enabled: true
|
||||
retention: ` + retention + `
|
||||
resume:
|
||||
enabled: true
|
||||
pipelines:
|
||||
` + pipelineID + `:
|
||||
input: seriatim
|
||||
chunk:
|
||||
module: generic
|
||||
options:
|
||||
max_units: 10
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithWorkspaceResumeAndProfileFile(pipelineID, workspaceDir, retention, profileFile string) string {
|
||||
return `version: 2
|
||||
scriptorium:
|
||||
profile_file: ` + profileFile + `
|
||||
workspace:
|
||||
directory: ` + workspaceDir + `
|
||||
diagnostics:
|
||||
enabled: true
|
||||
retention: ` + retention + `
|
||||
resume:
|
||||
enabled: true
|
||||
pipelines:
|
||||
` + pipelineID + `:
|
||||
input: seriatim
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithWorkspaceResumeLanes(pipelineID, workspaceDir, retention string, laneIDs ...string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("version: 2\n")
|
||||
b.WriteString("workspace:\n")
|
||||
b.WriteString(" directory: " + workspaceDir + "\n")
|
||||
b.WriteString(" diagnostics:\n")
|
||||
b.WriteString(" enabled: true\n")
|
||||
b.WriteString(" retention: " + retention + "\n")
|
||||
b.WriteString(" resume:\n")
|
||||
b.WriteString(" enabled: true\n")
|
||||
b.WriteString("pipelines:\n")
|
||||
b.WriteString(" " + pipelineID + ":\n")
|
||||
b.WriteString(" input: seriatim\n")
|
||||
b.WriteString(" artifacts:\n")
|
||||
for _, laneID := range laneIDs {
|
||||
b.WriteString(" " + laneID + ":\n")
|
||||
b.WriteString(" extract: dnd/spells\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func mvpConfigYAMLWithWorkspaceDiagnosticsDisabled(pipelineID, workspaceDir string) string {
|
||||
return `version: 2
|
||||
workspace:
|
||||
directory: ` + workspaceDir + `
|
||||
diagnostics:
|
||||
enabled: false
|
||||
pipelines:
|
||||
` + pipelineID + `:
|
||||
input: seriatim
|
||||
artifacts:
|
||||
spells:
|
||||
extract: dnd/spells
|
||||
`
|
||||
}
|
||||
|
||||
func writeSeriatimInput(t *testing.T) string {
|
||||
t.Helper()
|
||||
return writeFile(t, "source.json", `{
|
||||
@@ -3080,6 +3636,136 @@ func onlyChildDir(t *testing.T, root string) string {
|
||||
return children[0]
|
||||
}
|
||||
|
||||
func onlyCheckpointIdentityDir(t *testing.T, workspaceDir string) string {
|
||||
t.Helper()
|
||||
pipelineDir := onlyChildDir(t, filepath.Join(workspaceDir, "checkpoints"))
|
||||
inputDir := onlyChildDir(t, pipelineDir)
|
||||
return onlyChildDir(t, inputDir)
|
||||
}
|
||||
|
||||
func anyDiagnosticsFileContains(t *testing.T, runDirs []string, name string, want string) bool {
|
||||
t.Helper()
|
||||
for _, runDir := range runDirs {
|
||||
data, err := os.ReadFile(filepath.Join(runDir, name))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
t.Fatalf("read diagnostics artifact %q under %q: %v", name, runDir, err)
|
||||
}
|
||||
if strings.Contains(string(data), want) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func assertDistinctRoots(t *testing.T, roots ...string) {
|
||||
t.Helper()
|
||||
for i, first := range roots {
|
||||
for _, second := range roots[i+1:] {
|
||||
firstAbs, err := filepath.Abs(first)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve %q: %v", first, err)
|
||||
}
|
||||
secondAbs, err := filepath.Abs(second)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve %q: %v", second, err)
|
||||
}
|
||||
if firstAbs == secondAbs {
|
||||
t.Fatalf("workspace roots overlap exactly: %q", firstAbs)
|
||||
}
|
||||
firstRel, err := filepath.Rel(firstAbs, secondAbs)
|
||||
if err != nil {
|
||||
t.Fatalf("rel %q %q: %v", firstAbs, secondAbs, err)
|
||||
}
|
||||
secondRel, err := filepath.Rel(secondAbs, firstAbs)
|
||||
if err != nil {
|
||||
t.Fatalf("rel %q %q: %v", secondAbs, firstAbs, err)
|
||||
}
|
||||
if !strings.HasPrefix(firstRel, ".."+string(filepath.Separator)) && firstRel != ".." {
|
||||
t.Fatalf("workspace root %q contains %q", firstAbs, secondAbs)
|
||||
}
|
||||
if !strings.HasPrefix(secondRel, ".."+string(filepath.Separator)) && secondRel != ".." {
|
||||
t.Fatalf("workspace root %q contains %q", secondAbs, firstAbs)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var debugBase64FieldPattern = regexp.MustCompile(`"(?:content_base64|content)"\s*:\s*"([^"]*)"`)
|
||||
|
||||
func assertDebugTreeDoesNotContain(t *testing.T, root string, forbidden ...string) {
|
||||
t.Helper()
|
||||
if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
text := string(data)
|
||||
for _, value := range forbidden {
|
||||
if strings.Contains(text, value) {
|
||||
t.Fatalf("debug artifact %q contains forbidden value %q", path, value)
|
||||
}
|
||||
}
|
||||
for _, match := range debugBase64FieldPattern.FindAllStringSubmatch(text, -1) {
|
||||
decoded, err := base64.StdEncoding.DecodeString(match[1])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
decodedText := string(decoded)
|
||||
for _, value := range forbidden {
|
||||
if strings.Contains(decodedText, value) {
|
||||
t.Fatalf("debug artifact %q decoded content contains forbidden value %q", path, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("walk debug tree %q: %v", root, err)
|
||||
}
|
||||
}
|
||||
|
||||
func seedWorkspaceCheckpoint(t *testing.T, configPath string, inputPath string, extraArgs []string) {
|
||||
t.Helper()
|
||||
client := newFakeRunLLMClient(false)
|
||||
args := []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", t.TempDir()}
|
||||
args = append(args, extraArgs...)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := RunWithOptions(args, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(client, nil),
|
||||
})
|
||||
if code != 0 {
|
||||
t.Fatalf("seed RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
if client.calls == 0 {
|
||||
t.Fatal("seed LLM calls = 0, want checkpoint seed run to execute")
|
||||
}
|
||||
}
|
||||
|
||||
func runResumeWithClient(t *testing.T, configPath string, inputPath string, extraArgs []string) *fakeRunLLMClient {
|
||||
t.Helper()
|
||||
client := newFakeRunLLMClient(false)
|
||||
args := []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", t.TempDir(), "--resume"}
|
||||
args = append(args, extraArgs...)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
code := RunWithOptions(args, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(client, nil),
|
||||
})
|
||||
if code != 0 {
|
||||
t.Fatalf("resume RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
func childDirs(t *testing.T, root string) []string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(root)
|
||||
@@ -3098,6 +3784,13 @@ func childDirs(t *testing.T, root string) []string {
|
||||
return dirs
|
||||
}
|
||||
|
||||
func assertPathNotExist(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
if _, err := os.Stat(path); !os.IsNotExist(err) {
|
||||
t.Fatalf("path %q stat err = %v, want not exist", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
@@ -12,6 +15,7 @@ type Config struct {
|
||||
Pipelines map[string]pipeline.PipelineProfile `json:"pipelines"`
|
||||
Concurrency ConcurrencyConfig `json:"concurrency"`
|
||||
Diagnostics DiagnosticsConfig `json:"diagnostics"`
|
||||
Workspace WorkspaceConfig `json:"workspace"`
|
||||
}
|
||||
|
||||
type ScriptoriumConfig struct {
|
||||
@@ -28,6 +32,28 @@ type DiagnosticsConfig struct {
|
||||
Retention diagnostics.RetentionMode `json:"retention"`
|
||||
}
|
||||
|
||||
type WorkspaceConfig struct {
|
||||
Directory string `json:"directory,omitempty"`
|
||||
Diagnostics WorkspaceDiagnosticsConfig `json:"diagnostics"`
|
||||
Resume WorkspaceResumeConfig `json:"resume"`
|
||||
Debug WorkspaceDebugConfig `json:"debug"`
|
||||
}
|
||||
|
||||
type WorkspaceDiagnosticsConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Retention diagnostics.RetentionMode `json:"retention,omitempty"`
|
||||
enabledSet bool
|
||||
retentionSet bool
|
||||
}
|
||||
|
||||
type WorkspaceResumeConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
type WorkspaceDebugConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
func Default() Config {
|
||||
return Config{
|
||||
Pipelines: map[string]pipeline.PipelineProfile{},
|
||||
@@ -38,9 +64,41 @@ func Default() Config {
|
||||
WorkDir: "/tmp/notarius",
|
||||
Retention: diagnostics.RetentionAuto,
|
||||
},
|
||||
Workspace: WorkspaceConfig{
|
||||
Diagnostics: WorkspaceDiagnosticsConfig{
|
||||
Enabled: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Config) RecomputeEffectiveDiagnostics() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
if dir := c.workspaceDirectory(); dir != "" {
|
||||
c.Diagnostics.WorkDir = filepath.Join(dir, "diagnostics")
|
||||
}
|
||||
if c.Workspace.Diagnostics.retentionSet {
|
||||
c.Diagnostics.Retention = c.Workspace.Diagnostics.Retention
|
||||
}
|
||||
}
|
||||
|
||||
func (c Config) DiagnosticsEnabled() bool {
|
||||
if !c.Workspace.Diagnostics.enabledSet {
|
||||
return true
|
||||
}
|
||||
return c.Workspace.Diagnostics.Enabled
|
||||
}
|
||||
|
||||
func (c Config) workspaceDirectory() string {
|
||||
dir := strings.TrimSpace(c.Workspace.Directory)
|
||||
if dir == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Clean(dir)
|
||||
}
|
||||
|
||||
func cloneConfig(in Config) Config {
|
||||
out := in
|
||||
out.Pipelines = make(map[string]pipeline.PipelineProfile, len(in.Pipelines))
|
||||
|
||||
@@ -24,6 +24,21 @@ func TestDefaultValues(t *testing.T) {
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionAuto {
|
||||
t.Fatalf("unexpected diagnostics retention: %q", cfg.Diagnostics.Retention)
|
||||
}
|
||||
if cfg.Workspace.Directory != "" {
|
||||
t.Fatalf("unexpected workspace directory: %q", cfg.Workspace.Directory)
|
||||
}
|
||||
if !cfg.Workspace.Diagnostics.Enabled || !cfg.DiagnosticsEnabled() {
|
||||
t.Fatalf("expected workspace diagnostics enabled by default: %+v", cfg.Workspace.Diagnostics)
|
||||
}
|
||||
if cfg.Workspace.Diagnostics.Retention != "" {
|
||||
t.Fatalf("unexpected workspace diagnostics retention: %q", cfg.Workspace.Diagnostics.Retention)
|
||||
}
|
||||
if cfg.Workspace.Resume.Enabled {
|
||||
t.Fatalf("workspace resume should be disabled by default")
|
||||
}
|
||||
if cfg.Workspace.Debug.Enabled {
|
||||
t.Fatalf("workspace debug should be disabled by default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigMergesWithDefaults(t *testing.T) {
|
||||
|
||||
@@ -42,6 +42,36 @@ func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool))
|
||||
if raw, ok := lookup("NOTARIUS_DIAGNOSTICS_RETENTION"); ok {
|
||||
c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(raw))
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIR"); ok {
|
||||
c.Workspace.Directory = strings.TrimSpace(raw)
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED"); ok {
|
||||
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Workspace.Diagnostics.Enabled = value
|
||||
c.Workspace.Diagnostics.enabledSet = true
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION"); ok {
|
||||
c.Workspace.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(raw))
|
||||
c.Workspace.Diagnostics.retentionSet = true
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_RESUME_ENABLED"); ok {
|
||||
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_RESUME_ENABLED", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Workspace.Resume.Enabled = value
|
||||
}
|
||||
if raw, ok := lookup("NOTARIUS_WORKSPACE_DEBUG_ENABLED"); ok {
|
||||
value, err := parseBoolEnv("NOTARIUS_WORKSPACE_DEBUG_ENABLED", raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.Workspace.Debug.Enabled = value
|
||||
}
|
||||
c.RecomputeEffectiveDiagnostics()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -52,3 +82,11 @@ func parseIntEnv(name string, raw string) (int, error) {
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func parseBoolEnv(name string, raw string) (bool, error) {
|
||||
value, err := strconv.ParseBool(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s: must be a boolean", name)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
@@ -13,10 +13,15 @@ func TestApplyEnvOverridesOperationalValues(t *testing.T) {
|
||||
cfg.Pipelines["example"] = pipeline.PipelineProfile{ID: "example", Input: pipeline.Binding("before")}
|
||||
|
||||
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
|
||||
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "3",
|
||||
"NOTARIUS_WORK_DIR": "/tmp/notarius-env",
|
||||
"NOTARIUS_DIAGNOSTICS_RETENTION": "never",
|
||||
"NOTARIUS_PIPELINE_INPUT": "after",
|
||||
"NOTARIUS_TOTAL_LLM_CONCURRENCY": "3",
|
||||
"NOTARIUS_WORK_DIR": "/tmp/notarius-env",
|
||||
"NOTARIUS_DIAGNOSTICS_RETENTION": "never",
|
||||
"NOTARIUS_WORKSPACE_DIR": "/var/lib/notarius-env",
|
||||
"NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED": "false",
|
||||
"NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION": "always",
|
||||
"NOTARIUS_WORKSPACE_RESUME_ENABLED": "true",
|
||||
"NOTARIUS_WORKSPACE_DEBUG_ENABLED": "true",
|
||||
"NOTARIUS_PIPELINE_INPUT": "after",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyEnvOverrides: %v", err)
|
||||
@@ -28,9 +33,21 @@ func TestApplyEnvOverridesOperationalValues(t *testing.T) {
|
||||
if cfg.Concurrency.TotalLLM != 3 {
|
||||
t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM)
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/tmp/notarius-env" || cfg.Diagnostics.Retention != diagnostics.RetentionNever {
|
||||
if cfg.Workspace.Directory != "/var/lib/notarius-env" {
|
||||
t.Fatalf("unexpected workspace directory: %q", cfg.Workspace.Directory)
|
||||
}
|
||||
if cfg.DiagnosticsEnabled() {
|
||||
t.Fatalf("expected workspace diagnostics disabled")
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/var/lib/notarius-env/diagnostics" || cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
|
||||
t.Fatalf("unexpected diagnostics config: %+v", cfg.Diagnostics)
|
||||
}
|
||||
if !cfg.Workspace.Resume.Enabled {
|
||||
t.Fatalf("expected workspace resume enabled")
|
||||
}
|
||||
if !cfg.Workspace.Debug.Enabled {
|
||||
t.Fatalf("expected workspace debug enabled")
|
||||
}
|
||||
if cfg.Pipelines["example"].Input.Module != "before" {
|
||||
t.Fatalf("environment overrides must not change pipeline wiring: %+v", cfg.Pipelines["example"])
|
||||
}
|
||||
@@ -46,6 +63,41 @@ func TestApplyEnvOverridesRejectsInvalidIntegers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyEnvOverridesRejectsInvalidBooleans(t *testing.T) {
|
||||
for _, name := range []string{
|
||||
"NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED",
|
||||
"NOTARIUS_WORKSPACE_RESUME_ENABLED",
|
||||
"NOTARIUS_WORKSPACE_DEBUG_ENABLED",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := Default()
|
||||
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{name: "maybe"}))
|
||||
if err == nil || !strings.Contains(err.Error(), name) {
|
||||
t.Fatalf("expected named boolean error, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyEnvOverridesLegacyDiagnosticsRemainCompatibleWithoutWorkspace(t *testing.T) {
|
||||
cfg := Default()
|
||||
|
||||
err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
|
||||
"NOTARIUS_WORK_DIR": "/tmp/notarius-env",
|
||||
"NOTARIUS_DIAGNOSTICS_RETENTION": "never",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("ApplyEnvOverrides: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Diagnostics.WorkDir != "/tmp/notarius-env" {
|
||||
t.Fatalf("diagnostics work dir = %q, want legacy env", cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionNever {
|
||||
t.Fatalf("diagnostics retention = %q, want legacy env", cfg.Diagnostics.Retention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFromEnvUsesDefaultConfig(t *testing.T) {
|
||||
t.Setenv("NOTARIUS_TOTAL_LLM_CONCURRENCY", "2")
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ type FileConfig struct {
|
||||
Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
|
||||
Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"`
|
||||
Diagnostics *FileDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
||||
Workspace *FileWorkspaceConfig `yaml:"workspace,omitempty"`
|
||||
}
|
||||
|
||||
type FileScriptoriumConfig struct {
|
||||
@@ -50,6 +51,22 @@ type FileDiagnosticsConfig struct {
|
||||
Retention *string `yaml:"retention,omitempty"`
|
||||
}
|
||||
|
||||
type FileWorkspaceConfig struct {
|
||||
Directory *string `yaml:"directory,omitempty"`
|
||||
Diagnostics *FileWorkspaceDiagnosticsConfig `yaml:"diagnostics,omitempty"`
|
||||
Resume *FileWorkspaceEnabledConfig `yaml:"resume,omitempty"`
|
||||
Debug *FileWorkspaceEnabledConfig `yaml:"debug,omitempty"`
|
||||
}
|
||||
|
||||
type FileWorkspaceDiagnosticsConfig struct {
|
||||
Enabled *bool `yaml:"enabled,omitempty"`
|
||||
Retention *string `yaml:"retention,omitempty"`
|
||||
}
|
||||
|
||||
type FileWorkspaceEnabledConfig struct {
|
||||
Enabled *bool `yaml:"enabled,omitempty"`
|
||||
}
|
||||
|
||||
type fileModuleBinding struct {
|
||||
Module string
|
||||
LLMProfile string
|
||||
@@ -307,6 +324,28 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
|
||||
c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Diagnostics.Retention))
|
||||
}
|
||||
}
|
||||
if fileCfg.Workspace != nil {
|
||||
if fileCfg.Workspace.Directory != nil {
|
||||
c.Workspace.Directory = strings.TrimSpace(*fileCfg.Workspace.Directory)
|
||||
}
|
||||
if fileCfg.Workspace.Diagnostics != nil {
|
||||
if fileCfg.Workspace.Diagnostics.Enabled != nil {
|
||||
c.Workspace.Diagnostics.Enabled = *fileCfg.Workspace.Diagnostics.Enabled
|
||||
c.Workspace.Diagnostics.enabledSet = true
|
||||
}
|
||||
if fileCfg.Workspace.Diagnostics.Retention != nil {
|
||||
c.Workspace.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Workspace.Diagnostics.Retention))
|
||||
c.Workspace.Diagnostics.retentionSet = true
|
||||
}
|
||||
}
|
||||
if fileCfg.Workspace.Resume != nil && fileCfg.Workspace.Resume.Enabled != nil {
|
||||
c.Workspace.Resume.Enabled = *fileCfg.Workspace.Resume.Enabled
|
||||
}
|
||||
if fileCfg.Workspace.Debug != nil && fileCfg.Workspace.Debug.Enabled != nil {
|
||||
c.Workspace.Debug.Enabled = *fileCfg.Workspace.Debug.Enabled
|
||||
}
|
||||
}
|
||||
c.RecomputeEffectiveDiagnostics()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -554,6 +554,95 @@ diagnostics:
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigWorkspaceSection(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
enabled: false
|
||||
retention: never
|
||||
resume:
|
||||
enabled: true
|
||||
debug:
|
||||
enabled: true
|
||||
diagnostics:
|
||||
work_dir: /tmp/legacy
|
||||
retention: always
|
||||
`)
|
||||
|
||||
if cfg.Workspace.Directory != "/var/lib/notarius" {
|
||||
t.Fatalf("workspace directory = %q, want /var/lib/notarius", cfg.Workspace.Directory)
|
||||
}
|
||||
if cfg.DiagnosticsEnabled() {
|
||||
t.Fatalf("expected diagnostics disabled")
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/var/lib/notarius/diagnostics" {
|
||||
t.Fatalf("effective diagnostics work dir = %q, want workspace diagnostics root", cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionNever {
|
||||
t.Fatalf("effective diagnostics retention = %q, want workspace override", cfg.Diagnostics.Retention)
|
||||
}
|
||||
if !cfg.Workspace.Resume.Enabled {
|
||||
t.Fatalf("expected resume enabled")
|
||||
}
|
||||
if !cfg.Workspace.Debug.Enabled {
|
||||
t.Fatalf("expected debug enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigLegacyDiagnosticsRemainCompatible(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
diagnostics:
|
||||
work_dir: /tmp/legacy
|
||||
retention: always
|
||||
`)
|
||||
|
||||
if cfg.Workspace.Directory != "" {
|
||||
t.Fatalf("workspace directory = %q, want unset", cfg.Workspace.Directory)
|
||||
}
|
||||
if !cfg.DiagnosticsEnabled() {
|
||||
t.Fatalf("expected diagnostics enabled")
|
||||
}
|
||||
if cfg.Diagnostics.WorkDir != "/tmp/legacy" {
|
||||
t.Fatalf("effective diagnostics work dir = %q, want legacy", cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
|
||||
t.Fatalf("effective diagnostics retention = %q, want legacy", cfg.Diagnostics.Retention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFileConfigWorkspaceRetentionOverridesLegacyRetentionOnlyWhenSet(t *testing.T) {
|
||||
t.Run("legacy retained", func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
retention: never
|
||||
`)
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionNever {
|
||||
t.Fatalf("effective diagnostics retention = %q, want legacy", cfg.Diagnostics.Retention)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("workspace overrides", func(t *testing.T) {
|
||||
cfg := parseAndApplyConfig(t, `
|
||||
version: 2
|
||||
workspace:
|
||||
directory: /var/lib/notarius
|
||||
diagnostics:
|
||||
retention: always
|
||||
diagnostics:
|
||||
retention: never
|
||||
`)
|
||||
if cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
|
||||
t.Fatalf("effective diagnostics retention = %q, want workspace", cfg.Diagnostics.Retention)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func parseAndApplyConfig(t *testing.T, raw string) Config {
|
||||
t.Helper()
|
||||
fileCfg, err := ParseFileConfigYAML([]byte(raw))
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
func TestRedactedConfigCopiesScriptoriumConfig(t *testing.T) {
|
||||
cfg := Default()
|
||||
cfg.Scriptorium.ProfileDir = "./profiles"
|
||||
cfg.Workspace.Directory = "/var/lib/notarius"
|
||||
cfg.Workspace.Resume.Enabled = true
|
||||
|
||||
redacted := cfg.Redacted()
|
||||
|
||||
@@ -20,6 +22,13 @@ func TestRedactedConfigCopiesScriptoriumConfig(t *testing.T) {
|
||||
if cfg.Scriptorium.ProfileDir != "./profiles" {
|
||||
t.Fatalf("redaction mutated original config")
|
||||
}
|
||||
if redacted.Workspace.Directory != "/var/lib/notarius" || !redacted.Workspace.Resume.Enabled {
|
||||
t.Fatalf("expected workspace config preserved, got %+v", redacted.Workspace)
|
||||
}
|
||||
redacted.Workspace.Directory = "/changed"
|
||||
if cfg.Workspace.Directory != "/var/lib/notarius" {
|
||||
t.Fatalf("redaction mutated original workspace config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigRedactedDiagnosticsPayloadCopiesConfig(t *testing.T) {
|
||||
|
||||
@@ -12,6 +12,9 @@ func (c Config) Validate() error {
|
||||
if err := validateScriptorium(c.Scriptorium); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateWorkspace(c.Workspace); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateDiagnostics(c.Diagnostics); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -28,6 +31,17 @@ func validateScriptorium(cfg ScriptoriumConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateWorkspace(cfg WorkspaceConfig) error {
|
||||
if cfg.Diagnostics.retentionSet {
|
||||
switch cfg.Diagnostics.Retention {
|
||||
case "", diagnostics.RetentionAuto, diagnostics.RetentionAlways, diagnostics.RetentionNever:
|
||||
default:
|
||||
return fmt.Errorf("workspace diagnostics retention %q is not supported", cfg.Diagnostics.Retention)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDiagnostics(cfg DiagnosticsConfig) error {
|
||||
if strings.TrimSpace(cfg.WorkDir) == "" {
|
||||
return fmt.Errorf("diagnostics work dir must not be empty")
|
||||
|
||||
@@ -99,6 +99,17 @@ func TestValidateRejectsInvalidDiagnosticsRetention(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsInvalidWorkspaceDiagnosticsRetention(t *testing.T) {
|
||||
cfg := validConfig()
|
||||
cfg.Workspace.Diagnostics.Retention = diagnostics.RetentionMode("sometimes")
|
||||
cfg.Workspace.Diagnostics.retentionSet = true
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil || !strings.Contains(err.Error(), "workspace diagnostics retention") {
|
||||
t.Fatalf("expected workspace retention error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsInvalidReferenceMaps(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -5,6 +5,7 @@ const (
|
||||
ArtifactEffectiveConfig = "effective-config.json"
|
||||
ArtifactResolvedPipeline = "resolved-pipeline.json"
|
||||
ArtifactResolvedReferences = "resolved-references.json"
|
||||
ArtifactCheckpointEvents = "checkpoint-events.json"
|
||||
ArtifactSourceDocument = "source-document.json"
|
||||
ArtifactRunManifest = "run-manifest.json"
|
||||
ArtifactRunReport = "run-report.json"
|
||||
|
||||
@@ -51,6 +51,7 @@ type InvocationMetadata struct {
|
||||
Operation string `json:"operation"`
|
||||
PipelineID string `json:"pipeline_id,omitempty"`
|
||||
PipelineDigest string `json:"pipeline_digest,omitempty"`
|
||||
Resume bool `json:"resume,omitempty"`
|
||||
InputPath string `json:"input_path,omitempty"`
|
||||
ConfigPath string `json:"config_path,omitempty"`
|
||||
ConfigSource string `json:"config_source,omitempty"`
|
||||
@@ -153,6 +154,10 @@ func (r *RunDirectory) WriteResolvedReferences(payload any) error {
|
||||
return r.WriteJSONArtifact(ArtifactResolvedReferences, payload)
|
||||
}
|
||||
|
||||
func (r *RunDirectory) WriteCheckpointEvents(payload any) error {
|
||||
return r.WriteJSONArtifact(ArtifactCheckpointEvents, payload)
|
||||
}
|
||||
|
||||
func (r *RunDirectory) WriteSourceDocument(payload any) error {
|
||||
return r.WriteJSONArtifact(ArtifactSourceDocument, payload)
|
||||
}
|
||||
|
||||
114
internal/core/workspace/files.go
Normal file
114
internal/core/workspace/files.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func SafePath(root string, name string) (string, error) {
|
||||
root = strings.TrimSpace(root)
|
||||
if root == "" {
|
||||
return "", fmt.Errorf("workspace root must not be empty")
|
||||
}
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return "", fmt.Errorf("workspace artifact name must not be empty")
|
||||
}
|
||||
if strings.Contains(name, `\`) {
|
||||
return "", fmt.Errorf("workspace artifact name %q must use slash-separated relative paths", name)
|
||||
}
|
||||
if path.IsAbs(name) || filepath.IsAbs(name) {
|
||||
return "", fmt.Errorf("workspace artifact name %q must be relative", name)
|
||||
}
|
||||
if name == "." || strings.Contains(name, "..") {
|
||||
return "", fmt.Errorf("workspace artifact name %q must not contain ..", name)
|
||||
}
|
||||
cleaned := path.Clean(name)
|
||||
if cleaned != name {
|
||||
return "", fmt.Errorf("workspace artifact name %q must be clean", name)
|
||||
}
|
||||
|
||||
absRoot, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve workspace root %q: %w", root, err)
|
||||
}
|
||||
target, err := filepath.Abs(filepath.Join(absRoot, filepath.FromSlash(cleaned)))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve workspace artifact %q: %w", name, err)
|
||||
}
|
||||
rel, err := filepath.Rel(absRoot, target)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve workspace artifact %q: %w", name, err)
|
||||
}
|
||||
if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("workspace artifact name %q resolves outside workspace root", name)
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
|
||||
func WriteJSON(root string, name string, payload any) error {
|
||||
target, err := SafePath(root, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(payload, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal workspace artifact %q: %w", name, err)
|
||||
}
|
||||
data = append(data, '\n')
|
||||
if err := writeFileAtomic(target, data, 0o644); err != nil {
|
||||
return fmt.Errorf("write workspace artifact %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func WriteBytes(root string, name string, data []byte) error {
|
||||
target, err := SafePath(root, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeFileAtomic(target, data, 0o644); err != nil {
|
||||
return fmt.Errorf("write workspace artifact %q: %w", name, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeFileAtomic(target string, data []byte, perm os.FileMode) error {
|
||||
dir := filepath.Dir(target)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
temp, err := os.CreateTemp(dir, "."+filepath.Base(target)+".tmp-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tempPath := temp.Name()
|
||||
removeTemp := true
|
||||
defer func() {
|
||||
if removeTemp {
|
||||
_ = os.Remove(tempPath)
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := temp.Write(data); err != nil {
|
||||
_ = temp.Close()
|
||||
return err
|
||||
}
|
||||
if err := temp.Chmod(perm); err != nil {
|
||||
_ = temp.Close()
|
||||
return err
|
||||
}
|
||||
if err := temp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tempPath, target); err != nil {
|
||||
return err
|
||||
}
|
||||
removeTemp = false
|
||||
return nil
|
||||
}
|
||||
148
internal/core/workspace/files_test.go
Normal file
148
internal/core/workspace/files_test.go
Normal file
@@ -0,0 +1,148 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSafePathAcceptsCleanRelativePaths(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
|
||||
got, err := SafePath(root, "source/manifest.json")
|
||||
if err != nil {
|
||||
t.Fatalf("SafePath: %v", err)
|
||||
}
|
||||
|
||||
want := filepath.Join(root, "source", "manifest.json")
|
||||
if got != want {
|
||||
t.Fatalf("SafePath = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafePathRejectsUnsafeNames(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{name: "empty", path: " ", want: "empty"},
|
||||
{name: "absolute", path: filepath.Join(root, "artifact.json"), want: "relative"},
|
||||
{name: "parent segment", path: "../artifact.json", want: ".."},
|
||||
{name: "embedded parent", path: "source/../artifact.json", want: ".."},
|
||||
{name: "backslash", path: `source\artifact.json`, want: "slash-separated"},
|
||||
{name: "unclean", path: "source//artifact.json", want: "clean"},
|
||||
{name: "dot", path: ".", want: ".."},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := SafePath(root, tc.path)
|
||||
if err == nil {
|
||||
t.Fatalf("SafePath returned %q, want error", got)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("SafePath error = %v, want containing %q", err, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafePathRejectsEmptyRoot(t *testing.T) {
|
||||
got, err := SafePath(" ", "artifact.json")
|
||||
if err == nil {
|
||||
t.Fatalf("SafePath returned %q, want error", got)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "root") {
|
||||
t.Fatalf("SafePath error = %v, want root error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafePathDoesNotPermitEscapingRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for _, name := range []string{
|
||||
"..",
|
||||
"../outside.json",
|
||||
"nested/../../outside.json",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, err := SafePath(root, name)
|
||||
if err == nil {
|
||||
t.Fatalf("SafePath returned %q, want error", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteJSONWritesIndentedAtomicArtifact(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
|
||||
err := WriteJSON(root, "source/manifest.json", map[string]any{
|
||||
"status": "succeeded",
|
||||
"count": 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("WriteJSON: %v", err)
|
||||
}
|
||||
|
||||
got := string(readFile(t, filepath.Join(root, "source", "manifest.json")))
|
||||
if !strings.HasSuffix(got, "\n") {
|
||||
t.Fatalf("expected trailing newline, got %q", got)
|
||||
}
|
||||
if !strings.Contains(got, `"status": "succeeded"`) || !strings.Contains(got, `"count": 2`) {
|
||||
t.Fatalf("unexpected JSON: %s", got)
|
||||
}
|
||||
assertNoTempFiles(t, filepath.Join(root, "source"))
|
||||
}
|
||||
|
||||
func TestWriteBytesWritesNestedArtifact(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
|
||||
if err := WriteBytes(root, "chunk/chunks.json", []byte("payload")); err != nil {
|
||||
t.Fatalf("WriteBytes: %v", err)
|
||||
}
|
||||
|
||||
got := string(readFile(t, filepath.Join(root, "chunk", "chunks.json")))
|
||||
if got != "payload" {
|
||||
t.Fatalf("bytes = %q, want payload", got)
|
||||
}
|
||||
assertNoTempFiles(t, filepath.Join(root, "chunk"))
|
||||
}
|
||||
|
||||
func TestWritersRejectUnsafePaths(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
|
||||
if err := WriteBytes(root, "../outside.json", []byte("payload")); err == nil {
|
||||
t.Fatalf("WriteBytes accepted unsafe path")
|
||||
}
|
||||
if err := WriteJSON(root, `debug\trace.json`, map[string]string{"x": "y"}); err == nil {
|
||||
t.Fatalf("WriteJSON accepted unsafe path")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "..", "outside.json")); !os.IsNotExist(err) {
|
||||
t.Fatalf("outside path stat err = %v, want not exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %q: %v", path, err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func assertNoTempFiles(t *testing.T, dir string) {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read dir %q: %v", dir, err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if strings.Contains(entry.Name(), ".tmp-") {
|
||||
t.Fatalf("temporary file was not cleaned up: %s", entry.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
268
internal/core/workspace/identity.go
Normal file
268
internal/core/workspace/identity.go
Normal file
@@ -0,0 +1,268 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
const digestPrefixLength = 16
|
||||
|
||||
type Fingerprint struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type CheckpointIdentityInput struct {
|
||||
Pipeline pipeline.ResolvedPipeline
|
||||
InputKey string
|
||||
RawInputDigest string
|
||||
SourceDigest string
|
||||
SelectedLanes []string
|
||||
RuntimeOverrides []Fingerprint
|
||||
References []artifacts.ReferenceProvenance
|
||||
ProvenanceFingerprints []Fingerprint
|
||||
}
|
||||
|
||||
type CheckpointIdentity struct {
|
||||
Digest string `json:"digest"`
|
||||
PipelineID string `json:"pipeline_id"`
|
||||
PipelineDigest string `json:"pipeline_digest"`
|
||||
InputKey string `json:"input_key"`
|
||||
RawInputDigest string `json:"raw_input_digest,omitempty"`
|
||||
SourceDigest string `json:"source_digest,omitempty"`
|
||||
SelectedLanes []string `json:"selected_lanes,omitempty"`
|
||||
RuntimeOverrides []Fingerprint `json:"runtime_overrides,omitempty"`
|
||||
ReferenceDigests []Fingerprint `json:"reference_digests,omitempty"`
|
||||
ProvenanceFingerprints []Fingerprint `json:"provenance_fingerprints,omitempty"`
|
||||
}
|
||||
|
||||
func NewCheckpointIdentity(input CheckpointIdentityInput) (CheckpointIdentity, error) {
|
||||
pipelineID := strings.TrimSpace(input.Pipeline.ID)
|
||||
if pipelineID == "" {
|
||||
return CheckpointIdentity{}, fmt.Errorf("checkpoint identity pipeline id must not be empty")
|
||||
}
|
||||
pipelineDigest := strings.TrimSpace(input.Pipeline.Digest)
|
||||
if pipelineDigest == "" {
|
||||
return CheckpointIdentity{}, fmt.Errorf("checkpoint identity pipeline digest must not be empty")
|
||||
}
|
||||
inputKey := strings.TrimSpace(input.InputKey)
|
||||
if inputKey == "" {
|
||||
inputKey = strings.TrimSpace(input.Pipeline.Input.Module)
|
||||
}
|
||||
if inputKey == "" {
|
||||
return CheckpointIdentity{}, fmt.Errorf("checkpoint identity input key must not be empty")
|
||||
}
|
||||
|
||||
rawInputDigest := strings.TrimSpace(input.RawInputDigest)
|
||||
sourceDigest := strings.TrimSpace(input.SourceDigest)
|
||||
if rawInputDigest == "" && sourceDigest == "" {
|
||||
return CheckpointIdentity{}, fmt.Errorf("checkpoint identity raw input digest or source digest must be set")
|
||||
}
|
||||
|
||||
identity := CheckpointIdentity{
|
||||
PipelineID: pipelineID,
|
||||
PipelineDigest: pipelineDigest,
|
||||
InputKey: inputKey,
|
||||
RawInputDigest: rawInputDigest,
|
||||
SourceDigest: sourceDigest,
|
||||
SelectedLanes: normalizedLanes(input.SelectedLanes, input.Pipeline.ArtifactLanes),
|
||||
RuntimeOverrides: normalizeFingerprints(input.RuntimeOverrides),
|
||||
ReferenceDigests: referenceFingerprints(input.References),
|
||||
ProvenanceFingerprints: normalizeFingerprints(input.ProvenanceFingerprints),
|
||||
}
|
||||
digest, err := identityDigest(identity)
|
||||
if err != nil {
|
||||
return CheckpointIdentity{}, err
|
||||
}
|
||||
identity.Digest = digest
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func (s Settings) CheckpointDirectory(identity CheckpointIdentity) (string, error) {
|
||||
if !s.ResumeEnabled || strings.TrimSpace(s.CheckpointsRoot) == "" {
|
||||
return "", nil
|
||||
}
|
||||
relative, err := identity.RelativePath()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return SafePath(s.CheckpointsRoot, relative)
|
||||
}
|
||||
|
||||
func (i CheckpointIdentity) RelativePath() (string, error) {
|
||||
pipelineID, err := safePathComponent(i.PipelineID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("checkpoint identity pipeline id: %w", err)
|
||||
}
|
||||
inputKey, err := safePathComponent(i.InputKey)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("checkpoint identity input key: %w", err)
|
||||
}
|
||||
sourceDigest := digestPrefix(i.SourceDigest)
|
||||
if sourceDigest == "" {
|
||||
sourceDigest = digestPrefix(i.RawInputDigest)
|
||||
}
|
||||
if sourceDigest == "" {
|
||||
return "", fmt.Errorf("checkpoint identity source digest prefix must not be empty")
|
||||
}
|
||||
pipelineDigest := digestPrefix(i.PipelineDigest)
|
||||
if pipelineDigest == "" {
|
||||
return "", fmt.Errorf("checkpoint identity pipeline digest prefix must not be empty")
|
||||
}
|
||||
sourceComponent, err := safePathComponent(sourceDigest)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("checkpoint identity source digest: %w", err)
|
||||
}
|
||||
pipelineComponent, err := safePathComponent(pipelineDigest)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("checkpoint identity pipeline digest: %w", err)
|
||||
}
|
||||
return filepath.ToSlash(filepath.Join(pipelineID, inputKey+"-"+sourceComponent, pipelineComponent)), nil
|
||||
}
|
||||
|
||||
func identityDigest(identity CheckpointIdentity) (string, error) {
|
||||
payload := identity
|
||||
payload.Digest = ""
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal checkpoint identity: %w", err)
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return "sha256:" + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func normalizedLanes(selected []string, resolved []pipeline.ResolvedArtifactLane) []string {
|
||||
if len(selected) > 0 {
|
||||
return normalizeStrings(selected)
|
||||
}
|
||||
lanes := make([]string, 0, len(resolved))
|
||||
for _, lane := range resolved {
|
||||
lanes = append(lanes, lane.ID)
|
||||
}
|
||||
return normalizeStrings(lanes)
|
||||
}
|
||||
|
||||
func normalizeFingerprints(values []Fingerprint) []Fingerprint {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
byName := make(map[string]string, len(values))
|
||||
for _, value := range values {
|
||||
name := strings.TrimSpace(value.Name)
|
||||
fingerprint := strings.TrimSpace(value.Value)
|
||||
if name == "" || fingerprint == "" {
|
||||
continue
|
||||
}
|
||||
byName[name] = fingerprint
|
||||
}
|
||||
if len(byName) == 0 {
|
||||
return nil
|
||||
}
|
||||
names := make([]string, 0, len(byName))
|
||||
for name := range byName {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
out := make([]Fingerprint, 0, len(names))
|
||||
for _, name := range names {
|
||||
out = append(out, Fingerprint{Name: name, Value: byName[name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func referenceFingerprints(references []artifacts.ReferenceProvenance) []Fingerprint {
|
||||
if len(references) == 0 {
|
||||
return nil
|
||||
}
|
||||
values := make([]Fingerprint, 0, len(references))
|
||||
for _, reference := range references {
|
||||
digest := strings.TrimSpace(reference.Digest)
|
||||
if digest == "" {
|
||||
continue
|
||||
}
|
||||
parts := []string{
|
||||
strings.TrimSpace(reference.Stage),
|
||||
strings.TrimSpace(reference.LaneID),
|
||||
strings.TrimSpace(reference.SlotName),
|
||||
strings.TrimSpace(reference.OriginURI),
|
||||
}
|
||||
values = append(values, Fingerprint{
|
||||
Name: strings.Join(parts, ":"),
|
||||
Value: digest,
|
||||
})
|
||||
}
|
||||
return normalizeFingerprints(values)
|
||||
}
|
||||
|
||||
func normalizeStrings(values []string) []string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
continue
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
}
|
||||
if len(seen) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for value := range seen {
|
||||
out = append(out, value)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func digestPrefix(digest string) string {
|
||||
digest = strings.TrimSpace(digest)
|
||||
if digest == "" {
|
||||
return ""
|
||||
}
|
||||
if idx := strings.Index(digest, ":"); idx >= 0 {
|
||||
digest = digest[idx+1:]
|
||||
}
|
||||
digest = strings.TrimSpace(digest)
|
||||
if len(digest) > digestPrefixLength {
|
||||
return digest[:digestPrefixLength]
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
func safePathComponent(value string) (string, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("must not be empty")
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
b.WriteRune(r)
|
||||
case r >= 'A' && r <= 'Z':
|
||||
b.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '-' || r == '_' || r == '.':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteString(fmt.Sprintf("~%x", r))
|
||||
}
|
||||
}
|
||||
encoded := b.String()
|
||||
if encoded == "." || encoded == ".." || strings.Contains(encoded, "..") || strings.ContainsAny(encoded, `/\`) {
|
||||
return "", fmt.Errorf("%q is not filesystem safe", value)
|
||||
}
|
||||
return encoded, nil
|
||||
}
|
||||
242
internal/core/workspace/identity_test.go
Normal file
242
internal/core/workspace/identity_test.go
Normal file
@@ -0,0 +1,242 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestCheckpointIdentityIsDeterministic(t *testing.T) {
|
||||
first := mustIdentity(t, identityInput())
|
||||
second := mustIdentity(t, identityInput())
|
||||
|
||||
if first.Digest != second.Digest {
|
||||
t.Fatalf("digest changed for same input: %q != %q", first.Digest, second.Digest)
|
||||
}
|
||||
if !strings.HasPrefix(first.Digest, "sha256:") {
|
||||
t.Fatalf("digest = %q, want sha256 prefix", first.Digest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckpointIdentityChangesWhenInputsChange(t *testing.T) {
|
||||
base := mustIdentity(t, identityInput())
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(CheckpointIdentityInput) CheckpointIdentityInput
|
||||
}{
|
||||
{
|
||||
name: "pipeline digest",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.Pipeline.Digest = "sha256:pipeline-b"
|
||||
return input
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "raw input digest",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.RawInputDigest = "sha256:raw-b"
|
||||
return input
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "selected lanes",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.SelectedLanes = []string{"items"}
|
||||
return input
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "reference digest",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.References[0].Digest = "sha256:reference-b"
|
||||
return input
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "runtime override",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.RuntimeOverrides = []Fingerprint{{Name: "llm_profile", Value: "careful"}}
|
||||
return input
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "provenance fingerprint",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.ProvenanceFingerprints = []Fingerprint{{Name: "prompt:dnd.spells", Value: "sha256:prompt-b"}}
|
||||
return input
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
changed := mustIdentity(t, tc.mutate(identityInput()))
|
||||
if changed.Digest == base.Digest {
|
||||
t.Fatalf("digest did not change after %s mutation: %q", tc.name, changed.Digest)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckpointIdentityNormalizesOrder(t *testing.T) {
|
||||
input := identityInput()
|
||||
input.SelectedLanes = []string{"spells", "items", "spells"}
|
||||
input.RuntimeOverrides = []Fingerprint{
|
||||
{Name: "z", Value: "2"},
|
||||
{Name: "a", Value: "1"},
|
||||
}
|
||||
input.ProvenanceFingerprints = []Fingerprint{
|
||||
{Name: "schema", Value: "sha256:schema"},
|
||||
{Name: "prompt", Value: "sha256:prompt"},
|
||||
}
|
||||
|
||||
identity := mustIdentity(t, input)
|
||||
|
||||
if got := strings.Join(identity.SelectedLanes, ","); got != "items,spells" {
|
||||
t.Fatalf("selected lanes = %q, want sorted unique values", got)
|
||||
}
|
||||
if identity.RuntimeOverrides[0].Name != "a" || identity.ProvenanceFingerprints[0].Name != "prompt" {
|
||||
t.Fatalf("fingerprints not sorted: runtime=%+v provenance=%+v", identity.RuntimeOverrides, identity.ProvenanceFingerprints)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckpointIdentityPathIsFilesystemSafe(t *testing.T) {
|
||||
input := identityInput()
|
||||
input.Pipeline.ID = "campaign/main"
|
||||
input.InputKey = "seriatim/input"
|
||||
input.SourceDigest = "sha256:abcdef0123456789ffffffff"
|
||||
input.Pipeline.Digest = "sha256:1234567890abcdefeeeeeeee"
|
||||
identity := mustIdentity(t, input)
|
||||
|
||||
relative, err := identity.RelativePath()
|
||||
if err != nil {
|
||||
t.Fatalf("RelativePath: %v", err)
|
||||
}
|
||||
if strings.Contains(relative, `\`) || strings.Contains(relative, "..") {
|
||||
t.Fatalf("relative path is not filesystem safe: %q", relative)
|
||||
}
|
||||
if relative != "campaign~2fmain/seriatim~2finput-abcdef0123456789/1234567890abcdef" {
|
||||
t.Fatalf("relative path = %q", relative)
|
||||
}
|
||||
|
||||
root := t.TempDir()
|
||||
settings := Settings{
|
||||
CheckpointsRoot: filepath.Join(root, "checkpoints"),
|
||||
ResumeEnabled: true,
|
||||
}
|
||||
got, err := settings.CheckpointDirectory(identity)
|
||||
if err != nil {
|
||||
t.Fatalf("CheckpointDirectory: %v", err)
|
||||
}
|
||||
want := filepath.Join(root, "checkpoints", "campaign~2fmain", "seriatim~2finput-abcdef0123456789", "1234567890abcdef")
|
||||
if got != want {
|
||||
t.Fatalf("checkpoint directory = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckpointDirectoryDisabledReturnsEmptyPath(t *testing.T) {
|
||||
settings := Settings{CheckpointsRoot: filepath.Join(t.TempDir(), "checkpoints")}
|
||||
got, err := settings.CheckpointDirectory(mustIdentity(t, identityInput()))
|
||||
if err != nil {
|
||||
t.Fatalf("CheckpointDirectory: %v", err)
|
||||
}
|
||||
if got != "" {
|
||||
t.Fatalf("CheckpointDirectory = %q, want empty path", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCheckpointIdentityRequiresCoreInputs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(CheckpointIdentityInput) CheckpointIdentityInput
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "pipeline id",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.Pipeline.ID = ""
|
||||
return input
|
||||
},
|
||||
want: "pipeline id",
|
||||
},
|
||||
{
|
||||
name: "pipeline digest",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.Pipeline.Digest = ""
|
||||
return input
|
||||
},
|
||||
want: "pipeline digest",
|
||||
},
|
||||
{
|
||||
name: "input key",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.InputKey = ""
|
||||
input.Pipeline.Input.Module = ""
|
||||
return input
|
||||
},
|
||||
want: "input key",
|
||||
},
|
||||
{
|
||||
name: "input digest",
|
||||
mutate: func(input CheckpointIdentityInput) CheckpointIdentityInput {
|
||||
input.RawInputDigest = ""
|
||||
input.SourceDigest = ""
|
||||
return input
|
||||
},
|
||||
want: "raw input digest or source digest",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := NewCheckpointIdentity(tc.mutate(identityInput()))
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.want, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func identityInput() CheckpointIdentityInput {
|
||||
return CheckpointIdentityInput{
|
||||
Pipeline: pipeline.ResolvedPipeline{
|
||||
ID: "dnd-session",
|
||||
Digest: "sha256:pipeline-a",
|
||||
Input: pipeline.Binding("seriatim"),
|
||||
ArtifactLanes: []pipeline.ResolvedArtifactLane{
|
||||
{ID: "spells"},
|
||||
{ID: "items"},
|
||||
},
|
||||
},
|
||||
InputKey: "seriatim",
|
||||
RawInputDigest: "sha256:raw-a",
|
||||
SelectedLanes: []string{"spells"},
|
||||
RuntimeOverrides: []Fingerprint{
|
||||
{Name: "llm_profile", Value: "fast"},
|
||||
},
|
||||
References: []artifacts.ReferenceProvenance{
|
||||
{
|
||||
Stage: "extract",
|
||||
LaneID: "spells",
|
||||
SlotName: "party",
|
||||
OriginURI: "file:///party.yml",
|
||||
Digest: "sha256:reference-a",
|
||||
},
|
||||
},
|
||||
ProvenanceFingerprints: []Fingerprint{
|
||||
{Name: "prompt:dnd.spells", Value: "sha256:prompt-a"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func mustIdentity(t *testing.T, input CheckpointIdentityInput) CheckpointIdentity {
|
||||
t.Helper()
|
||||
identity, err := NewCheckpointIdentity(input)
|
||||
if err != nil {
|
||||
t.Fatalf("NewCheckpointIdentity: %v", err)
|
||||
}
|
||||
return identity
|
||||
}
|
||||
82
internal/core/workspace/manifest.go
Normal file
82
internal/core/workspace/manifest.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package workspace
|
||||
|
||||
import "time"
|
||||
|
||||
const WorkspaceSchemaVersion = "notarius.workspace.v1"
|
||||
|
||||
type StageName string
|
||||
|
||||
const (
|
||||
StageSource StageName = "source"
|
||||
StageChunk StageName = "chunk"
|
||||
StageExtract StageName = "extract"
|
||||
StageMerge StageName = "merge"
|
||||
StageNormalize StageName = "normalize"
|
||||
)
|
||||
|
||||
type StageStatus string
|
||||
|
||||
const (
|
||||
StatusPending StageStatus = "pending"
|
||||
StatusRunning StageStatus = "running"
|
||||
StatusSucceeded StageStatus = "succeeded"
|
||||
StatusSucceededWithRejections StageStatus = "succeeded_with_rejections"
|
||||
StatusFailed StageStatus = "failed"
|
||||
StatusInvalidated StageStatus = "invalidated"
|
||||
)
|
||||
|
||||
type StageManifest struct {
|
||||
WorkspaceSchemaVersion string `json:"workspace_schema_version"`
|
||||
Stage StageName `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
DependencyFingerprints []Fingerprint `json:"dependency_fingerprints,omitempty"`
|
||||
Status StageStatus `json:"status"`
|
||||
OutputDigests []Fingerprint `json:"output_digests,omitempty"`
|
||||
ValidationStatus string `json:"validation_status,omitempty"`
|
||||
Rejections []RejectionSummary `json:"rejections,omitempty"`
|
||||
StartedAt *time.Time `json:"started_at,omitempty"`
|
||||
CompletedAt *time.Time `json:"completed_at,omitempty"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type RejectionSummary struct {
|
||||
ValidatorName string `json:"validator_name,omitempty"`
|
||||
ReasonCode string `json:"reason_code,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Count int `json:"count,omitempty"`
|
||||
}
|
||||
|
||||
type SourceManifest struct {
|
||||
StageManifest
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
}
|
||||
|
||||
type ChunkManifest struct {
|
||||
StageManifest
|
||||
ChunkCount int `json:"chunk_count,omitempty"`
|
||||
}
|
||||
|
||||
type ExtractLaneManifest struct {
|
||||
StageManifest
|
||||
ChunkCount int `json:"chunk_count,omitempty"`
|
||||
OutputCount int `json:"output_count,omitempty"`
|
||||
}
|
||||
|
||||
type MergeLaneManifest struct {
|
||||
StageManifest
|
||||
InputCount int `json:"input_count,omitempty"`
|
||||
}
|
||||
|
||||
type NormalizeLaneManifest struct {
|
||||
StageManifest
|
||||
InputCount int `json:"input_count,omitempty"`
|
||||
}
|
||||
|
||||
func NewStageManifest(stage StageName, status StageStatus) StageManifest {
|
||||
return StageManifest{
|
||||
WorkspaceSchemaVersion: WorkspaceSchemaVersion,
|
||||
Stage: stage,
|
||||
Status: status,
|
||||
}
|
||||
}
|
||||
143
internal/core/workspace/manifest_test.go
Normal file
143
internal/core/workspace/manifest_test.go
Normal file
@@ -0,0 +1,143 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStageManifestDefaults(t *testing.T) {
|
||||
manifest := NewStageManifest(StageExtract, StatusRunning)
|
||||
|
||||
if manifest.WorkspaceSchemaVersion != WorkspaceSchemaVersion {
|
||||
t.Fatalf("schema version = %q, want %q", manifest.WorkspaceSchemaVersion, WorkspaceSchemaVersion)
|
||||
}
|
||||
if manifest.Stage != StageExtract {
|
||||
t.Fatalf("stage = %q, want extract", manifest.Stage)
|
||||
}
|
||||
if manifest.Status != StatusRunning {
|
||||
t.Fatalf("status = %q, want running", manifest.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestJSONRoundTrips(t *testing.T) {
|
||||
started := time.Unix(100, 0).UTC()
|
||||
completed := time.Unix(200, 0).UTC()
|
||||
|
||||
t.Run("source", func(t *testing.T) {
|
||||
manifest := SourceManifest{
|
||||
StageManifest: populatedManifest(StageSource, "", "seriatim", started, completed),
|
||||
SourceID: "source-1",
|
||||
}
|
||||
var got SourceManifest
|
||||
roundTripManifest(t, manifest, &got)
|
||||
if got.SourceID != manifest.SourceID || got.Stage != StageSource {
|
||||
t.Fatalf("round trip source manifest = %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("chunk", func(t *testing.T) {
|
||||
manifest := ChunkManifest{
|
||||
StageManifest: populatedManifest(StageChunk, "", "generic", started, completed),
|
||||
ChunkCount: 3,
|
||||
}
|
||||
var got ChunkManifest
|
||||
roundTripManifest(t, manifest, &got)
|
||||
if got.ChunkCount != manifest.ChunkCount || got.Stage != StageChunk {
|
||||
t.Fatalf("round trip chunk manifest = %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("extract", func(t *testing.T) {
|
||||
manifest := ExtractLaneManifest{
|
||||
StageManifest: populatedManifest(StageExtract, "spells", "dnd/spells", started, completed),
|
||||
ChunkCount: 3,
|
||||
OutputCount: 2,
|
||||
}
|
||||
var got ExtractLaneManifest
|
||||
roundTripManifest(t, manifest, &got)
|
||||
if got.LaneID != "spells" || got.OutputCount != manifest.OutputCount || got.Stage != StageExtract {
|
||||
t.Fatalf("round trip extract manifest = %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("merge", func(t *testing.T) {
|
||||
manifest := MergeLaneManifest{
|
||||
StageManifest: populatedManifest(StageMerge, "spells", "appendorder", started, completed),
|
||||
InputCount: 2,
|
||||
}
|
||||
var got MergeLaneManifest
|
||||
roundTripManifest(t, manifest, &got)
|
||||
if got.InputCount != manifest.InputCount || got.Stage != StageMerge {
|
||||
t.Fatalf("round trip merge manifest = %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("normalize", func(t *testing.T) {
|
||||
manifest := NormalizeLaneManifest{
|
||||
StageManifest: populatedManifest(StageNormalize, "spells", "noop", started, completed),
|
||||
InputCount: 1,
|
||||
}
|
||||
var got NormalizeLaneManifest
|
||||
roundTripManifest(t, manifest, &got)
|
||||
if got.InputCount != manifest.InputCount || got.Stage != StageNormalize {
|
||||
t.Fatalf("round trip normalize manifest = %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestStatusValues(t *testing.T) {
|
||||
values := []StageStatus{
|
||||
StatusPending,
|
||||
StatusRunning,
|
||||
StatusSucceeded,
|
||||
StatusSucceededWithRejections,
|
||||
StatusFailed,
|
||||
StatusInvalidated,
|
||||
}
|
||||
want := []string{
|
||||
"pending",
|
||||
"running",
|
||||
"succeeded",
|
||||
"succeeded_with_rejections",
|
||||
"failed",
|
||||
"invalidated",
|
||||
}
|
||||
for i, value := range values {
|
||||
if string(value) != want[i] {
|
||||
t.Fatalf("status[%d] = %q, want %q", i, value, want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func populatedManifest(stage StageName, laneID string, moduleKey string, started time.Time, completed time.Time) StageManifest {
|
||||
manifest := NewStageManifest(stage, StatusSucceededWithRejections)
|
||||
manifest.LaneID = laneID
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = []Fingerprint{{Name: "source", Value: "sha256:source"}}
|
||||
manifest.OutputDigests = []Fingerprint{{Name: "output", Value: "sha256:output"}}
|
||||
manifest.ValidationStatus = "approved_with_warnings"
|
||||
manifest.Rejections = []RejectionSummary{
|
||||
{
|
||||
ValidatorName: "shape",
|
||||
ReasonCode: "invalid_shape",
|
||||
Message: "invalid output shape",
|
||||
Count: 1,
|
||||
},
|
||||
}
|
||||
manifest.StartedAt = &started
|
||||
manifest.CompletedAt = &completed
|
||||
manifest.Metadata = map[string]string{"attempt": "1"}
|
||||
return manifest
|
||||
}
|
||||
|
||||
func roundTripManifest(t *testing.T, in any, out any) {
|
||||
t.Helper()
|
||||
data, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal manifest: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
t.Fatalf("unmarshal manifest: %v", err)
|
||||
}
|
||||
}
|
||||
76
internal/core/workspace/settings.go
Normal file
76
internal/core/workspace/settings.go
Normal file
@@ -0,0 +1,76 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
)
|
||||
|
||||
type Settings struct {
|
||||
RootDir string
|
||||
DiagnosticsRoot string
|
||||
CheckpointsRoot string
|
||||
DebugRoot string
|
||||
DiagnosticsEnabled bool
|
||||
ResumeEnabled bool
|
||||
DebugEnabled bool
|
||||
}
|
||||
|
||||
func FromConfig(cfg config.Config) Settings {
|
||||
root := cleanPath(cfg.Workspace.Directory)
|
||||
settings := Settings{
|
||||
RootDir: root,
|
||||
DiagnosticsEnabled: cfg.DiagnosticsEnabled(),
|
||||
}
|
||||
if settings.DiagnosticsEnabled {
|
||||
settings.DiagnosticsRoot = cleanPath(cfg.Diagnostics.WorkDir)
|
||||
}
|
||||
if root == "" {
|
||||
return settings
|
||||
}
|
||||
|
||||
settings.CheckpointsRoot = filepath.Join(root, "checkpoints")
|
||||
settings.DebugRoot = filepath.Join(root, "debug")
|
||||
settings.ResumeEnabled = cfg.Workspace.Resume.Enabled
|
||||
settings.DebugEnabled = cfg.Workspace.Debug.Enabled
|
||||
return settings
|
||||
}
|
||||
|
||||
func (s Settings) DiagnosticsRunDirectory(runID string) (string, error) {
|
||||
if !s.DiagnosticsEnabled || strings.TrimSpace(s.DiagnosticsRoot) == "" {
|
||||
return "", nil
|
||||
}
|
||||
return safeSingleDirectory(s.DiagnosticsRoot, runID, "diagnostics run ID")
|
||||
}
|
||||
|
||||
func (s Settings) CheckpointIdentityDirectory(identity string) (string, error) {
|
||||
if !s.ResumeEnabled || strings.TrimSpace(s.CheckpointsRoot) == "" {
|
||||
return "", nil
|
||||
}
|
||||
return SafePath(s.CheckpointsRoot, identity)
|
||||
}
|
||||
|
||||
func (s Settings) DebugRunDirectory(runID string) (string, error) {
|
||||
if !s.DebugEnabled || strings.TrimSpace(s.DebugRoot) == "" {
|
||||
return "", nil
|
||||
}
|
||||
return safeSingleDirectory(s.DebugRoot, runID, "debug run ID")
|
||||
}
|
||||
|
||||
func cleanPath(path string) string {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return ""
|
||||
}
|
||||
return filepath.Clean(path)
|
||||
}
|
||||
|
||||
func safeSingleDirectory(root string, name string, label string) (string, error) {
|
||||
name = strings.TrimSpace(name)
|
||||
if strings.Contains(name, "/") || strings.Contains(name, `\`) {
|
||||
return "", fmt.Errorf("%s %q must be a single directory name", label, name)
|
||||
}
|
||||
return SafePath(root, name)
|
||||
}
|
||||
127
internal/core/workspace/settings_test.go
Normal file
127
internal/core/workspace/settings_test.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
)
|
||||
|
||||
func TestFromConfigBuildsWorkspaceRoots(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.Workspace.Directory = "/var/lib/notarius"
|
||||
cfg.Workspace.Resume.Enabled = true
|
||||
cfg.Workspace.Debug.Enabled = true
|
||||
cfg.RecomputeEffectiveDiagnostics()
|
||||
|
||||
settings := FromConfig(cfg)
|
||||
|
||||
if settings.RootDir != "/var/lib/notarius" {
|
||||
t.Fatalf("RootDir = %q, want /var/lib/notarius", settings.RootDir)
|
||||
}
|
||||
if settings.DiagnosticsRoot != "/var/lib/notarius/diagnostics" || !settings.DiagnosticsEnabled {
|
||||
t.Fatalf("diagnostics settings = %+v, want workspace diagnostics root enabled", settings)
|
||||
}
|
||||
if settings.CheckpointsRoot != "/var/lib/notarius/checkpoints" || !settings.ResumeEnabled {
|
||||
t.Fatalf("checkpoint settings = %+v, want workspace checkpoints root enabled", settings)
|
||||
}
|
||||
if settings.DebugRoot != "/var/lib/notarius/debug" || !settings.DebugEnabled {
|
||||
t.Fatalf("debug settings = %+v, want workspace debug root enabled", settings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFromConfigKeepsLegacyDiagnosticsRootWithoutWorkspaceRoot(t *testing.T) {
|
||||
cfg := config.Default()
|
||||
cfg.Diagnostics.WorkDir = "/tmp/notarius-legacy"
|
||||
cfg.Workspace.Resume.Enabled = true
|
||||
cfg.Workspace.Debug.Enabled = true
|
||||
|
||||
settings := FromConfig(cfg)
|
||||
|
||||
if settings.RootDir != "" {
|
||||
t.Fatalf("RootDir = %q, want empty", settings.RootDir)
|
||||
}
|
||||
if settings.DiagnosticsRoot != "/tmp/notarius-legacy" || !settings.DiagnosticsEnabled {
|
||||
t.Fatalf("diagnostics settings = %+v, want legacy diagnostics root enabled", settings)
|
||||
}
|
||||
if settings.CheckpointsRoot != "" || settings.ResumeEnabled {
|
||||
t.Fatalf("checkpoint settings = %+v, want disabled empty root", settings)
|
||||
}
|
||||
if settings.DebugRoot != "" || settings.DebugEnabled {
|
||||
t.Fatalf("debug settings = %+v, want disabled empty root", settings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPathConstructors(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
settings := Settings{
|
||||
RootDir: root,
|
||||
DiagnosticsRoot: filepath.Join(root, "diagnostics"),
|
||||
CheckpointsRoot: filepath.Join(root, "checkpoints"),
|
||||
DebugRoot: filepath.Join(root, "debug"),
|
||||
DiagnosticsEnabled: true,
|
||||
ResumeEnabled: true,
|
||||
DebugEnabled: true,
|
||||
}
|
||||
|
||||
diagnosticsDir, err := settings.DiagnosticsRunDirectory("run-123")
|
||||
if err != nil {
|
||||
t.Fatalf("DiagnosticsRunDirectory: %v", err)
|
||||
}
|
||||
if diagnosticsDir != filepath.Join(root, "diagnostics", "run-123") {
|
||||
t.Fatalf("diagnostics dir = %q", diagnosticsDir)
|
||||
}
|
||||
|
||||
checkpointDir, err := settings.CheckpointIdentityDirectory("pipeline/input-digest/pipeline-digest")
|
||||
if err != nil {
|
||||
t.Fatalf("CheckpointIdentityDirectory: %v", err)
|
||||
}
|
||||
if checkpointDir != filepath.Join(root, "checkpoints", "pipeline", "input-digest", "pipeline-digest") {
|
||||
t.Fatalf("checkpoint dir = %q", checkpointDir)
|
||||
}
|
||||
|
||||
debugDir, err := settings.DebugRunDirectory("run-456")
|
||||
if err != nil {
|
||||
t.Fatalf("DebugRunDirectory: %v", err)
|
||||
}
|
||||
if debugDir != filepath.Join(root, "debug", "run-456") {
|
||||
t.Fatalf("debug dir = %q", debugDir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisabledPathConstructorsReturnEmptyPaths(t *testing.T) {
|
||||
settings := Settings{}
|
||||
|
||||
for name, call := range map[string]func() (string, error){
|
||||
"diagnostics": func() (string, error) { return settings.DiagnosticsRunDirectory("run-1") },
|
||||
"checkpoint": func() (string, error) { return settings.CheckpointIdentityDirectory("identity") },
|
||||
"debug": func() (string, error) { return settings.DebugRunDirectory("run-1") },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, err := call()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got != "" {
|
||||
t.Fatalf("path = %q, want empty", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDirectoryConstructorsRejectNestedNames(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
settings := Settings{
|
||||
DiagnosticsRoot: filepath.Join(root, "diagnostics"),
|
||||
DebugRoot: filepath.Join(root, "debug"),
|
||||
DiagnosticsEnabled: true,
|
||||
DebugEnabled: true,
|
||||
}
|
||||
|
||||
if got, err := settings.DiagnosticsRunDirectory("run-1/nested"); err == nil {
|
||||
t.Fatalf("DiagnosticsRunDirectory returned %q, want error", got)
|
||||
}
|
||||
if got, err := settings.DebugRunDirectory("run-1/nested"); err == nil {
|
||||
t.Fatalf("DebugRunDirectory returned %q, want error", got)
|
||||
}
|
||||
}
|
||||
345
internal/framework/checkpoint/loader.go
Normal file
345
internal/framework/checkpoint/loader.go
Normal file
@@ -0,0 +1,345 @@
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
type WorkspaceLoader struct {
|
||||
root string
|
||||
identityDigest string
|
||||
}
|
||||
|
||||
func NewWorkspaceLoader(settings coreworkspace.Settings, identity coreworkspace.CheckpointIdentity) (pipeline.CheckpointLoader, error) {
|
||||
root, err := settings.CheckpointDirectory(identity)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return pipeline.NoopCheckpointLoader(), nil
|
||||
}
|
||||
return &WorkspaceLoader{root: root, identityDigest: identity.Digest}, nil
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) Enabled() bool {
|
||||
return l != nil && strings.TrimSpace(l.root) != ""
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) Source(moduleKey string) (pipeline.SourceCheckpoint, pipeline.CheckpointDecision) {
|
||||
var manifest coreworkspace.SourceManifest
|
||||
if decision := l.readJSON("source/manifest.json", &manifest); !decision.Reused {
|
||||
return pipeline.SourceCheckpoint{}, decision
|
||||
}
|
||||
if decision := l.validateManifest(manifest.StageManifest, coreworkspace.StageSource, "", moduleKey, coreworkspace.StatusSucceeded, nil); !decision.Reused {
|
||||
return pipeline.SourceCheckpoint{}, decision
|
||||
}
|
||||
var payload sourceDocumentEnvelope
|
||||
if decision := l.readJSON("source/source-document.json", &payload); !decision.Reused {
|
||||
return pipeline.SourceCheckpoint{}, decision
|
||||
}
|
||||
doc := cloneSourceDocument(payload.Document)
|
||||
if err := source.ValidateDocument(&doc); err != nil {
|
||||
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint document is invalid: %v", err)
|
||||
}
|
||||
if strings.TrimSpace(manifest.SourceID) != "" && manifest.SourceID != doc.ID {
|
||||
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint source id does not match payload")
|
||||
}
|
||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), digestFingerprints("source_document", doc.Digest)) {
|
||||
return pipeline.SourceCheckpoint{}, invalidDecision("source checkpoint output digest does not match payload")
|
||||
}
|
||||
return pipeline.SourceCheckpoint{Document: &doc}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) Chunk(moduleKey string, sourceDigest string) (pipeline.ChunkCheckpoint, pipeline.CheckpointDecision) {
|
||||
expectedDependencies := digestFingerprints("source_document", sourceDigest)
|
||||
var manifest coreworkspace.ChunkManifest
|
||||
if decision := l.readJSON("chunk/manifest.json", &manifest); !decision.Reused {
|
||||
return pipeline.ChunkCheckpoint{}, decision
|
||||
}
|
||||
if decision := l.validateManifest(manifest.StageManifest, coreworkspace.StageChunk, "", moduleKey, coreworkspace.StatusSucceeded, expectedDependencies); !decision.Reused {
|
||||
return pipeline.ChunkCheckpoint{}, decision
|
||||
}
|
||||
var payload chunksEnvelope
|
||||
if decision := l.readJSON("chunk/chunks.json", &payload); !decision.Reused {
|
||||
return pipeline.ChunkCheckpoint{}, decision
|
||||
}
|
||||
chunks, err := sourceChunksFromEnvelope(payload.Chunks)
|
||||
if err != nil {
|
||||
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint payload is invalid: %v", err)
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint payload has no chunks")
|
||||
}
|
||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), chunkOutputDigests(chunks)) {
|
||||
return pipeline.ChunkCheckpoint{}, invalidDecision("chunk checkpoint output digests do not match payload")
|
||||
}
|
||||
return pipeline.ChunkCheckpoint{Chunks: chunks, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) Extract(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.ExtractCheckpoint, pipeline.CheckpointDecision) {
|
||||
var manifest coreworkspace.ExtractLaneManifest
|
||||
if decision := l.readJSON(laneManifestPath("extract", laneID), &manifest); !decision.Reused {
|
||||
return pipeline.ExtractCheckpoint{}, decision
|
||||
}
|
||||
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageExtract, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded, coreworkspace.StatusSucceededWithRejections); !decision.Reused {
|
||||
return pipeline.ExtractCheckpoint{}, decision
|
||||
}
|
||||
var payload extractOutputsEnvelope
|
||||
if decision := l.readJSON(lanePayloadPath("extract", laneID, "outputs.json"), &payload); !decision.Reused {
|
||||
return pipeline.ExtractCheckpoint{}, decision
|
||||
}
|
||||
outputs, err := extractOutputsFromEnvelope(payload.Outputs)
|
||||
if err != nil {
|
||||
return pipeline.ExtractCheckpoint{}, invalidDecision("extract checkpoint payload is invalid: %v", err)
|
||||
}
|
||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests(extractPayloads(outputs))) {
|
||||
return pipeline.ExtractCheckpoint{}, invalidDecision("extract checkpoint output digests do not match payload")
|
||||
}
|
||||
return pipeline.ExtractCheckpoint{
|
||||
Outputs: outputs,
|
||||
Rejected: cloneRejectedOutputs(payload.Rejected),
|
||||
Warnings: cloneWarnings(payload.Warnings),
|
||||
}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) Merge(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.MergeCheckpoint, pipeline.CheckpointDecision) {
|
||||
var manifest coreworkspace.MergeLaneManifest
|
||||
if decision := l.readJSON(laneManifestPath("merge", laneID), &manifest); !decision.Reused {
|
||||
return pipeline.MergeCheckpoint{}, decision
|
||||
}
|
||||
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageMerge, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !decision.Reused {
|
||||
return pipeline.MergeCheckpoint{}, decision
|
||||
}
|
||||
var payload mergeOutputEnvelope
|
||||
if decision := l.readJSON(lanePayloadPath("merge", laneID, "output.json"), &payload); !decision.Reused {
|
||||
return pipeline.MergeCheckpoint{}, decision
|
||||
}
|
||||
output, err := mergeOutputFromEnvelope(payload.Output)
|
||||
if err != nil {
|
||||
return pipeline.MergeCheckpoint{}, invalidDecision("merge checkpoint payload is invalid: %v", err)
|
||||
}
|
||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests([]contracts.RawPayload{output.Payload})) {
|
||||
return pipeline.MergeCheckpoint{}, invalidDecision("merge checkpoint output digest does not match payload")
|
||||
}
|
||||
return pipeline.MergeCheckpoint{Output: output, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) Normalize(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) (pipeline.NormalizeCheckpoint, pipeline.CheckpointDecision) {
|
||||
var manifest coreworkspace.NormalizeLaneManifest
|
||||
if decision := l.readJSON(laneManifestPath("normalize", laneID), &manifest); !decision.Reused {
|
||||
return pipeline.NormalizeCheckpoint{}, decision
|
||||
}
|
||||
if decision := l.validateLaneManifest(manifest.StageManifest, coreworkspace.StageNormalize, laneID, moduleKey, dependencies, coreworkspace.StatusSucceeded); !decision.Reused {
|
||||
return pipeline.NormalizeCheckpoint{}, decision
|
||||
}
|
||||
var payload normalizeOutputEnvelope
|
||||
if decision := l.readJSON(lanePayloadPath("normalize", laneID, "output.json"), &payload); !decision.Reused {
|
||||
return pipeline.NormalizeCheckpoint{}, decision
|
||||
}
|
||||
output, err := normalizeOutputFromEnvelope(payload.Output)
|
||||
if err != nil {
|
||||
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize checkpoint payload is invalid: %v", err)
|
||||
}
|
||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.OutputDigests), rawOutputDigests([]contracts.RawPayload{output.Payload})) {
|
||||
return pipeline.NormalizeCheckpoint{}, invalidDecision("normalize checkpoint output digest does not match payload")
|
||||
}
|
||||
return pipeline.NormalizeCheckpoint{Output: output, Warnings: cloneWarnings(payload.Warnings)}, reusedDecision()
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) readJSON(name string, out any) pipeline.CheckpointDecision {
|
||||
if !l.Enabled() {
|
||||
return pipeline.CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
target, err := coreworkspace.SafePath(l.root, name)
|
||||
if err != nil {
|
||||
return invalidDecision("checkpoint path is invalid: %v", err)
|
||||
}
|
||||
data, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return pipeline.CheckpointDecision{Reason: "checkpoint artifact is missing"}
|
||||
}
|
||||
return invalidDecision("read checkpoint artifact: %v", err)
|
||||
}
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
return invalidDecision("decode checkpoint artifact: %v", err)
|
||||
}
|
||||
return reusedDecision()
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) validateManifest(manifest coreworkspace.StageManifest, stage coreworkspace.StageName, laneID string, moduleKey string, status coreworkspace.StageStatus, dependencies []pipeline.CheckpointFingerprint) pipeline.CheckpointDecision {
|
||||
return l.validateLaneManifest(manifest, stage, laneID, moduleKey, dependencies, status)
|
||||
}
|
||||
|
||||
func (l *WorkspaceLoader) validateLaneManifest(manifest coreworkspace.StageManifest, stage coreworkspace.StageName, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, statuses ...coreworkspace.StageStatus) pipeline.CheckpointDecision {
|
||||
if manifest.WorkspaceSchemaVersion != coreworkspace.WorkspaceSchemaVersion {
|
||||
return invalidDecision("checkpoint workspace schema version %q is not supported", manifest.WorkspaceSchemaVersion)
|
||||
}
|
||||
if strings.TrimSpace(l.identityDigest) != "" && manifest.Metadata["checkpoint_identity_digest"] != l.identityDigest {
|
||||
return invalidDecision("checkpoint identity digest does not match current invocation")
|
||||
}
|
||||
if manifest.Stage != stage {
|
||||
return invalidDecision("checkpoint stage %q does not match %q", manifest.Stage, stage)
|
||||
}
|
||||
if strings.TrimSpace(laneID) != "" && manifest.LaneID != laneID {
|
||||
return invalidDecision("checkpoint lane %q does not match %q", manifest.LaneID, laneID)
|
||||
}
|
||||
if strings.TrimSpace(moduleKey) != "" && manifest.ModuleKey != moduleKey {
|
||||
return invalidDecision("checkpoint module %q does not match %q", manifest.ModuleKey, moduleKey)
|
||||
}
|
||||
statusOK := false
|
||||
for _, status := range statuses {
|
||||
if manifest.Status == status {
|
||||
statusOK = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !statusOK {
|
||||
return invalidDecision("checkpoint status %q cannot be reused", manifest.Status)
|
||||
}
|
||||
if !fingerprintsEqual(coreworkspaceToPipelineFingerprints(manifest.DependencyFingerprints), dependencies) {
|
||||
return invalidDecision("checkpoint dependency fingerprints do not match")
|
||||
}
|
||||
return reusedDecision()
|
||||
}
|
||||
|
||||
func sourceChunksFromEnvelope(values []chunkEnvelope) ([]contracts.SourceChunk, error) {
|
||||
if len(values) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
out := make([]contracts.SourceChunk, 0, len(values))
|
||||
for _, value := range values {
|
||||
content, err := contentFromEnvelope(value.Content)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, contracts.SourceChunk{
|
||||
ID: value.ID,
|
||||
SourceID: value.SourceID,
|
||||
Index: value.Index,
|
||||
StartUnitID: value.StartUnitID,
|
||||
EndUnitID: value.EndUnitID,
|
||||
Content: content,
|
||||
MediaType: value.Content.MediaType,
|
||||
Units: cloneSourceUnits(value.Units),
|
||||
Metadata: cloneMetadata(value.Metadata),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func extractOutputsFromEnvelope(values []extractOutputEnvelope) ([]contracts.ExtractOutput, error) {
|
||||
if len(values) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
out := make([]contracts.ExtractOutput, 0, len(values))
|
||||
for _, value := range values {
|
||||
payload, err := rawPayloadFromEnvelope(value.Payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, contracts.ExtractOutput{
|
||||
LaneID: value.LaneID,
|
||||
ExtractorKey: value.ExtractorKey,
|
||||
SourceID: value.SourceID,
|
||||
ChunkID: value.ChunkID,
|
||||
ChunkIndex: value.ChunkIndex,
|
||||
Schema: value.Schema,
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func mergeOutputFromEnvelope(value mergeOutputPayload) (contracts.MergeOutput, error) {
|
||||
payload, err := rawPayloadFromEnvelope(value.Payload)
|
||||
if err != nil {
|
||||
return contracts.MergeOutput{}, err
|
||||
}
|
||||
return contracts.MergeOutput{
|
||||
LaneID: value.LaneID,
|
||||
MergerKey: value.MergerKey,
|
||||
SourceID: value.SourceID,
|
||||
Schema: value.Schema,
|
||||
Payload: payload,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func normalizeOutputFromEnvelope(value normalizeOutputPayload) (contracts.NormalizeOutput, error) {
|
||||
payload, err := rawPayloadFromEnvelope(value.Payload)
|
||||
if err != nil {
|
||||
return contracts.NormalizeOutput{}, err
|
||||
}
|
||||
return contracts.NormalizeOutput{
|
||||
LaneID: value.LaneID,
|
||||
NormalizerKey: value.NormalizerKey,
|
||||
SourceID: value.SourceID,
|
||||
Schema: value.Schema,
|
||||
Payload: payload,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func rawPayloadFromEnvelope(value binaryEnvelope) (contracts.RawPayload, error) {
|
||||
content, err := contentFromEnvelope(value)
|
||||
if err != nil {
|
||||
return contracts.RawPayload{}, err
|
||||
}
|
||||
return contracts.RawPayload{
|
||||
Content: content,
|
||||
MediaType: value.MediaType,
|
||||
Metadata: cloneMetadata(value.Metadata),
|
||||
Warnings: cloneWarnings(value.Warnings),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func contentFromEnvelope(value binaryEnvelope) ([]byte, error) {
|
||||
content, err := base64.StdEncoding.DecodeString(value.ContentBase64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode content_base64: %w", err)
|
||||
}
|
||||
if digest := strings.TrimSpace(value.ContentDigest); digest != "" && digest != contentDigest(content) {
|
||||
return nil, fmt.Errorf("content digest mismatch")
|
||||
}
|
||||
return content, nil
|
||||
}
|
||||
|
||||
func coreworkspaceToPipelineFingerprints(values []coreworkspace.Fingerprint) []pipeline.CheckpointFingerprint {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]pipeline.CheckpointFingerprint, 0, len(values))
|
||||
for _, value := range values {
|
||||
out = append(out, pipeline.CheckpointFingerprint{Name: value.Name, Value: value.Value})
|
||||
}
|
||||
return normalizeFingerprints(out)
|
||||
}
|
||||
|
||||
func fingerprintsEqual(a []pipeline.CheckpointFingerprint, b []pipeline.CheckpointFingerprint) bool {
|
||||
a = normalizeFingerprints(a)
|
||||
b = normalizeFingerprints(b)
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func reusedDecision() pipeline.CheckpointDecision {
|
||||
return pipeline.CheckpointDecision{Reused: true, Reason: "checkpoint is valid"}
|
||||
}
|
||||
|
||||
func invalidDecision(format string, args ...any) pipeline.CheckpointDecision {
|
||||
return pipeline.CheckpointDecision{Reason: fmt.Sprintf(format, args...)}
|
||||
}
|
||||
634
internal/framework/checkpoint/recorder.go
Normal file
634
internal/framework/checkpoint/recorder.go
Normal file
@@ -0,0 +1,634 @@
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
type WorkspaceRecorder struct {
|
||||
root string
|
||||
identityDigest string
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewWorkspaceRecorder(settings coreworkspace.Settings, identity coreworkspace.CheckpointIdentity) (pipeline.CheckpointRecorder, error) {
|
||||
root, err := settings.CheckpointDirectory(identity)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return pipeline.NoopCheckpointRecorder(), nil
|
||||
}
|
||||
return &WorkspaceRecorder{root: root, identityDigest: identity.Digest, now: time.Now}, nil
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) SourceRunning(moduleKey string) error {
|
||||
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusRunning)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) SourceSucceeded(moduleKey string, doc *source.SourceDocument) error {
|
||||
if doc == nil {
|
||||
return fmt.Errorf("checkpoint source document must not be nil")
|
||||
}
|
||||
if err := r.writePayload("source/source-document.json", sourceDocumentEnvelope{Document: cloneSourceDocument(*doc)}); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusSucceeded)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.OutputDigests = workspaceFingerprints(digestFingerprints("source_document", doc.Digest))
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{
|
||||
StageManifest: manifest,
|
||||
SourceID: doc.ID,
|
||||
})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) SourceFailed(moduleKey string, err error) error {
|
||||
manifest := r.newStageManifest(coreworkspace.StageSource, coreworkspace.StatusFailed)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest("source/manifest.json", coreworkspace.SourceManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ChunkRunning(moduleKey string, sourceDigest string) error {
|
||||
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusRunning)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ChunkSucceeded(moduleKey string, sourceDigest string, chunks []contracts.SourceChunk, warnings []contracts.Warning) error {
|
||||
payload := chunksEnvelope{Chunks: chunkEnvelopes(chunks), Warnings: cloneWarnings(warnings)}
|
||||
if err := r.writePayload("chunk/chunks.json", payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceeded)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
||||
manifest.OutputDigests = workspaceFingerprints(chunkOutputDigests(chunks))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{
|
||||
StageManifest: manifest,
|
||||
ChunkCount: len(chunks),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error {
|
||||
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusSucceededWithRejections)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
||||
manifest.ValidationStatus = "rejected"
|
||||
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ChunkFailed(moduleKey string, sourceDigest string, err error) error {
|
||||
manifest := r.newStageManifest(coreworkspace.StageChunk, coreworkspace.StatusFailed)
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(digestFingerprints("source_document", sourceDigest))
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest("chunk/manifest.json", coreworkspace.ChunkManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ExtractRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ExtractSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, outputs []contracts.ExtractOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error {
|
||||
payload := extractOutputsEnvelope{
|
||||
Outputs: extractOutputEnvelopes(outputs),
|
||||
Rejected: cloneRejectedOutputs(rejected),
|
||||
Warnings: cloneWarnings(warnings),
|
||||
}
|
||||
if err := r.writePayload(lanePayloadPath("extract", laneID, "outputs.json"), payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.laneManifest(coreworkspace.StageExtract, statusForRejected(rejected), laneID, moduleKey, dependencies)
|
||||
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests(extractPayloads(outputs)))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, rejected)
|
||||
manifest.Rejections = rejectionSummaries(rejected)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{
|
||||
StageManifest: manifest,
|
||||
ChunkCount: len(outputs) + len(rejected),
|
||||
OutputCount: len(outputs),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) ExtractFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageExtract, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest(laneManifestPath("extract", laneID), coreworkspace.ExtractLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) MergeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) MergeSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output contracts.MergeOutput, warnings []contracts.Warning) error {
|
||||
payload := mergeOutputEnvelope{Output: mergeOutputEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}
|
||||
if err := r.writePayload(lanePayloadPath("merge", laneID, "output.json"), payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
|
||||
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{
|
||||
StageManifest: manifest,
|
||||
InputCount: len(dependencies),
|
||||
})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) MergeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
|
||||
manifest.ValidationStatus = "rejected"
|
||||
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) MergeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageMerge, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest(laneManifestPath("merge", laneID), coreworkspace.MergeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) NormalizeRunning(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusRunning, laneID, moduleKey, dependencies)
|
||||
manifest.StartedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) NormalizeSucceeded(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, output contracts.NormalizeOutput, warnings []contracts.Warning) error {
|
||||
payload := normalizeOutputEnvelope{Output: normalizeOutputEnvelopeFromOutput(output), Warnings: cloneWarnings(warnings)}
|
||||
if err := r.writePayload(lanePayloadPath("normalize", laneID, "output.json"), payload); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceeded, laneID, moduleKey, dependencies)
|
||||
manifest.OutputDigests = workspaceFingerprints(rawOutputDigests([]contracts.RawPayload{output.Payload}))
|
||||
manifest.ValidationStatus = validationStatusString(warnings, nil)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) NormalizeRejected(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, rejected contracts.RejectedOutput) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusSucceededWithRejections, laneID, moduleKey, dependencies)
|
||||
manifest.ValidationStatus = "rejected"
|
||||
manifest.Rejections = rejectionSummaries([]contracts.RejectedOutput{rejected})
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest, InputCount: len(dependencies)})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) NormalizeFailed(laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint, err error) error {
|
||||
manifest := r.laneManifest(coreworkspace.StageNormalize, coreworkspace.StatusFailed, laneID, moduleKey, dependencies)
|
||||
manifest.CompletedAt = timePtr(r.timestamp())
|
||||
manifest.Metadata = errorMetadata(err)
|
||||
return r.writeManifest(laneManifestPath("normalize", laneID), coreworkspace.NormalizeLaneManifest{StageManifest: manifest})
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) writeManifest(name string, payload any) error {
|
||||
return r.writeJSON(name, payload)
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) writePayload(name string, payload any) error {
|
||||
return r.writeJSON(name, payload)
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) writeJSON(name string, payload any) error {
|
||||
if r == nil || strings.TrimSpace(r.root) == "" {
|
||||
return nil
|
||||
}
|
||||
return coreworkspace.WriteJSON(r.root, name, payload)
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) timestamp() time.Time {
|
||||
if r == nil || r.now == nil {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
return r.now().UTC()
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) newStageManifest(stage coreworkspace.StageName, status coreworkspace.StageStatus) coreworkspace.StageManifest {
|
||||
manifest := coreworkspace.NewStageManifest(stage, status)
|
||||
if strings.TrimSpace(r.identityDigest) != "" {
|
||||
manifest.Metadata = map[string]string{"checkpoint_identity_digest": r.identityDigest}
|
||||
}
|
||||
return manifest
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) laneManifest(stage coreworkspace.StageName, status coreworkspace.StageStatus, laneID string, moduleKey string, dependencies []pipeline.CheckpointFingerprint) coreworkspace.StageManifest {
|
||||
manifest := r.newStageManifest(stage, status)
|
||||
manifest.LaneID = laneID
|
||||
manifest.ModuleKey = moduleKey
|
||||
manifest.DependencyFingerprints = workspaceFingerprints(dependencies)
|
||||
return manifest
|
||||
}
|
||||
|
||||
type sourceDocumentEnvelope struct {
|
||||
Document source.SourceDocument `json:"document"`
|
||||
}
|
||||
|
||||
type chunksEnvelope struct {
|
||||
Chunks []chunkEnvelope `json:"chunks"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type chunkEnvelope struct {
|
||||
ID string `json:"id"`
|
||||
SourceID string `json:"source_id"`
|
||||
Index int `json:"index"`
|
||||
StartUnitID int `json:"start_unit_id"`
|
||||
EndUnitID int `json:"end_unit_id"`
|
||||
Content binaryEnvelope `json:"content"`
|
||||
Units []source.SourceUnit `json:"units,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type extractOutputsEnvelope struct {
|
||||
Outputs []extractOutputEnvelope `json:"outputs"`
|
||||
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type extractOutputEnvelope struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
SourceID string `json:"source_id"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ChunkIndex int `json:"chunk_index"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload binaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type mergeOutputEnvelope struct {
|
||||
Output mergeOutputPayload `json:"output"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type mergeOutputPayload struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
MergerKey string `json:"merger_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload binaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type normalizeOutputEnvelope struct {
|
||||
Output normalizeOutputPayload `json:"output"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type normalizeOutputPayload struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
NormalizerKey string `json:"normalizer_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload binaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type binaryEnvelope struct {
|
||||
ContentBase64 string `json:"content_base64,omitempty"`
|
||||
ContentDigest string `json:"content_digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
func chunkEnvelopes(chunks []contracts.SourceChunk) []chunkEnvelope {
|
||||
if len(chunks) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]chunkEnvelope, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
out = append(out, chunkEnvelope{
|
||||
ID: chunk.ID,
|
||||
SourceID: chunk.SourceID,
|
||||
Index: chunk.Index,
|
||||
StartUnitID: chunk.StartUnitID,
|
||||
EndUnitID: chunk.EndUnitID,
|
||||
Content: binaryEnvelopeFromContent(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
|
||||
Units: cloneSourceUnits(chunk.Units),
|
||||
Metadata: cloneMetadata(chunk.Metadata),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func extractOutputEnvelopes(outputs []contracts.ExtractOutput) []extractOutputEnvelope {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]extractOutputEnvelope, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
out = append(out, extractOutputEnvelope{
|
||||
LaneID: output.LaneID,
|
||||
ExtractorKey: output.ExtractorKey,
|
||||
SourceID: output.SourceID,
|
||||
ChunkID: output.ChunkID,
|
||||
ChunkIndex: output.ChunkIndex,
|
||||
Schema: schemaEnvelope(output.Schema),
|
||||
Payload: binaryEnvelopeFromPayload(output.Payload),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mergeOutputEnvelopeFromOutput(output contracts.MergeOutput) mergeOutputPayload {
|
||||
return mergeOutputPayload{
|
||||
LaneID: output.LaneID,
|
||||
MergerKey: output.MergerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: schemaEnvelope(output.Schema),
|
||||
Payload: binaryEnvelopeFromPayload(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeOutputEnvelopeFromOutput(output contracts.NormalizeOutput) normalizeOutputPayload {
|
||||
return normalizeOutputPayload{
|
||||
LaneID: output.LaneID,
|
||||
NormalizerKey: output.NormalizerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: schemaEnvelope(output.Schema),
|
||||
Payload: binaryEnvelopeFromPayload(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func schemaEnvelope(schema contracts.ResponseSchema) contracts.ResponseSchema {
|
||||
schema.JSONSchema = nil
|
||||
return schema
|
||||
}
|
||||
|
||||
func binaryEnvelopeFromPayload(payload contracts.RawPayload) binaryEnvelope {
|
||||
return binaryEnvelopeFromContent(payload.Content, payload.MediaType, payload.Metadata, payload.Warnings)
|
||||
}
|
||||
|
||||
func binaryEnvelopeFromContent(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) binaryEnvelope {
|
||||
return binaryEnvelope{
|
||||
ContentBase64: base64.StdEncoding.EncodeToString(content),
|
||||
ContentDigest: contentDigest(content),
|
||||
MediaType: mediaType,
|
||||
Metadata: cloneMetadata(metadata),
|
||||
Warnings: cloneWarnings(warnings),
|
||||
}
|
||||
}
|
||||
|
||||
func cloneSourceDocument(doc source.SourceDocument) source.SourceDocument {
|
||||
doc.Units = cloneSourceUnits(doc.Units)
|
||||
doc.Metadata = cloneMetadata(doc.Metadata)
|
||||
return doc
|
||||
}
|
||||
|
||||
func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
|
||||
if len(units) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]source.SourceUnit, 0, len(units))
|
||||
for _, unit := range units {
|
||||
out = append(out, source.SourceUnit{
|
||||
ID: unit.ID,
|
||||
Kind: unit.Kind,
|
||||
Text: unit.Text,
|
||||
Metadata: cloneMetadata(unit.Metadata),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
|
||||
if len(warnings) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]contracts.Warning(nil), warnings...)
|
||||
}
|
||||
|
||||
func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
|
||||
if len(rejected) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]contracts.RejectedOutput(nil), rejected...)
|
||||
}
|
||||
|
||||
func cloneMetadata(metadata map[string]any) map[string]any {
|
||||
if len(metadata) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(metadata))
|
||||
for key, value := range metadata {
|
||||
out[key] = value
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rawOutputDigests(payloads []contracts.RawPayload) []pipeline.CheckpointFingerprint {
|
||||
values := make([]pipeline.CheckpointFingerprint, 0, len(payloads))
|
||||
for i, payload := range payloads {
|
||||
values = append(values, pipeline.CheckpointFingerprint{
|
||||
Name: fmt.Sprintf("payload[%d]", i),
|
||||
Value: contentDigest(payload.Content),
|
||||
})
|
||||
}
|
||||
return normalizeFingerprints(values)
|
||||
}
|
||||
|
||||
func extractPayloads(outputs []contracts.ExtractOutput) []contracts.RawPayload {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
payloads := make([]contracts.RawPayload, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
payloads = append(payloads, output.Payload)
|
||||
}
|
||||
return payloads
|
||||
}
|
||||
|
||||
func chunkOutputDigests(chunks []contracts.SourceChunk) []pipeline.CheckpointFingerprint {
|
||||
values := make([]pipeline.CheckpointFingerprint, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
values = append(values, pipeline.CheckpointFingerprint{
|
||||
Name: chunk.ID,
|
||||
Value: contentDigest(chunk.Content),
|
||||
})
|
||||
}
|
||||
return normalizeFingerprints(values)
|
||||
}
|
||||
|
||||
func digestFingerprints(name string, digest string) []pipeline.CheckpointFingerprint {
|
||||
digest = strings.TrimSpace(digest)
|
||||
if digest == "" {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{{Name: name, Value: digest}}
|
||||
}
|
||||
|
||||
func workspaceFingerprints(values []pipeline.CheckpointFingerprint) []coreworkspace.Fingerprint {
|
||||
normalized := normalizeFingerprints(values)
|
||||
if len(normalized) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]coreworkspace.Fingerprint, 0, len(normalized))
|
||||
for _, value := range normalized {
|
||||
out = append(out, coreworkspace.Fingerprint{Name: value.Name, Value: value.Value})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeFingerprints(values []pipeline.CheckpointFingerprint) []pipeline.CheckpointFingerprint {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
byName := make(map[string]string, len(values))
|
||||
for _, value := range values {
|
||||
name := strings.TrimSpace(value.Name)
|
||||
fingerprint := strings.TrimSpace(value.Value)
|
||||
if name == "" || fingerprint == "" {
|
||||
continue
|
||||
}
|
||||
byName[name] = fingerprint
|
||||
}
|
||||
if len(byName) == 0 {
|
||||
return nil
|
||||
}
|
||||
names := make([]string, 0, len(byName))
|
||||
for name := range byName {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
out := make([]pipeline.CheckpointFingerprint, 0, len(names))
|
||||
for _, name := range names {
|
||||
out = append(out, pipeline.CheckpointFingerprint{Name: name, Value: byName[name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rejectionSummaries(rejected []contracts.RejectedOutput) []coreworkspace.RejectionSummary {
|
||||
if len(rejected) == 0 {
|
||||
return nil
|
||||
}
|
||||
type key struct {
|
||||
validatorName string
|
||||
reasonCode string
|
||||
message string
|
||||
}
|
||||
counts := make(map[key]int, len(rejected))
|
||||
for _, item := range rejected {
|
||||
k := key{validatorName: item.ValidatorName, reasonCode: item.ReasonCode, message: item.Message}
|
||||
counts[k]++
|
||||
}
|
||||
keys := make([]key, 0, len(counts))
|
||||
for k := range counts {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].validatorName != keys[j].validatorName {
|
||||
return keys[i].validatorName < keys[j].validatorName
|
||||
}
|
||||
if keys[i].reasonCode != keys[j].reasonCode {
|
||||
return keys[i].reasonCode < keys[j].reasonCode
|
||||
}
|
||||
return keys[i].message < keys[j].message
|
||||
})
|
||||
out := make([]coreworkspace.RejectionSummary, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
out = append(out, coreworkspace.RejectionSummary{
|
||||
ValidatorName: k.validatorName,
|
||||
ReasonCode: k.reasonCode,
|
||||
Message: k.message,
|
||||
Count: counts[k],
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func statusForRejected(rejected []contracts.RejectedOutput) coreworkspace.StageStatus {
|
||||
if len(rejected) > 0 {
|
||||
return coreworkspace.StatusSucceededWithRejections
|
||||
}
|
||||
return coreworkspace.StatusSucceeded
|
||||
}
|
||||
|
||||
func validationStatusString(warnings []contracts.Warning, rejected []contracts.RejectedOutput) string {
|
||||
if len(rejected) > 0 {
|
||||
return "rejected"
|
||||
}
|
||||
if len(warnings) > 0 {
|
||||
return "approved_with_warnings"
|
||||
}
|
||||
return "approved"
|
||||
}
|
||||
|
||||
func errorMetadata(err error) map[string]string {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{"error": err.Error()}
|
||||
}
|
||||
|
||||
func laneManifestPath(stage string, laneID string) string {
|
||||
return lanePayloadPath(stage, laneID, "manifest.json")
|
||||
}
|
||||
|
||||
func lanePayloadPath(stage string, laneID string, file string) string {
|
||||
return path.Join(stage, checkpointPathComponent(laneID), file)
|
||||
}
|
||||
|
||||
func checkpointPathComponent(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return "_"
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
b.WriteRune(r)
|
||||
case r >= 'A' && r <= 'Z':
|
||||
b.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '-' || r == '_' || r == '.':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteString(fmt.Sprintf("~%x", r))
|
||||
}
|
||||
}
|
||||
out := b.String()
|
||||
if out == "." || out == ".." || strings.Contains(out, "..") {
|
||||
return "_"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func contentDigest(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func timePtr(t time.Time) *time.Time {
|
||||
return &t
|
||||
}
|
||||
344
internal/framework/checkpoint/recorder_test.go
Normal file
344
internal/framework/checkpoint/recorder_test.go
Normal file
@@ -0,0 +1,344 @@
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
func TestWorkspaceRecorderWritesSuccessfulCheckpointFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
doc := &source.SourceDocument{
|
||||
ID: "source-1",
|
||||
Kind: "document",
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:source",
|
||||
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello"}},
|
||||
}
|
||||
chunks := []contracts.SourceChunk{
|
||||
{
|
||||
ID: "chunk-1",
|
||||
SourceID: "source-1",
|
||||
Index: 0,
|
||||
StartUnitID: 1,
|
||||
EndUnitID: 1,
|
||||
Content: []byte("chunk content"),
|
||||
MediaType: "text/plain",
|
||||
Units: doc.Units,
|
||||
},
|
||||
}
|
||||
|
||||
if err := recorder.SourceRunning("seriatim"); err != nil {
|
||||
t.Fatalf("SourceRunning: %v", err)
|
||||
}
|
||||
assertManifestStatus(t, filepath.Join(root, "source", "manifest.json"), coreworkspace.StatusRunning)
|
||||
if err := recorder.SourceSucceeded("seriatim", doc); err != nil {
|
||||
t.Fatalf("SourceSucceeded: %v", err)
|
||||
}
|
||||
assertManifestStatus(t, filepath.Join(root, "source", "manifest.json"), coreworkspace.StatusSucceeded)
|
||||
if _, err := os.Stat(filepath.Join(root, "source", "source-document.json")); err != nil {
|
||||
t.Fatalf("expected source checkpoint payload: %v", err)
|
||||
}
|
||||
|
||||
if err := recorder.ChunkRunning("generic", doc.Digest); err != nil {
|
||||
t.Fatalf("ChunkRunning: %v", err)
|
||||
}
|
||||
if err := recorder.ChunkSucceeded("generic", doc.Digest, chunks, nil); err != nil {
|
||||
t.Fatalf("ChunkSucceeded: %v", err)
|
||||
}
|
||||
assertManifestStatus(t, filepath.Join(root, "chunk", "manifest.json"), coreworkspace.StatusSucceeded)
|
||||
var chunkPayload struct {
|
||||
Chunks []struct {
|
||||
Content struct {
|
||||
ContentBase64 string `json:"content_base64"`
|
||||
ContentDigest string `json:"content_digest"`
|
||||
} `json:"content"`
|
||||
} `json:"chunks"`
|
||||
}
|
||||
readJSON(t, filepath.Join(root, "chunk", "chunks.json"), &chunkPayload)
|
||||
if len(chunkPayload.Chunks) != 1 {
|
||||
t.Fatalf("checkpoint chunks = %#v, want one", chunkPayload.Chunks)
|
||||
}
|
||||
decoded, err := base64.StdEncoding.DecodeString(chunkPayload.Chunks[0].Content.ContentBase64)
|
||||
if err != nil {
|
||||
t.Fatalf("decode chunk content: %v", err)
|
||||
}
|
||||
if string(decoded) != "chunk content" {
|
||||
t.Fatalf("chunk content = %q, want original content", decoded)
|
||||
}
|
||||
if got, want := chunkPayload.Chunks[0].Content.ContentDigest, contentDigest([]byte("chunk content")); got != want {
|
||||
t.Fatalf("content digest = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceLoaderReusesSuccessfulCheckpointFiles(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
loader := &WorkspaceLoader{root: root}
|
||||
doc := &source.SourceDocument{
|
||||
ID: "source-1",
|
||||
Kind: "document",
|
||||
Format: "text/plain",
|
||||
Digest: "sha256:source",
|
||||
Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "hello"}},
|
||||
}
|
||||
chunks := []contracts.SourceChunk{
|
||||
{
|
||||
ID: "chunk-1",
|
||||
SourceID: "source-1",
|
||||
Index: 0,
|
||||
StartUnitID: 1,
|
||||
EndUnitID: 1,
|
||||
Content: []byte("chunk content"),
|
||||
MediaType: "text/plain",
|
||||
Units: doc.Units,
|
||||
},
|
||||
}
|
||||
extractOutput := contracts.ExtractOutput{
|
||||
LaneID: "spells",
|
||||
ExtractorKey: "dnd/spells",
|
||||
SourceID: doc.ID,
|
||||
ChunkID: "chunk-1",
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"spell":"cure wounds"}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
mergeOutput := contracts.MergeOutput{
|
||||
LaneID: "spells",
|
||||
MergerKey: "appendorder",
|
||||
SourceID: doc.ID,
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"merged":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
normalizeOutput := contracts.NormalizeOutput{
|
||||
LaneID: "spells",
|
||||
NormalizerKey: "noop",
|
||||
SourceID: doc.ID,
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"normalized":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
|
||||
if err := recorder.SourceSucceeded("seriatim", doc); err != nil {
|
||||
t.Fatalf("SourceSucceeded: %v", err)
|
||||
}
|
||||
if err := recorder.ChunkSucceeded("generic", doc.Digest, chunks, nil); err != nil {
|
||||
t.Fatalf("ChunkSucceeded: %v", err)
|
||||
}
|
||||
extractDeps := []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}
|
||||
if err := recorder.ExtractSucceeded("spells", "dnd/spells", extractDeps, []contracts.ExtractOutput{extractOutput}, nil, nil); err != nil {
|
||||
t.Fatalf("ExtractSucceeded: %v", err)
|
||||
}
|
||||
mergeDeps := rawOutputDigests([]contracts.RawPayload{extractOutput.Payload})
|
||||
if err := recorder.MergeSucceeded("spells", "appendorder", mergeDeps, mergeOutput, nil); err != nil {
|
||||
t.Fatalf("MergeSucceeded: %v", err)
|
||||
}
|
||||
normalizeDeps := rawOutputDigests([]contracts.RawPayload{mergeOutput.Payload})
|
||||
if err := recorder.NormalizeSucceeded("spells", "noop", normalizeDeps, normalizeOutput, nil); err != nil {
|
||||
t.Fatalf("NormalizeSucceeded: %v", err)
|
||||
}
|
||||
|
||||
sourceCheckpoint, decision := loader.Source("seriatim")
|
||||
if !decision.Reused || sourceCheckpoint.Document.ID != "source-1" {
|
||||
t.Fatalf("source decision = %#v checkpoint=%#v, want reused", decision, sourceCheckpoint)
|
||||
}
|
||||
chunkCheckpoint, decision := loader.Chunk("generic", doc.Digest)
|
||||
if !decision.Reused || len(chunkCheckpoint.Chunks) != 1 || string(chunkCheckpoint.Chunks[0].Content) != "chunk content" {
|
||||
t.Fatalf("chunk decision = %#v checkpoint=%#v, want reused", decision, chunkCheckpoint)
|
||||
}
|
||||
extractCheckpoint, decision := loader.Extract("spells", "dnd/spells", extractDeps)
|
||||
if !decision.Reused || len(extractCheckpoint.Outputs) != 1 || string(extractCheckpoint.Outputs[0].Payload.Content) != `{"spell":"cure wounds"}` {
|
||||
t.Fatalf("extract decision = %#v checkpoint=%#v, want reused", decision, extractCheckpoint)
|
||||
}
|
||||
mergeCheckpoint, decision := loader.Merge("spells", "appendorder", mergeDeps)
|
||||
if !decision.Reused || string(mergeCheckpoint.Output.Payload.Content) != `{"merged":true}` {
|
||||
t.Fatalf("merge decision = %#v checkpoint=%#v, want reused", decision, mergeCheckpoint)
|
||||
}
|
||||
normalizeCheckpoint, decision := loader.Normalize("spells", "noop", normalizeDeps)
|
||||
if !decision.Reused || string(normalizeCheckpoint.Output.Payload.Content) != `{"normalized":true}` {
|
||||
t.Fatalf("normalize decision = %#v checkpoint=%#v, want reused", decision, normalizeCheckpoint)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceLoaderInvalidatesMissingCorruptAndMismatchedCheckpoints(t *testing.T) {
|
||||
t.Run("missing", func(t *testing.T) {
|
||||
loader := &WorkspaceLoader{root: t.TempDir()}
|
||||
if _, decision := loader.Source("seriatim"); decision.Reused || !strings.Contains(decision.Reason, "missing") {
|
||||
t.Fatalf("decision = %#v, want missing invalidation", decision)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("dependency mismatch", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
chunks := []contracts.SourceChunk{{
|
||||
ID: "chunk-1",
|
||||
SourceID: "source-1",
|
||||
Content: []byte("chunk content"),
|
||||
MediaType: "text/plain",
|
||||
}}
|
||||
if err := recorder.ChunkSucceeded("generic", "sha256:source-a", chunks, nil); err != nil {
|
||||
t.Fatalf("ChunkSucceeded: %v", err)
|
||||
}
|
||||
loader := &WorkspaceLoader{root: root}
|
||||
if _, decision := loader.Chunk("generic", "sha256:source-b"); decision.Reused || !strings.Contains(decision.Reason, "dependency") {
|
||||
t.Fatalf("decision = %#v, want dependency invalidation", decision)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("corrupt payload", func(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
chunks := []contracts.SourceChunk{{
|
||||
ID: "chunk-1",
|
||||
SourceID: "source-1",
|
||||
Content: []byte("chunk content"),
|
||||
MediaType: "text/plain",
|
||||
}}
|
||||
if err := recorder.ChunkSucceeded("generic", "sha256:source", chunks, nil); err != nil {
|
||||
t.Fatalf("ChunkSucceeded: %v", err)
|
||||
}
|
||||
payloadPath := filepath.Join(root, "chunk", "chunks.json")
|
||||
data := strings.ReplaceAll(string(readFile(t, payloadPath)), contentDigest([]byte("chunk content")), "sha256:bad")
|
||||
if err := os.WriteFile(payloadPath, []byte(data), 0o644); err != nil {
|
||||
t.Fatalf("corrupt chunk payload: %v", err)
|
||||
}
|
||||
loader := &WorkspaceLoader{root: root}
|
||||
if _, decision := loader.Chunk("generic", "sha256:source"); decision.Reused || !strings.Contains(decision.Reason, "invalid") {
|
||||
t.Fatalf("decision = %#v, want corrupt payload invalidation", decision)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestWorkspaceRecorderRecordsRejectedExtractOutputs(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
rejected := []contracts.RejectedOutput{
|
||||
{
|
||||
Stage: string(pipeline.StageExtract),
|
||||
LaneID: "spells",
|
||||
ModuleKey: "dnd/spells",
|
||||
ChunkID: "chunk-1",
|
||||
ValidatorName: "shape",
|
||||
ReasonCode: "invalid_shape",
|
||||
Message: "bad shape",
|
||||
},
|
||||
}
|
||||
|
||||
if err := recorder.ExtractRunning("spells", "dnd/spells", []pipeline.CheckpointFingerprint{{Name: "chunks", Value: "sha256:chunks"}}); err != nil {
|
||||
t.Fatalf("ExtractRunning: %v", err)
|
||||
}
|
||||
if err := recorder.ExtractSucceeded("spells", "dnd/spells", nil, nil, rejected, nil); err != nil {
|
||||
t.Fatalf("ExtractSucceeded: %v", err)
|
||||
}
|
||||
|
||||
var manifest coreworkspace.ExtractLaneManifest
|
||||
readJSON(t, filepath.Join(root, "extract", "spells", "manifest.json"), &manifest)
|
||||
if manifest.Status != coreworkspace.StatusSucceededWithRejections || manifest.ValidationStatus != "rejected" {
|
||||
t.Fatalf("extract manifest status = %q validation=%q", manifest.Status, manifest.ValidationStatus)
|
||||
}
|
||||
if len(manifest.Rejections) != 1 || manifest.Rejections[0].Count != 1 || manifest.Rejections[0].ReasonCode != "invalid_shape" {
|
||||
t.Fatalf("rejections = %#v", manifest.Rejections)
|
||||
}
|
||||
var payload struct {
|
||||
Rejected []contracts.RejectedOutput `json:"rejected"`
|
||||
}
|
||||
readJSON(t, filepath.Join(root, "extract", "spells", "outputs.json"), &payload)
|
||||
if len(payload.Rejected) != 1 || payload.Rejected[0].ChunkID != "chunk-1" {
|
||||
t.Fatalf("checkpoint rejected payload = %#v", payload.Rejected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceRecorderRecordsFailedStages(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
|
||||
if err := recorder.MergeRunning("spells", "appendorder", nil); err != nil {
|
||||
t.Fatalf("MergeRunning: %v", err)
|
||||
}
|
||||
if err := recorder.MergeFailed("spells", "appendorder", nil, assertErr("merge failed")); err != nil {
|
||||
t.Fatalf("MergeFailed: %v", err)
|
||||
}
|
||||
|
||||
var manifest coreworkspace.MergeLaneManifest
|
||||
readJSON(t, filepath.Join(root, "merge", "spells", "manifest.json"), &manifest)
|
||||
if manifest.Status != coreworkspace.StatusFailed {
|
||||
t.Fatalf("status = %q, want failed", manifest.Status)
|
||||
}
|
||||
if !strings.Contains(manifest.Metadata["error"], "merge failed") {
|
||||
t.Fatalf("metadata = %#v, want error", manifest.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceRecorderRecordsWarningOnlyValidation(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
recorder := newTestRecorder(t, root)
|
||||
output := contracts.NormalizeOutput{
|
||||
LaneID: "spells",
|
||||
NormalizerKey: "noop",
|
||||
SourceID: "source-1",
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"ok":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
warnings := []contracts.Warning{{ReasonCode: "note", Message: "warning"}}
|
||||
|
||||
if err := recorder.NormalizeSucceeded("spells", "noop", nil, output, warnings); err != nil {
|
||||
t.Fatalf("NormalizeSucceeded: %v", err)
|
||||
}
|
||||
|
||||
var manifest coreworkspace.NormalizeLaneManifest
|
||||
readJSON(t, filepath.Join(root, "normalize", "spells", "manifest.json"), &manifest)
|
||||
if manifest.Status != coreworkspace.StatusSucceeded || manifest.ValidationStatus != "approved_with_warnings" {
|
||||
t.Fatalf("normalize manifest status = %q validation=%q", manifest.Status, manifest.ValidationStatus)
|
||||
}
|
||||
}
|
||||
|
||||
func newTestRecorder(t *testing.T, root string) *WorkspaceRecorder {
|
||||
t.Helper()
|
||||
return &WorkspaceRecorder{root: root}
|
||||
}
|
||||
|
||||
func assertManifestStatus(t *testing.T, path string, want coreworkspace.StageStatus) {
|
||||
t.Helper()
|
||||
var manifest coreworkspace.StageManifest
|
||||
readJSON(t, path, &manifest)
|
||||
if manifest.Status != want {
|
||||
t.Fatalf("%s status = %q, want %q", path, manifest.Status, want)
|
||||
}
|
||||
}
|
||||
|
||||
func readJSON(t *testing.T, path string, out any) {
|
||||
t.Helper()
|
||||
data := readFile(t, path)
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
t.Fatalf("decode %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read %q: %v", path, err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
type assertErr string
|
||||
|
||||
func (e assertErr) Error() string { return string(e) }
|
||||
34
internal/framework/debug/recorder.go
Normal file
34
internal/framework/debug/recorder.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package debug
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
coreworkspace "gitea.maximumdirect.net/eric/notarius/internal/core/workspace"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
type WorkspaceRecorder struct {
|
||||
root string
|
||||
}
|
||||
|
||||
func NewWorkspaceRecorder(settings coreworkspace.Settings, runID string) (pipeline.DebugRecorder, error) {
|
||||
root, err := settings.DebugRunDirectory(runID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(root) == "" {
|
||||
return pipeline.NoopDebugRecorder(), nil
|
||||
}
|
||||
return &WorkspaceRecorder{root: root}, nil
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) Enabled() bool {
|
||||
return r != nil && strings.TrimSpace(r.root) != ""
|
||||
}
|
||||
|
||||
func (r *WorkspaceRecorder) WriteJSON(name string, payload any) error {
|
||||
if !r.Enabled() {
|
||||
return nil
|
||||
}
|
||||
return coreworkspace.WriteJSON(r.root, name, payload)
|
||||
}
|
||||
227
internal/framework/pipeline/checkpoint.go
Normal file
227
internal/framework/pipeline/checkpoint.go
Normal file
@@ -0,0 +1,227 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type CheckpointFingerprint struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
type CheckpointRecorder interface {
|
||||
SourceRunning(moduleKey string) error
|
||||
SourceSucceeded(moduleKey string, doc *source.SourceDocument) error
|
||||
SourceFailed(moduleKey string, err error) error
|
||||
ChunkRunning(moduleKey string, sourceDigest string) error
|
||||
ChunkSucceeded(moduleKey string, sourceDigest string, chunks []contracts.SourceChunk, warnings []contracts.Warning) error
|
||||
ChunkRejected(moduleKey string, sourceDigest string, rejected contracts.RejectedOutput) error
|
||||
ChunkFailed(moduleKey string, sourceDigest string, err error) error
|
||||
ExtractRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
ExtractSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, outputs []contracts.ExtractOutput, rejected []contracts.RejectedOutput, warnings []contracts.Warning) error
|
||||
ExtractFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
MergeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
MergeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output contracts.MergeOutput, warnings []contracts.Warning) error
|
||||
MergeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
|
||||
MergeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
NormalizeRunning(laneID string, moduleKey string, dependencies []CheckpointFingerprint) error
|
||||
NormalizeSucceeded(laneID string, moduleKey string, dependencies []CheckpointFingerprint, output contracts.NormalizeOutput, warnings []contracts.Warning) error
|
||||
NormalizeRejected(laneID string, moduleKey string, dependencies []CheckpointFingerprint, rejected contracts.RejectedOutput) error
|
||||
NormalizeFailed(laneID string, moduleKey string, dependencies []CheckpointFingerprint, err error) error
|
||||
}
|
||||
|
||||
type CheckpointDecision struct {
|
||||
Reused bool `json:"reused"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
type CheckpointEvent struct {
|
||||
Stage string `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
Action string `json:"action"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
type SourceCheckpoint struct {
|
||||
Document *source.SourceDocument
|
||||
}
|
||||
|
||||
type ChunkCheckpoint struct {
|
||||
Chunks []contracts.SourceChunk
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type ExtractCheckpoint struct {
|
||||
Outputs []contracts.ExtractOutput
|
||||
Rejected []contracts.RejectedOutput
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type MergeCheckpoint struct {
|
||||
Output contracts.MergeOutput
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type NormalizeCheckpoint struct {
|
||||
Output contracts.NormalizeOutput
|
||||
Warnings []contracts.Warning
|
||||
}
|
||||
|
||||
type CheckpointLoader interface {
|
||||
Enabled() bool
|
||||
Source(moduleKey string) (SourceCheckpoint, CheckpointDecision)
|
||||
Chunk(moduleKey string, sourceDigest string) (ChunkCheckpoint, CheckpointDecision)
|
||||
Extract(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision)
|
||||
Merge(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision)
|
||||
Normalize(laneID string, moduleKey string, dependencies []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision)
|
||||
}
|
||||
|
||||
type noopCheckpointRecorder struct{}
|
||||
type noopCheckpointLoader struct{}
|
||||
|
||||
func NoopCheckpointRecorder() CheckpointRecorder { return noopCheckpointRecorder{} }
|
||||
func NoopCheckpointLoader() CheckpointLoader { return noopCheckpointLoader{} }
|
||||
|
||||
func (noopCheckpointRecorder) SourceRunning(string) error { return nil }
|
||||
func (noopCheckpointRecorder) SourceSucceeded(string, *source.SourceDocument) error { return nil }
|
||||
func (noopCheckpointRecorder) SourceFailed(string, error) error { return nil }
|
||||
func (noopCheckpointRecorder) ChunkRunning(string, string) error { return nil }
|
||||
func (noopCheckpointRecorder) ChunkSucceeded(string, string, []contracts.SourceChunk, []contracts.Warning) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) ChunkRejected(string, string, contracts.RejectedOutput) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) ChunkFailed(string, string, error) error { return nil }
|
||||
func (noopCheckpointRecorder) ExtractRunning(string, string, []CheckpointFingerprint) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) ExtractSucceeded(string, string, []CheckpointFingerprint, []contracts.ExtractOutput, []contracts.RejectedOutput, []contracts.Warning) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) ExtractFailed(string, string, []CheckpointFingerprint, error) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) MergeRunning(string, string, []CheckpointFingerprint) error { return nil }
|
||||
func (noopCheckpointRecorder) MergeSucceeded(string, string, []CheckpointFingerprint, contracts.MergeOutput, []contracts.Warning) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) MergeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) MergeFailed(string, string, []CheckpointFingerprint, error) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) NormalizeRunning(string, string, []CheckpointFingerprint) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) NormalizeSucceeded(string, string, []CheckpointFingerprint, contracts.NormalizeOutput, []contracts.Warning) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) NormalizeRejected(string, string, []CheckpointFingerprint, contracts.RejectedOutput) error {
|
||||
return nil
|
||||
}
|
||||
func (noopCheckpointRecorder) NormalizeFailed(string, string, []CheckpointFingerprint, error) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (noopCheckpointLoader) Enabled() bool { return false }
|
||||
func (noopCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) {
|
||||
return SourceCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
func (noopCheckpointLoader) Chunk(string, string) (ChunkCheckpoint, CheckpointDecision) {
|
||||
return ChunkCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
func (noopCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||
return ExtractCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
func (noopCheckpointLoader) Merge(string, string, []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
|
||||
return MergeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
func (noopCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
|
||||
return NormalizeCheckpoint{}, CheckpointDecision{Reason: "checkpoint loading disabled"}
|
||||
}
|
||||
|
||||
func rawOutputDigests(payloads []contracts.RawPayload) []CheckpointFingerprint {
|
||||
values := make([]CheckpointFingerprint, 0, len(payloads))
|
||||
for i, payload := range payloads {
|
||||
values = append(values, CheckpointFingerprint{
|
||||
Name: fmt.Sprintf("payload[%d]", i),
|
||||
Value: checkpointContentDigest(payload.Content),
|
||||
})
|
||||
}
|
||||
return normalizeCheckpointFingerprints(values)
|
||||
}
|
||||
|
||||
func extractPayloads(outputs []contracts.ExtractOutput) []contracts.RawPayload {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
payloads := make([]contracts.RawPayload, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
payloads = append(payloads, output.Payload)
|
||||
}
|
||||
return payloads
|
||||
}
|
||||
|
||||
func digestFingerprints(name string, digest string) []CheckpointFingerprint {
|
||||
digest = strings.TrimSpace(digest)
|
||||
if digest == "" {
|
||||
return nil
|
||||
}
|
||||
return []CheckpointFingerprint{{Name: name, Value: digest}}
|
||||
}
|
||||
|
||||
func joinedChunkDigest(chunks []contracts.SourceChunk) string {
|
||||
if len(chunks) == 0 {
|
||||
return ""
|
||||
}
|
||||
values := make([]string, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
values = append(values, chunk.ID+"="+checkpointContentDigest(chunk.Content))
|
||||
}
|
||||
sort.Strings(values)
|
||||
sum := sha256.Sum256([]byte(strings.Join(values, "\n")))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func normalizeCheckpointFingerprints(values []CheckpointFingerprint) []CheckpointFingerprint {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
byName := make(map[string]string, len(values))
|
||||
for _, value := range values {
|
||||
name := strings.TrimSpace(value.Name)
|
||||
fingerprint := strings.TrimSpace(value.Value)
|
||||
if name == "" || fingerprint == "" {
|
||||
continue
|
||||
}
|
||||
byName[name] = fingerprint
|
||||
}
|
||||
if len(byName) == 0 {
|
||||
return nil
|
||||
}
|
||||
names := make([]string, 0, len(byName))
|
||||
for name := range byName {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
out := make([]CheckpointFingerprint, 0, len(names))
|
||||
for _, name := range names {
|
||||
out = append(out, CheckpointFingerprint{Name: name, Value: byName[name]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func checkpointContentDigest(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
563
internal/framework/pipeline/debug.go
Normal file
563
internal/framework/pipeline/debug.go
Normal file
@@ -0,0 +1,563 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type DebugRecorder interface {
|
||||
Enabled() bool
|
||||
WriteJSON(name string, payload any) error
|
||||
}
|
||||
|
||||
type noopDebugRecorder struct{}
|
||||
|
||||
func NoopDebugRecorder() DebugRecorder { return noopDebugRecorder{} }
|
||||
|
||||
func (noopDebugRecorder) Enabled() bool { return false }
|
||||
func (noopDebugRecorder) WriteJSON(string, any) error { return nil }
|
||||
func debugPathComponent(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return "_"
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range value {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
b.WriteRune(r)
|
||||
case r >= 'A' && r <= 'Z':
|
||||
b.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '-' || r == '_' || r == '.':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteString(fmt.Sprintf("~%x", r))
|
||||
}
|
||||
}
|
||||
out := b.String()
|
||||
if out == "." || out == ".." || strings.Contains(out, "..") {
|
||||
return "_"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type debugTimedEnvelope struct {
|
||||
Stage string `json:"stage,omitempty"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key,omitempty"`
|
||||
Attempt int `json:"attempt,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
CompletedAt time.Time `json:"completed_at"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Payload any `json:"payload,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type debugBinaryEnvelope struct {
|
||||
ContentBase64 string `json:"content_base64,omitempty"`
|
||||
ContentDigest string `json:"content_digest,omitempty"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
type debugRawPayload struct {
|
||||
Content debugBinaryEnvelope `json:"content"`
|
||||
}
|
||||
|
||||
type debugSourceInput struct {
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Raw debugBinaryEnvelope `json:"raw,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type debugSourceDocument struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Format string `json:"format,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Units []source.SourceUnit `json:"units,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type debugSourceChunk struct {
|
||||
ID string `json:"id"`
|
||||
SourceID string `json:"source_id"`
|
||||
Index int `json:"index"`
|
||||
StartUnitID int `json:"start_unit_id"`
|
||||
EndUnitID int `json:"end_unit_id"`
|
||||
Content debugBinaryEnvelope `json:"content"`
|
||||
Units []source.SourceUnit `json:"units,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type debugExtractOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
ExtractorKey string `json:"extractor_key"`
|
||||
SourceID string `json:"source_id"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ChunkIndex int `json:"chunk_index"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload debugBinaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type debugMergeOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
MergerKey string `json:"merger_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload debugBinaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type debugNormalizeOutput struct {
|
||||
LaneID string `json:"lane_id"`
|
||||
NormalizerKey string `json:"normalizer_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload debugBinaryEnvelope `json:"payload"`
|
||||
}
|
||||
|
||||
type debugLLMInputMaterial struct {
|
||||
Name string `json:"name"`
|
||||
MediaType string `json:"media_type,omitempty"`
|
||||
Content string `json:"content_base64,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
OriginURI string `json:"origin_uri,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes,omitempty"`
|
||||
}
|
||||
|
||||
type debugStructuredCompletionRequest struct {
|
||||
StageName string `json:"stage_name"`
|
||||
PromptID string `json:"prompt_id,omitempty"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Inputs map[string]debugLLMInputMaterial `json:"inputs,omitempty"`
|
||||
Vars map[string]any `json:"vars,omitempty"`
|
||||
}
|
||||
|
||||
type debugStructuredCompletionResponse struct {
|
||||
Content string `json:"content,omitempty"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
PromptTokens int `json:"prompt_tokens,omitempty"`
|
||||
CompletionTokens int `json:"completion_tokens,omitempty"`
|
||||
TotalTokens int `json:"total_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type debugStructuredLLMCall struct {
|
||||
Request debugStructuredCompletionRequest `json:"request"`
|
||||
Response debugStructuredCompletionResponse `json:"response,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type debugValidationRequest struct {
|
||||
Stage string `json:"stage"`
|
||||
LaneID string `json:"lane_id,omitempty"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
SourceID string `json:"source_id,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
Metadata map[string]any `json:"metadata,omitempty"`
|
||||
Schema contracts.ResponseSchema `json:"schema,omitempty"`
|
||||
Payload *debugBinaryEnvelope `json:"payload,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
Chunk *debugSourceChunk `json:"chunk,omitempty"`
|
||||
Chunks []debugSourceChunk `json:"chunks,omitempty"`
|
||||
ExtractOutputs []debugExtractOutput `json:"extract_outputs,omitempty"`
|
||||
MergeOutput *debugMergeOutput `json:"merge_output,omitempty"`
|
||||
}
|
||||
|
||||
type debugValidationCall struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Request debugValidationRequest `json:"request"`
|
||||
Result contracts.ValidationResult `json:"result,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type debugLLMClient struct {
|
||||
inner contracts.StructuredLLMClient
|
||||
recorder DebugRecorder
|
||||
counter int
|
||||
}
|
||||
|
||||
func wrapDebugLLMClient(client contracts.StructuredLLMClient, recorder DebugRecorder) contracts.StructuredLLMClient {
|
||||
if client == nil || recorder == nil || !recorder.Enabled() {
|
||||
return client
|
||||
}
|
||||
return &debugLLMClient{inner: client, recorder: recorder}
|
||||
}
|
||||
|
||||
func (client *debugLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.counter++
|
||||
started := time.Now().UTC()
|
||||
response, err := client.inner.CompleteStructured(ctx, req, out)
|
||||
completed := time.Now().UTC()
|
||||
payload := debugStructuredLLMCall{
|
||||
Request: debugCompletionRequest(req),
|
||||
Response: debugCompletionResponse(response),
|
||||
}
|
||||
if err != nil {
|
||||
payload.Error = err.Error()
|
||||
}
|
||||
writeErr := writeDebugTimed(client.recorder, path.Join("llm", fmt.Sprintf("call-%04d.json", client.counter)), debugTimedEnvelope{
|
||||
Stage: req.StageName,
|
||||
ModuleKey: req.StageName,
|
||||
StartedAt: started,
|
||||
CompletedAt: completed,
|
||||
DurationMS: completed.Sub(started).Milliseconds(),
|
||||
Payload: payload,
|
||||
Error: payload.Error,
|
||||
})
|
||||
if err != nil {
|
||||
return response, err
|
||||
}
|
||||
if writeErr != nil {
|
||||
return response, fmt.Errorf("write LLM debug artifact: %w", writeErr)
|
||||
}
|
||||
return response, err
|
||||
}
|
||||
|
||||
func (client *debugLLMClient) LLMProfileManifests() []artifacts.LLMProfileManifest {
|
||||
provider, ok := client.inner.(contracts.LLMProfileManifestProvider)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return provider.LLMProfileManifests()
|
||||
}
|
||||
|
||||
func writeDebugTimed(recorder DebugRecorder, name string, envelope debugTimedEnvelope) error {
|
||||
if recorder == nil || !recorder.Enabled() {
|
||||
return nil
|
||||
}
|
||||
if envelope.CompletedAt.IsZero() {
|
||||
envelope.CompletedAt = time.Now().UTC()
|
||||
}
|
||||
if envelope.StartedAt.IsZero() {
|
||||
envelope.StartedAt = envelope.CompletedAt
|
||||
}
|
||||
if envelope.DurationMS == 0 {
|
||||
envelope.DurationMS = envelope.CompletedAt.Sub(envelope.StartedAt).Milliseconds()
|
||||
}
|
||||
return recorder.WriteJSON(name, envelope)
|
||||
}
|
||||
|
||||
func debugContentEnvelope(content []byte, mediaType string, metadata map[string]any, warnings []contracts.Warning) debugBinaryEnvelope {
|
||||
content = redactSecretBytes(content)
|
||||
return debugBinaryEnvelope{
|
||||
ContentBase64: base64.StdEncoding.EncodeToString(content),
|
||||
ContentDigest: debugContentDigest(content),
|
||||
MediaType: mediaType,
|
||||
Metadata: redactSensitiveMap(metadata),
|
||||
Warnings: cloneWarnings(warnings),
|
||||
}
|
||||
}
|
||||
|
||||
func debugPayloadEnvelope(payload contracts.RawPayload) debugBinaryEnvelope {
|
||||
return debugContentEnvelope(payload.Content, payload.MediaType, payload.Metadata, payload.Warnings)
|
||||
}
|
||||
|
||||
func debugSourceDocumentEnvelope(doc *source.SourceDocument) *debugSourceDocument {
|
||||
if doc == nil {
|
||||
return nil
|
||||
}
|
||||
return &debugSourceDocument{
|
||||
ID: doc.ID,
|
||||
Kind: doc.Kind,
|
||||
Format: doc.Format,
|
||||
Digest: doc.Digest,
|
||||
Units: cloneSourceUnits(doc.Units),
|
||||
Metadata: redactSensitiveMap(doc.Metadata),
|
||||
}
|
||||
}
|
||||
|
||||
func debugSourceChunkEnvelope(chunk contracts.SourceChunk) debugSourceChunk {
|
||||
return debugSourceChunk{
|
||||
ID: chunk.ID,
|
||||
SourceID: chunk.SourceID,
|
||||
Index: chunk.Index,
|
||||
StartUnitID: chunk.StartUnitID,
|
||||
EndUnitID: chunk.EndUnitID,
|
||||
Content: debugContentEnvelope(chunk.Content, chunk.MediaType, chunk.Metadata, nil),
|
||||
Units: cloneSourceUnits(chunk.Units),
|
||||
Metadata: redactSensitiveMap(chunk.Metadata),
|
||||
}
|
||||
}
|
||||
|
||||
func debugSourceChunkEnvelopes(chunks []contracts.SourceChunk) []debugSourceChunk {
|
||||
if len(chunks) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]debugSourceChunk, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
out = append(out, debugSourceChunkEnvelope(chunk))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugExtractOutputEnvelope(output contracts.ExtractOutput) debugExtractOutput {
|
||||
output.Schema.JSONSchema = nil
|
||||
return debugExtractOutput{
|
||||
LaneID: output.LaneID,
|
||||
ExtractorKey: output.ExtractorKey,
|
||||
SourceID: output.SourceID,
|
||||
ChunkID: output.ChunkID,
|
||||
ChunkIndex: output.ChunkIndex,
|
||||
Schema: output.Schema,
|
||||
Payload: debugPayloadEnvelope(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func debugExtractOutputEnvelopes(outputs []contracts.ExtractOutput) []debugExtractOutput {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]debugExtractOutput, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
out = append(out, debugExtractOutputEnvelope(output))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugMergeOutputEnvelope(output contracts.MergeOutput) debugMergeOutput {
|
||||
output.Schema.JSONSchema = nil
|
||||
return debugMergeOutput{
|
||||
LaneID: output.LaneID,
|
||||
MergerKey: output.MergerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: output.Schema,
|
||||
Payload: debugPayloadEnvelope(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func debugNormalizeOutputEnvelope(output contracts.NormalizeOutput) debugNormalizeOutput {
|
||||
output.Schema.JSONSchema = nil
|
||||
return debugNormalizeOutput{
|
||||
LaneID: output.LaneID,
|
||||
NormalizerKey: output.NormalizerKey,
|
||||
SourceID: output.SourceID,
|
||||
Schema: output.Schema,
|
||||
Payload: debugPayloadEnvelope(output.Payload),
|
||||
}
|
||||
}
|
||||
|
||||
func debugNormalizeOutputEnvelopes(outputs []contracts.NormalizeOutput) []debugNormalizeOutput {
|
||||
if len(outputs) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]debugNormalizeOutput, 0, len(outputs))
|
||||
for _, output := range outputs {
|
||||
out = append(out, debugNormalizeOutputEnvelope(output))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type debugOutputFile struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
Content debugBinaryEnvelope `json:"content"`
|
||||
}
|
||||
|
||||
func debugOutputFiles(files []contracts.OutputFile) []debugOutputFile {
|
||||
if len(files) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]debugOutputFile, 0, len(files))
|
||||
for _, file := range files {
|
||||
out = append(out, debugOutputFile{
|
||||
Name: file.Name,
|
||||
ContentType: file.ContentType,
|
||||
Content: debugContentEnvelope(file.Bytes, file.ContentType, nil, nil),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugCompletionRequest(req contracts.StructuredCompletionRequest) debugStructuredCompletionRequest {
|
||||
inputs := make(map[string]debugLLMInputMaterial, len(req.Inputs))
|
||||
for key, material := range req.Inputs {
|
||||
inputs[key] = debugLLMInputMaterial{
|
||||
Name: material.Name,
|
||||
MediaType: material.MediaType,
|
||||
Content: base64.StdEncoding.EncodeToString(redactSecretBytes(material.Content)),
|
||||
Digest: material.Digest,
|
||||
OriginURI: material.OriginURI,
|
||||
SizeBytes: material.SizeBytes,
|
||||
}
|
||||
}
|
||||
if len(inputs) == 0 {
|
||||
inputs = nil
|
||||
}
|
||||
return debugStructuredCompletionRequest{
|
||||
StageName: req.StageName,
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
Inputs: inputs,
|
||||
Vars: redactSensitiveMap(req.Vars),
|
||||
}
|
||||
}
|
||||
|
||||
func debugCompletionResponse(response contracts.StructuredCompletionResponse) debugStructuredCompletionResponse {
|
||||
return debugStructuredCompletionResponse{
|
||||
Content: base64.StdEncoding.EncodeToString(redactSecretBytes(response.Content)),
|
||||
Provider: response.Provider,
|
||||
Model: response.Model,
|
||||
ProfileID: response.ProfileID,
|
||||
PromptTokens: response.PromptTokens,
|
||||
CompletionTokens: response.CompletionTokens,
|
||||
TotalTokens: response.TotalTokens,
|
||||
}
|
||||
}
|
||||
|
||||
func debugValidationRequestEnvelope(req contracts.ValidationRequest) debugValidationRequest {
|
||||
req.Schema.JSONSchema = nil
|
||||
out := debugValidationRequest{
|
||||
Stage: req.Stage,
|
||||
LaneID: req.LaneID,
|
||||
ModuleKey: req.ModuleKey,
|
||||
SourceID: req.SourceID,
|
||||
SessionID: req.SessionID,
|
||||
LLMProfile: req.LLMProfile,
|
||||
Options: redactSensitiveMap(req.Options),
|
||||
Metadata: redactSensitiveMap(req.Metadata),
|
||||
Schema: req.Schema,
|
||||
ChunkID: req.ChunkID,
|
||||
ChunkIndex: req.ChunkIndex,
|
||||
}
|
||||
payload := debugPayloadEnvelope(req.Payload)
|
||||
out.Payload = &payload
|
||||
if req.Chunk != nil {
|
||||
chunk := debugSourceChunkEnvelope(*req.Chunk)
|
||||
out.Chunk = &chunk
|
||||
}
|
||||
out.Chunks = debugSourceChunkEnvelopes(req.Chunks)
|
||||
out.ExtractOutputs = debugExtractOutputEnvelopes(req.ExtractOutputs)
|
||||
if len(req.MergeOutput.Payload.Content) > 0 || req.MergeOutput.LaneID != "" {
|
||||
merge := debugMergeOutputEnvelope(req.MergeOutput)
|
||||
out.MergeOutput = &merge
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugValidationResultEnvelope(result contracts.ValidationResult) contracts.ValidationResult {
|
||||
result.Message = string(redactSecretBytes([]byte(result.Message)))
|
||||
result.DiagnosticArtifactPath = string(redactSecretBytes([]byte(result.DiagnosticArtifactPath)))
|
||||
for i := range result.Warnings {
|
||||
result.Warnings[i].Message = string(redactSecretBytes([]byte(result.Warnings[i].Message)))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func debugRejectedOutputEnvelope(rejected contracts.RejectedOutput) contracts.RejectedOutput {
|
||||
rejected.Message = string(redactSecretBytes([]byte(rejected.Message)))
|
||||
rejected.DiagnosticArtifactPath = string(redactSecretBytes([]byte(rejected.DiagnosticArtifactPath)))
|
||||
return rejected
|
||||
}
|
||||
|
||||
func debugRejectedOutputPtr(rejected *contracts.RejectedOutput) any {
|
||||
if rejected == nil {
|
||||
return nil
|
||||
}
|
||||
out := debugRejectedOutputEnvelope(*rejected)
|
||||
return out
|
||||
}
|
||||
|
||||
func debugRejectedOutputEnvelopes(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
|
||||
if len(rejected) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]contracts.RejectedOutput, 0, len(rejected))
|
||||
for _, item := range rejected {
|
||||
out = append(out, debugRejectedOutputEnvelope(item))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func debugContentDigest(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
var secretPatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)bearer\s+[a-z0-9._~+/=-]{8,}`),
|
||||
regexp.MustCompile(`(?i)sk-[a-z0-9_-]{8,}`),
|
||||
}
|
||||
|
||||
func redactSecretBytes(content []byte) []byte {
|
||||
if len(content) == 0 || !utf8.Valid(content) {
|
||||
return append([]byte(nil), content...)
|
||||
}
|
||||
text := string(content)
|
||||
for _, pattern := range secretPatterns {
|
||||
text = pattern.ReplaceAllString(text, "[REDACTED]")
|
||||
}
|
||||
return []byte(text)
|
||||
}
|
||||
|
||||
func redactSensitiveMap(values map[string]any) map[string]any {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]any, len(values))
|
||||
for key, value := range values {
|
||||
if sensitiveKey(key) {
|
||||
out[key] = "[REDACTED]"
|
||||
continue
|
||||
}
|
||||
out[key] = redactSensitiveValue(value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func redactSensitiveValue(value any) any {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return string(redactSecretBytes([]byte(typed)))
|
||||
case map[string]any:
|
||||
return redactSensitiveMap(typed)
|
||||
case map[string]string:
|
||||
out := make(map[string]string, len(typed))
|
||||
for key, value := range typed {
|
||||
if sensitiveKey(key) {
|
||||
out[key] = "[REDACTED]"
|
||||
} else {
|
||||
out[key] = string(redactSecretBytes([]byte(value)))
|
||||
}
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func sensitiveKey(key string) bool {
|
||||
key = strings.ToLower(key)
|
||||
return strings.Contains(key, "api_key") ||
|
||||
strings.Contains(key, "apikey") ||
|
||||
strings.Contains(key, "authorization") ||
|
||||
strings.Contains(key, "bearer") ||
|
||||
strings.Contains(key, "password") ||
|
||||
strings.Contains(key, "secret") ||
|
||||
strings.Contains(key, "token")
|
||||
}
|
||||
@@ -48,6 +48,9 @@ type RunInput struct {
|
||||
LLMProfiles []artifacts.LLMProfileManifest
|
||||
Metadata map[string]any
|
||||
Warnings []contracts.Warning
|
||||
Checkpoints CheckpointRecorder
|
||||
Checkpoint CheckpointLoader
|
||||
Debug DebugRecorder
|
||||
}
|
||||
|
||||
type RunOutput struct {
|
||||
@@ -56,6 +59,7 @@ type RunOutput struct {
|
||||
Rejected []contracts.RejectedOutput `json:"rejected,omitempty"`
|
||||
Warnings []contracts.Warning `json:"warnings,omitempty"`
|
||||
OutputFiles []contracts.OutputFile `json:"-"`
|
||||
CheckpointEvents []CheckpointEvent `json:"checkpoint_events,omitempty"`
|
||||
}
|
||||
|
||||
func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) {
|
||||
@@ -70,29 +74,94 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
}
|
||||
|
||||
output.Manifest = manifestFromPipeline(input)
|
||||
checkpoints := input.Checkpoints
|
||||
if checkpoints == nil {
|
||||
checkpoints = NoopCheckpointRecorder()
|
||||
}
|
||||
checkpointLoader := input.Checkpoint
|
||||
if checkpointLoader == nil {
|
||||
checkpointLoader = NoopCheckpointLoader()
|
||||
}
|
||||
debugRecorder := input.Debug
|
||||
if debugRecorder == nil {
|
||||
debugRecorder = NoopDebugRecorder()
|
||||
}
|
||||
input.Debug = debugRecorder
|
||||
input.LLMClient = wrapDebugLLMClient(input.LLMClient, debugRecorder)
|
||||
defer func() {
|
||||
output.Manifest.LLMProfiles = mergeLLMProfileManifests(input.LLMProfiles, llmProfileManifests(input.LLMClient))
|
||||
}()
|
||||
output.Warnings = append(output.Warnings, cloneWarnings(input.Warnings)...)
|
||||
if err := writeDebugTimed(debugRecorder, "run.json", debugTimedEnvelope{
|
||||
Stage: "run",
|
||||
StartedAt: startedTime(input.StartedAt),
|
||||
Payload: map[string]any{
|
||||
"pipeline_id": input.Pipeline.ID,
|
||||
"pipeline_digest": input.Pipeline.Digest,
|
||||
"run_id": output.Manifest.RunID,
|
||||
},
|
||||
}); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write debug run artifact: %w", err)
|
||||
}
|
||||
|
||||
adapter, err := r.registries.Inputs.Build(input.Pipeline.Input.Module)
|
||||
if err != nil {
|
||||
return failOutput(output), fmt.Errorf("build input adapter %q: %w", input.Pipeline.Input.Module, err)
|
||||
}
|
||||
attachModuleManifestMetadata(&output, "input", adapter)
|
||||
doc, err := adapter.Parse(ctx, contracts.ParseRequest{
|
||||
SourceID: input.SourceID,
|
||||
Path: input.Path,
|
||||
Raw: input.RawInput,
|
||||
LLMProfile: input.Pipeline.Input.LLMProfile,
|
||||
Options: cloneOptions(input.Pipeline.Input.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err)
|
||||
sourceCheckpoint, sourceDecision := checkpointLoader.Source(adapter.Key())
|
||||
recordCheckpointEvent(&output, checkpointLoader, "source", "", adapter.Key(), sourceDecision)
|
||||
doc := sourceCheckpoint.Document
|
||||
sourceStarted := time.Now().UTC()
|
||||
if err := writeDebugTimed(debugRecorder, "source/input.json", debugTimedEnvelope{
|
||||
Stage: "source",
|
||||
ModuleKey: adapter.Key(),
|
||||
StartedAt: sourceStarted,
|
||||
Payload: debugSourceInput{
|
||||
SourceID: input.SourceID,
|
||||
Path: input.Path,
|
||||
Raw: debugContentEnvelope(input.RawInput, sourceInputMediaType(input.Path), nil, nil),
|
||||
Options: redactSensitiveMap(input.Pipeline.Input.Options),
|
||||
Metadata: redactSensitiveMap(input.Metadata),
|
||||
},
|
||||
}); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
|
||||
}
|
||||
if err := source.ValidateDocument(doc); err != nil {
|
||||
return failOutput(output), fmt.Errorf("validate source document: %w", err)
|
||||
if !sourceDecision.Reused {
|
||||
if err := checkpoints.SourceRunning(adapter.Key()); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
|
||||
}
|
||||
doc, err = adapter.Parse(ctx, contracts.ParseRequest{
|
||||
SourceID: input.SourceID,
|
||||
Path: input.Path,
|
||||
Raw: input.RawInput,
|
||||
LLMProfile: input.Pipeline.Input.LLMProfile,
|
||||
Options: cloneOptions(input.Pipeline.Input.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
_ = checkpoints.SourceFailed(adapter.Key(), err)
|
||||
return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err)
|
||||
}
|
||||
if err := source.ValidateDocument(doc); err != nil {
|
||||
_ = checkpoints.SourceFailed(adapter.Key(), err)
|
||||
return failOutput(output), fmt.Errorf("validate source document: %w", err)
|
||||
}
|
||||
if err := checkpoints.SourceSucceeded(adapter.Key(), doc); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
|
||||
}
|
||||
}
|
||||
if err := writeDebugTimed(debugRecorder, "source/output.json", debugTimedEnvelope{
|
||||
Stage: "source",
|
||||
ModuleKey: adapter.Key(),
|
||||
StartedAt: sourceStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": sourceDecision.Reused,
|
||||
"decision": sourceDecision,
|
||||
"document": debugSourceDocumentEnvelope(doc),
|
||||
},
|
||||
}); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
|
||||
}
|
||||
sourceInput := sourceInputMaterial(input.Path, input.RawInput)
|
||||
sessionID := resolvedSessionID(input.SessionID, doc.ID)
|
||||
@@ -106,47 +175,131 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
attachModuleManifestMetadata(&output, "chunker", chunker)
|
||||
var canonicalChunks []contracts.SourceChunk
|
||||
var chunkWarnings []contracts.Warning
|
||||
chunksAccepted, chunkRejection, err := runWithRetry(ctx, input.Pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
||||
Source: doc,
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(input.Pipeline.ChunkReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: input.Pipeline.Chunk.LLMProfile,
|
||||
Options: cloneOptions(input.Pipeline.Chunk.Options),
|
||||
Metadata: input.Metadata,
|
||||
chunkCheckpoint, chunkDecision := checkpointLoader.Chunk(chunker.Key(), doc.Digest)
|
||||
recordCheckpointEvent(&output, checkpointLoader, string(StageChunk), "", chunker.Key(), chunkDecision)
|
||||
chunkStarted := time.Now().UTC()
|
||||
if err := writeDebugTimed(debugRecorder, "chunk/input.json", debugTimedEnvelope{
|
||||
Stage: string(StageChunk),
|
||||
ModuleKey: chunker.Key(),
|
||||
StartedAt: chunkStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": chunkDecision.Reused,
|
||||
"decision": chunkDecision,
|
||||
"source": debugSourceDocumentEnvelope(doc),
|
||||
"source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil),
|
||||
"options": redactSensitiveMap(input.Pipeline.Chunk.Options),
|
||||
"metadata": redactSensitiveMap(input.Metadata),
|
||||
},
|
||||
}); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
|
||||
}
|
||||
chunksAccepted := chunkDecision.Reused
|
||||
var chunkRejection *contracts.RejectedOutput
|
||||
if chunkDecision.Reused {
|
||||
canonicalChunks = cloneSourceChunks(chunkCheckpoint.Chunks)
|
||||
chunkWarnings = cloneWarnings(chunkCheckpoint.Warnings)
|
||||
output.Warnings = append(output.Warnings, chunkWarnings...)
|
||||
} else {
|
||||
if err := checkpoints.ChunkRunning(chunker.Key(), doc.Digest); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
|
||||
}
|
||||
chunksAccepted, chunkRejection, err = runWithRetry(ctx, input.Pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
attemptStarted := time.Now().UTC()
|
||||
chunkResult, err := chunker.Chunk(ctx, contracts.ChunkRequest{
|
||||
Source: doc,
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(input.Pipeline.ChunkReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: input.Pipeline.Chunk.LLMProfile,
|
||||
Options: cloneOptions(input.Pipeline.Chunk.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
_ = writeDebugTimed(debugRecorder, path.Join("chunk", fmt.Sprintf("attempt-%02d.json", attempt)), debugTimedEnvelope{
|
||||
Stage: string(StageChunk),
|
||||
ModuleKey: chunker.Key(),
|
||||
Attempt: attempt,
|
||||
StartedAt: attemptStarted,
|
||||
Error: err.Error(),
|
||||
})
|
||||
return false, nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), err)
|
||||
}
|
||||
if len(chunkResult.Chunks) == 0 {
|
||||
return false, nil, fmt.Errorf("chunker %q returned no chunks", chunker.Key())
|
||||
}
|
||||
chunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
|
||||
}
|
||||
validationWarnings, rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.Pipeline.ChunkReferences.ReferenceSet, input.LLMClient, input.Metadata, input.Pipeline.ValidatorChains, attempt, input.Debug)
|
||||
if err != nil || rejection != nil {
|
||||
_ = writeDebugTimed(debugRecorder, path.Join("chunk", fmt.Sprintf("attempt-%02d.json", attempt)), debugTimedEnvelope{
|
||||
Stage: string(StageChunk),
|
||||
ModuleKey: chunker.Key(),
|
||||
Attempt: attempt,
|
||||
StartedAt: attemptStarted,
|
||||
Payload: map[string]any{
|
||||
"chunks": debugSourceChunkEnvelopes(chunks),
|
||||
"warnings": append(cloneWarnings(chunkResult.Warnings), validationWarnings...),
|
||||
"rejection": debugRejectedOutputPtr(rejection),
|
||||
},
|
||||
})
|
||||
return false, rejection, err
|
||||
}
|
||||
canonicalChunks = chunks
|
||||
chunkWarnings = append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
|
||||
if err := writeDebugTimed(debugRecorder, path.Join("chunk", fmt.Sprintf("attempt-%02d.json", attempt)), debugTimedEnvelope{
|
||||
Stage: string(StageChunk),
|
||||
ModuleKey: chunker.Key(),
|
||||
Attempt: attempt,
|
||||
StartedAt: attemptStarted,
|
||||
Payload: map[string]any{
|
||||
"chunks": debugSourceChunkEnvelopes(chunks),
|
||||
"warnings": chunkWarnings,
|
||||
},
|
||||
}); err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), err)
|
||||
_ = checkpoints.ChunkFailed(chunker.Key(), doc.Digest, err)
|
||||
return failOutput(output), err
|
||||
}
|
||||
if len(chunkResult.Chunks) == 0 {
|
||||
return false, nil, fmt.Errorf("chunker %q returned no chunks", chunker.Key())
|
||||
if !chunksAccepted {
|
||||
output.Rejected = append(output.Rejected, *chunkRejection)
|
||||
if err := checkpoints.ChunkRejected(chunker.Key(), doc.Digest, *chunkRejection); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
|
||||
}
|
||||
} else {
|
||||
output.Warnings = append(output.Warnings, chunkWarnings...)
|
||||
if err := checkpoints.ChunkSucceeded(chunker.Key(), doc.Digest, canonicalChunks, chunkWarnings); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
|
||||
}
|
||||
}
|
||||
chunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
|
||||
}
|
||||
validationWarnings, rejection, err := r.validateChunksRaw(ctx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.Pipeline.ChunkReferences.ReferenceSet, input.LLMClient, input.Metadata, input.Pipeline.ValidatorChains, attempt)
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
}
|
||||
canonicalChunks = chunks
|
||||
chunkWarnings = append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return failOutput(output), err
|
||||
}
|
||||
if !chunksAccepted {
|
||||
output.Rejected = append(output.Rejected, *chunkRejection)
|
||||
} else {
|
||||
output.Warnings = append(output.Warnings, chunkWarnings...)
|
||||
chunkDebugPayload := map[string]any{
|
||||
"reused": chunkDecision.Reused,
|
||||
"accepted": chunksAccepted,
|
||||
"chunks": debugSourceChunkEnvelopes(canonicalChunks),
|
||||
"warnings": chunkWarnings,
|
||||
}
|
||||
if chunkRejection != nil {
|
||||
chunkDebugPayload["rejection"] = debugRejectedOutputEnvelope(*chunkRejection)
|
||||
}
|
||||
if err := writeDebugTimed(debugRecorder, "chunk/output.json", debugTimedEnvelope{
|
||||
Stage: string(StageChunk),
|
||||
ModuleKey: chunker.Key(),
|
||||
StartedAt: chunkStarted,
|
||||
Payload: chunkDebugPayload,
|
||||
}); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
|
||||
}
|
||||
|
||||
if chunksAccepted {
|
||||
for _, lane := range input.Pipeline.ArtifactLanes {
|
||||
if err := r.runLane(ctx, input, doc, sourceInput, sessionID, canonicalChunks, lane, &output); err != nil {
|
||||
if err := r.runLane(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, canonicalChunks, lane, &output); err != nil {
|
||||
return failOutput(output), err
|
||||
}
|
||||
}
|
||||
@@ -165,6 +318,22 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
return failOutput(output), fmt.Errorf("build output encoder %q: %w", input.Pipeline.Output.Module, err)
|
||||
}
|
||||
attachModuleManifestMetadata(&output, "output", encoder)
|
||||
outputStarted := time.Now().UTC()
|
||||
if err := writeDebugTimed(debugRecorder, "output/input.json", debugTimedEnvelope{
|
||||
Stage: string(StageOutput),
|
||||
ModuleKey: encoder.Key(),
|
||||
StartedAt: outputStarted,
|
||||
Payload: map[string]any{
|
||||
"manifest": output.Manifest,
|
||||
"normalize_outputs": debugNormalizeOutputEnvelopes(output.NormalizeOutputs),
|
||||
"rejected": debugRejectedOutputEnvelopes(output.Rejected),
|
||||
"warnings": output.Warnings,
|
||||
"options": redactSensitiveMap(input.Pipeline.Output.Options),
|
||||
"metadata": redactSensitiveMap(input.Metadata),
|
||||
},
|
||||
}); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write output debug artifact: %w", err)
|
||||
}
|
||||
encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
|
||||
Manifest: output.Manifest,
|
||||
NormalizeOutputs: cloneNormalizeOutputs(output.NormalizeOutputs),
|
||||
@@ -183,11 +352,22 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
|
||||
return failOutput(output), fmt.Errorf("validate output files from encoder %q: %w", encoder.Key(), err)
|
||||
}
|
||||
output.OutputFiles = files
|
||||
if err := writeDebugTimed(debugRecorder, "output/output.json", debugTimedEnvelope{
|
||||
Stage: string(StageOutput),
|
||||
ModuleKey: encoder.Key(),
|
||||
StartedAt: outputStarted,
|
||||
Payload: map[string]any{
|
||||
"files": debugOutputFiles(files),
|
||||
"warnings": encoded.Warnings,
|
||||
},
|
||||
}); err != nil {
|
||||
return failOutput(output), fmt.Errorf("write output debug artifact: %w", err)
|
||||
}
|
||||
|
||||
return output, nil
|
||||
}
|
||||
|
||||
func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []contracts.SourceChunk, lane ResolvedArtifactLane, output *RunOutput) error {
|
||||
func (r *Runner) runLane(ctx context.Context, input RunInput, checkpoints CheckpointRecorder, checkpointLoader CheckpointLoader, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string, chunks []contracts.SourceChunk, lane ResolvedArtifactLane, output *RunOutput) error {
|
||||
extractor, err := r.registries.Extractors.Build(lane.Extract.Module)
|
||||
if err != nil {
|
||||
return fmt.Errorf("build extractor %q for lane %q: %w", lane.Extract.Module, lane.ID, err)
|
||||
@@ -203,67 +383,120 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
|
||||
setLaneManifestMetadata(output, lane.ID, extractor, merger, normalizer)
|
||||
|
||||
extractOutputs := make([]contracts.ExtractOutput, 0, len(chunks))
|
||||
for index := range chunks {
|
||||
chunk := chunks[index]
|
||||
var acceptedOutput contracts.ExtractOutput
|
||||
var acceptedWarnings []contracts.Warning
|
||||
accepted, rejection, err := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
|
||||
Source: doc,
|
||||
Chunk: &chunk,
|
||||
SourceInput: chunkInputMaterial(sourceInput, chunk),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: lane.Extract.LLMProfile,
|
||||
Options: cloneOptions(lane.Extract.Options),
|
||||
Metadata: input.Metadata,
|
||||
extractWarnings := []contracts.Warning{}
|
||||
extractRejectedStart := len(output.Rejected)
|
||||
extractDependencies := digestFingerprints("chunks", joinedChunkDigest(chunks))
|
||||
extractCheckpoint, extractDecision := checkpointLoader.Extract(lane.ID, extractor.Key(), extractDependencies)
|
||||
recordCheckpointEvent(output, checkpointLoader, string(StageExtract), lane.ID, extractor.Key(), extractDecision)
|
||||
extractStarted := time.Now().UTC()
|
||||
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
|
||||
Stage: string(StageExtract),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: extractor.Key(),
|
||||
StartedAt: extractStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": extractDecision.Reused,
|
||||
"decision": extractDecision,
|
||||
"source": debugSourceDocumentEnvelope(doc),
|
||||
"chunks": debugSourceChunkEnvelopes(chunks),
|
||||
"options": redactSensitiveMap(lane.Extract.Options),
|
||||
"metadata": redactSensitiveMap(input.Metadata),
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write extract debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
if extractDecision.Reused {
|
||||
extractOutputs = cloneExtractOutputs(extractCheckpoint.Outputs)
|
||||
extractWarnings = cloneWarnings(extractCheckpoint.Warnings)
|
||||
output.Rejected = append(output.Rejected, cloneRejectedOutputs(extractCheckpoint.Rejected)...)
|
||||
output.Warnings = append(output.Warnings, extractWarnings...)
|
||||
} else {
|
||||
if err := checkpoints.ExtractRunning(lane.ID, extractor.Key(), extractDependencies); err != nil {
|
||||
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
for index := range chunks {
|
||||
chunk := chunks[index]
|
||||
var acceptedOutput contracts.ExtractOutput
|
||||
var acceptedWarnings []contracts.Warning
|
||||
accepted, rejection, err := runWithRetry(ctx, lane.Extract.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
result, err := extractor.Extract(ctx, contracts.ExtractionRequest{
|
||||
Source: doc,
|
||||
Chunk: &chunk,
|
||||
SourceInput: chunkInputMaterial(sourceInput, chunk),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.ExtractReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: lane.Extract.LLMProfile,
|
||||
Options: cloneOptions(lane.Extract.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, extractor.Key(), err)
|
||||
}
|
||||
extractOutput := result.Output
|
||||
extractOutput.LaneID = lane.ID
|
||||
extractOutput.ExtractorKey = extractor.Key()
|
||||
extractOutput.SourceID = doc.ID
|
||||
extractOutput.ChunkID = chunk.ID
|
||||
extractOutput.ChunkIndex = chunk.Index
|
||||
extractOutput.Payload.Warnings = append(extractOutput.Payload.Warnings, result.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageExtract,
|
||||
laneID: lane.ID,
|
||||
moduleKey: extractor.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
chunkID: chunk.ID,
|
||||
chunkIndex: chunk.Index,
|
||||
chunk: &chunk,
|
||||
sourceInput: chunkInputMaterial(sourceInput, chunk),
|
||||
sessionID: sessionID,
|
||||
references: lane.ExtractReferences.ReferenceSet,
|
||||
llmClient: input.LLMClient,
|
||||
schema: extractOutput.Schema,
|
||||
payload: extractOutput.Payload,
|
||||
metadata: input.Metadata,
|
||||
chains: input.Pipeline.ValidatorChains,
|
||||
attempt: attempt,
|
||||
debug: input.Debug,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
}
|
||||
acceptedOutput = cloneExtractOutput(extractOutput)
|
||||
acceptedWarnings = append(cloneWarnings(result.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("extract lane %q chunk %q with extractor %q: %w", lane.ID, chunk.ID, extractor.Key(), err)
|
||||
_ = checkpoints.ExtractFailed(lane.ID, extractor.Key(), extractDependencies, err)
|
||||
return err
|
||||
}
|
||||
extractOutput := result.Output
|
||||
extractOutput.LaneID = lane.ID
|
||||
extractOutput.ExtractorKey = extractor.Key()
|
||||
extractOutput.SourceID = doc.ID
|
||||
extractOutput.ChunkID = chunk.ID
|
||||
extractOutput.ChunkIndex = chunk.Index
|
||||
extractOutput.Payload.Warnings = append(extractOutput.Payload.Warnings, result.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageExtract,
|
||||
laneID: lane.ID,
|
||||
moduleKey: extractor.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
chunkID: chunk.ID,
|
||||
chunkIndex: chunk.Index,
|
||||
chunk: &chunk,
|
||||
sourceInput: chunkInputMaterial(sourceInput, chunk),
|
||||
sessionID: sessionID,
|
||||
references: lane.ExtractReferences.ReferenceSet,
|
||||
llmClient: input.LLMClient,
|
||||
schema: extractOutput.Schema,
|
||||
payload: extractOutput.Payload,
|
||||
metadata: input.Metadata,
|
||||
chains: input.Pipeline.ValidatorChains,
|
||||
attempt: attempt,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
if !accepted {
|
||||
output.Rejected = append(output.Rejected, *rejection)
|
||||
continue
|
||||
}
|
||||
acceptedOutput = cloneExtractOutput(extractOutput)
|
||||
acceptedWarnings = append(cloneWarnings(result.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
output.Warnings = append(output.Warnings, acceptedWarnings...)
|
||||
extractWarnings = append(extractWarnings, acceptedWarnings...)
|
||||
extractOutputs = append(extractOutputs, acceptedOutput)
|
||||
}
|
||||
if !accepted {
|
||||
output.Rejected = append(output.Rejected, *rejection)
|
||||
continue
|
||||
extractRejected := cloneRejectedOutputs(output.Rejected[extractRejectedStart:])
|
||||
if err := checkpoints.ExtractSucceeded(lane.ID, extractor.Key(), extractDependencies, extractOutputs, extractRejected, extractWarnings); err != nil {
|
||||
return fmt.Errorf("write extract checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
output.Warnings = append(output.Warnings, acceptedWarnings...)
|
||||
extractOutputs = append(extractOutputs, acceptedOutput)
|
||||
}
|
||||
if err := writeDebugTimed(input.Debug, path.Join("extract", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
|
||||
Stage: string(StageExtract),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: extractor.Key(),
|
||||
StartedAt: extractStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": extractDecision.Reused,
|
||||
"outputs": debugExtractOutputEnvelopes(extractOutputs),
|
||||
"rejected": debugRejectedOutputEnvelopes(output.Rejected[extractRejectedStart:]),
|
||||
"warnings": extractWarnings,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write extract debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
|
||||
if len(extractOutputs) == 0 {
|
||||
@@ -272,115 +505,243 @@ func (r *Runner) runLane(ctx context.Context, input RunInput, doc *source.Source
|
||||
|
||||
var acceptedMerge contracts.MergeOutput
|
||||
var mergeWarnings []contracts.Warning
|
||||
mergeAccepted, mergeRejection, err := runWithRetry(ctx, lane.Merge.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
mergeResult, err := merger.Merge(ctx, contracts.MergeRequest{
|
||||
Source: doc,
|
||||
LaneID: lane.ID,
|
||||
ExtractOutputs: cloneExtractOutputs(extractOutputs),
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.MergeReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: lane.Merge.LLMProfile,
|
||||
Options: cloneOptions(lane.Merge.Options),
|
||||
Metadata: input.Metadata,
|
||||
mergeDependencies := rawOutputDigests(extractPayloads(extractOutputs))
|
||||
mergeCheckpoint, mergeDecision := checkpointLoader.Merge(lane.ID, merger.Key(), mergeDependencies)
|
||||
recordCheckpointEvent(output, checkpointLoader, string(StageMerge), lane.ID, merger.Key(), mergeDecision)
|
||||
mergeStarted := time.Now().UTC()
|
||||
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
|
||||
Stage: string(StageMerge),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: merger.Key(),
|
||||
StartedAt: mergeStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": mergeDecision.Reused,
|
||||
"decision": mergeDecision,
|
||||
"source": debugSourceDocumentEnvelope(doc),
|
||||
"extract_outputs": debugExtractOutputEnvelopes(extractOutputs),
|
||||
"options": redactSensitiveMap(lane.Merge.Options),
|
||||
"metadata": redactSensitiveMap(input.Metadata),
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
if mergeDecision.Reused {
|
||||
acceptedMerge = cloneMergeOutput(mergeCheckpoint.Output)
|
||||
mergeWarnings = cloneWarnings(mergeCheckpoint.Warnings)
|
||||
output.Warnings = append(output.Warnings, mergeWarnings...)
|
||||
} else {
|
||||
if err := checkpoints.MergeRunning(lane.ID, merger.Key(), mergeDependencies); err != nil {
|
||||
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
mergeAccepted, mergeRejection, err := runWithRetry(ctx, lane.Merge.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
mergeResult, err := merger.Merge(ctx, contracts.MergeRequest{
|
||||
Source: doc,
|
||||
LaneID: lane.ID,
|
||||
ExtractOutputs: cloneExtractOutputs(extractOutputs),
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.MergeReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: lane.Merge.LLMProfile,
|
||||
Options: cloneOptions(lane.Merge.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("merge lane %q with merger %q: %w", lane.ID, merger.Key(), err)
|
||||
}
|
||||
mergeOutput := mergeResult.Output
|
||||
mergeOutput.LaneID = lane.ID
|
||||
mergeOutput.MergerKey = merger.Key()
|
||||
mergeOutput.SourceID = doc.ID
|
||||
mergeOutput.Payload.Warnings = append(mergeOutput.Payload.Warnings, mergeResult.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageMerge,
|
||||
laneID: lane.ID,
|
||||
moduleKey: merger.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
sourceInput: sourceInput.Clone(),
|
||||
sessionID: sessionID,
|
||||
references: lane.MergeReferences.ReferenceSet,
|
||||
llmClient: input.LLMClient,
|
||||
schema: mergeOutput.Schema,
|
||||
payload: mergeOutput.Payload,
|
||||
extractOutputs: extractOutputs,
|
||||
metadata: input.Metadata,
|
||||
chains: input.Pipeline.ValidatorChains,
|
||||
attempt: attempt,
|
||||
debug: input.Debug,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
}
|
||||
acceptedMerge = cloneMergeOutput(mergeOutput)
|
||||
mergeWarnings = append(cloneWarnings(mergeResult.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("merge lane %q with merger %q: %w", lane.ID, merger.Key(), err)
|
||||
_ = checkpoints.MergeFailed(lane.ID, merger.Key(), mergeDependencies, err)
|
||||
return err
|
||||
}
|
||||
mergeOutput := mergeResult.Output
|
||||
mergeOutput.LaneID = lane.ID
|
||||
mergeOutput.MergerKey = merger.Key()
|
||||
mergeOutput.SourceID = doc.ID
|
||||
mergeOutput.Payload.Warnings = append(mergeOutput.Payload.Warnings, mergeResult.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageMerge,
|
||||
laneID: lane.ID,
|
||||
moduleKey: merger.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
sourceInput: sourceInput.Clone(),
|
||||
sessionID: sessionID,
|
||||
references: lane.MergeReferences.ReferenceSet,
|
||||
llmClient: input.LLMClient,
|
||||
schema: mergeOutput.Schema,
|
||||
payload: mergeOutput.Payload,
|
||||
extractOutputs: extractOutputs,
|
||||
metadata: input.Metadata,
|
||||
chains: input.Pipeline.ValidatorChains,
|
||||
attempt: attempt,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
if !mergeAccepted {
|
||||
output.Rejected = append(output.Rejected, *mergeRejection)
|
||||
if err := checkpoints.MergeRejected(lane.ID, merger.Key(), mergeDependencies, *mergeRejection); err != nil {
|
||||
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
|
||||
Stage: string(StageMerge),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: merger.Key(),
|
||||
StartedAt: mergeStarted,
|
||||
Payload: map[string]any{
|
||||
"accepted": false,
|
||||
"rejection": debugRejectedOutputEnvelope(*mergeRejection),
|
||||
"warnings": mergeWarnings,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
output.Warnings = append(output.Warnings, mergeWarnings...)
|
||||
if err := checkpoints.MergeSucceeded(lane.ID, merger.Key(), mergeDependencies, acceptedMerge, mergeWarnings); err != nil {
|
||||
return fmt.Errorf("write merge checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
acceptedMerge = cloneMergeOutput(mergeOutput)
|
||||
mergeWarnings = append(cloneWarnings(mergeResult.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !mergeAccepted {
|
||||
output.Rejected = append(output.Rejected, *mergeRejection)
|
||||
return nil
|
||||
if err := writeDebugTimed(input.Debug, path.Join("merge", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
|
||||
Stage: string(StageMerge),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: merger.Key(),
|
||||
StartedAt: mergeStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": mergeDecision.Reused,
|
||||
"accepted": true,
|
||||
"output": debugMergeOutputEnvelope(acceptedMerge),
|
||||
"warnings": mergeWarnings,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write merge debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
output.Warnings = append(output.Warnings, mergeWarnings...)
|
||||
|
||||
var acceptedNormalize contracts.NormalizeOutput
|
||||
var normalizeWarnings []contracts.Warning
|
||||
normalizeAccepted, normalizeRejection, err := runWithRetry(ctx, lane.Normalize.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
normalizeResult, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
|
||||
Source: doc,
|
||||
LaneID: lane.ID,
|
||||
MergeOutput: cloneMergeOutput(acceptedMerge),
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: lane.Normalize.LLMProfile,
|
||||
Options: cloneOptions(lane.Normalize.Options),
|
||||
Metadata: input.Metadata,
|
||||
normalizeDependencies := rawOutputDigests([]contracts.RawPayload{acceptedMerge.Payload})
|
||||
normalizeCheckpoint, normalizeDecision := checkpointLoader.Normalize(lane.ID, normalizer.Key(), normalizeDependencies)
|
||||
recordCheckpointEvent(output, checkpointLoader, string(StageNormalize), lane.ID, normalizer.Key(), normalizeDecision)
|
||||
normalizeStarted := time.Now().UTC()
|
||||
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "input.json"), debugTimedEnvelope{
|
||||
Stage: string(StageNormalize),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: normalizer.Key(),
|
||||
StartedAt: normalizeStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": normalizeDecision.Reused,
|
||||
"decision": normalizeDecision,
|
||||
"source": debugSourceDocumentEnvelope(doc),
|
||||
"merge_output": debugMergeOutputEnvelope(acceptedMerge),
|
||||
"options": redactSensitiveMap(lane.Normalize.Options),
|
||||
"metadata": redactSensitiveMap(input.Metadata),
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
if normalizeDecision.Reused {
|
||||
acceptedNormalize = cloneNormalizeOutput(normalizeCheckpoint.Output)
|
||||
normalizeWarnings = cloneWarnings(normalizeCheckpoint.Warnings)
|
||||
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
||||
} else {
|
||||
if err := checkpoints.NormalizeRunning(lane.ID, normalizer.Key(), normalizeDependencies); err != nil {
|
||||
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
normalizeAccepted, normalizeRejection, err := runWithRetry(ctx, lane.Normalize.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
|
||||
normalizeResult, err := normalizer.Normalize(ctx, contracts.NormalizeRequest{
|
||||
Source: doc,
|
||||
LaneID: lane.ID,
|
||||
MergeOutput: cloneMergeOutput(acceptedMerge),
|
||||
SourceInput: sourceInput.Clone(),
|
||||
SessionID: sessionID,
|
||||
References: CloneReferenceSet(lane.NormalizeReferences.ReferenceSet),
|
||||
LLMClient: input.LLMClient,
|
||||
LLMProfile: lane.Normalize.LLMProfile,
|
||||
Options: cloneOptions(lane.Normalize.Options),
|
||||
Metadata: input.Metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, normalizer.Key(), err)
|
||||
}
|
||||
normalizeOutput := normalizeResult.Output
|
||||
normalizeOutput.LaneID = lane.ID
|
||||
normalizeOutput.NormalizerKey = normalizer.Key()
|
||||
normalizeOutput.SourceID = doc.ID
|
||||
normalizeOutput.Payload.Warnings = append(normalizeOutput.Payload.Warnings, normalizeResult.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageNormalize,
|
||||
laneID: lane.ID,
|
||||
moduleKey: normalizer.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
sourceInput: sourceInput.Clone(),
|
||||
sessionID: sessionID,
|
||||
references: lane.NormalizeReferences.ReferenceSet,
|
||||
llmClient: input.LLMClient,
|
||||
schema: normalizeOutput.Schema,
|
||||
payload: normalizeOutput.Payload,
|
||||
mergeOutput: acceptedMerge,
|
||||
metadata: input.Metadata,
|
||||
chains: input.Pipeline.ValidatorChains,
|
||||
attempt: attempt,
|
||||
debug: input.Debug,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
}
|
||||
acceptedNormalize = cloneNormalizeOutput(normalizeOutput)
|
||||
normalizeWarnings = append(cloneWarnings(normalizeResult.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return false, nil, fmt.Errorf("normalize lane %q with normalizer %q: %w", lane.ID, normalizer.Key(), err)
|
||||
_ = checkpoints.NormalizeFailed(lane.ID, normalizer.Key(), normalizeDependencies, err)
|
||||
return err
|
||||
}
|
||||
normalizeOutput := normalizeResult.Output
|
||||
normalizeOutput.LaneID = lane.ID
|
||||
normalizeOutput.NormalizerKey = normalizer.Key()
|
||||
normalizeOutput.SourceID = doc.ID
|
||||
normalizeOutput.Payload.Warnings = append(normalizeOutput.Payload.Warnings, normalizeResult.Warnings...)
|
||||
validationWarnings, rejection, err := r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageNormalize,
|
||||
laneID: lane.ID,
|
||||
moduleKey: normalizer.Key(),
|
||||
source: doc,
|
||||
sourceID: doc.ID,
|
||||
sourceInput: sourceInput.Clone(),
|
||||
sessionID: sessionID,
|
||||
references: lane.NormalizeReferences.ReferenceSet,
|
||||
llmClient: input.LLMClient,
|
||||
schema: normalizeOutput.Schema,
|
||||
payload: normalizeOutput.Payload,
|
||||
mergeOutput: acceptedMerge,
|
||||
metadata: input.Metadata,
|
||||
chains: input.Pipeline.ValidatorChains,
|
||||
attempt: attempt,
|
||||
})
|
||||
if err != nil || rejection != nil {
|
||||
return false, rejection, err
|
||||
if !normalizeAccepted {
|
||||
output.Rejected = append(output.Rejected, *normalizeRejection)
|
||||
if err := checkpoints.NormalizeRejected(lane.ID, normalizer.Key(), normalizeDependencies, *normalizeRejection); err != nil {
|
||||
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
|
||||
Stage: string(StageNormalize),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: normalizer.Key(),
|
||||
StartedAt: normalizeStarted,
|
||||
Payload: map[string]any{
|
||||
"accepted": false,
|
||||
"rejection": debugRejectedOutputEnvelope(*normalizeRejection),
|
||||
"warnings": normalizeWarnings,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
||||
if err := checkpoints.NormalizeSucceeded(lane.ID, normalizer.Key(), normalizeDependencies, acceptedNormalize, normalizeWarnings); err != nil {
|
||||
return fmt.Errorf("write normalize checkpoint for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
acceptedNormalize = cloneNormalizeOutput(normalizeOutput)
|
||||
normalizeWarnings = append(cloneWarnings(normalizeResult.Warnings), validationWarnings...)
|
||||
return true, nil, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !normalizeAccepted {
|
||||
output.Rejected = append(output.Rejected, *normalizeRejection)
|
||||
return nil
|
||||
if err := writeDebugTimed(input.Debug, path.Join("normalize", debugPathComponent(lane.ID), "output.json"), debugTimedEnvelope{
|
||||
Stage: string(StageNormalize),
|
||||
LaneID: lane.ID,
|
||||
ModuleKey: normalizer.Key(),
|
||||
StartedAt: normalizeStarted,
|
||||
Payload: map[string]any{
|
||||
"reused": normalizeDecision.Reused,
|
||||
"accepted": true,
|
||||
"output": debugNormalizeOutputEnvelope(acceptedNormalize),
|
||||
"warnings": normalizeWarnings,
|
||||
},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("write normalize debug artifact for lane %q: %w", lane.ID, err)
|
||||
}
|
||||
output.Warnings = append(output.Warnings, normalizeWarnings...)
|
||||
output.NormalizeOutputs = append(output.NormalizeOutputs, acceptedNormalize)
|
||||
return nil
|
||||
}
|
||||
@@ -406,6 +767,7 @@ type rawValidationTarget struct {
|
||||
metadata map[string]any
|
||||
chains []ResolvedValidatorChain
|
||||
attempt int
|
||||
debug DebugRecorder
|
||||
}
|
||||
|
||||
func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool, *contracts.RejectedOutput, error)) (bool, *contracts.RejectedOutput, error) {
|
||||
@@ -455,7 +817,7 @@ func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool,
|
||||
return false, lastRejection, nil
|
||||
}
|
||||
|
||||
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, chains []ResolvedValidatorChain, attempt int) ([]contracts.Warning, *contracts.RejectedOutput, error) {
|
||||
func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []contracts.SourceChunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, llmClient contracts.StructuredLLMClient, metadata map[string]any, chains []ResolvedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
|
||||
return r.validateRaw(ctx, rawValidationTarget{
|
||||
stage: StageChunk,
|
||||
moduleKey: moduleKey,
|
||||
@@ -469,6 +831,7 @@ func (r *Runner) validateChunksRaw(ctx context.Context, doc *source.SourceDocume
|
||||
metadata: metadata,
|
||||
chains: chains,
|
||||
attempt: attempt,
|
||||
debug: debug,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -488,7 +851,27 @@ func (r *Runner) validateRaw(ctx context.Context, target rawValidationTarget) ([
|
||||
return nil, nil, fmt.Errorf("build validator %q: %w", validatorBinding.Binding.Module, err)
|
||||
}
|
||||
request := target.validationRequest(validatorBinding.Binding)
|
||||
started := time.Now().UTC()
|
||||
result, err := validator.Validate(ctx, request)
|
||||
debugPayload := debugValidationCall{
|
||||
ValidatorName: validator.Name(),
|
||||
Request: debugValidationRequestEnvelope(request),
|
||||
Result: debugValidationResultEnvelope(result),
|
||||
}
|
||||
if err != nil {
|
||||
debugPayload.Error = err.Error()
|
||||
}
|
||||
if debugErr := writeDebugTimed(target.debug, path.Join("validate", debugPathComponent(string(target.stage)), debugPathComponent(target.laneID), debugPathComponent(target.moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d.json", len(warnings)+1, debugPathComponent(validator.Name()), target.attempt)), debugTimedEnvelope{
|
||||
Stage: string(target.stage),
|
||||
LaneID: target.laneID,
|
||||
ModuleKey: target.moduleKey,
|
||||
Attempt: target.attempt,
|
||||
StartedAt: started,
|
||||
Payload: debugPayload,
|
||||
Error: debugPayload.Error,
|
||||
}); debugErr != nil {
|
||||
return nil, nil, fmt.Errorf("write validation debug artifact: %w", debugErr)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("validate raw %s output with validator %q: %w", target.stage, validator.Name(), err)
|
||||
}
|
||||
@@ -630,6 +1013,13 @@ func validateRunInput(input RunInput) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func startedTime(t time.Time) time.Time {
|
||||
if t.IsZero() {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
return t.UTC()
|
||||
}
|
||||
|
||||
func manifestFromPipeline(input RunInput) artifacts.RunManifest {
|
||||
startedAt := input.StartedAt
|
||||
if startedAt.IsZero() {
|
||||
@@ -702,6 +1092,23 @@ func failOutput(output RunOutput) RunOutput {
|
||||
return output
|
||||
}
|
||||
|
||||
func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage string, laneID string, moduleKey string, decision CheckpointDecision) {
|
||||
if output == nil || loader == nil || !loader.Enabled() {
|
||||
return
|
||||
}
|
||||
action := "executed"
|
||||
if decision.Reused {
|
||||
action = "reused"
|
||||
}
|
||||
output.CheckpointEvents = append(output.CheckpointEvents, CheckpointEvent{
|
||||
Stage: stage,
|
||||
LaneID: laneID,
|
||||
ModuleKey: moduleKey,
|
||||
Action: action,
|
||||
Reason: decision.Reason,
|
||||
})
|
||||
}
|
||||
|
||||
func populateRawOutputManifest(output *RunOutput) {
|
||||
if output == nil {
|
||||
return
|
||||
|
||||
@@ -992,6 +992,158 @@ func TestRunPassesChunkContentAndMediaTypeToExtractors(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunDoesNotPassCheckpointPathsToModules(t *testing.T) {
|
||||
modules := defaultRunnerModules()
|
||||
|
||||
_, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
|
||||
Pipeline: resolvedPipeline(),
|
||||
Checkpoints: NoopCheckpointRecorder(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
for _, req := range modules.input.requests {
|
||||
assertNoCheckpointMetadata(t, req.Metadata)
|
||||
}
|
||||
for _, req := range modules.chunker.requests {
|
||||
assertNoCheckpointMetadata(t, req.Metadata)
|
||||
}
|
||||
for _, req := range modules.extractors["extract-alpha"].requests {
|
||||
assertNoCheckpointMetadata(t, req.Metadata)
|
||||
}
|
||||
for _, req := range modules.mergers["merge"].requests {
|
||||
assertNoCheckpointMetadata(t, req.Metadata)
|
||||
}
|
||||
for _, req := range modules.normalizers["normalize"].requests {
|
||||
assertNoCheckpointMetadata(t, req.Metadata)
|
||||
}
|
||||
for _, req := range modules.output.requests {
|
||||
assertNoCheckpointMetadata(t, req.Metadata)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReusesCheckpointedWorkflowOutputs(t *testing.T) {
|
||||
modules := defaultRunnerModules()
|
||||
doc := validSourceDocument()
|
||||
chunks := []contracts.SourceChunk{sourceChunkWithID("chunk-0", 0)}
|
||||
extractOutput := contracts.ExtractOutput{
|
||||
LaneID: "alpha",
|
||||
ExtractorKey: "extract-alpha",
|
||||
SourceID: doc.ID,
|
||||
ChunkID: "chunk-0",
|
||||
ChunkIndex: 0,
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"cached_extract":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
mergeOutput := contracts.MergeOutput{
|
||||
LaneID: "alpha",
|
||||
MergerKey: "merge",
|
||||
SourceID: doc.ID,
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"cached_merge":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
normalizeOutput := contracts.NormalizeOutput{
|
||||
LaneID: "alpha",
|
||||
NormalizerKey: "normalize",
|
||||
SourceID: doc.ID,
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"cached_normalize":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
loader := &runnerCheckpointLoader{
|
||||
source: SourceCheckpoint{Document: doc},
|
||||
chunk: ChunkCheckpoint{Chunks: chunks},
|
||||
extract: ExtractCheckpoint{Outputs: []contracts.ExtractOutput{extractOutput}},
|
||||
merge: MergeCheckpoint{Output: mergeOutput},
|
||||
normalize: NormalizeCheckpoint{Output: normalizeOutput},
|
||||
reuse: map[string]bool{
|
||||
"source": true,
|
||||
"chunk": true,
|
||||
"extract": true,
|
||||
"merge": true,
|
||||
"normalize": true,
|
||||
},
|
||||
}
|
||||
|
||||
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
|
||||
Pipeline: resolvedPipeline(),
|
||||
Checkpoint: loader,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if len(modules.input.requests) != 0 || len(modules.chunker.requests) != 0 || len(modules.extractors["extract-alpha"].requests) != 0 || len(modules.mergers["merge"].requests) != 0 || len(modules.normalizers["normalize"].requests) != 0 {
|
||||
t.Fatalf("module requests = input:%d chunk:%d extract:%d merge:%d normalize:%d, want all skipped", len(modules.input.requests), len(modules.chunker.requests), len(modules.extractors["extract-alpha"].requests), len(modules.mergers["merge"].requests), len(modules.normalizers["normalize"].requests))
|
||||
}
|
||||
if len(output.NormalizeOutputs) != 1 || string(output.NormalizeOutputs[0].Payload.Content) != `{"cached_normalize":true}` {
|
||||
t.Fatalf("NormalizeOutputs = %#v, want cached normalize output", output.NormalizeOutputs)
|
||||
}
|
||||
if len(output.CheckpointEvents) != 5 {
|
||||
t.Fatalf("checkpoint events = %#v, want one per reusable workflow step", output.CheckpointEvents)
|
||||
}
|
||||
for _, event := range output.CheckpointEvents {
|
||||
if event.Action != "reused" {
|
||||
t.Fatalf("checkpoint event = %#v, want reused", event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPreservesCheckpointedExtractRejections(t *testing.T) {
|
||||
modules := defaultRunnerModules()
|
||||
extractOutput := contracts.ExtractOutput{
|
||||
LaneID: "alpha",
|
||||
ExtractorKey: "extract-alpha",
|
||||
SourceID: "source-1",
|
||||
ChunkID: "chunk-1",
|
||||
ChunkIndex: 1,
|
||||
Payload: contracts.RawPayload{
|
||||
Content: []byte(`{"cached_extract":true}`),
|
||||
MediaType: "application/json",
|
||||
},
|
||||
}
|
||||
rejected := contracts.RejectedOutput{
|
||||
Stage: string(StageExtract),
|
||||
LaneID: "alpha",
|
||||
ModuleKey: "extract-alpha",
|
||||
ChunkID: "chunk-0",
|
||||
ReasonCode: "invalid_shape",
|
||||
Message: "invalid extract",
|
||||
}
|
||||
loader := &runnerCheckpointLoader{
|
||||
extract: ExtractCheckpoint{
|
||||
Outputs: []contracts.ExtractOutput{extractOutput},
|
||||
Rejected: []contracts.RejectedOutput{rejected},
|
||||
},
|
||||
reuse: map[string]bool{"extract": true},
|
||||
}
|
||||
|
||||
output, err := New(newRunnerRegistries(t, modules)).Run(context.Background(), RunInput{
|
||||
Pipeline: resolvedPipeline(),
|
||||
Checkpoint: loader,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
|
||||
if len(modules.extractors["extract-alpha"].requests) != 0 {
|
||||
t.Fatalf("extract requests = %d, want reused checkpoint", len(modules.extractors["extract-alpha"].requests))
|
||||
}
|
||||
if len(output.Rejected) != 1 || output.Rejected[0].ChunkID != "chunk-0" {
|
||||
t.Fatalf("rejected outputs = %#v, want checkpointed extract rejection", output.Rejected)
|
||||
}
|
||||
mergeRequests := modules.mergers["merge"].requests
|
||||
if len(mergeRequests) != 1 || len(mergeRequests[0].ExtractOutputs) != 1 || mergeRequests[0].ExtractOutputs[0].ChunkID != "chunk-1" {
|
||||
t.Fatalf("merge extract outputs = %#v, want only checkpointed accepted extract", mergeRequests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunOmitsRejectedExtractOutputsFromMerge(t *testing.T) {
|
||||
modules := defaultRunnerModules()
|
||||
validator := &runnerChainValidator{name: "chain-extract", approved: []bool{false, true}, reason: "bad_extract", message: "extract rejected"}
|
||||
@@ -2058,6 +2210,54 @@ func (encoder *runnerOutputEncoder) ManifestMetadata() map[string]any {
|
||||
return encoder.manifestMetadata
|
||||
}
|
||||
|
||||
type runnerCheckpointLoader struct {
|
||||
source SourceCheckpoint
|
||||
chunk ChunkCheckpoint
|
||||
extract ExtractCheckpoint
|
||||
merge MergeCheckpoint
|
||||
normalize NormalizeCheckpoint
|
||||
reuse map[string]bool
|
||||
}
|
||||
|
||||
func (loader *runnerCheckpointLoader) Enabled() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (loader *runnerCheckpointLoader) Source(string) (SourceCheckpoint, CheckpointDecision) {
|
||||
if loader.reuse["source"] {
|
||||
return loader.source, CheckpointDecision{Reused: true, Reason: "test checkpoint"}
|
||||
}
|
||||
return SourceCheckpoint{}, CheckpointDecision{Reason: "test checkpoint missing"}
|
||||
}
|
||||
|
||||
func (loader *runnerCheckpointLoader) Chunk(string, string) (ChunkCheckpoint, CheckpointDecision) {
|
||||
if loader.reuse["chunk"] {
|
||||
return loader.chunk, CheckpointDecision{Reused: true, Reason: "test checkpoint"}
|
||||
}
|
||||
return ChunkCheckpoint{}, CheckpointDecision{Reason: "test checkpoint missing"}
|
||||
}
|
||||
|
||||
func (loader *runnerCheckpointLoader) Extract(string, string, []CheckpointFingerprint) (ExtractCheckpoint, CheckpointDecision) {
|
||||
if loader.reuse["extract"] {
|
||||
return loader.extract, CheckpointDecision{Reused: true, Reason: "test checkpoint"}
|
||||
}
|
||||
return ExtractCheckpoint{}, CheckpointDecision{Reason: "test checkpoint missing"}
|
||||
}
|
||||
|
||||
func (loader *runnerCheckpointLoader) Merge(string, string, []CheckpointFingerprint) (MergeCheckpoint, CheckpointDecision) {
|
||||
if loader.reuse["merge"] {
|
||||
return loader.merge, CheckpointDecision{Reused: true, Reason: "test checkpoint"}
|
||||
}
|
||||
return MergeCheckpoint{}, CheckpointDecision{Reason: "test checkpoint missing"}
|
||||
}
|
||||
|
||||
func (loader *runnerCheckpointLoader) Normalize(string, string, []CheckpointFingerprint) (NormalizeCheckpoint, CheckpointDecision) {
|
||||
if loader.reuse["normalize"] {
|
||||
return loader.normalize, CheckpointDecision{Reused: true, Reason: "test checkpoint"}
|
||||
}
|
||||
return NormalizeCheckpoint{}, CheckpointDecision{Reason: "test checkpoint missing"}
|
||||
}
|
||||
|
||||
type fakeLLMClient struct{}
|
||||
|
||||
func (client fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
@@ -2181,6 +2381,25 @@ func warningReasons(warnings []contracts.Warning) []string {
|
||||
return reasons
|
||||
}
|
||||
|
||||
func assertNoCheckpointMetadata(t *testing.T, metadata map[string]any) {
|
||||
t.Helper()
|
||||
|
||||
for key, value := range metadata {
|
||||
lowerKey := strings.ToLower(key)
|
||||
if strings.Contains(lowerKey, "checkpoint") || strings.Contains(lowerKey, "workspace") {
|
||||
t.Fatalf("metadata key %q exposes checkpoint/workspace state", key)
|
||||
}
|
||||
text, ok := value.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
lowerValue := strings.ToLower(text)
|
||||
if strings.Contains(lowerValue, "checkpoint") || strings.Contains(lowerValue, "workspace") {
|
||||
t.Fatalf("metadata value for %q exposes checkpoint/workspace state: %q", key, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertRunError(t *testing.T, err error, want string) {
|
||||
t.Helper()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user