From b1fe9dc5a79ccf7a02899aef692358b7862710f2 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 17 Jul 2026 02:56:31 +0000 Subject: [PATCH] Reorganize user and operator documentation --- README.md | 6 +- docs/cli.md | 120 +++++-------------- docs/config.md | 135 ++++++---------------- docs/integrations/dnd-spell-artifacts.md | 94 ++++----------- docs/integrations/json-output.md | 110 ++++++------------ docs/integrations/seriatim.md | 108 +++++------------ docs/operations.md | 121 +++++++------------ examples/dnd-spells-production.config.yml | 25 ++++ examples/dnd-spells.config.yml | 22 ---- internal/cli/run_test.go | 32 ++--- 10 files changed, 234 insertions(+), 539 deletions(-) create mode 100644 examples/dnd-spells-production.config.yml diff --git a/README.md b/README.md index 13845d4..e5487c2 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ go run ./cmd/notarius run dnd-session \ --input examples/seriatim-minimal-transcript.json ``` -The maintained example uses Scriptorium's built-in `mistral-small-3` profile, -which reads `OPENROUTER_API_KEY`. Outputs are written under -`./notarius-output//` unless `--output-dir` is provided. +This invocation uses the maintained example configuration and input. See the +configuration and operations references for profile selection, credentials, and +run artifacts. Useful references: diff --git a/docs/cli.md b/docs/cli.md index 5b22d92..9052841 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -3,27 +3,15 @@ This is the canonical reference for the implemented Notarius command-line interface. -## Quick Run - -```sh -OPENROUTER_API_KEY=... \ -go run ./cmd/notarius run dnd-session \ - --config examples/dnd-spells.config.yml \ - --input examples/seriatim-minimal-transcript.json -``` - -The maintained example uses prompt defaults and Scriptorium's built-in -`mistral-small-3` profile, which reads `OPENROUTER_API_KEY`. To use another -endpoint or model, configure a Scriptorium profile source and select its profile -ID in config or with `--llm-profile`. +For the minimal end-to-end invocation, see the [README](../README.md). ## Commands ```text notarius help -notarius run --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector] -notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b] -notarius pipelines list --config path/to/config.yml [--json] +notarius run --input path/to/source.json [--config path/to/config.yml] [--only lane-a,lane-b] [--output-dir path] [--diagnostics-dir path] [--llm-profile id] [--resume] [--session-id id] [--reference selector=path] [--without-reference selector] +notarius config validate [--config path/to/config.yml] [--pipeline pipeline-id] [--only lane-a,lane-b] +notarius pipelines list [--config path/to/config.yml] [--json] ``` Running `notarius` with no arguments, `notarius help`, `notarius --help`, or @@ -37,20 +25,18 @@ file. Flags: - `--input path`: required source input file. -- `--config path`: config file path. If omitted, Notarius checks - `NOTARIUS_CONFIG`, then `/usr/local/etc/notarius/config.yml`. +- `--config path`: config file path. If omitted, Notarius uses the discovery + rules in [Configuration](config.md#discovery). - `--only lane-a,lane-b`: run only the named artifact lanes. Values are comma-separated and must be non-empty. -- `--resume`: reuse valid workspace checkpoints for this invocation. Requires - an effective workspace directory and `workspace.resume.enabled: true`. -- `--output-dir path`: output root. The run writes to `//`. - Defaults to `./notarius-output`. +- `--resume`: request checkpoint reuse for this invocation. See + [Operations](operations.md#checkpoints) for prerequisites and reuse behavior. +- `--output-dir path`: output root. Defaults to `./notarius-output`. - `--diagnostics-dir path`: diagnostics work directory override for this invocation. It does not change the workspace directory. - `--llm-profile id`: override every effective LLM-capable pipeline module - binding to use one Scriptorium profile ID. Validator-specific profiles are - not overridden. Configured LLM-backed validators with explicit profiles are - validated against the configured Scriptorium profile source. + binding with one Scriptorium profile ID. Validator-specific profiles are not + overridden. - `--session-id id`: pass a stable prompt session identifier through LLM-backed module calls. - `--reference selector=path`: bind a reference path to a chunk, extractor, @@ -63,23 +49,11 @@ On success, the command prints the completed pipeline ID, normalized output and rejected output counts, and the output directory. If the run completes with warnings, the warning count is printed to stderr. -Reference flags are resolved against selected chunk, extractor, merger, and normalizer -targets before the run starts. Flat slot names are accepted only when exactly -one selected target declares that slot. Bound reference files are read before -pipeline work starts, validated as UTF-8 text, and recorded as provenance for -the target that declares the slot. Runtime reference content is passed to the -chunker, extractor, merger, or normalizer target that declares the slot. Notarius infers -reference media types from file extensions for provenance and for optional slot -checks. Reference content is not written to diagnostics, logs, errors, or -manifests. - -Reference binding precedence is: - -1. pipeline-level config `references`; -2. target-local config references, including legacy lane-level extractor - `references`; -3. `--reference` run flags; -4. `--without-reference` run flags. +Reference flags are resolved against selected chunk, extractor, merger, and +normalizer targets before the run starts. Flat slot names are accepted only +when exactly one selected target declares that slot. For configured reference +bindings, precedence, path resolution, and validation, see +[Configuration](config.md#pipelines). `--reference` binds or replaces one slot for one selected target. Selectors are: @@ -142,8 +116,7 @@ go run ./cmd/notarius run dnd-session \ --session-id campaign-17-session-04 ``` -Use `--resume` to reuse valid checkpoints from a previous compatible -invocation: +The resume flag can be added to an otherwise identical run invocation: ```sh go run ./cmd/notarius run dnd-session \ @@ -152,13 +125,8 @@ go run ./cmd/notarius run dnd-session \ --resume ``` -Plain `run` does not skip completed work. It executes the pipeline normally and -refreshes checkpoints when checkpointing is enabled. `--resume` verifies each -checkpoint before reuse and executes any missing, corrupt, or incompatible step -normally. - -For durable output, diagnostics, retention, and failure inspection, see -[Operations](operations.md). +For checkpoint behavior, durable output, diagnostics, retention, and failure +inspection, see [Operations](operations.md). ## `config validate` @@ -166,8 +134,8 @@ For durable output, diagnostics, retention, and failure inspection, see Flags: -- `--config path`: config file path. If omitted, discovery uses - `NOTARIUS_CONFIG`, then `/usr/local/etc/notarius/config.yml`. +- `--config path`: config file path. If omitted, Notarius uses the discovery + rules in [Configuration](config.md#discovery). - `--pipeline pipeline-id`: additionally resolve one configured pipeline against the production module catalog. - `--only lane-a,lane-b`: validate resolution for selected artifact lanes. This @@ -191,8 +159,8 @@ go run ./cmd/notarius config validate \ Flags: -- `--config path`: config file path. If omitted, discovery uses - `NOTARIUS_CONFIG`, then `/usr/local/etc/notarius/config.yml`. +- `--config path`: config file path. If omitted, Notarius uses the discovery + rules in [Configuration](config.md#discovery). - `--json`: print `{"pipelines":[...]}` instead of one ID per line. Examples: @@ -215,42 +183,6 @@ go run ./cmd/notarius pipelines list \ - `2`: command syntax was invalid, a command was unknown, a required argument was missing, or a flag value was malformed. -## Implemented Production Pipeline Modules - -The production CLI currently registers these module keys: - -- input: `seriatim` -- chunk: `generic`, `dnd/scenes` -- extract: `dnd/spells` -- merge: `appendorder` -- normalize: `noop` -- output: `json` - -## Implemented Production Validators - -The production CLI currently registers these validator keys: - -- `generic/always_accept` -- `generic/always_reject` -- `generic/valid_json` -- `generic/valid_json_schema` -- `extract/dnd/spells/shape` -- `extract/dnd/spells/source_refs` -- `extract/dnd/spells/source_relatedness` - -The production default chain for the `dnd/spells` extractor is: - -1. `generic/valid_json` -2. `generic/valid_json_schema` -3. `extract/dnd/spells/shape` -4. `extract/dnd/spells/source_refs` -5. `extract/dnd/spells/source_relatedness` - -Validator chain overrides are configured on `chunk`, lane `extract`, lane -`merge`, and lane `normalize` bindings. Omitted overrides use production -defaults, `validators: []` disables validation for that binding, and non-empty -lists replace the default chain in configured order. Validator keys are resolved -against the registered validator catalog. - -For YAML structure, Scriptorium profile sources, environment overrides, and -module binding syntax, see [Configuration](config.md). +For YAML structure, defaults, Scriptorium profile sources, environment +overrides, and selectable module and validator keys, see +[Configuration](config.md). diff --git a/docs/config.md b/docs/config.md index 562b46a..b8fa41c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -15,26 +15,13 @@ Commands that accept `--config` load configuration in this order: If none is available, the command fails with a config file not found error. -## Minimal Example +## Maintained Examples -```yaml -version: 2 -pipelines: - dnd-session: - input: seriatim - references: - party: ./dnd-spells-roster.txt - glossary: ./dnd-spells-glossary.txt - chunk: - module: generic - options: - max_units: 50 - artifacts: - spells: - extract: dnd/spells -``` +- [Minimal D&D spell configuration](../examples/dnd-spells.config.yml) +- [Production-oriented D&D spell configuration](../examples/dnd-spells-production.config.yml) -The maintained fixture is [examples/dnd-spells.config.yml](../examples/dnd-spells.config.yml). +Both complete files are validated by the CLI test suite. The fragments below +illustrate individual fields and are not alternate complete configurations. ## Top-Level Fields @@ -52,31 +39,21 @@ rejected; execution profiles now come from Scriptorium. Built-in defaults: -```yaml -concurrency: - total_llm: 1 -diagnostics: - work_dir: /tmp/notarius - retention: auto -workspace: - diagnostics: - enabled: true - resume: - enabled: false - debug: - enabled: false -``` - -`workspace.directory` is unset by default. Without a workspace directory, -diagnostics continue to use `/tmp/notarius`, and checkpoint and debug workspace -features have no storage root. +- `concurrency.total_llm`: `1` +- `diagnostics.work_dir`: `/tmp/notarius` +- `diagnostics.retention`: `auto` +- `workspace.directory`: unset +- `workspace.diagnostics.enabled`: `true` +- `workspace.resume.enabled`: `false` +- `workspace.debug.enabled`: `false` No pipelines are built in. A run requires a configured pipeline. If `scriptorium` is omitted, Notarius uses Scriptorium's built-in profile catalog. Prompt definitions may also name default profile IDs. The current D&D -scene and spell prompts use Scriptorium prompt defaults when a module binding -does not set `llm_profile`. +scene and spell prompts default to the built-in `mistral-small-3` profile when a +module binding does not set `llm_profile`. That built-in profile reads its +credential from `OPENROUTER_API_KEY`. ## Scriptorium Profiles @@ -159,23 +136,20 @@ Artifact lane fields: - `references`: optional compatibility alias for extractor reference bindings. Lane bindings override pipeline-level bindings for the same slot. -`notarius run` and `notarius config validate --pipeline` resolve the pipeline -against the production module catalog and fail fast for unknown or incompatible -module keys. +Commands that resolve a pipeline fail for unknown or incompatible module keys. +See [CLI Reference](cli.md) for command syntax. Reference bindings are validated against reference slots declared by eligible chunk, extract, merge, and normalize targets during pipeline resolution. Required slots must be bound after config defaults, target-local references, lane-level -compatibility bindings, and run-time `--reference` or `--without-reference` -overrides are applied. Config-relative paths are resolved relative to the config -file; CLI reference paths are resolved relative to the current working -directory. Materialized bound files must be UTF-8 text. Materialized reference -provenance is recorded for chunk, extractor, merger, and normalizer targets, and runtime -reference content is passed to the target that declares the slot. Reference -media types are inferred from file extensions, recorded as canonical base media -types, and checked only when a module declares `AcceptedMediaTypes`; unknown -extensions are recorded as `application/octet-stream`. Reference content is not -written to diagnostics, logs, errors, or manifests. +compatibility bindings, and command-line reference overrides are applied. +Config-relative paths are resolved relative to the config file; command-line +reference paths are resolved relative to the current working directory. Bound +files must be UTF-8 text. Reference media types are inferred from file +extensions and checked when a module restricts accepted types; unknown +extensions use `application/octet-stream`. See [CLI Reference](cli.md#run) for +command-line selectors and [Operations](operations.md) for recorded provenance +and sensitive-data handling. Pipeline-level `references` are defaults. They are valid when at least one eligible target in the full configured pipeline declares the slot, including @@ -252,7 +226,7 @@ Binding fields: - `llm_profile`: optional Scriptorium profile ID. Empty or omitted lets the Scriptorium prompt default select the profile. - `retries`: non-negative retry count for extra runtime attempts after the - first attempt. The runner applies retries to `chunk`, `extract`, `merge`, and + first attempt. Default: `0`. Supported on `chunk`, `extract`, `merge`, and `normalize` bindings. - `options`: optional module-specific settings. - `references`: optional reference bindings. Supported only for `chunk`, @@ -275,11 +249,6 @@ Validator bindings reject `references`, `retries`, and nested `validators`. During resolution, deterministic validators reject explicit `llm_profile` values. -The `--llm-profile` run flag overrides every effective LLM-capable module -binding to use one Scriptorium profile ID: chunk, every selected lane extract, -merge, and normalize binding. It does not override validator-specific -`llm_profile` values. - Configured LLM-backed validators with explicit `llm_profile` values are validated against the configured Scriptorium profile source. Deterministic production validators do not call the LLM and must not set `llm_profile`. @@ -350,46 +319,13 @@ casts still must be present in the source transcript. `workspace` fields: - `directory`: optional workspace root for Notarius-owned local state. -- `diagnostics.enabled`: set to `false` to skip diagnostics run directories and - diagnostics artifact writes. Default: `true`. -- `diagnostics.retention`: `auto`, `always`, or `never`. - `resume.enabled`: boolean resume checkpointing setting. Default: `false`. - `debug.enabled`: boolean debug artifact setting. Default: `false`. - -Use `/var/lib/notarius` as the standard production workspace directory. For -local development, prefer a project-local ignored path such as -`./.notarius/workspace`. - -```yaml -workspace: - directory: /var/lib/notarius - diagnostics: - enabled: true - retention: auto - resume: - enabled: false - debug: - enabled: false -``` - -When `workspace.directory` is set, diagnostics are written under -`/diagnostics/`. - -When both `workspace.directory` and `workspace.resume.enabled` are set, runs -write stage-owned checkpoint artifacts under -`/checkpoints/`. `notarius run --resume` can reuse valid -checkpoints from a compatible invocation. Checkpoints may contain source text, -intermediate raw outputs, rejected outputs, metadata, and warnings. Protect the -workspace as sensitive local state. - -When both `workspace.directory` and `workspace.debug.enabled` are set, runs -write per-invocation debug artifacts under -`/debug//`. Debug artifacts may contain source -material, reference material, prompt inputs, model outputs, validation payloads, -and other sensitive content. Debug is disabled by default. +- `diagnostics`: optional diagnostics settings defined below. `workspace.resume.enabled` and `workspace.debug.enabled` are independent. -Enabling one does not enable the other. +Enabling one does not enable the other. For directory layout, state lifecycle, +permissions, and sensitive content, see [Operations](operations.md). ## Diagnostics @@ -399,9 +335,8 @@ Preferred workspace diagnostics fields: - `workspace.diagnostics.enabled`: set to `false` to skip creating diagnostics run directories and diagnostics artifacts. Default: `true`. - `workspace.diagnostics.retention`: `auto`, `always`, or `never`. + When unset, the effective diagnostics retention default is `auto`. -When `workspace.directory` is set, diagnostics use -`/diagnostics` as their work directory. `workspace.diagnostics.retention` overrides legacy diagnostics retention when set. @@ -417,13 +352,9 @@ and `NOTARIUS_DIAGNOSTICS_RETENTION` inputs remain supported for compatibility. New configuration should use `workspace.directory` and `workspace.diagnostics.retention` instead. -`auto` retains diagnostics for failed runs and successful runs with warnings. -`always` retains diagnostics for every run. `never` removes diagnostics for -successful runs without regard to warnings; failed runs are retained. - -The `--diagnostics-dir` run flag overrides the effective diagnostics work -directory for that invocation. It affects diagnostics only and does not change -the workspace directory. +For retention behavior and the physical diagnostics layout, see +[Operations](operations.md#retention). For the invocation-specific diagnostics +override, see [CLI Reference](cli.md#run). ## Validation diff --git a/docs/integrations/dnd-spell-artifacts.md b/docs/integrations/dnd-spell-artifacts.md index 7f38610..1d2da01 100644 --- a/docs/integrations/dnd-spell-artifacts.md +++ b/docs/integrations/dnd-spell-artifacts.md @@ -13,100 +13,48 @@ This document is the durable raw output contract for the implemented - Response schema version: `v1` - Media type: `application/json` -The extractor requires source chunks and transcript source capability. It -returns canonical spell-cast JSON derived from the structured LLM response. The -extractor assigns source IDs deterministically and keeps source-unit ranges as -model-authored evidence locations. The default `appendorder` merger passes a -single chunk output through and concatenates multiple -`spell_casts` arrays in chunk order. The default `noop` normalizer passes the -merge output through unchanged. +The output contains canonical spell casts derived from transcript evidence. +Source IDs are assigned from the input identity; source-unit ranges identify +the evidence location. ## Output Shape -For a single chunk, `lanes/spells.json` has this shape: +`lanes/spells.json` is a JSON object with one required top-level array. Its +structure is: -```json -{ - "spell_casts": [ - { - "caster": "Aria", - "spell": "Cure Wounds", - "effect": "heals an injured ally", - "narrative_description": "Aria raises her holy symbol and casts Cure Wounds.", - "source_refs": [ - { - "source_id": "session-alpha", - "start_unit_id": 1, - "end_unit_id": 1 - } - ] - } - ] -} +```text +{"spell_casts": [, ...]} ``` `spell_casts` must be present. It may be empty when no spell casts are found. - -For multiple chunks with the default merger, the lane output keeps the same -top-level shape and concatenates `spell_casts` in chunk order: - -```json -{ - "spell_casts": [ - { - "caster": "Aria", - "spell": "Cure Wounds", - "effect": "heals an injured ally", - "narrative_description": "Aria raises her holy symbol and casts Cure Wounds.", - "source_refs": [ - { - "source_id": "session-alpha", - "start_unit_id": 1, - "end_unit_id": 1 - } - ] - } - ] -} -``` +With the default configured merger, multiple chunk results are concatenated in +chunk order. ## Spell-Cast Fields -Each spell cast contains: +Each spell cast contains exactly these required fields: - `caster`: in-world character or creature casting the spell; - `spell`: spell name; - `effect`: concise spell effect in the scene; - `narrative_description`: short description of the spell cast in context; - `source_refs`: transcript source references with extractor-assigned source - IDs and model-supplied unit ranges. + IDs and evidence unit ranges. It must contain at least one entry. -`caster` is the in-world caster, not the transcript speaker. +All four string fields must be non-empty. `caster` is the in-world caster, not +the transcript speaker. Unknown fields are rejected. ## Source References -Each source reference uses the generic source-reference shape: +Each source reference contains exactly three required fields: `source_id`, +`start_unit_id`, and `end_unit_id`. The source ID must match the input identity. +The unit IDs must be positive integers present in the input, and the start unit +must not appear after the end unit. Unknown fields are rejected. -- `source_id` -- `start_unit_id` -- `end_unit_id` - -The LLM-facing prompt schema asks only for integer `start_unit_id` and -`end_unit_id` values matching source-unit IDs. `source_id` is assigned by the -extractor from the source document ID before validation and output, and is -required in this durable output contract. - -## References - -The extractor accepts optional UTF-8 text references: - -- `players` -- `party` -- `glossary` -- `roster`, a deprecated compatibility alias for `party` - -References are supporting disambiguation material only. They are not source -evidence and are not addressable through `source_refs`. +Reference slot keys and accepted file types are defined in +[Configuration](../config.md#implemented-production-modules). References are +supporting disambiguation material, not source evidence, and are not +addressable through `source_refs`. ## Manifest Metadata diff --git a/docs/integrations/json-output.md b/docs/integrations/json-output.md index 7a99f04..83994a7 100644 --- a/docs/integrations/json-output.md +++ b/docs/integrations/json-output.md @@ -3,16 +3,9 @@ This document is the durable JSON output file-format contract produced by the implemented `json` output module and written by the CLI. -## Output Directory - -The CLI writes logical output files under: - -```text -// -``` - -The default output root is `./notarius-output`. Operational behavior is covered -in [Operations](../operations.md). +The output module produces the logical bundle described here. The CLI's +physical placement and lifecycle for that bundle are defined in +[Operations](../operations.md#output-directory). ## Files @@ -25,6 +18,7 @@ The `json` output module writes: - `warnings.json` Files are pretty-printed JSON with a trailing newline when the payload is JSON. +Logical file paths are relative, slash-separated, and may not contain `..`. ## `index.json` @@ -58,22 +52,21 @@ sanitizing the lane ID: - empty sanitized names are rejected; - two lanes that sanitize to the same output file are rejected. +`manifest_file`, `rejected_file`, and `warnings_file` contain the fixed paths +shown above. Each `output_files` entry requires `lane_id` and `file`. It also +contains the normalized payload `media_type`, normalizer `module_key`, and +response `schema_id`, `schema_name`, and `schema_version` when those values are +available. + ## `manifest.json` -`manifest.json` contains a run manifest: +`manifest.json` contains a run manifest. This abridged example shows its core +structure: ```json { "run_id": "run-123", "pipeline_id": "dnd-session", - "pipeline_digest": "sha256:...", - "input_module": "seriatim", - "chunker": "dnd/scenes", - "source_digests": ["sha256:..."], - "extractors": ["dnd/spells"], - "merger": "appendorder", - "normalizer": "noop", - "output_encoder": "json", "artifact_lanes": [ { "id": "spells", @@ -82,35 +75,6 @@ sanitizing the lane ID: "normalizer": "noop" } ], - "validator_chains": [ - { - "stage": "extract", - "lane_id": "spells", - "module_key": "dnd/spells", - "validators": [ - { - "key": "generic/valid_json", - "execution_class": "deterministic" - }, - { - "key": "generic/valid_json_schema", - "execution_class": "deterministic" - }, - { - "key": "extract/dnd/spells/shape", - "execution_class": "deterministic" - }, - { - "key": "extract/dnd/spells/source_refs", - "execution_class": "deterministic" - }, - { - "key": "extract/dnd/spells/source_relatedness", - "execution_class": "deterministic" - } - ] - } - ], "validation_status": "approved", "started_at": "2026-01-01T00:00:00Z", "completed_at": "2026-01-01T00:00:01Z" @@ -119,6 +83,23 @@ sanitizing the lane ID: Fields with empty values may be omitted by JSON encoding. +The manifest fields are: + +- `run_id`, `pipeline_id`, and `pipeline_digest`: run and resolved-pipeline + identity; +- `input_module`, `chunker`, `extractors`, `merger`, `normalizer`, and + `output_encoder`: resolved module keys; +- `module_metadata` and `artifact_lanes`: module and per-lane provenance, + including prompt and response-schema provenance when provided; +- `validator_chains`: resolved validation points and validators; +- `source_digests` and `references`: source and reference provenance; +- `normalized_outputs` and `rejected_outputs`: payload-free result summaries; +- `llm_profiles`: selected profile IDs and provider or model names when + available; +- `metadata`: the effective prompt `session_id`; +- `validation_status`: `approved` or `rejected`; +- `started_at` and `completed_at`: UTC run timestamps. + `source_digests` contains source document digests only. Bound references are recorded separately under `references`, which contains provenance only: target stage, lane ID when present, slot name, origin type and URI, digest, media @@ -152,27 +133,8 @@ Each normalized raw output is written to `lanes/.json`. The JSON output encoder accepts only `application/json` normalized outputs. The file contains the raw JSON payload pretty-printed. -For the current D&D spell extractor, `lanes/spells.json` has this shape: - -```json -{ - "spell_casts": [ - { - "caster": "Aria", - "spell": "Cure Wounds", - "effect": "heals an injured ally", - "narrative_description": "Aria raises her holy symbol and casts Cure Wounds.", - "source_refs": [ - { - "source_id": "session-alpha", - "start_unit_id": 1, - "end_unit_id": 1 - } - ] - } - ] -} -``` +The schema of each lane payload is owned by that artifact contract. For the +current D&D spell lane, see [D&D Spell Raw Output](dnd-spell-artifacts.md). ## `rejected.json` @@ -184,10 +146,10 @@ Shape: } ``` -When raw output validation rejects an output, entries use the -`contracts.RejectedOutput` shape, including stage, lane ID, module key, -validator name, reason code, message, attempt count, and optional diagnostic -artifact path. +When raw output validation rejects an output, each entry contains `stage` and +`message`. It includes `lane_id`, `module_key`, `chunk_id`, `chunk_index`, +`validator_name`, `reason_code`, `attempt_count`, and +`diagnostic_artifact_path` when applicable. ## `warnings.json` @@ -206,3 +168,5 @@ Shape: ``` `warnings` is an empty array when no warnings are reported. +Each warning requires `reason_code` and `message`; `scope` is omitted when it is +empty. diff --git a/docs/integrations/seriatim.md b/docs/integrations/seriatim.md index 4a33fe1..51a2031 100644 --- a/docs/integrations/seriatim.md +++ b/docs/integrations/seriatim.md @@ -6,109 +6,55 @@ input adapter. ## Adapter - Module key: `seriatim` -- Document kind: `transcript` -- Unit kind: `transcript_segment` - Source format: `application/vnd.seriatim+json` -The adapter parses raw Seriatim JSON into a generic source document. It owns -transcript-specific JSON parsing and metadata mapping; core source and pipeline -code stay source-format agnostic. - ## Accepted Shape The input must be one JSON object with top-level `metadata` and `segments` fields. This covers the maintained minimal fixture and Seriatim intermediate output that provides the same required segment fields. -```json -{ - "metadata": { - "id": "session-alpha", - "title": "Synthetic D&D spell session" - }, - "segments": [ - { - "id": 1, - "start": 0, - "end": 4, - "speaker": "Aria", - "text": "Aria raises her holy symbol and casts Cure Wounds." - } - ] -} -``` - The maintained example is [examples/seriatim-minimal-transcript.json](../../examples/seriatim-minimal-transcript.json). -Top-level metadata entries are preserved. Other segment fields, such as -`categories`, are ignored. +Required top-level fields: + +- `metadata`: an object. Its entries are accepted as source metadata. +- `segments`: a non-empty array of segment objects. + +Required segment fields: + +- `id`: a positive integer JSON number or canonical decimal string without + leading zeros or surrounding whitespace; +- `start`: a finite, non-negative JSON number or numeric string; +- `end`: a finite, non-negative JSON number or numeric string that is not less + than `start`; +- `speaker`: a non-empty string; +- `text`: a non-empty string. + +Other top-level and segment fields, such as `categories`, are ignored. Multiple top-level JSON values are rejected. ## Validation -The adapter rejects: - -- empty raw input; -- malformed JSON; -- top-level JSON that is not an object; -- missing, null, or non-object `metadata`; -- missing, null, non-array, or empty `segments`; -- segment values that are not objects; -- segment `id` values that are not positive integer JSON numbers or numeric - strings; -- non-string `speaker` or `text`; -- duplicate segment IDs; -- missing or empty `speaker`; -- missing, empty, invalid, non-finite, or negative `start`; -- missing, empty, invalid, non-finite, or negative `end`; -- `end` values before `start`; -- missing or empty `text`. +The adapter rejects empty input, malformed JSON, multiple top-level JSON values, +non-object segment values, duplicate segment IDs, and any violation of the +shape or field constraints above. Segment text is preserved as provided, but it must not be empty after trimming. -## Source Mapping +## Derived Identity -The adapter maps input to `SourceDocument`: +Notarius identifies the parsed source in this order: -- `metadata` becomes `SourceDocument.Metadata`; -- `SourceDocument.Kind` is `transcript`; -- `SourceDocument.Format` is `application/vnd.seriatim+json`; -- `SourceDocument.Digest` is `sha256:` of the exact raw input bytes. +1. `metadata.id`, when it is a non-empty string after trimming; +2. `metadata.source_id`, when it is a non-empty string after trimming; +3. `seriatim:`. -`SourceDocument.ID` is selected in this order: - -1. the parse request source ID, after trimming; -2. `metadata.id`, when it is a non-empty string after trimming; -3. `metadata.source_id`, when it is a non-empty string after trimming; -4. `seriatim:`. - -Each segment becomes one `SourceUnit`: - -- `segment.id` becomes integer `SourceUnit.ID`; -- `segment.text` becomes `SourceUnit.Text`; -- `SourceUnit.Kind` is `transcript_segment`; -- `speaker`, `start`, and `end` are stored in source-unit metadata. - -## Metadata Keys - -Seriatim unit metadata uses these keys: - -- `speaker`: string speaker label; -- `start`: `json.Number` start value; -- `end`: `json.Number` end value. - -The `internal/modules/input/seriatim` package exposes typed accessors for these -values. - -## Capabilities - -The module declares these provided capabilities: - -- `source.transcript` -- `transcript.speaker` -- `transcript.timestamps` +The source digest recorded in output provenance is `sha256:` of the exact +raw input bytes. Segment IDs become the unit IDs used by artifact source +references. ## Compatibility Limit diff --git a/docs/operations.md b/docs/operations.md index 13f65eb..2408774 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -4,24 +4,16 @@ This is the canonical reference for operating implemented Notarius runs. ## Normal Run -A run reads one source file, resolves one configured pipeline, calls the -configured Scriptorium-backed LLM runtime, writes durable JSON output, and -writes diagnostics for inspection. +A run reads one source file, resolves one configured pipeline, executes its +modules, writes durable output, and writes diagnostics when enabled. Start with +the [README quickstart](../README.md), then use the [CLI reference](cli.md) for +invocation options. -```sh -go run ./cmd/notarius run dnd-session \ - --config examples/dnd-spells.config.yml \ - --input examples/seriatim-minimal-transcript.json \ - --output-dir ./notarius-output \ - --diagnostics-dir /tmp/notarius -``` - -The command prints a success line with the pipeline ID, normalized output count, -rejected output count, and the output path. - -For production, configure a workspace such as `/var/lib/notarius` and ensure the -Notarius process can create files below it. For local development, prefer an -ignored project-local workspace such as `./.notarius/workspace`. +For production, configure an application-owned workspace such as +`/var/lib/notarius` and ensure the Notarius process can create files below it. +For local development, prefer an ignored project-local workspace such as +`./.notarius/workspace`. See [Configuration](config.md#workspace) for workspace +fields. ## Output Directory @@ -31,23 +23,10 @@ Durable output is written to: // ``` -The default output root is `./notarius-output`. Use `--output-dir` to choose a -different root. - -The `json` output module writes these files: - -- `index.json`: file index with paths to the manifest, lane output files, - rejected outputs, and warnings. -- `manifest.json`: run manifest with resolved pipeline provenance, top-level - module metadata, module keys, reference provenance, validation status, and - timing. -- `lanes/.json`: normalized raw JSON output payloads, one file per - lane. For the current D&D spell extractor, this includes `lanes/spells.json`. -- `rejected.json`: rejected raw output records. -- `warnings.json`: warnings reported by pipeline modules or the output encoder. - -Output writes are atomic per file. Logical output file names must be clean, -relative, slash-separated paths and must not contain `..`. +The output root and its invocation-specific override are defined in the +[CLI reference](cli.md#run). Output writes are atomic per file. The +[JSON output contract](integrations/json-output.md) defines the logical files, +paths, schemas, and media types inside each run directory. ## Diagnostics Directory @@ -57,17 +36,15 @@ Diagnostics are written under: // ``` -The default diagnostics work directory is `/tmp/notarius`. It can be set with -`workspace.directory`, `NOTARIUS_WORKSPACE_DIR`, legacy -`diagnostics.work_dir`, legacy `NOTARIUS_WORK_DIR`, or `--diagnostics-dir`. -When a workspace directory is set, diagnostics are written under -`/diagnostics//` unless `--diagnostics-dir` -overrides the diagnostics work directory for that invocation. +When a workspace directory is configured, diagnostics are written under +`/diagnostics//`. An invocation-specific override +changes only the diagnostics root, not the workspace root. Configuration and +environment controls are defined in [Configuration](config.md); the override +flag is defined in the [CLI reference](cli.md#run). -Set `workspace.diagnostics.enabled: false` or -`NOTARIUS_WORKSPACE_DIAGNOSTICS_ENABLED=false` to skip diagnostics directory -creation and diagnostics artifact writes. Concise failures are still printed to -stderr. +Diagnostics can be disabled through configuration. When disabled, Notarius +does not create a diagnostics run directory or write diagnostics artifacts; +concise failures are still printed to stderr. Implemented diagnostics artifacts: @@ -87,9 +64,6 @@ Implemented diagnostics artifacts: - `error.log`: failure message, written after diagnostics directory creation when a run fails. -`source-document.json` is supported by the diagnostics writer but is not written -by the current CLI run workflow. - ## Checkpoints When `workspace.resume.enabled: true` and `workspace.directory` is set, runs @@ -104,19 +78,15 @@ root-level checkpoint summary. Ordinary `notarius run` invocations execute the pipeline normally and refresh checkpoints. `notarius run --resume` reuses valid checkpoints and executes any missing, invalid, or incompatible step normally. -Checkpoint payloads preserve byte content with base64 envelopes, media type, -metadata, warnings, and content digests where applicable. Checkpoints do not -include raw prompts, raw reference contents, raw LLM request payloads, or debug -traces. They can still contain source text, intermediate extracted content, -rejected outputs, metadata, and warnings. Treat checkpoint directories as -sensitive local state. +Checkpoints do not include raw prompts, raw reference contents, raw LLM request +payloads, or debug traces. They can still contain source text, intermediate +extracted content, rejected outputs, metadata, warnings, and content digests. +Treat checkpoint directories as sensitive local state. -A checkpoint is reused only when its workspace schema version, checkpoint -identity digest, step status, dependency fingerprints, payload files, and -payload digests match the current invocation. Changes to input bytes, resolved -pipeline digest, selected lanes, runtime LLM profile override, materialized -reference digests, or other identity material invalidate reuse and use a -separate checkpoint directory. +A checkpoint is reused only when its stored status, dependencies, payloads, and +digests match the current invocation. Changes to input bytes, the resolved +pipeline, selected lanes, the runtime LLM profile override, or bound reference +content invalidate reuse. Plain `notarius run` does not reuse checkpoints. It executes the workflow and refreshes checkpoint files when checkpointing is enabled. `notarius run @@ -135,10 +105,10 @@ Debug output is per invocation. It is independent of checkpointing and is not used for resume. Enabling debug does not write checkpoints, and enabling resume checkpointing does not write debug output. -Debug artifacts include framework-boundary inputs and outputs for source, -chunk, extract, merge, normalize, and output work, structured LLM request and -response data from Notarius contracts, validator requests and results, timing, -and retry attempt metadata. LLM calls made inside a retry or validator attempt +Debug artifacts include inputs and outputs for source, chunk, extract, merge, +normalize, and output work, structured LLM request and response data, validator +requests and results, timing, and retry attempt metadata. LLM calls made inside +a retry or validator attempt write `prompt-000N.json`, `response-000N.json`, and `response-content-000N.*` files under that attempt directory and are linked from the attempt `llm_calls` array. Prompt content is written inline in the prompt @@ -147,22 +117,20 @@ response body is written separately as pretty-printed JSON when possible or as raw text otherwise. Debug artifacts may contain source material, reference material, prompt inputs, model outputs, and other sensitive data. API keys are not written, and obvious credential-shaped values and sensitive map keys are -redacted in framework envelopes, but debug directories should still be protected -as sensitive local state. +redacted, but debug directories should still be protected as sensitive local +state. ## Retention -Diagnostics retention is configured with `workspace.diagnostics.retention`, -`NOTARIUS_WORKSPACE_DIAGNOSTICS_RETENTION`, legacy `diagnostics.retention`, -legacy `NOTARIUS_DIAGNOSTICS_RETENTION`, or the default `auto`. +Diagnostics retention uses the effective mode selected through configuration; +see [Configuration](config.md#diagnostics) for the fields, environment +overrides, precedence, and default. - `auto`: keep failed runs and successful runs with warnings; remove successful warning-free runs. - `always`: keep every diagnostics run directory. - `never`: remove successful run directories; failed runs are still retained. -Unknown retention values are rejected during config validation. - ## Failures Failures before diagnostics directory creation, such as a missing config file or @@ -183,11 +151,8 @@ retained for inspection and may include `run-manifest.json`, `warnings.json`, A successful run with warnings exits with code `0`, prints a warning count to stderr, and writes warnings to durable output and diagnostics when retained. -The run manifest `validation_status` indicates whether raw outputs were -approved or rejected after validation. - -Reference-related warnings include empty bound reference files. Empty references -are still passed to extractors so optional slots can be intentionally blank. +The [JSON output contract](integrations/json-output.md) defines durable warning +and validation-status fields. ## Cleanup @@ -216,9 +181,9 @@ If `--resume` cannot reuse a checkpoint, Notarius executes that step and writes a fresh checkpoint when checkpointing is enabled. Provider retries and timeouts are handled by Scriptorium according to the -selected execution profile. Pipeline module retries are controlled by module -binding `retries` values in config for chunk, extract, merge, and normalize. -There is no separate CLI retry command. +selected execution profile. Pipeline module retry settings are defined in +[Configuration](config.md#module-bindings). There is no separate CLI retry +command. Notarius writes local files only. Remote storage and archive management are not part of the implemented CLI. diff --git a/examples/dnd-spells-production.config.yml b/examples/dnd-spells-production.config.yml new file mode 100644 index 0000000..2ccfc9e --- /dev/null +++ b/examples/dnd-spells-production.config.yml @@ -0,0 +1,25 @@ +version: 2 +concurrency: + total_llm: 1 +workspace: + directory: /var/lib/notarius + diagnostics: + enabled: true + retention: auto + resume: + enabled: false + debug: + enabled: false +pipelines: + dnd-session: + input: seriatim + references: + party: ./dnd-spells-roster.txt + glossary: ./dnd-spells-glossary.txt + chunk: + module: generic + options: + max_units: 50 + artifacts: + spells: + extract: dnd/spells diff --git a/examples/dnd-spells.config.yml b/examples/dnd-spells.config.yml index ff1aee9..a7b4089 100644 --- a/examples/dnd-spells.config.yml +++ b/examples/dnd-spells.config.yml @@ -1,29 +1,7 @@ version: 2 -# For production runs, use a writable application-owned workspace such as: -# -# workspace: -# directory: /var/lib/notarius -# diagnostics: -# retention: auto -# resume: -# enabled: false -# debug: -# enabled: false -# -# For local development, use a project-local ignored path such as: -# -# workspace: -# directory: ./.notarius/workspace pipelines: dnd-session: input: seriatim - references: - party: ./dnd-spells-roster.txt - glossary: ./dnd-spells-glossary.txt - chunk: - module: generic - options: - max_units: 50 artifacts: spells: extract: dnd/spells diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index e1ff8da..157d561 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -2647,23 +2647,29 @@ func TestRunPipelineDiagnosticsDirFlagOverridesWorkspaceDiagnosticsOnly(t *testi } func TestExampleFixtureConfigValidateAndPipelinesList(t *testing.T) { - configPath := fixturePath(t, "examples/dnd-spells.config.yml") + for _, path := range []string{ + "examples/dnd-spells.config.yml", + "examples/dnd-spells-production.config.yml", + } { + path := path + t.Run("validate "+filepath.Base(path), func(t *testing.T) { + configPath := fixturePath(t, path) + var stdout bytes.Buffer + var stderr bytes.Buffer - t.Run("validate configured pipeline", func(t *testing.T) { - var stdout bytes.Buffer - var stderr bytes.Buffer + code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{}) - code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{}) - - if code != 0 { - t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) - } - if !strings.Contains(stdout.String(), "dnd-session") { - t.Fatalf("stdout = %q, want pipeline ID", stdout.String()) - } - }) + if code != 0 { + t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "dnd-session") { + t.Fatalf("stdout = %q, want pipeline ID", stdout.String()) + } + }) + } t.Run("list configured pipelines", func(t *testing.T) { + configPath := fixturePath(t, "examples/dnd-spells.config.yml") var stdout bytes.Buffer var stderr bytes.Buffer